fix(push-host-keys): detect non-interactive stdin, direct to SUDO_PASS

read exits non-zero when stdin is not a terminal (set -e killed the
script silently). Catch that and emit a clear error pointing to the
SUDO_PASS environment variable rather than crashing with no output.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

flake.nix

@@ -96,9 +96,9 @@
         proxmox-pxe-boot = mkTarget { platform = "proxmox"; buildType = "pxe-boot"; hostPath = ./hosts/pxe-boot/host.nix; };
         lxc-pxe-boot = mkTarget { platform = "lxc"; buildType = "pxe-boot"; hostPath = ./hosts/pxe-boot/host.nix; };
 
-        linode-tailscale-exit-node = mkTarget { platform = "linode"; buildType = "tailscale-exit-node"; hostPath = ./hosts/tailscale-exit-node/host.nix; };
-        proxmox-tailscale-exit-node = mkTarget { platform = "proxmox"; buildType = "tailscale-exit-node"; hostPath = ./hosts/tailscale-exit-node/host.nix; };
-        lxc-tailscale-exit-node = mkTarget { platform = "lxc"; buildType = "tailscale-exit-node"; hostPath = ./hosts/tailscale-exit-node/host.nix; };
+        linode-tailscale-subnet-router = mkTarget { platform = "linode"; buildType = "tailscale-subnet-router"; hostPath = ./hosts/tailscale-subnet-router/host.nix; };
+        proxmox-tailscale-subnet-router = mkTarget { platform = "proxmox"; buildType = "tailscale-subnet-router"; hostPath = ./hosts/tailscale-subnet-router/host.nix; };
+        lxc-tailscale-subnet-router = mkTarget { platform = "lxc"; buildType = "tailscale-subnet-router"; hostPath = ./hosts/tailscale-subnet-router/host.nix; };
 
         lxc-tor-relay = mkTarget { platform = "lxc"; buildType = "tor-relay"; hostPath = ./hosts/tor-relay/host.nix; };
       };

hosts/tailscale-exit-node/host.nix

@@ -1,12 +0,0 @@
-_:
-
-{
-  networking.hostName = "exit-node";
-
-  # No networking.hostId: only ZFS-touching hosts (server, docker) need one
-  # for pool-import safety, and this host does neither.
-
-  # A genuinely new host (not a pre-refactor carry-over), so it tracks the
-  # flake's current nixpkgs release rather than being pinned to an older one.
-  system.stateVersion = "26.05";
-}

hosts/tailscale-subnet-router/host.nix

@@ -0,0 +1,10 @@
+_:
+
+{
+  networking.hostName = "tailscale-router";
+
+  # No networking.hostId: only ZFS-touching hosts (server, docker) need one
+  # for pool-import safety, and this host does neither.
+
+  system.stateVersion = "26.05";
+}

modules/build-types/tailscale-exit-node.nix

@@ -1,21 +0,0 @@
-{ ... }:
-
-{
-  imports = [
-    ../tailscale/exit-node.nix
-  ];
-
-  # "server", not "both": this build type only ever advertises itself as an
-  # exit node (see ../tailscale/exit-node.nix) -- it doesn't advertise LAN
-  # subnet routes, so it doesn't need the "client"-side loose reverse-path
-  # filtering that "both" would also turn on. Deliberately left unbundled
-  # from LAN-subnet-route advertisement so this build type stays valid on
-  # every platform, including linode (a remote VPS with no network path to
-  # the home LAN at all).
-  services.tailscale.useRoutingFeatures = "server";
-
-  # Forwarded exit-node traffic arrives on tailscale0 already
-  # tailscale-authenticated -- the firewall's normal per-port allow-list
-  # would otherwise drop it. Standard NixOS/Tailscale exit-node guidance.
-  networking.firewall.trustedInterfaces = [ "tailscale0" ];
-}

modules/build-types/tailscale-subnet-router.nix

@@ -0,0 +1,19 @@
+{ ... }:
+
+{
+  imports = [
+    ../tailscale/subnet-router.nix
+  ];
+
+  # "server", not "both": this build type advertises LAN subnet routes but
+  # doesn't use another tailscale exit node itself, so it doesn't need the
+  # "client"-side loose reverse-path filtering that "both" would also enable.
+  # Deliberately kept explicit here (not just relying on subnet-router.nix's
+  # own setting) so the intent is clear at the build-type level.
+  services.tailscale.useRoutingFeatures = "server";
+
+  # Forwarded subnet-router traffic arrives on tailscale0 already
+  # tailscale-authenticated -- the firewall's normal per-port allow-list
+  # would otherwise drop it. Standard NixOS/Tailscale subnet-router guidance.
+  networking.firewall.trustedInterfaces = [ "tailscale0" ];
+}

modules/tailscale/exit-node.nix

@@ -1,27 +0,0 @@
-_:
-
-{
-  imports = [ ./enable-service.nix ];
-
-  services.tailscale = {
-    # Enables the sysctl forwarding settings exit nodes/subnet routers need;
-    # without this, --advertise-exit-node has no effect.
-    useRoutingFeatures = "server";
-
-    # Lets peers reach this node directly over the tailscale UDP port
-    # instead of relaying through DERP.
-    openFirewall = true;
-
-    # extraSetFlags (tailscale set, via the always-on tailscaled-set
-    # service), not extraUpFlags -- extraUpFlags is only ever applied by
-    # tailscaled-autoconnect, which itself only runs when
-    # services.tailscale.authKeyFile is set (nothing in this repo sets one,
-    # so tailscale up is a manual, one-time operator step on every host that
-    # uses this service). extraSetFlags has no such gate, so
-    # --advertise-exit-node self-reapplies on every boot once the operator
-    # has authenticated the node once.
-    extraSetFlags = [
-      "--advertise-exit-node"
-    ];
-  };
-}

modules/tailscale/subnet-router.nix

@@ -0,0 +1,15 @@
+_:
+
+{
+  imports = [ ./enable-service.nix ];
+
+  services.tailscale = {
+    # Enables the sysctl forwarding settings subnet routers need;
+    # without this, --advertise-routes has no effect.
+    useRoutingFeatures = "server";
+
+    # Lets peers reach this node directly over the tailscale UDP port
+    # instead of relaying through DERP.
+    openFirewall = true;
+  };
+}

scripts/recover-hosts.sh

@@ -12,6 +12,9 @@
 # Fixes applied automatically (then prompts before rebuilding):
 #   1. SSH host key drift — live key no longer matches host-keys/<target>_ssh_host_ed25519_key
 #      Fix: scp the registered key back and restore it (needs sudo once per host).
+#      To push new keys proactively (before drift, e.g. right after
+#      sync-host-keys.sh --regenerate-all-keys), use instead:
+#        scripts/secrets/push-host-keys.sh --all
 #   2. Stale/invalid GitHub access token — the rendered nix-github-token.conf has
 #      a token GitHub rejects (401), blocking any rebuild that fetches disko or
 #      other public GitHub flake inputs.
@@ -131,7 +134,7 @@ for host in "${HOSTNAMES[@]}"; do
     if [ "$live" = "$want" ]; then
       info "SSH host key OK"
     else
-      warn "SSH host key MISMATCH (live ≠ host-keys/)"
+      warn "SSH host key MISMATCH (live ≠ host-keys/) -- use push-host-keys.sh proactively next time"
       echo "       live:       $live"
       echo "       registered: $want"
       host_broken=true

scripts/secrets/push-host-keys.sh

@@ -0,0 +1,318 @@
+#!/usr/bin/env bash
+# Pushes newly-generated SSH host keys from host-keys/ to already-running
+# NixOS hosts, so they can decrypt sops secrets after a nixos-rebuild
+# following scripts/secrets/sync-host-keys.sh --regenerate-all-keys.
+#
+# Before pushing any key, verifies that .sops.yaml and secrets/*.yaml are
+# committed and pushed to the remote -- hosts rebuild from the remote Gitea
+# flake, so recipient changes must land there before any rebuild, not just
+# before the key push.
+#
+#   push-host-keys.sh --all [--dry-run] [--skip-git-check]
+#   push-host-keys.sh <target> [--dry-run] [--skip-git-check]
+#
+#   --all              Push to every reachable managed host. Default when no
+#                      target is given.
+#   <target>           Push to one flake target only (e.g. lxc-server).
+#   --dry-run          Print what would be done; write nothing.
+#   --skip-git-check   Skip the commit/push check. Use only when the remote
+#                      already has the current .sops.yaml/secrets/*.yaml.
+#
+# SSH: connects as SSH_USER@<hostname> (default: nixos, the user with the
+# admin authorized key), then installs files via sudo -S (reads the sudo
+# password from stdin). The password is prompted once at startup and reused
+# for every host -- no PTY or terminal required on the remote side.
+# Hosts are reached at their bare hostname (relies on LAN DNS/mDNS).
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
+keydir="${repo_root}/host-keys"
+
+# shellcheck source=../env.sh
+source "${repo_root}/scripts/env.sh"
+# shellcheck source=../lib/nix-eval.sh
+source "${repo_root}/scripts/lib/nix-eval.sh"
+
+: "${SSH_USER:=nixos}"
+SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=5)
+
+dry_run=0
+skip_git_check=0
+sudo_password=""
+
+usage() {
+  cat <<EOF
+Usage: $0 [--all | <target>] [--dry-run] [--skip-git-check]
+
+  --all              Push to every reachable managed host. Default when no
+                     target is given.
+  <target>           Push to one flake target only (e.g. lxc-server).
+  --dry-run          Print what would be done; write nothing.
+  --skip-git-check   Skip the check that .sops.yaml/secrets/*.yaml are
+                     committed and pushed to the remote repo.
+
+Environment:
+  SSH_USER           SSH username (default: nixos).
+  SUDO_PASS          Sudo password (skips the interactive prompt; useful
+                     when calling from another script).
+EOF
+}
+
+# Prompt for the sudo password once; store it for all _do_push calls.
+# Accepts SUDO_PASS from the environment to allow non-interactive callers.
+prompt_sudo_password() {
+  [[ "$dry_run" -eq 1 ]] && return
+  if [[ -n "${SUDO_PASS:-}" ]]; then
+    sudo_password="$SUDO_PASS"
+    return
+  fi
+  # read exits non-zero when stdin is not a terminal (e.g. CI, background
+  # agents). Catch that and give a clear message rather than a silent exit.
+  if ! read -r -s -p "sudo password for ${SSH_USER} on remote hosts: " sudo_password; then
+    echo >&2
+    echo "ERROR: stdin is not a terminal -- cannot prompt for sudo password." >&2
+    echo "       Set SUDO_PASS=<password> in the environment and re-run." >&2
+    exit 1
+  fi
+  echo >&2
+}
+
+locally_managed_hosts() {
+  for f in "${keydir}"/*_ssh_host_ed25519_key.pub; do
+    [[ -e "$f" ]] || continue
+    basename "$f" _ssh_host_ed25519_key.pub
+  done
+}
+
+# --- git state check/fix --------------------------------------------------
+# Hosts rebuild from the remote Gitea flake:
+#   nixos-rebuild switch --flake "git+https://<gitea>/nixos.git#<target>"
+# so .sops.yaml (updated recipients) and secrets/*.yaml (re-encrypted DEKs)
+# must be committed and pushed before any rebuild can succeed. This check
+# catches the common case where --regenerate-all-keys was just run but the
+# resulting diff hasn't been committed/pushed yet.
+ensure_remote_current() {
+  [[ "$skip_git_check" -eq 1 ]] && return
+
+  cd "$repo_root"
+
+  local dirty_unstaged dirty_staged
+  dirty_unstaged="$(git diff --name-only -- .sops.yaml secrets/ 2>/dev/null || true)"
+  dirty_staged="$(git diff --cached --name-only -- .sops.yaml secrets/ 2>/dev/null || true)"
+
+  if [[ -n "$dirty_unstaged" || -n "$dirty_staged" ]]; then
+    echo "Uncommitted changes in sops-managed files:"
+    [[ -n "$dirty_unstaged" ]] && sed 's/^/  (unstaged) /' <<<"$dirty_unstaged"
+    [[ -n "$dirty_staged"   ]] && sed 's/^/  (staged)   /' <<<"$dirty_staged"
+    echo
+    if [[ "$dry_run" -eq 1 ]]; then
+      echo "[dry-run] would prompt to commit .sops.yaml/secrets/ before continuing."
+    else
+      read -rp "Commit .sops.yaml + secrets/ now? [y/N]: " ans
+      if [[ "$ans" =~ ^[Yy]$ ]]; then
+        git add -- .sops.yaml secrets/
+        git commit -m "secrets: update recipients and re-encrypt for host key changes"
+        echo "Committed."
+      else
+        echo "Continuing with uncommitted changes -- the remote won't have the"
+        echo "updated recipients until you commit and push."
+      fi
+    fi
+    echo
+  fi
+
+  # Check if we're ahead of the remote tracking branch
+  local ahead
+  ahead="$(git rev-list --count '@{upstream}..HEAD' 2>/dev/null || echo "")"
+  if [[ -z "$ahead" ]]; then
+    echo "NOTE: no remote tracking branch found -- skipping push check."
+    echo "      Ensure the remote has the current .sops.yaml/secrets/ before"
+    echo "      triggering nixos-rebuild on any host."
+    echo
+    return
+  fi
+
+  if [[ "$ahead" -gt 0 ]]; then
+    echo "Local branch is ${ahead} commit(s) ahead of remote."
+    if [[ "$dry_run" -eq 1 ]]; then
+      echo "[dry-run] would prompt to push before continuing."
+    else
+      read -rp "Push to remote now? [y/N]: " ans
+      if [[ "$ans" =~ ^[Yy]$ ]]; then
+        git push
+        echo "Pushed."
+      else
+        echo "Continuing without pushing -- remember to push before running"
+        echo "nixos-rebuild on any of these hosts."
+      fi
+    fi
+    echo
+  fi
+}
+
+# --- key installation (shared) -------------------------------------------
+_do_push() {
+  local hostname="$1" target="$2"
+  local keyfile="${keydir}/${target}_ssh_host_ed25519_key"
+  local pubfile="${keyfile}.pub"
+
+  if [[ "$dry_run" -eq 1 ]]; then
+    echo "  [dry-run] would scp host-keys/${target}_ssh_host_ed25519_key{,.pub} to /tmp/"
+    echo "  [dry-run] would: sudo -S install -m 0600/0644 to /etc/ssh/ and rm /tmp copies"
+    return
+  fi
+
+  # Upload to /tmp (writable as nixos, no privilege needed)
+  scp -o StrictHostKeyChecking=no \
+    "$keyfile" "${SSH_USER}@${hostname}:/tmp/push_ed25519_key"
+  scp -o StrictHostKeyChecking=no \
+    "$pubfile" "${SSH_USER}@${hostname}:/tmp/push_ed25519_key.pub"
+
+  # Install via sudo -S: the password is piped via herestring so no PTY is
+  # needed on either side. -p '' suppresses sudo's own prompt string.
+  ssh -o StrictHostKeyChecking=no "${SSH_USER}@${hostname}" \
+    "sudo -S -p '' bash -c '
+      install -m 0600 /tmp/push_ed25519_key     /etc/ssh/ssh_host_ed25519_key
+      install -m 0644 /tmp/push_ed25519_key.pub /etc/ssh/ssh_host_ed25519_key.pub
+      rm -f /tmp/push_ed25519_key /tmp/push_ed25519_key.pub
+      echo \"  [ok] host key installed\"
+    '" <<< "$sudo_password"
+
+  # Drop the stale known_hosts entry for this host (public key just changed)
+  ssh-keygen -R "$hostname" 2>/dev/null || true
+
+  echo "  Done. Run nixos-rebuild switch on ${hostname} to activate."
+}
+
+# --- single named target --------------------------------------------------
+push_target() {
+  local target="$1"
+  local keyfile="${keydir}/${target}_ssh_host_ed25519_key"
+
+  if [[ ! -f "$keyfile" ]]; then
+    echo "ERROR: host-keys/${target}_ssh_host_ed25519_key not found." >&2
+    echo "       This target may not be locally managed (e.g. &${target} was" >&2
+    echo "       registered from the host's real SSH key, not generated here)." >&2
+    exit 1
+  fi
+
+  local hostname
+  hostname="$(flake_target_hostname "$repo_root" "$target")"
+  if [[ -z "$hostname" ]]; then
+    echo "ERROR: cannot resolve hostname for '${target}' from the flake." >&2
+    exit 1
+  fi
+
+  echo "==> ${target} (→ ${hostname})"
+
+  if ! ssh "${SSH_OPTS[@]}" "${SSH_USER}@${hostname}" true 2>/dev/null; then
+    echo "  SKIP: ${SSH_USER}@${hostname} unreachable."
+    return
+  fi
+
+  # Sanity-check that /etc/flake-target on the host agrees
+  local live_target
+  live_target="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${hostname}" \
+    "cat /etc/flake-target 2>/dev/null || true")"
+  if [[ -n "$live_target" && "$live_target" != "$target" ]]; then
+    echo "  WARN: host reports /etc/flake-target='${live_target}', not '${target}'."
+    echo "       Pushing the key you specified (${target}) anyway."
+  fi
+
+  _do_push "$hostname" "$target"
+}
+
+# --- all managed hosts ----------------------------------------------------
+# For each unique hostname derived from managed targets, SSHes in and reads
+# /etc/flake-target to determine which key to push -- handles the case where
+# multiple targets share a hostname (e.g. lxc-server and proxmox-server both
+# resolve to "server"; only one is actually running).
+push_all() {
+  mapfile -t managed < <(locally_managed_hosts)
+  if [[ "${#managed[@]}" -eq 0 ]]; then
+    echo "No managed keys in host-keys/ -- nothing to push."
+    return
+  fi
+
+  echo "Pushing to all reachable managed hosts..."
+  echo
+
+  declare -A seen_hostnames=()
+  local t hostname
+  for t in "${managed[@]}"; do
+    hostname="$(flake_target_hostname "$repo_root" "$t" 2>/dev/null || true)"
+    [[ -z "$hostname" ]] && continue
+    [[ -n "${seen_hostnames[$hostname]+x}" ]] && continue
+    seen_hostnames["$hostname"]=1
+
+    echo "==> checking ${hostname}"
+
+    if ! ssh "${SSH_OPTS[@]}" "${SSH_USER}@${hostname}" true 2>/dev/null; then
+      echo "  SKIP: ${SSH_USER}@${hostname} unreachable."
+      continue
+    fi
+
+    # Ask the host which flake target it actually is
+    local live_target
+    live_target="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${hostname}" \
+      "cat /etc/flake-target 2>/dev/null || true")"
+
+    if [[ -z "$live_target" ]]; then
+      echo "  SKIP: no /etc/flake-target on host -- can't determine which key to push."
+      continue
+    fi
+
+    local live_keyfile="${keydir}/${live_target}_ssh_host_ed25519_key"
+    if [[ ! -f "$live_keyfile" ]]; then
+      echo "  SKIP: host is '${live_target}' but no host-keys/${live_target}_... (hand-registered key, not managed here)."
+      continue
+    fi
+
+    echo "  target: ${live_target}"
+    _do_push "$hostname" "$live_target"
+  done
+}
+
+# --- main -----------------------------------------------------------------
+mode="all"
+target_arg=""
+extra_args=()
+
+for arg in "$@"; do
+  case "$arg" in
+    --dry-run)        dry_run=1 ;;
+    --skip-git-check) skip_git_check=1 ;;
+    --all)            mode="all" ;;
+    -h|--help)        usage; exit 0 ;;
+    --*)              echo "Unknown option: $arg" >&2; usage >&2; exit 1 ;;
+    *)                extra_args+=("$arg") ;;
+  esac
+done
+
+if [[ "${#extra_args[@]}" -gt 1 ]]; then
+  echo "ERROR: specify at most one target (or --all)." >&2
+  usage >&2; exit 1
+elif [[ "${#extra_args[@]}" -eq 1 ]]; then
+  mode="single"
+  target_arg="${extra_args[0]}"
+fi
+
+[[ "$dry_run" -eq 1 ]] && { echo "[dry-run] no changes will be made"; echo; }
+
+nix_extra_opts
+ensure_remote_current
+prompt_sudo_password
+
+if [[ "$mode" == "single" ]]; then
+  push_target "$target_arg"
+else
+  push_all
+fi
+
+echo
+if [[ "$dry_run" -eq 1 ]]; then
+  echo "[dry-run] Nothing was changed. Re-run without --dry-run to apply."
+else
+  echo "Key push complete. For each updated host, run nixos-rebuild switch to"
+  echo "apply the config and let sops-nix decrypt secrets with the new key."
+fi

scripts/secrets/sync-host-keys.sh

@@ -359,8 +359,11 @@ cmd_regenerate_all() {
   echo "This will remove and freshly regenerate ALL locally-managed keys:"
   printf '  %s\n' "${hosts[@]}"
   echo
-  echo "Every host above will need its new key baked into a rebuilt install"
-  echo "image/tarball before it can decrypt secrets again."
+  echo "After regenerating, each host needs its new key before it can decrypt secrets:"
+  echo "  • Already running: push the key before rebuilding:"
+  echo "      scripts/secrets/push-host-keys.sh --all"
+  echo "  • Not yet deployed: rebuild the install image with the new keys baked in"
+  echo "      (see docs/auto-installer.md)."
 
   if [[ "$dry_run" -ne 1 ]]; then
     if ! confirm_typed "REGENERATE" "Type REGENERATE to confirm: "; then
@@ -397,6 +400,16 @@ cmd_regenerate_all() {
   echo
   echo "Regenerating fresh keys for every current flake target..."
   cmd_all
+
+  echo
+  echo "Next steps:"
+  echo "  1. Commit and push .sops.yaml + secrets/ so the remote flake is current."
+  echo "  2. Push the new host key to each already-running managed host:"
+  echo "       scripts/secrets/push-host-keys.sh --all"
+  echo "     (this also prompts to commit/push if step 1 wasn't done yet)"
+  echo "  3. Run nixos-rebuild switch on each updated host."
+  echo "  4. For hosts not yet deployed, rebuild the install image (see"
+  echo "     docs/auto-installer.md)."
 }
 
 main() {