From d74efd9f66a1bc8cb04bdf76bd84628ce7592baf Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Sat, 25 Jul 2026 15:29:16 +1000 Subject: [PATCH] feat(secrets): add push-host-keys.sh; integrate into sync/recover scripts New script: scripts/secrets/push-host-keys.sh - Pushes newly-generated SSH host keys from host-keys/ to already-running NixOS hosts after sync-host-keys.sh --regenerate-all-keys. - Before pushing any key, checks that .sops.yaml and secrets/*.yaml are committed and pushed to the remote Gitea flake (hosts rebuild from there, so recipient changes must land first); offers to auto-commit/push if not. - Reads /etc/flake-target from each host to confirm which key to install, handling the case where multiple flake targets share a hostname. - Deduplicates by hostname in --all mode; skips hand-registered targets that have no host-keys/ entry. - --dry-run, --skip-git-check, SSH_USER override (default: nixos). sync-host-keys.sh --regenerate-all-keys: - Updated pre-confirmation warning to distinguish already-running hosts (need push-host-keys.sh) from not-yet-deployed hosts (need installer image rebuild). - Added next-steps block after regeneration completes pointing to push-host-keys.sh --all. recover-hosts.sh: - Header and SSH host key mismatch warn now cross-reference push-host-keys.sh as the proactive (pre-drift) alternative. Co-Authored-By: Claude Sonnet 4.6 --- scripts/recover-hosts.sh | 5 +- scripts/secrets/push-host-keys.sh | 294 ++++++++++++++++++++++++++++++ scripts/secrets/sync-host-keys.sh | 17 +- 3 files changed, 313 insertions(+), 3 deletions(-) create mode 100755 scripts/secrets/push-host-keys.sh diff --git a/scripts/recover-hosts.sh b/scripts/recover-hosts.sh index 702a58f..5f8dcf2 100755 --- a/scripts/recover-hosts.sh +++ b/scripts/recover-hosts.sh @@ -12,6 +12,9 @@ # Fixes applied automatically (then prompts before rebuilding): # 1. SSH host key drift — live key no longer matches host-keys/_ssh_host_ed25519_key # Fix: scp the registered key back and restore it (needs sudo once per host). +# To push new keys proactively (before drift, e.g. right after +# sync-host-keys.sh --regenerate-all-keys), use instead: +# scripts/secrets/push-host-keys.sh --all # 2. Stale/invalid GitHub access token — the rendered nix-github-token.conf has # a token GitHub rejects (401), blocking any rebuild that fetches disko or # other public GitHub flake inputs. @@ -131,7 +134,7 @@ for host in "${HOSTNAMES[@]}"; do if [ "$live" = "$want" ]; then info "SSH host key OK" else - warn "SSH host key MISMATCH (live ≠ host-keys/)" + warn "SSH host key MISMATCH (live ≠ host-keys/) -- use push-host-keys.sh proactively next time" echo " live: $live" echo " registered: $want" host_broken=true diff --git a/scripts/secrets/push-host-keys.sh b/scripts/secrets/push-host-keys.sh new file mode 100755 index 0000000..a74d73f --- /dev/null +++ b/scripts/secrets/push-host-keys.sh @@ -0,0 +1,294 @@ +#!/usr/bin/env bash +# Pushes newly-generated SSH host keys from host-keys/ to already-running +# NixOS hosts, so they can decrypt sops secrets after a nixos-rebuild +# following scripts/secrets/sync-host-keys.sh --regenerate-all-keys. +# +# Before pushing any key, verifies that .sops.yaml and secrets/*.yaml are +# committed and pushed to the remote -- hosts rebuild from the remote Gitea +# flake, so recipient changes must land there before any rebuild, not just +# before the key push. +# +# push-host-keys.sh --all [--dry-run] [--skip-git-check] +# push-host-keys.sh [--dry-run] [--skip-git-check] +# +# --all Push to every reachable managed host. Default when no +# target is given. +# Push to one flake target only (e.g. lxc-server). +# --dry-run Print what would be done; write nothing. +# --skip-git-check Skip the commit/push check. Use only when the remote +# already has the current .sops.yaml/secrets/*.yaml. +# +# SSH: connects as SSH_USER@ (default: nixos, the user with the +# admin authorized key), then installs files via sudo. You will be prompted +# for the sudo password once per host. +# Hosts are reached at their bare hostname (relies on LAN DNS/mDNS). +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +keydir="${repo_root}/host-keys" + +# shellcheck source=../env.sh +source "${repo_root}/scripts/env.sh" +# shellcheck source=../lib/nix-eval.sh +source "${repo_root}/scripts/lib/nix-eval.sh" + +: "${SSH_USER:=nixos}" +SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=5) + +dry_run=0 +skip_git_check=0 + +usage() { + cat <] [--dry-run] [--skip-git-check] + + --all Push to every reachable managed host. Default when no + target is given. + Push to one flake target only (e.g. lxc-server). + --dry-run Print what would be done; write nothing. + --skip-git-check Skip the check that .sops.yaml/secrets/*.yaml are + committed and pushed to the remote repo. + +Environment: + SSH_USER SSH username (default: nixos). +EOF +} + +locally_managed_hosts() { + for f in "${keydir}"/*_ssh_host_ed25519_key.pub; do + [[ -e "$f" ]] || continue + basename "$f" _ssh_host_ed25519_key.pub + done +} + +# --- git state check/fix -------------------------------------------------- +# Hosts rebuild from the remote Gitea flake: +# nixos-rebuild switch --flake "git+https:///nixos.git#" +# so .sops.yaml (updated recipients) and secrets/*.yaml (re-encrypted DEKs) +# must be committed and pushed before any rebuild can succeed. This check +# catches the common case where --regenerate-all-keys was just run but the +# resulting diff hasn't been committed/pushed yet. +ensure_remote_current() { + [[ "$skip_git_check" -eq 1 ]] && return + + cd "$repo_root" + + local dirty_unstaged dirty_staged + dirty_unstaged="$(git diff --name-only -- .sops.yaml secrets/ 2>/dev/null || true)" + dirty_staged="$(git diff --cached --name-only -- .sops.yaml secrets/ 2>/dev/null || true)" + + if [[ -n "$dirty_unstaged" || -n "$dirty_staged" ]]; then + echo "Uncommitted changes in sops-managed files:" + [[ -n "$dirty_unstaged" ]] && sed 's/^/ (unstaged) /' <<<"$dirty_unstaged" + [[ -n "$dirty_staged" ]] && sed 's/^/ (staged) /' <<<"$dirty_staged" + echo + if [[ "$dry_run" -eq 1 ]]; then + echo "[dry-run] would prompt to commit .sops.yaml/secrets/ before continuing." + else + read -rp "Commit .sops.yaml + secrets/ now? [y/N]: " ans + if [[ "$ans" =~ ^[Yy]$ ]]; then + git add -- .sops.yaml secrets/ + git commit -m "secrets: update recipients and re-encrypt for host key changes" + echo "Committed." + else + echo "Continuing with uncommitted changes -- the remote won't have the" + echo "updated recipients until you commit and push." + fi + fi + echo + fi + + # Check if we're ahead of the remote tracking branch + local ahead + ahead="$(git rev-list --count '@{upstream}..HEAD' 2>/dev/null || echo "")" + if [[ -z "$ahead" ]]; then + echo "NOTE: no remote tracking branch found -- skipping push check." + echo " Ensure the remote has the current .sops.yaml/secrets/ before" + echo " triggering nixos-rebuild on any host." + echo + return + fi + + if [[ "$ahead" -gt 0 ]]; then + echo "Local branch is ${ahead} commit(s) ahead of remote." + if [[ "$dry_run" -eq 1 ]]; then + echo "[dry-run] would prompt to push before continuing." + else + read -rp "Push to remote now? [y/N]: " ans + if [[ "$ans" =~ ^[Yy]$ ]]; then + git push + echo "Pushed." + else + echo "Continuing without pushing -- remember to push before running" + echo "nixos-rebuild on any of these hosts." + fi + fi + echo + fi +} + +# --- key installation (shared) ------------------------------------------- +_do_push() { + local hostname="$1" target="$2" + local keyfile="${keydir}/${target}_ssh_host_ed25519_key" + local pubfile="${keyfile}.pub" + + if [[ "$dry_run" -eq 1 ]]; then + echo " [dry-run] would scp host-keys/${target}_ssh_host_ed25519_key{,.pub} to /tmp/" + echo " [dry-run] would: sudo install -m 0600/0644 to /etc/ssh/ and rm /tmp copies" + return + fi + + # Upload to /tmp (writable as nixos, no privilege needed) + scp -o StrictHostKeyChecking=no \ + "$keyfile" "${SSH_USER}@${hostname}:/tmp/push_ed25519_key" + scp -o StrictHostKeyChecking=no \ + "$pubfile" "${SSH_USER}@${hostname}:/tmp/push_ed25519_key.pub" + + # Install with correct permissions in one interactive sudo session + echo " (sudo password may be required)" + ssh -t -o StrictHostKeyChecking=no "${SSH_USER}@${hostname}" \ + "sudo bash -s" <<'REMOTE' +install -m 0600 /tmp/push_ed25519_key /etc/ssh/ssh_host_ed25519_key +install -m 0644 /tmp/push_ed25519_key.pub /etc/ssh/ssh_host_ed25519_key.pub +rm -f /tmp/push_ed25519_key /tmp/push_ed25519_key.pub +echo " [ok] host key installed" +REMOTE + + # Drop the stale known_hosts entry for this host (public key just changed) + ssh-keygen -R "$hostname" 2>/dev/null || true + + echo " Done. Run nixos-rebuild switch on ${hostname} to activate." +} + +# --- single named target -------------------------------------------------- +push_target() { + local target="$1" + local keyfile="${keydir}/${target}_ssh_host_ed25519_key" + + if [[ ! -f "$keyfile" ]]; then + echo "ERROR: host-keys/${target}_ssh_host_ed25519_key not found." >&2 + echo " This target may not be locally managed (e.g. &${target} was" >&2 + echo " registered from the host's real SSH key, not generated here)." >&2 + exit 1 + fi + + local hostname + hostname="$(flake_target_hostname "$repo_root" "$target")" + if [[ -z "$hostname" ]]; then + echo "ERROR: cannot resolve hostname for '${target}' from the flake." >&2 + exit 1 + fi + + echo "==> ${target} (→ ${hostname})" + + if ! ssh "${SSH_OPTS[@]}" "${SSH_USER}@${hostname}" true 2>/dev/null; then + echo " SKIP: ${SSH_USER}@${hostname} unreachable." + return + fi + + # Sanity-check that /etc/flake-target on the host agrees + local live_target + live_target="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${hostname}" \ + "cat /etc/flake-target 2>/dev/null || true")" + if [[ -n "$live_target" && "$live_target" != "$target" ]]; then + echo " WARN: host reports /etc/flake-target='${live_target}', not '${target}'." + echo " Pushing the key you specified (${target}) anyway." + fi + + _do_push "$hostname" "$target" +} + +# --- all managed hosts ---------------------------------------------------- +# For each unique hostname derived from managed targets, SSHes in and reads +# /etc/flake-target to determine which key to push -- handles the case where +# multiple targets share a hostname (e.g. lxc-server and proxmox-server both +# resolve to "server"; only one is actually running). +push_all() { + mapfile -t managed < <(locally_managed_hosts) + if [[ "${#managed[@]}" -eq 0 ]]; then + echo "No managed keys in host-keys/ -- nothing to push." + return + fi + + echo "Pushing to all reachable managed hosts..." + echo + + declare -A seen_hostnames=() + local t hostname + for t in "${managed[@]}"; do + hostname="$(flake_target_hostname "$repo_root" "$t" 2>/dev/null || true)" + [[ -z "$hostname" ]] && continue + [[ -n "${seen_hostnames[$hostname]+x}" ]] && continue + seen_hostnames["$hostname"]=1 + + echo "==> checking ${hostname}" + + if ! ssh "${SSH_OPTS[@]}" "${SSH_USER}@${hostname}" true 2>/dev/null; then + echo " SKIP: ${SSH_USER}@${hostname} unreachable." + continue + fi + + # Ask the host which flake target it actually is + local live_target + live_target="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${hostname}" \ + "cat /etc/flake-target 2>/dev/null || true")" + + if [[ -z "$live_target" ]]; then + echo " SKIP: no /etc/flake-target on host -- can't determine which key to push." + continue + fi + + local live_keyfile="${keydir}/${live_target}_ssh_host_ed25519_key" + if [[ ! -f "$live_keyfile" ]]; then + echo " SKIP: host is '${live_target}' but no host-keys/${live_target}_... (hand-registered key, not managed here)." + continue + fi + + echo " target: ${live_target}" + _do_push "$hostname" "$live_target" + done +} + +# --- main ----------------------------------------------------------------- +mode="all" +target_arg="" +extra_args=() + +for arg in "$@"; do + case "$arg" in + --dry-run) dry_run=1 ;; + --skip-git-check) skip_git_check=1 ;; + --all) mode="all" ;; + -h|--help) usage; exit 0 ;; + --*) echo "Unknown option: $arg" >&2; usage >&2; exit 1 ;; + *) extra_args+=("$arg") ;; + esac +done + +if [[ "${#extra_args[@]}" -gt 1 ]]; then + echo "ERROR: specify at most one target (or --all)." >&2 + usage >&2; exit 1 +elif [[ "${#extra_args[@]}" -eq 1 ]]; then + mode="single" + target_arg="${extra_args[0]}" +fi + +[[ "$dry_run" -eq 1 ]] && { echo "[dry-run] no changes will be made"; echo; } + +nix_extra_opts +ensure_remote_current + +if [[ "$mode" == "single" ]]; then + push_target "$target_arg" +else + push_all +fi + +echo +if [[ "$dry_run" -eq 1 ]]; then + echo "[dry-run] Nothing was changed. Re-run without --dry-run to apply." +else + echo "Key push complete. For each updated host, run nixos-rebuild switch to" + echo "apply the config and let sops-nix decrypt secrets with the new key." +fi diff --git a/scripts/secrets/sync-host-keys.sh b/scripts/secrets/sync-host-keys.sh index eda2915..fe52536 100755 --- a/scripts/secrets/sync-host-keys.sh +++ b/scripts/secrets/sync-host-keys.sh @@ -359,8 +359,11 @@ cmd_regenerate_all() { echo "This will remove and freshly regenerate ALL locally-managed keys:" printf ' %s\n' "${hosts[@]}" echo - echo "Every host above will need its new key baked into a rebuilt install" - echo "image/tarball before it can decrypt secrets again." + echo "After regenerating, each host needs its new key before it can decrypt secrets:" + echo " • Already running: push the key before rebuilding:" + echo " scripts/secrets/push-host-keys.sh --all" + echo " • Not yet deployed: rebuild the install image with the new keys baked in" + echo " (see docs/auto-installer.md)." if [[ "$dry_run" -ne 1 ]]; then if ! confirm_typed "REGENERATE" "Type REGENERATE to confirm: "; then @@ -397,6 +400,16 @@ cmd_regenerate_all() { echo echo "Regenerating fresh keys for every current flake target..." cmd_all + + echo + echo "Next steps:" + echo " 1. Commit and push .sops.yaml + secrets/ so the remote flake is current." + echo " 2. Push the new host key to each already-running managed host:" + echo " scripts/secrets/push-host-keys.sh --all" + echo " (this also prompts to commit/push if step 1 wasn't done yet)" + echo " 3. Run nixos-rebuild switch on each updated host." + echo " 4. For hosts not yet deployed, rebuild the install image (see" + echo " docs/auto-installer.md)." } main() {