From dce3788499afe6b129e1fe994d9f50d140dfe997 Mon Sep 17 00:00:00 2001
From: beatzaplenty <wayne.bennett@live.com>
Date: Fri, 24 Jul 2026 09:17:58 +1000
Subject: [PATCH] fix(lxc): prevent SSH host key deletion on every rebuild; add
 recovery script

NixOS's etc activation removes files that were in a previous generation's
environment.etc but absent from the current one -- even real copies, not
only symlinks.  LXC tarballs bake the host key into environment.etc (via
NIXOS_HOST_KEYS_DIR), but every subsequent nixos-rebuild switch lacks that
env var, so the key is removed as "obsolete".  sops-nix derives its age
decryption key from /etc/ssh/ssh_host_ed25519_key, so deletion cascades
into "Error getting data key: 0 successful groups required, got 0" for
every sops secret on the host.

Fix: two activation scripts bracket the etc step.
  preserveSshHostKey (no deps, runs before etc): copies the live key to
    /run (tmpfs) before etc can delete it.
  restoreSshHostKey (deps=[etc], runs after etc): reinstalls via `install`
    if etc removed the key.  The resulting file is not tracked in either
    generation's environment.etc, so subsequent rebuilds leave it alone.

scripts/recover-hosts.sh: restore both private and public key files (not
just the private key), use install(1) for atomic mode setting, and add a
post-rebuild sops-nix verification step to confirm success.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014zT1L6hmsq6i1evAEH7dmi
---
 modules/platforms/lxc.nix |  34 ++++++
 scripts/recover-hosts.sh  | 250 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 284 insertions(+)
 create mode 100755 scripts/recover-hosts.sh

diff --git a/modules/platforms/lxc.nix b/modules/platforms/lxc.nix
index ac24108..7d6cee4 100644
--- a/modules/platforms/lxc.nix
+++ b/modules/platforms/lxc.nix
@@ -106,6 +106,40 @@ in
     };
   };
 
+  # NixOS's etc activation removes any /etc file that was in the previous
+  # generation's environment.etc but is absent from the current one — even
+  # real (non-symlink) copies.  On every routine nixos-rebuild switch/test that
+  # lacks NIXOS_HOST_KEYS_DIR the key is absent from environment.etc, so it
+  # gets removed as "obsolete".  sops-nix derives its age decryption key from
+  # /etc/ssh/ssh_host_ed25519_key; deletion cascades into every sops secret
+  # failing with "Error getting data key: 0 successful groups required, got 0".
+  #
+  # Fix: two activation scripts that bracket the etc step.
+  #   preserveSshHostKey — no deps, runs before etc — saves the live key to
+  #                        /run (tmpfs) before etc can delete it.
+  #   restoreSshHostKey  — deps=[etc], runs after etc — reinstalls the key via
+  #                        `install` (atomic, sets mode) if etc removed it.
+  #                        The resulting file is not registered in environment.etc
+  #                        for either the previous or current generation, so
+  #                        subsequent rebuilds leave it alone permanently.
+  system.activationScripts.preserveSshHostKey = ''
+    if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
+      cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
+      cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
+    fi
+  '';
+
+  system.activationScripts.restoreSshHostKey = {
+    deps = [ "etc" ];
+    text = ''
+      if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
+        install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
+        install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
+      fi
+      rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
+    '';
+  };
+
   # virtualisation/proxmox-lxc.nix (imported above) registers the Nix
   # store DB via a systemd service (register-nix-paths) -- it never runs
   # an activation script at all. Confirmed live this means neither
diff --git a/scripts/recover-hosts.sh b/scripts/recover-hosts.sh
new file mode 100755
index 0000000..702a58f
--- /dev/null
+++ b/scripts/recover-hosts.sh
@@ -0,0 +1,250 @@
+#!/usr/bin/env bash
+# recover-hosts.sh — Fix sops/SSH-key/GitHub-token issues on deployed NixOS hosts
+# and trigger a Switch-nix rebuild on each.
+#
+# Run from the repo root on the workstation (nixos@nixos):
+#   bash scripts/recover-hosts.sh [<hostname> ...]
+#
+# With no args it discovers and checks every known hostname.
+# With args it checks only those hostnames:
+#   bash scripts/recover-hosts.sh tor-relay
+#
+# Fixes applied automatically (then prompts before rebuilding):
+#   1. SSH host key drift — live key no longer matches host-keys/<target>_ssh_host_ed25519_key
+#      Fix: scp the registered key back and restore it (needs sudo once per host).
+#   2. Stale/invalid GitHub access token — the rendered nix-github-token.conf has
+#      a token GitHub rejects (401), blocking any rebuild that fetches disko or
+#      other public GitHub flake inputs.
+#      Fix: empty the rendered file so nix makes unauthenticated requests instead.
+#      Public repos (disko, nixpkgs, etc.) work fine without auth. sops-nix
+#      re-renders the correct new token automatically after the first successful
+#      rebuild.
+#
+# Both fixes need one interactive sudo session per host. The script opens a
+# single ssh -t per broken host so you enter the password once and all steps
+# run in sequence.
+
+set -euo pipefail
+cd "$(dirname "$0")/.."
+source scripts/env.sh 2>/dev/null || true
+
+SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=5)
+SSH_USER=nixos
+
+# Known flake-target → ssh hostname map for all currently-defined hosts.
+# Add new hosts here as they are deployed.
+declare -A TARGET_HOST=(
+  [lxc-docker]=docker
+  [lxc-nix-cache]=nix-cache
+  [lxc-pxe-boot]=pxe-boot
+  [lxc-tor-relay]=tor-relay
+  [lxc-minimal]=nix-minimal
+  [proxmox-server]=server
+  [baremetal-gui]=nixos
+)
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+
+info() { echo "  [✓] $*"; }
+warn() { echo "  [!] $*"; }
+step() { echo "==> $*"; }
+
+ssh_host_age() {
+  ssh-keyscan -t ed25519 "$1" 2>/dev/null \
+    | nix shell nixpkgs#ssh-to-age --command ssh-to-age 2>/dev/null \
+    | head -1 || true
+}
+
+registered_age() {
+  local keyfile="host-keys/${1}_ssh_host_ed25519_key.pub"
+  [ -f "$keyfile" ] || return 0
+  nix shell nixpkgs#ssh-to-age --command ssh-to-age < "$keyfile" 2>/dev/null \
+    | head -1 || true
+}
+
+github_token_valid() {
+  local host=$1
+  local raw token code
+  raw=$(ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" \
+    "cat /run/secrets/rendered/nix-github-token.conf 2>/dev/null || true")
+  token=$(echo "$raw" | grep -oP '(?<=github\.com=)\S+' || true)
+  if [ -z "$token" ]; then
+    return 0  # no token = unauthenticated, works for public repos
+  fi
+  code=$(curl -s -o /dev/null -w "%{http_code}" \
+    -H "Authorization: token $token" \
+    "https://api.github.com/repos/nix-community/disko" 2>/dev/null || echo 000)
+  [ "$code" = "200" ]
+}
+
+# ── discover hosts ────────────────────────────────────────────────────────────
+
+if [ $# -gt 0 ]; then
+  HOSTNAMES=("$@")
+else
+  HOSTNAMES=()
+  seen=()
+  for target in "${!TARGET_HOST[@]}"; do
+    h="${TARGET_HOST[$target]}"
+    # deduplicate (e.g. proxmox-server and lxc-server both map to "server")
+    if [[ ! " ${seen[*]:-} " =~ " $h " ]]; then
+      seen+=("$h")
+      if ssh "${SSH_OPTS[@]}" "$SSH_USER@$h" "true" 2>/dev/null; then
+        HOSTNAMES+=("$h")
+      fi
+    fi
+  done
+fi
+
+if [ ${#HOSTNAMES[@]} -eq 0 ]; then
+  echo "No reachable hosts found. Pass hostnames explicitly or check SSH."
+  exit 1
+fi
+
+echo ""
+echo "Hosts to check: ${HOSTNAMES[*]}"
+echo ""
+
+# ── check phase ───────────────────────────────────────────────────────────────
+
+NEEDS_FIX=()
+
+for host in "${HOSTNAMES[@]}"; do
+  step "$host"
+
+  if ! ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "true" 2>/dev/null; then
+    warn "SSH unreachable — clearing stale known_hosts entry"
+    ssh-keygen -R "$host" 2>/dev/null || true
+    continue
+  fi
+
+  flake_target=$(ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" \
+    "cat /etc/flake-target 2>/dev/null || true")
+  echo "  flake-target: ${flake_target:-unknown}"
+
+  host_broken=false
+
+  # SSH host key
+  if [ -n "$flake_target" ] && [ -f "host-keys/${flake_target}_ssh_host_ed25519_key.pub" ]; then
+    live=$(ssh_host_age "$host")
+    want=$(registered_age "$flake_target")
+    if [ "$live" = "$want" ]; then
+      info "SSH host key OK"
+    else
+      warn "SSH host key MISMATCH (live ≠ host-keys/)"
+      echo "       live:       $live"
+      echo "       registered: $want"
+      host_broken=true
+    fi
+  else
+    echo "  [~] No host-keys/ entry for ${flake_target:-unknown} — skipping key check"
+  fi
+
+  # GitHub token
+  if github_token_valid "$host"; then
+    info "GitHub token OK"
+  else
+    warn "GitHub token invalid (rebuild will fail with 401)"
+    host_broken=true
+  fi
+
+  # sops-nix result
+  sops_result=$(ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" \
+    "systemctl show sops-nix --property=Result --value 2>/dev/null || echo unknown")
+  if [ "$sops_result" = "success" ]; then
+    info "sops-nix: success"
+  else
+    warn "sops-nix: $sops_result"
+  fi
+
+  $host_broken && NEEDS_FIX+=("$host")
+  echo ""
+done
+
+# ── fix phase ─────────────────────────────────────────────────────────────────
+
+if [ ${#NEEDS_FIX[@]} -eq 0 ]; then
+  echo "All hosts healthy — nothing to fix."
+  exit 0
+fi
+
+echo "Hosts needing fixes: ${NEEDS_FIX[*]}"
+echo ""
+echo "Each fix requires one sudo session per host. You will be prompted for"
+echo "the nixos sudo password once per host; all steps run in that session."
+echo ""
+read -r -p "Proceed with fixes + Switch-nix on each broken host? [y/N] " confirm
+[[ "$confirm" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; }
+echo ""
+
+for host in "${NEEDS_FIX[@]}"; do
+  step "Fixing $host"
+  flake_target=$(ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" \
+    "cat /etc/flake-target 2>/dev/null || true")
+
+  fix_script=""
+
+  # Fix 1: restore SSH host key
+  live=$(ssh_host_age "$host")
+  want=$(registered_age "${flake_target:-}")
+  if [ -n "$want" ] && [ "$live" != "$want" ]; then
+    echo "  Uploading registered SSH host key (private + public)..."
+    scp -o StrictHostKeyChecking=no \
+      "host-keys/${flake_target}_ssh_host_ed25519_key" \
+      "$SSH_USER@$host:/tmp/recover_ed25519_key"
+    scp -o StrictHostKeyChecking=no \
+      "host-keys/${flake_target}_ssh_host_ed25519_key.pub" \
+      "$SSH_USER@$host:/tmp/recover_ed25519_key.pub"
+    fix_script+='
+echo "[fix] Restoring SSH host key..."
+install -m 0600 /tmp/recover_ed25519_key /etc/ssh/ssh_host_ed25519_key
+install -m 0644 /tmp/recover_ed25519_key.pub /etc/ssh/ssh_host_ed25519_key.pub
+rm -f /tmp/recover_ed25519_key /tmp/recover_ed25519_key.pub
+echo "      Done."
+'
+    ssh-keygen -R "$host" 2>/dev/null || true
+  fi
+
+  # Fix 2: clear invalid GitHub token
+  if ! github_token_valid "$host"; then
+    fix_script+='
+echo "[fix] Clearing stale GitHub token (nix will use unauthenticated access)..."
+echo "" > /run/secrets/rendered/nix-github-token.conf
+systemctl restart nix-daemon 2>/dev/null || true
+echo "      Done."
+'
+  fi
+
+  # Fix 3: rebuild
+  fix_script+='
+echo "[fix] Running nixos-rebuild switch..."
+nixos-rebuild switch \
+  --no-write-lock-file \
+  --refresh \
+  --flake "git+https://gitea.lan.ddnsgeek.com/beatzaplenty/nixos.git#$(cat /etc/flake-target)"
+echo "[fix] Rebuild complete."
+'
+
+  echo "  Opening SSH session (enter sudo password when prompted)..."
+  if ssh -t -o StrictHostKeyChecking=no "$SSH_USER@$host" \
+    "sudo bash -s" <<< "$fix_script"; then
+    echo ""
+    info "$host fixed and rebuilt"
+  else
+    rc=$?
+    echo ""
+    warn "$host: rebuild exited with code $rc (may still have succeeded — check sops-nix below)"
+  fi
+
+  # Verify: re-check sops-nix result post-rebuild
+  sops_result_after=$(ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" \
+    "systemctl show sops-nix --property=Result --value 2>/dev/null || echo unknown" 2>/dev/null || echo "ssh-failed")
+  if [ "$sops_result_after" = "success" ]; then
+    info "$host sops-nix: success post-rebuild"
+  else
+    warn "$host sops-nix: $sops_result_after post-rebuild (may need another pass)"
+  fi
+  echo ""
+done
+
+echo "Recovery complete."