Merge pull request 'Fix nix-cache remote-builder trust: stale host key + wrong sshKey path' (#21) from worktree-fix-nix-cache-host-key into main

Reviewed-on: #21

CLAUDE.md

@@ -116,6 +116,22 @@ lift the shared part into `scripts/lib/` instead of copying it.
   Purely a local filesystem copy — never touches `.sops.yaml`/
   `secrets/*.yaml` or the repo at all. The resulting file is exactly what
   `rotate-admin-key.sh` expects as its backup-key argument.
+- `scripts/secrets/sync-nix-cache-host-key.sh [--check] [--dry-run]
+  [--host <name>]` — detects drift between the ed25519 SSH host key
+  nix-cache is actually serving right now (via `ssh-keyscan`) and
+  `vars.nixCacheHostKey` (`variables.nix`), the value
+  `modules/nix-cache/remote-builder-client.nix` bakes into every real
+  client's declarative `programs.ssh.knownHosts` and
+  `configure-nix-cache-client.sh` hardcodes as its own default for
+  non-NixOS clients. That value has no automatic source of truth — it's
+  set once from whatever nix-cache's host key happened to be at the time,
+  and silently goes stale if the host is ever rebuilt/recreated with a new
+  key, breaking every client's distributed-build SSH trust with no error
+  that points back here. `--check` (used by `codex-maintenance.sh`, which
+  treats an unreachable nix-cache — e.g. from a non-LAN CI runner — as a
+  silent skip rather than a failure) only reports drift; the no-flags form
+  updates both files in place. Declarative clients still need a rebuild to
+  pick up the fix.
 
 ### `scripts/proxmox/`
 

docs/nix-cache.md

@@ -46,18 +46,26 @@ the new key up automatically on next activation — no more manual
 
 ## Remote builder SSH keys
 
-On each client, install the private key used to authenticate as `nixremote`:
+Each client authenticates as `nixremote` using its **own default root SSH
+identity** (`/root/.ssh/id_ed25519`) — not a separately-named or shared
+keypair. If a client doesn't have one yet:
 
 ```bash
-sudo install -d -m 0700 /root/.ssh
-sudo install -m 0600 ./nixremote /root/.ssh/nixremote
-sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version
+sudo ssh-keygen -t ed25519 -N '' -f /root/.ssh/id_ed25519
 ```
 
-On `nix-cache`, install the matching public key used by `nixremote` authorized keys.
+Then add its `.pub` contents as a new entry in `vars.remoteBuilderAuthorizedKeys`
+(`variables.nix`) and rebuild `nix-cache` to pick it up (that list is
+declarative — an imperative `ssh-copy-id nixremote@nix-cache` won't stick;
+it gets overwritten on every rebuild). Verify with:
 
-The committed `nixremote` authorized keys are public SSH keys only. Keep the
-matching private keys on client hosts and out of the repository.
+```bash
+sudo ssh -i /root/.ssh/id_ed25519 nixremote@nix-cache nix-store --version
+```
+
+The committed `remoteBuilderAuthorizedKeys` entries are public SSH keys
+only. Keep the matching private keys on client hosts and out of the
+repository.
 
 nix-cache's own SSH *host* key is trusted declaratively via
 `programs.ssh.knownHosts` in `modules/nix-cache/remote-builder-client.nix`,
@@ -76,8 +84,8 @@ After deployment:
 curl http://nix-cache/nix-cache-info
 nix store ping --store http://nix-cache
 nix show-config | grep -E 'substituters|trusted-public-keys|builders-use-substitutes'
-sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version
-nix build nixpkgs#hello --builders 'ssh://nixremote@nix-cache x86_64-linux /root/.ssh/nixremote 4 2 big-parallel,kvm,nixos-test,benchmark' -L
+sudo ssh -i /root/.ssh/id_ed25519 nixremote@nix-cache nix-store --version
+nix build nixpkgs#hello --builders 'ssh://nixremote@nix-cache x86_64-linux /root/.ssh/id_ed25519 4 2 big-parallel,kvm,nixos-test,benchmark' -L
 nix path-info -r nixpkgs#hello
 curl -I "http://nix-cache/$(basename "$(nix path-info nixpkgs#hello)").narinfo"
 ```

modules/nix-cache/remote-builder-client.nix

@@ -1,10 +1,14 @@
 { pkgs, vars, ... }:
 
 {
-  # Install the remote builder key on each client host (do not commit private keys):
-  #   sudo install -d -m 0700 /root/.ssh
-  #   sudo install -m 0600 ./nixremote /root/.ssh/nixremote
-  #   sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version
+  # Authenticate as nixremote using the client host's own default root SSH
+  # identity (/root/.ssh/id_ed25519) rather than a separately-named key --
+  # matches vars.remoteBuilderAuthorizedKeys, which already authorizes
+  # each host's own default key (one entry per host, not a shared
+  # dedicated keypair). If this host doesn't have one yet:
+  #   sudo -u root ssh-keygen -t ed25519 -N '' -f /root/.ssh/id_ed25519
+  #   # then add its .pub to vars.remoteBuilderAuthorizedKeys and rebuild nix-cache
+  #   sudo ssh -i /root/.ssh/id_ed25519 nixremote@nix-cache nix-store --version
   # Trust nix-cache's SSH host key declaratively so the nix-daemon (root)
   # can connect the first time without a manual ssh-keyscan/known_hosts
   # step on every new client.
@@ -20,7 +24,7 @@
       {
         hostName = vars.nixCacheHost;
         sshUser = vars.remoteBuilderUser;
-        sshKey = "/root/.ssh/${vars.remoteBuilderUser}";
+        sshKey = "/root/.ssh/id_ed25519";
         inherit (pkgs.stdenv.hostPlatform) system;
         maxJobs = 4;
         speedFactor = 2;

scripts/codex-maintenance.sh

@@ -41,6 +41,20 @@ echo
 echo "Running statix lint..."
 nix run "${NIX_EVAL_FLAGS[@]}" github:NixOS/nixpkgs/nixos-25.11#statix -- check .
 
+echo
+echo "Checking nix-cache host key for drift..."
+if bash "${script_dir}/secrets/sync-nix-cache-host-key.sh" --check; then
+  :
+else
+  drift_status=$?
+  if [[ "$drift_status" -eq 2 ]]; then
+    echo "nix-cache unreachable from here -- skipping host-key drift check."
+  else
+    echo "WARNING: nix-cache's host key has drifted from variables.nix (see above)." >&2
+    echo "  Run 'bash scripts/secrets/sync-nix-cache-host-key.sh' to fix." >&2
+  fi
+fi
+
 echo
 echo "Evaluating host toplevel derivations..."
 for host in $hosts; do

scripts/proxmox/configure-nix-cache-client.sh

@@ -15,6 +15,16 @@
 # this repo -- update both if nix-cache is ever rebuilt with a new host
 # key or the cache signing key is rotated (see docs/nix-cache.md).
 #
+# REMOTE_BUILDER_KEY defaults to this machine's own default root SSH
+# identity (matches modules/nix-cache/remote-builder-client.nix's
+# convention for real NixOS clients: authenticate as nixremote with the
+# host's own default key, added individually to
+# vars.remoteBuilderAuthorizedKeys, rather than a separately-named or
+# shared keypair) -- generate one with
+# `ssh-keygen -t ed25519 -N '' -f /root/.ssh/id_ed25519` if this machine
+# doesn't have one yet, then add its .pub to vars.remoteBuilderAuthorizedKeys
+# and rebuild nix-cache.
+#
 # Usage:
 #   sudo ./configure-nix-cache-client.sh [--dry-run] [--no-remote-builder] [--no-restart]
 #
@@ -24,9 +34,9 @@
 set -euo pipefail
 
 : "${NIX_CACHE_HOST:=nix-cache}"
-: "${NIX_CACHE_HOST_KEY:=ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHrMKZlIGUd3pH9G3AqbsruqUGjxIXMAZw52u9MwiBCn lxc-nix-cache}"
+: "${NIX_CACHE_HOST_KEY:=ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPeWgMsdaiz4axT/deFc1+0B5bN+GX/NOeW9bbQ0c/IT lxc-nix-cache}"
 : "${REMOTE_BUILDER_USER:=nixremote}"
-: "${REMOTE_BUILDER_KEY:=/root/.ssh/nixremote}"
+: "${REMOTE_BUILDER_KEY:=/root/.ssh/id_ed25519}"
 
 CACHE_PUB_KEY="cache.local-1:usoWYanY3Kpq2+kDIS2nhWoLZiRxanmdysdzqCFBHW4="
 FALLBACK_URL="https://cache.nixos.org/"

scripts/secrets/sync-nix-cache-host-key.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Detects and fixes drift between the ed25519 SSH host key nix-cache is
+# actually serving right now and vars.nixCacheHostKey (variables.nix) --
+# the value modules/nix-cache/remote-builder-client.nix bakes into every
+# client's declarative programs.ssh.knownHosts, and
+# scripts/proxmox/configure-nix-cache-client.sh hardcodes as its own
+# default for non-NixOS clients.
+#
+# This value has no automatic source of truth: nix-cache's host key is
+# generated once (first boot / container recreate) and never touches this
+# repo again unless someone remembers to update it by hand afterwards. It
+# drifted silently once already -- confirmed live: variables.nix recorded
+# a key that no longer matched what nix-cache actually presented, which
+# would fail every real client's SSH host-key verification for
+# distributed builds without ever producing an obvious error pointing
+# back here (a client just sees "Host key verification failed" against
+# *some* key, with no hint that the trusted value itself was stale).
+#
+# codex-maintenance.sh runs this in --check mode on every invocation so
+# that drift surfaces as a warning instead of a future debugging session.
+#
+# Usage:
+#   scripts/secrets/sync-nix-cache-host-key.sh [--check] [--dry-run] [--host <name>]
+#
+#   --check     Only report drift (exit 1 if found, 2 if nix-cache is
+#               unreachable); never writes. For CI/maintenance use.
+#   --dry-run   Show what would change; never writes.
+#   --host      Override the hostname to scan (default: variables.nix's
+#               nixCacheHost / env.sh's NIX_CACHE_HOST).
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
+variables_nix="${repo_root}/variables.nix"
+client_script="${repo_root}/scripts/proxmox/configure-nix-cache-client.sh"
+
+# shellcheck source=../env.sh
+source "${repo_root}/scripts/env.sh"
+
+check_only=0
+dry_run=0
+host="${NIX_CACHE_HOST}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --check) check_only=1; shift ;;
+    --dry-run) dry_run=1; shift ;;
+    --host)
+      host="${2:?--host requires a hostname}"
+      shift 2
+      ;;
+    -h|--help)
+      sed -n '2,23p' "$0"
+      exit 0
+      ;;
+    *)
+      echo "ERROR: unknown argument: $1" >&2
+      exit 1
+      ;;
+  esac
+done
+
+current_value="$(grep -oE 'nixCacheHostKey = "[^"]+"' "$variables_nix" | sed -E 's/nixCacheHostKey = "(.*)"/\1/')"
+if [[ -z "$current_value" ]]; then
+  echo "ERROR: couldn't find nixCacheHostKey in $variables_nix" >&2
+  exit 1
+fi
+current_type_blob="$(awk '{print $1, $2}' <<<"$current_value")"
+current_label="$(awk '{print $3}' <<<"$current_value")"
+
+echo "Scanning ${host} for its current ed25519 SSH host key..."
+nix_extra_opts
+scanned="$(nix-shell "${NIX_OPTS[@]}" -p openssh --run "ssh-keyscan -t ed25519 -T 5 '${host}'" 2>/dev/null | grep -v '^#' | head -1 || true)"
+if [[ -z "$scanned" ]]; then
+  echo "ERROR: couldn't reach ${host} (or got no ed25519 host key back) via ssh-keyscan." >&2
+  exit 2
+fi
+scanned_type_blob="$(awk '{print $2, $3}' <<<"$scanned")"
+
+if [[ "$current_type_blob" == "$scanned_type_blob" ]]; then
+  echo "Up to date: ${host}'s host key matches variables.nix's nixCacheHostKey."
+  exit 0
+fi
+
+echo "DRIFT DETECTED:"
+echo "  variables.nix has: $current_type_blob"
+echo "  ${host} is now:    $scanned_type_blob"
+
+if [[ "$check_only" -eq 1 ]]; then
+  echo
+  echo "Run 'scripts/secrets/sync-nix-cache-host-key.sh' (no flags) to fix." >&2
+  exit 1
+fi
+
+new_value="${scanned_type_blob} ${current_label}"
+
+if [[ "$dry_run" -eq 1 ]]; then
+  echo "(--dry-run: would update variables.nix and ${client_script##*/} to:)"
+  echo "  $new_value"
+  exit 0
+fi
+
+sed -i "s|nixCacheHostKey = \"[^\"]*\"|nixCacheHostKey = \"${new_value}\"|" "$variables_nix"
+sed -i "s|NIX_CACHE_HOST_KEY:=[^}]*}|NIX_CACHE_HOST_KEY:=${new_value}}|" "$client_script"
+
+echo "Updated variables.nix and ${client_script##*/} to:"
+echo "  $new_value"
+echo
+echo "This only takes effect on already-deployed NixOS clients after their"
+echo "next rebuild (programs.ssh.knownHosts is declarative). Review with"
+echo "'git diff', then run 'bash scripts/codex-maintenance.sh' before committing."

variables.nix

@@ -26,7 +26,7 @@
   # fresh client that has never manually ssh'd to nix-cache before. Update
   # this if nix-cache's host key is ever rotated or the host is rebuilt
   # from scratch.
-  nixCacheHostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHrMKZlIGUd3pH9G3AqbsruqUGjxIXMAZw52u9MwiBCn lxc-nix-cache";
+  nixCacheHostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPeWgMsdaiz4axT/deFc1+0B5bN+GX/NOeW9bbQ0c/IT lxc-nix-cache";
 
   # Public keys authorized to SSH in as remoteBuilderUser on the nix-cache
   # host (modules/nix-cache/server.nix) — one per client host that's allowed