diff --git a/CLAUDE.md b/CLAUDE.md index 52d2589..d3fabaf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -116,6 +116,22 @@ lift the shared part into `scripts/lib/` instead of copying it. Purely a local filesystem copy — never touches `.sops.yaml`/ `secrets/*.yaml` or the repo at all. The resulting file is exactly what `rotate-admin-key.sh` expects as its backup-key argument. +- `scripts/secrets/sync-nix-cache-host-key.sh [--check] [--dry-run] + [--host ]` — detects drift between the ed25519 SSH host key + nix-cache is actually serving right now (via `ssh-keyscan`) and + `vars.nixCacheHostKey` (`variables.nix`), the value + `modules/nix-cache/remote-builder-client.nix` bakes into every real + client's declarative `programs.ssh.knownHosts` and + `configure-nix-cache-client.sh` hardcodes as its own default for + non-NixOS clients. That value has no automatic source of truth — it's + set once from whatever nix-cache's host key happened to be at the time, + and silently goes stale if the host is ever rebuilt/recreated with a new + key, breaking every client's distributed-build SSH trust with no error + that points back here. `--check` (used by `codex-maintenance.sh`, which + treats an unreachable nix-cache — e.g. from a non-LAN CI runner — as a + silent skip rather than a failure) only reports drift; the no-flags form + updates both files in place. Declarative clients still need a rebuild to + pick up the fix. ### `scripts/proxmox/` diff --git a/docs/nix-cache.md b/docs/nix-cache.md index 51ee44c..66b7bc3 100644 --- a/docs/nix-cache.md +++ b/docs/nix-cache.md @@ -46,18 +46,26 @@ the new key up automatically on next activation — no more manual ## Remote builder SSH keys -On each client, install the private key used to authenticate as `nixremote`: +Each client authenticates as `nixremote` using its **own default root SSH +identity** (`/root/.ssh/id_ed25519`) — not a separately-named or shared +keypair. If a client doesn't have one yet: ```bash -sudo install -d -m 0700 /root/.ssh -sudo install -m 0600 ./nixremote /root/.ssh/nixremote -sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version +sudo ssh-keygen -t ed25519 -N '' -f /root/.ssh/id_ed25519 ``` -On `nix-cache`, install the matching public key used by `nixremote` authorized keys. +Then add its `.pub` contents as a new entry in `vars.remoteBuilderAuthorizedKeys` +(`variables.nix`) and rebuild `nix-cache` to pick it up (that list is +declarative — an imperative `ssh-copy-id nixremote@nix-cache` won't stick; +it gets overwritten on every rebuild). Verify with: -The committed `nixremote` authorized keys are public SSH keys only. Keep the -matching private keys on client hosts and out of the repository. +```bash +sudo ssh -i /root/.ssh/id_ed25519 nixremote@nix-cache nix-store --version +``` + +The committed `remoteBuilderAuthorizedKeys` entries are public SSH keys +only. Keep the matching private keys on client hosts and out of the +repository. nix-cache's own SSH *host* key is trusted declaratively via `programs.ssh.knownHosts` in `modules/nix-cache/remote-builder-client.nix`, @@ -76,8 +84,8 @@ After deployment: curl http://nix-cache/nix-cache-info nix store ping --store http://nix-cache nix show-config | grep -E 'substituters|trusted-public-keys|builders-use-substitutes' -sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version -nix build nixpkgs#hello --builders 'ssh://nixremote@nix-cache x86_64-linux /root/.ssh/nixremote 4 2 big-parallel,kvm,nixos-test,benchmark' -L +sudo ssh -i /root/.ssh/id_ed25519 nixremote@nix-cache nix-store --version +nix build nixpkgs#hello --builders 'ssh://nixremote@nix-cache x86_64-linux /root/.ssh/id_ed25519 4 2 big-parallel,kvm,nixos-test,benchmark' -L nix path-info -r nixpkgs#hello curl -I "http://nix-cache/$(basename "$(nix path-info nixpkgs#hello)").narinfo" ``` diff --git a/modules/nix-cache/remote-builder-client.nix b/modules/nix-cache/remote-builder-client.nix index a3a641a..e705c74 100644 --- a/modules/nix-cache/remote-builder-client.nix +++ b/modules/nix-cache/remote-builder-client.nix @@ -1,10 +1,14 @@ { pkgs, vars, ... }: { - # Install the remote builder key on each client host (do not commit private keys): - # sudo install -d -m 0700 /root/.ssh - # sudo install -m 0600 ./nixremote /root/.ssh/nixremote - # sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version + # Authenticate as nixremote using the client host's own default root SSH + # identity (/root/.ssh/id_ed25519) rather than a separately-named key -- + # matches vars.remoteBuilderAuthorizedKeys, which already authorizes + # each host's own default key (one entry per host, not a shared + # dedicated keypair). If this host doesn't have one yet: + # sudo -u root ssh-keygen -t ed25519 -N '' -f /root/.ssh/id_ed25519 + # # then add its .pub to vars.remoteBuilderAuthorizedKeys and rebuild nix-cache + # sudo ssh -i /root/.ssh/id_ed25519 nixremote@nix-cache nix-store --version # Trust nix-cache's SSH host key declaratively so the nix-daemon (root) # can connect the first time without a manual ssh-keyscan/known_hosts # step on every new client. @@ -20,7 +24,7 @@ { hostName = vars.nixCacheHost; sshUser = vars.remoteBuilderUser; - sshKey = "/root/.ssh/${vars.remoteBuilderUser}"; + sshKey = "/root/.ssh/id_ed25519"; inherit (pkgs.stdenv.hostPlatform) system; maxJobs = 4; speedFactor = 2; diff --git a/scripts/codex-maintenance.sh b/scripts/codex-maintenance.sh index 8e5f845..58c034e 100755 --- a/scripts/codex-maintenance.sh +++ b/scripts/codex-maintenance.sh @@ -41,6 +41,20 @@ echo echo "Running statix lint..." nix run "${NIX_EVAL_FLAGS[@]}" github:NixOS/nixpkgs/nixos-25.11#statix -- check . +echo +echo "Checking nix-cache host key for drift..." +if bash "${script_dir}/secrets/sync-nix-cache-host-key.sh" --check; then + : +else + drift_status=$? + if [[ "$drift_status" -eq 2 ]]; then + echo "nix-cache unreachable from here -- skipping host-key drift check." + else + echo "WARNING: nix-cache's host key has drifted from variables.nix (see above)." >&2 + echo " Run 'bash scripts/secrets/sync-nix-cache-host-key.sh' to fix." >&2 + fi +fi + echo echo "Evaluating host toplevel derivations..." for host in $hosts; do diff --git a/scripts/proxmox/configure-nix-cache-client.sh b/scripts/proxmox/configure-nix-cache-client.sh index 6c6c66f..b191c11 100755 --- a/scripts/proxmox/configure-nix-cache-client.sh +++ b/scripts/proxmox/configure-nix-cache-client.sh @@ -15,6 +15,16 @@ # this repo -- update both if nix-cache is ever rebuilt with a new host # key or the cache signing key is rotated (see docs/nix-cache.md). # +# REMOTE_BUILDER_KEY defaults to this machine's own default root SSH +# identity (matches modules/nix-cache/remote-builder-client.nix's +# convention for real NixOS clients: authenticate as nixremote with the +# host's own default key, added individually to +# vars.remoteBuilderAuthorizedKeys, rather than a separately-named or +# shared keypair) -- generate one with +# `ssh-keygen -t ed25519 -N '' -f /root/.ssh/id_ed25519` if this machine +# doesn't have one yet, then add its .pub to vars.remoteBuilderAuthorizedKeys +# and rebuild nix-cache. +# # Usage: # sudo ./configure-nix-cache-client.sh [--dry-run] [--no-remote-builder] [--no-restart] # @@ -24,9 +34,9 @@ set -euo pipefail : "${NIX_CACHE_HOST:=nix-cache}" -: "${NIX_CACHE_HOST_KEY:=ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHrMKZlIGUd3pH9G3AqbsruqUGjxIXMAZw52u9MwiBCn lxc-nix-cache}" +: "${NIX_CACHE_HOST_KEY:=ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPeWgMsdaiz4axT/deFc1+0B5bN+GX/NOeW9bbQ0c/IT lxc-nix-cache}" : "${REMOTE_BUILDER_USER:=nixremote}" -: "${REMOTE_BUILDER_KEY:=/root/.ssh/nixremote}" +: "${REMOTE_BUILDER_KEY:=/root/.ssh/id_ed25519}" CACHE_PUB_KEY="cache.local-1:usoWYanY3Kpq2+kDIS2nhWoLZiRxanmdysdzqCFBHW4=" FALLBACK_URL="https://cache.nixos.org/" diff --git a/scripts/secrets/sync-nix-cache-host-key.sh b/scripts/secrets/sync-nix-cache-host-key.sh new file mode 100755 index 0000000..8382f50 --- /dev/null +++ b/scripts/secrets/sync-nix-cache-host-key.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Detects and fixes drift between the ed25519 SSH host key nix-cache is +# actually serving right now and vars.nixCacheHostKey (variables.nix) -- +# the value modules/nix-cache/remote-builder-client.nix bakes into every +# client's declarative programs.ssh.knownHosts, and +# scripts/proxmox/configure-nix-cache-client.sh hardcodes as its own +# default for non-NixOS clients. +# +# This value has no automatic source of truth: nix-cache's host key is +# generated once (first boot / container recreate) and never touches this +# repo again unless someone remembers to update it by hand afterwards. It +# drifted silently once already -- confirmed live: variables.nix recorded +# a key that no longer matched what nix-cache actually presented, which +# would fail every real client's SSH host-key verification for +# distributed builds without ever producing an obvious error pointing +# back here (a client just sees "Host key verification failed" against +# *some* key, with no hint that the trusted value itself was stale). +# +# codex-maintenance.sh runs this in --check mode on every invocation so +# that drift surfaces as a warning instead of a future debugging session. +# +# Usage: +# scripts/secrets/sync-nix-cache-host-key.sh [--check] [--dry-run] [--host ] +# +# --check Only report drift (exit 1 if found, 2 if nix-cache is +# unreachable); never writes. For CI/maintenance use. +# --dry-run Show what would change; never writes. +# --host Override the hostname to scan (default: variables.nix's +# nixCacheHost / env.sh's NIX_CACHE_HOST). +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +variables_nix="${repo_root}/variables.nix" +client_script="${repo_root}/scripts/proxmox/configure-nix-cache-client.sh" + +# shellcheck source=../env.sh +source "${repo_root}/scripts/env.sh" + +check_only=0 +dry_run=0 +host="${NIX_CACHE_HOST}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --check) check_only=1; shift ;; + --dry-run) dry_run=1; shift ;; + --host) + host="${2:?--host requires a hostname}" + shift 2 + ;; + -h|--help) + sed -n '2,23p' "$0" + exit 0 + ;; + *) + echo "ERROR: unknown argument: $1" >&2 + exit 1 + ;; + esac +done + +current_value="$(grep -oE 'nixCacheHostKey = "[^"]+"' "$variables_nix" | sed -E 's/nixCacheHostKey = "(.*)"/\1/')" +if [[ -z "$current_value" ]]; then + echo "ERROR: couldn't find nixCacheHostKey in $variables_nix" >&2 + exit 1 +fi +current_type_blob="$(awk '{print $1, $2}' <<<"$current_value")" +current_label="$(awk '{print $3}' <<<"$current_value")" + +echo "Scanning ${host} for its current ed25519 SSH host key..." +nix_extra_opts +scanned="$(nix-shell "${NIX_OPTS[@]}" -p openssh --run "ssh-keyscan -t ed25519 -T 5 '${host}'" 2>/dev/null | grep -v '^#' | head -1 || true)" +if [[ -z "$scanned" ]]; then + echo "ERROR: couldn't reach ${host} (or got no ed25519 host key back) via ssh-keyscan." >&2 + exit 2 +fi +scanned_type_blob="$(awk '{print $2, $3}' <<<"$scanned")" + +if [[ "$current_type_blob" == "$scanned_type_blob" ]]; then + echo "Up to date: ${host}'s host key matches variables.nix's nixCacheHostKey." + exit 0 +fi + +echo "DRIFT DETECTED:" +echo " variables.nix has: $current_type_blob" +echo " ${host} is now: $scanned_type_blob" + +if [[ "$check_only" -eq 1 ]]; then + echo + echo "Run 'scripts/secrets/sync-nix-cache-host-key.sh' (no flags) to fix." >&2 + exit 1 +fi + +new_value="${scanned_type_blob} ${current_label}" + +if [[ "$dry_run" -eq 1 ]]; then + echo "(--dry-run: would update variables.nix and ${client_script##*/} to:)" + echo " $new_value" + exit 0 +fi + +sed -i "s|nixCacheHostKey = \"[^\"]*\"|nixCacheHostKey = \"${new_value}\"|" "$variables_nix" +sed -i "s|NIX_CACHE_HOST_KEY:=[^}]*}|NIX_CACHE_HOST_KEY:=${new_value}}|" "$client_script" + +echo "Updated variables.nix and ${client_script##*/} to:" +echo " $new_value" +echo +echo "This only takes effect on already-deployed NixOS clients after their" +echo "next rebuild (programs.ssh.knownHosts is declarative). Review with" +echo "'git diff', then run 'bash scripts/codex-maintenance.sh' before committing." diff --git a/variables.nix b/variables.nix index d9e339e..231d93c 100644 --- a/variables.nix +++ b/variables.nix @@ -26,7 +26,7 @@ # fresh client that has never manually ssh'd to nix-cache before. Update # this if nix-cache's host key is ever rotated or the host is rebuilt # from scratch. - nixCacheHostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHrMKZlIGUd3pH9G3AqbsruqUGjxIXMAZw52u9MwiBCn lxc-nix-cache"; + nixCacheHostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPeWgMsdaiz4axT/deFc1+0B5bN+GX/NOeW9bbQ0c/IT lxc-nix-cache"; # Public keys authorized to SSH in as remoteBuilderUser on the nix-cache # host (modules/nix-cache/server.nix) — one per client host that's allowed