From 78bb78426550b071887b83f414876b4c6166e16f Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Sat, 25 Jul 2026 17:14:25 +1000 Subject: [PATCH 1/4] fix(provision): block build until sops changes are committed, guard missing host keys Three ordering-related fixes to the Proxmox provisioning flow: 1. prepare-host-key.sh: make idempotent -- if the key already exists, print a note and exit 0 instead of erroring. The caller (create-proxmox-resource.sh) already guards standalone calls, but the script itself should be safe to run directly on a host that was already keyed. 2. create-proxmox-resource.sh: after sync-host-keys.sh updates .sops.yaml / secrets/, detect uncommitted changes and block with a prompt until the operator confirms they've committed and pushed. The PVE node's git pull only picks up committed+pushed state; without this gate, a new host's sops recipient is missing from the secrets files the image build uses, so the host can't decrypt secrets on first boot. 3. create-proxmox-resource.sh: add an explicit existence check for the host key in both the LXC and VM remote build heredocs, before it's passed as --pre-format-files / NIXOS_HOST_KEYS_DIR input. Gives a clear error pointing at sync-host-keys.sh instead of a raw `cp: cannot stat` from disko deep in the build. Co-Authored-By: Claude Sonnet 4.6 --- scripts/proxmox/create-proxmox-resource.sh | 35 ++++++++++++++++++++++ scripts/secrets/prepare-host-key.sh | 5 ++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/scripts/proxmox/create-proxmox-resource.sh b/scripts/proxmox/create-proxmox-resource.sh index 84903f4..35d6aa1 100755 --- a/scripts/proxmox/create-proxmox-resource.sh +++ b/scripts/proxmox/create-proxmox-resource.sh @@ -502,6 +502,29 @@ sync_args=("$flake_target") [[ "$dry_run" -eq 1 ]] && sync_args+=(--dry-run) bash "$sync_keys" "${sync_args[@]}" +# If sync-host-keys.sh changed .sops.yaml or secrets/, those changes must be +# committed and pushed before the remote `git pull` below picks them up -- +# the PVE node builds from whatever HEAD is checked out there, not the local +# working tree. Detect uncommitted changes and block until the operator +# confirms they've pushed, so the build never runs against a stale flake. +if [[ "$dry_run" -eq 0 ]]; then + _sops_dirty="$(git -C "$repo_root" status --porcelain -- .sops.yaml secrets/ 2>/dev/null || true)" + if [[ -n "$_sops_dirty" ]]; then + echo + echo "==> COMMIT + PUSH REQUIRED before the remote build can succeed:" + echo " sync-host-keys.sh modified .sops.yaml / secrets/ to register the" + echo " new host's sops recipient. The PVE node builds from the git-tracked" + echo " flake, so these changes must be committed and pushed first -- otherwise" + echo " the image build will succeed but the host cannot decrypt secrets on" + echo " first boot (its age key isn't in the encrypted secrets files yet)." + echo + git -C "$repo_root" status --short -- .sops.yaml secrets/ || true + echo + read -rp " Commit and push those changes, then press Enter to continue (Ctrl-C to abort): " + fi + unset _sops_dirty +fi + # --- VMID: pick one, and refuse to touch anything that already exists --- echo if [[ -z "$vmid" ]]; then @@ -707,6 +730,12 @@ cd "$repo_dir" # right after a successful install. . scripts/lib/nix-bootstrap.sh ensure_nix_profile +if [[ ! -f "host-keys/${target}_ssh_host_ed25519_key" ]]; then + echo "ERROR: host-keys/${target}_ssh_host_ed25519_key not found in ${repo_dir}." >&2 + echo "Generate the key locally (scripts/secrets/sync-host-keys.sh ${target})" >&2 + echo "and ensure it was synced here before starting the build." >&2 + exit 1 +fi NIXOS_HOST_KEYS_DIR="$(pwd)/host-keys" nix build --impure \ --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \ ".#nixosConfigurations.${target}.config.system.build.tarball" \ @@ -747,6 +776,12 @@ declare -a NIX_OPTS=() cd "$repo_dir" . scripts/lib/nix-bootstrap.sh ensure_nix_profile +if [[ ! -f "host-keys/${target}_ssh_host_ed25519_key" ]]; then + echo "ERROR: host-keys/${target}_ssh_host_ed25519_key not found in ${repo_dir}." >&2 + echo "Generate the key locally (scripts/secrets/sync-host-keys.sh ${target})" >&2 + echo "and ensure it was synced here before starting the build." >&2 + exit 1 +fi nix build --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \ ".#nixosConfigurations.${target}.config.system.build.diskoImagesScript" \ --out-link "result-${target}" diff --git a/scripts/secrets/prepare-host-key.sh b/scripts/secrets/prepare-host-key.sh index a148cf0..a72e192 100755 --- a/scripts/secrets/prepare-host-key.sh +++ b/scripts/secrets/prepare-host-key.sh @@ -41,8 +41,9 @@ mkdir -p "$keydir" keyfile="${keydir}/${hostname}_ssh_host_ed25519_key" if [[ -f "$keyfile" ]]; then - echo "ERROR: $keyfile already exists. Remove it first if you want to regenerate." >&2 - exit 1 + echo "Key already exists: ${keyfile}" + echo "Reusing the existing key. Remove it first if you want to regenerate." + exit 0 fi nix_extra_opts -- 2.54.0 From a9745b594b7baf82569efe20e94a4e136f804e86 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Sat, 25 Jul 2026 17:29:29 +1000 Subject: [PATCH 2/4] feat(flake): add clan-core 26.05 as a flake input (Phase 1, no behavior change) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces clan-core pinned to its 26.05 release alongside the existing nixpkgs 26.05 input. No host configuration is changed — this is a pure dependency addition so Phase 2 (per-host vars/secret management migration) has the input available without a separate flake.lock bump. clan-core.inputs.nixpkgs.follows = "nixpkgs" keeps a single nixpkgs closure. sops-nix remains as a flake input; vars layers on top of it rather than replacing it (clan's sops storage backend still needs sops-nix). All hosts evaluate cleanly (codex-maintenance.sh --full-check equivalent triggered by the flake.nix change). Co-Authored-By: Claude Sonnet 4.6 --- flake.lock | 194 ++++++++++++++++++++++++++++++++++++++++++++++++++++- flake.nix | 4 ++ 2 files changed, 195 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 0137e45..ade6df1 100644 --- a/flake.lock +++ b/flake.lock @@ -1,6 +1,80 @@ { "nodes": { + "clan-core": { + "inputs": { + "data-mesher": "data-mesher", + "disko": "disko", + "flake-parts": "flake-parts", + "nix-darwin": "nix-darwin", + "nix-select": "nix-select", + "nixpkgs": [ + "nixpkgs" + ], + "sops-nix": "sops-nix", + "systems": "systems", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1783497933, + "narHash": "sha256-TxmwEews6URFPqOWEHNychtXbFDgLZjbOfEXtvtOm6U=", + "rev": "3dc0221ca09033599fe98055e9bbc81bdf32732a", + "type": "tarball", + "url": "https://git.clan.lol/api/v1/repos/clan/clan-core/archive/3dc0221ca09033599fe98055e9bbc81bdf32732a.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz" + } + }, + "data-mesher": { + "inputs": { + "flake-parts": [ + "clan-core", + "flake-parts" + ], + "nixpkgs": [ + "clan-core", + "nixpkgs" + ], + "treefmt-nix": [ + "clan-core", + "treefmt-nix" + ] + }, + "locked": { + "lastModified": 1778718524, + "narHash": "sha256-pXLoI6Ax0EnUK6r34UM1vibVC7CfTu6j72R2692ZzPs=", + "rev": "12c552ad547d87254f33f33bddd1a2cdbeac754d", + "type": "tarball", + "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/12c552ad547d87254f33f33bddd1a2cdbeac754d.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://git.clan.lol/clan/data-mesher/archive/main.tar.gz" + } + }, "disko": { + "inputs": { + "nixpkgs": [ + "clan-core", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1780290312, + "narHash": "sha256-eTAlX0CwgB84Ts3GaBd944A3DRXVMzgA0EqroZBISUo=", + "owner": "nix-community", + "repo": "disko", + "rev": "115e5211780054d8a890b41f0b7734cafad54dfe", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "disko", + "type": "github" + } + }, + "disko_2": { "inputs": { "nixpkgs": [ "nixpkgs" @@ -51,9 +125,30 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "clan-core", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1778716662, + "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "inputs": { - "systems": "systems" + "systems": "systems_2" }, "locked": { "lastModified": 1694529238, @@ -109,6 +204,40 @@ "type": "github" } }, + "nix-darwin": { + "inputs": { + "nixpkgs": [ + "clan-core", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1779036909, + "narHash": "sha256-zXcwYQGCT6pzinK+1dBB2ekTVtfxGZAapb3Evdcu4fY=", + "owner": "nix-darwin", + "repo": "nix-darwin", + "rev": "56c666e108467d87d13508936aade6d567f2a501", + "type": "github" + }, + "original": { + "owner": "nix-darwin", + "repo": "nix-darwin", + "type": "github" + } + }, + "nix-select": { + "locked": { + "lastModified": 1763303120, + "narHash": "sha256-yxcNOha7Cfv2nhVpz9ZXSNKk0R7wt4AiBklJ8D24rVg=", + "rev": "3d1e3860bef36857a01a2ddecba7cdb0a14c35a9", + "type": "tarball", + "url": "https://git.clan.lol/api/v1/repos/clan/nix-select/archive/3d1e3860bef36857a01a2ddecba7cdb0a14c35a9.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://git.clan.lol/clan/nix-select/archive/main.tar.gz" + } + }, "nixos-conf-editor": { "inputs": { "flake-compat": "flake-compat", @@ -163,11 +292,12 @@ }, "root": { "inputs": { - "disko": "disko", + "clan-core": "clan-core", + "disko": "disko_2", "home-manager": "home-manager", "nixos-conf-editor": "nixos-conf-editor", "nixpkgs": "nixpkgs_2", - "sops-nix": "sops-nix" + "sops-nix": "sops-nix_2" } }, "snowfall-lib": { @@ -194,6 +324,27 @@ } }, "sops-nix": { + "inputs": { + "nixpkgs": [ + "clan-core", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1780547341, + "narHash": "sha256-Gq8KNx5A7hBB3uGJaj6eQfLDIz5YdLu92gqBcvHvoUo=", + "owner": "Mic92", + "repo": "sops-nix", + "rev": "9ed65852b6257fbeae4355bc24ecfea307ca759a", + "type": "github" + }, + "original": { + "owner": "Mic92", + "repo": "sops-nix", + "type": "github" + } + }, + "sops-nix_2": { "inputs": { "nixpkgs": [ "nixpkgs" @@ -214,6 +365,22 @@ } }, "systems": { + "locked": { + "lastModified": 1774449309, + "narHash": "sha256-brhZ8DmuGtzkCYHJg4HEd602amKm89Y9ytsFZ5uWD1w=", + "owner": "nix-systems", + "repo": "default", + "rev": "c29398b59d2048c4ab79345812849c9bd15e9150", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "ref": "future-26.11", + "repo": "default", + "type": "github" + } + }, + "systems_2": { "locked": { "lastModified": 1681028828, "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", @@ -227,6 +394,27 @@ "repo": "default", "type": "github" } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "clan-core", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1780220602, + "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "db947814a175b7ca6ded66e21383d938df01c227", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 67a0d35..5d67e9f 100644 --- a/flake.nix +++ b/flake.nix @@ -16,6 +16,10 @@ url = "github:Mic92/sops-nix"; inputs.nixpkgs.follows = "nixpkgs"; }; + clan-core = { + url = "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = { self, nixpkgs, nixos-conf-editor, home-manager, sops-nix, ... } @ inputs: -- 2.54.0 From a63e1c70c3c9ef86304c2f5ceb2f7a295a38933c Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Sat, 25 Jul 2026 18:22:33 +1000 Subject: [PATCH 3/4] =?UTF-8?q?feat(provision):=20Phase=202=20=E2=80=94=20?= =?UTF-8?q?migrate=20SSH=20host=20keys=20to=20clan=20vars?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the gitignored host-keys/ directory with clan vars as the authoritative storage for SSH host keys. Keys are now generated as sops-binary-encrypted clan var files (admin-key only) and checked into vars/per-machine//openssh/, eliminating the plaintext private key that previously had to live outside the repo. Changes: - modules/clan/ssh-host-key.nix: clan vars generator for the ed25519 SSH host key pair (neededFor="activation" — not mapped to sops.secrets, delivered via tarball baking for LXC or --pre-format-files for VMs) - flake.nix: add clanCore module + required settings to every mkTarget; deduplicate bundled disko/sops-nix via follows; all 27 hosts eval clean - flake.lock: updated to reflect the new follows constraints - scripts/lib/clan-vars.sh: new helper library with clan_ssh_key_exists / clan_ssh_pubkey_path / clan_decrypt_ssh_key / clan_generate_ssh_key for use by the provisioning and sync scripts - scripts/secrets/sync-host-keys.sh: queue_host_sync() now checks clan vars first; generates via clan_generate_ssh_key if no key exists; derives age fingerprint from clan pub key for .sops.yaml registration - scripts/proxmox/create-proxmox-resource.sh: key management simplified (sync-host-keys.sh now generates the key if missing, so the inline prepare-host-key.sh call is gone); sync_remote_host_keys() decrypts the clan key into a temp dir and scps just the two files to the node when a clan key exists, falling back to the old host-keys/ scp for any remaining legacy entries Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01B2EJ4qTsM5KUqhS5c3GAwx --- flake.lock | 54 ++--------- flake.nix | 27 +++++- modules/clan/ssh-host-key.nix | 30 ++++++ scripts/lib/clan-vars.sh | 106 +++++++++++++++++++++ scripts/proxmox/create-proxmox-resource.sh | 51 ++++++---- scripts/secrets/sync-host-keys.sh | 25 +++-- 6 files changed, 218 insertions(+), 75 deletions(-) create mode 100644 modules/clan/ssh-host-key.nix create mode 100644 scripts/lib/clan-vars.sh diff --git a/flake.lock b/flake.lock index ade6df1..d5f6616 100644 --- a/flake.lock +++ b/flake.lock @@ -3,14 +3,18 @@ "clan-core": { "inputs": { "data-mesher": "data-mesher", - "disko": "disko", + "disko": [ + "disko" + ], "flake-parts": "flake-parts", "nix-darwin": "nix-darwin", "nix-select": "nix-select", "nixpkgs": [ "nixpkgs" ], - "sops-nix": "sops-nix", + "sops-nix": [ + "sops-nix" + ], "systems": "systems", "treefmt-nix": "treefmt-nix" }, @@ -54,27 +58,6 @@ } }, "disko": { - "inputs": { - "nixpkgs": [ - "clan-core", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1780290312, - "narHash": "sha256-eTAlX0CwgB84Ts3GaBd944A3DRXVMzgA0EqroZBISUo=", - "owner": "nix-community", - "repo": "disko", - "rev": "115e5211780054d8a890b41f0b7734cafad54dfe", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "disko", - "type": "github" - } - }, - "disko_2": { "inputs": { "nixpkgs": [ "nixpkgs" @@ -293,11 +276,11 @@ "root": { "inputs": { "clan-core": "clan-core", - "disko": "disko_2", + "disko": "disko", "home-manager": "home-manager", "nixos-conf-editor": "nixos-conf-editor", "nixpkgs": "nixpkgs_2", - "sops-nix": "sops-nix_2" + "sops-nix": "sops-nix" } }, "snowfall-lib": { @@ -324,27 +307,6 @@ } }, "sops-nix": { - "inputs": { - "nixpkgs": [ - "clan-core", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1780547341, - "narHash": "sha256-Gq8KNx5A7hBB3uGJaj6eQfLDIz5YdLu92gqBcvHvoUo=", - "owner": "Mic92", - "repo": "sops-nix", - "rev": "9ed65852b6257fbeae4355bc24ecfea307ca759a", - "type": "github" - }, - "original": { - "owner": "Mic92", - "repo": "sops-nix", - "type": "github" - } - }, - "sops-nix_2": { "inputs": { "nixpkgs": [ "nixpkgs" diff --git a/flake.nix b/flake.nix index 5d67e9f..47ac295 100644 --- a/flake.nix +++ b/flake.nix @@ -18,7 +18,16 @@ }; clan-core = { url = "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz"; - inputs.nixpkgs.follows = "nixpkgs"; + # Deduplicate modules: clan-core bundles its own disko and sops-nix + # (both imported by nixosModules.clanCore). Without follows, we'd get + # two different versions of each, and disko's _module.args.diskoLib + # unique option would conflict. With follows, clan-core uses the same + # store paths as us, so NixOS deduplicates the imports. + inputs = { + nixpkgs.follows = "nixpkgs"; + disko.follows = "disko"; + sops-nix.follows = "sops-nix"; + }; }; }; @@ -45,6 +54,22 @@ modules = [ inputs.disko.nixosModules.disko sops-nix.nixosModules.sops + inputs.clan-core.nixosModules.clanCore + { + # Required clan settings. directory is the flake root (where + # vars/ and sops/ directories live); machine.name is the flake + # target name (matches what clan vars generate uses as the key + # under vars/per-machine/). enableRecommendedDefaults = false + # is mandatory: without it, clan unconditionally enables + # networking.useNetworkd, adds packages, and tweaks nix settings + # -- none of which belong here. + clan.core = { + settings.directory = self; + settings.machine.name = flakeTarget; + enableRecommendedDefaults = false; + }; + } + ./modules/clan/ssh-host-key.nix ./modules/common/configuration.nix ./modules/platforms/${platform}.nix ./modules/build-types/${buildType}.nix diff --git a/modules/clan/ssh-host-key.nix b/modules/clan/ssh-host-key.nix new file mode 100644 index 0000000..9c2483c --- /dev/null +++ b/modules/clan/ssh-host-key.nix @@ -0,0 +1,30 @@ +{ pkgs, ... }: { + # Defines the SSH host key as a clan vars generator so that: + # - `clan vars generate ` creates and encrypts the key pair + # - The private key lives at vars/per-machine//openssh/ssh_host_ed25519_key/secret + # (sops binary-encrypted, admin-key-only; decrypted by the build script) + # - The public key lives at vars/per-machine//openssh/ssh_host_ed25519_key.pub/value + # (plaintext; used by sync-host-keys.sh to derive the sops age fingerprint) + # + # neededFor = "activation" means clan's deployment tool would upload this + # before running nixos-rebuild/nixos-install (for VM/baremetal via + # nixos-anywhere). For lxc-* hosts, the build script bakes it into the + # tarball directly via NIXOS_HOST_KEYS_DIR -- the neededFor value here + # simply ensures it is NOT mapped to sops.secrets (which would try to + # decrypt it at runtime as a regular service secret, which is wrong: the + # SSH host key reaches the container via the tarball, not sops). + clan.core.vars.generators.openssh = { + files."ssh_host_ed25519_key" = { + secret = true; + neededFor = "activation"; + }; + files."ssh_host_ed25519_key.pub" = { + secret = false; + neededFor = "activation"; + }; + runtimeInputs = [ pkgs.openssh ]; + script = '' + ssh-keygen -t ed25519 -N "" -C "" -f "$out/ssh_host_ed25519_key" + ''; + }; +} diff --git a/scripts/lib/clan-vars.sh b/scripts/lib/clan-vars.sh new file mode 100644 index 0000000..e0624b6 --- /dev/null +++ b/scripts/lib/clan-vars.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Clan vars helpers: manage SSH host keys stored as clan vars (sops-encrypted +# binary files under vars/per-machine//openssh/) instead of the +# gitignored host-keys/ directory. +# +# Layout (per clan's convention): +# vars/per-machine//openssh/ssh_host_ed25519_key/secret -- sops binary (admin-encrypted) +# vars/per-machine//openssh/ssh_host_ed25519_key.pub/value -- plaintext SSH pubkey +# +# Sourced by create-proxmox-resource.sh and sync-host-keys.sh. +# Depends on sops-age.sh and ssh-host-keys.sh being sourced first (for +# sops_yaml_admin_pubkey, ssh_pubkey_to_age, and NIX_OPTS). + +if ! declare -p NIX_OPTS >/dev/null 2>&1; then + declare -a NIX_OPTS=() +fi + +# clan_ssh_key_exists +# Returns 0 if clan vars hold a SSH host key for , 1 otherwise. +clan_ssh_key_exists() { + local target="$1" repo_root="$2" + [[ -f "${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key/secret" ]] +} + +# clan_ssh_pubkey_path +# Prints the path to the plaintext SSH public key value file. +clan_ssh_pubkey_path() { + local target="$1" repo_root="$2" + echo "${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key.pub/value" +} + +# clan_decrypt_ssh_key +# Decrypts the sops-encrypted SSH host private key for into , +# naming it _ssh_host_ed25519_key (to match NIXOS_HOST_KEYS_DIR +# conventions that lxc.nix and the disko build already expect). Also copies +# the plaintext public key. The caller is responsible for protecting and +# cleaning up . +clan_decrypt_ssh_key() { + local target="$1" repo_root="$2" dest_dir="$3" + local secret="${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key/secret" + local pubval="${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key.pub/value" + local dest_priv="${dest_dir}/${target}_ssh_host_ed25519_key" + local dest_pub="${dest_dir}/${target}_ssh_host_ed25519_key.pub" + + nix-shell "${NIX_OPTS[@]}" -p sops --run \ + "sops -d --output-type binary '${secret}'" > "$dest_priv" + chmod 0600 "$dest_priv" + cp "$pubval" "$dest_pub" +} + +# clan_generate_ssh_key +# Generates a new SSH host key pair and stores it in clan vars format: +# - private key: sops binary-encrypted for the admin age key +# - public key: plaintext value file +# Idempotent: if the secret already exists, prints a note and returns 0. +# Requires sops_yaml_admin_pubkey (from sops-age.sh) to be available. +clan_generate_ssh_key() { + local target="$1" repo_root="$2" + local var_base="${repo_root}/vars/per-machine/${target}/openssh" + local secret_dir="${var_base}/ssh_host_ed25519_key" + local pubval_dir="${var_base}/ssh_host_ed25519_key.pub" + + if [[ -f "${secret_dir}/secret" ]]; then + echo "Clan SSH host key for ${target} already exists -- skipping generation." + return 0 + fi + + # Resolve admin age public key from .sops.yaml + local admin_pubkey + admin_pubkey="$(sops_yaml_admin_pubkey "${repo_root}/.sops.yaml")" + if [[ -z "$admin_pubkey" ]]; then + echo "ERROR: Could not find &admin age key in ${repo_root}/.sops.yaml" >&2 + return 1 + fi + + # Generate the SSH key pair in a secure temp directory + local tmpdir + tmpdir="$(mktemp -d)" + local priv_tmp="${tmpdir}/ssh_host_ed25519_key" + + # shellcheck disable=SC2064 + trap "rm -rf '${tmpdir}'" RETURN + + nix-shell "${NIX_OPTS[@]}" -p openssh --run \ + "ssh-keygen -t ed25519 -N '' -C '${target}' -f '${priv_tmp}'" >/dev/null + + # Create a minimal sops config that uses only the admin age key -- this + # prevents sops from merging in ALL recipients from .sops.yaml (which + # would unnecessarily encrypt for every host's key, not just admin). + local sops_cfg="${tmpdir}/sops-config.json" + printf '{"creation_rules":[{"key_groups":[{"age":["%s"]}]}]}\n' \ + "$admin_pubkey" > "$sops_cfg" + + # Encrypt the private key in sops binary format (admin-only recipient) + mkdir -p "$secret_dir" "$pubval_dir" + nix-shell "${NIX_OPTS[@]}" -p sops --run \ + "sops -e --config '${sops_cfg}' --input-type binary '${priv_tmp}'" \ + > "${secret_dir}/secret" + + # Store the public key as a plaintext value file + cp "${priv_tmp}.pub" "${pubval_dir}/value" + + echo "Generated and stored clan SSH host key for ${target}." + echo " Private key: ${secret_dir}/secret (sops binary, admin-key encrypted)" + echo " Public key: ${pubval_dir}/value" +} diff --git a/scripts/proxmox/create-proxmox-resource.sh b/scripts/proxmox/create-proxmox-resource.sh index 35d6aa1..6a231d6 100755 --- a/scripts/proxmox/create-proxmox-resource.sh +++ b/scripts/proxmox/create-proxmox-resource.sh @@ -59,6 +59,10 @@ source "${repo_root}/scripts/env.sh" source "${repo_root}/scripts/lib/nix-eval.sh" # shellcheck source=../lib/confirm.sh source "${repo_root}/scripts/lib/confirm.sh" +# shellcheck source=../lib/sops-age.sh +source "${repo_root}/scripts/lib/sops-age.sh" +# shellcheck source=../lib/clan-vars.sh +source "${repo_root}/scripts/lib/clan-vars.sh" sync_keys="${repo_root}/scripts/secrets/sync-host-keys.sh" @@ -485,19 +489,10 @@ echo "Target: ${flake_target} (host=${host}, type=${type}) -> Proxmox resource ' nix_extra_opts # --- make sure this target has a registered host key -------------------- +# sync-host-keys.sh is idempotent and generates the key (via clan vars) if +# no key exists yet -- the old inline prepare-host-key.sh call is gone. echo echo "==> Ensuring host key exists and is registered..." -_host_keyfile="${repo_root}/host-keys/${flake_target}_ssh_host_ed25519_key" -if [[ ! -f "$_host_keyfile" ]]; then - echo " No host key found for ${flake_target}; generating one via prepare-host-key.sh..." - _prepare_host_key="${repo_root}/scripts/secrets/prepare-host-key.sh" - if [[ "$dry_run" -eq 1 ]]; then - echo "[dry-run] would run: bash ${_prepare_host_key} ${flake_target}" - else - bash "$_prepare_host_key" "$flake_target" - fi -fi -unset _host_keyfile _prepare_host_key sync_args=("$flake_target") [[ "$dry_run" -eq 1 ]] && sync_args+=(--dry-run) bash "$sync_keys" "${sync_args[@]}" @@ -647,21 +642,37 @@ ensure_remote_repo() { fi } -# --- sync locally-managed host-keys/ to the node --------------------------- -# Gitignored (see .gitignore), so `git pull` above never carries it -- both -# build paths need it present as NIXOS_HOST_KEYS_DIR / --pre-format-files -# input on the node itself now that the build runs there. scp (not rsync, -# not already a dependency anywhere else in this repo) mirrors how this -# script already transfers the --image case below. +# --- sync host key to the node --------------------------------------------- +# Clan-managed keys live in vars/per-machine/ (committed, sops-encrypted), +# so they arrive on the node via `git pull`. But the build scripts expect a +# plaintext key file in host-keys/ (NIXOS_HOST_KEYS_DIR for LXC, or +# --pre-format-files for VM). For clan keys, decrypt locally and scp just the +# two files for this target; for legacy host-keys/ entries, scp the whole dir. sync_remote_host_keys() { echo - echo "==> Syncing host-keys/ to ${node}..." + echo "==> Syncing host key for ${flake_target} to ${node}..." if [[ "$dry_run" -eq 1 ]]; then - echo "[dry-run] would copy ${repo_root}/host-keys/ to ${ssh_target}:${remote_repo_dir}/host-keys/" + if clan_ssh_key_exists "$flake_target" "$repo_root"; then + echo "[dry-run] would decrypt clan SSH key for ${flake_target} and copy to ${ssh_target}:${remote_repo_dir}/host-keys/" + else + echo "[dry-run] would copy ${repo_root}/host-keys/ to ${ssh_target}:${remote_repo_dir}/host-keys/" + fi return fi ssh "$ssh_target" "mkdir -p '${remote_repo_dir}/host-keys'" - scp -pr "${repo_root}/host-keys/." "${ssh_target}:${remote_repo_dir}/host-keys/" + if clan_ssh_key_exists "$flake_target" "$repo_root"; then + local tmpdir + tmpdir="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '${tmpdir}'" RETURN + echo " Decrypting clan SSH key for ${flake_target}..." + clan_decrypt_ssh_key "$flake_target" "$repo_root" "$tmpdir" + scp -p "${tmpdir}/${flake_target}_ssh_host_ed25519_key" \ + "${tmpdir}/${flake_target}_ssh_host_ed25519_key.pub" \ + "${ssh_target}:${remote_repo_dir}/host-keys/" + else + scp -pr "${repo_root}/host-keys/." "${ssh_target}:${remote_repo_dir}/host-keys/" + fi } # --- build (or reuse an image already on the node) ------------------------ diff --git a/scripts/secrets/sync-host-keys.sh b/scripts/secrets/sync-host-keys.sh index fe52536..e089e7e 100755 --- a/scripts/secrets/sync-host-keys.sh +++ b/scripts/secrets/sync-host-keys.sh @@ -39,6 +39,8 @@ source "${repo_root}/scripts/lib/ssh-host-keys.sh" source "${repo_root}/scripts/lib/sops-age.sh" # shellcheck source=../lib/confirm.sh source "${repo_root}/scripts/lib/confirm.sh" +# shellcheck source=../lib/clan-vars.sh +source "${repo_root}/scripts/lib/clan-vars.sh" mkdir -p "$keydir" @@ -146,13 +148,15 @@ dry_run=0 queue_host_sync() { local host="$1" local keyfile="${keydir}/${host}_ssh_host_ed25519_key" - local has_local_key=0 has_anchor=0 + local has_local_key=0 has_clan_key=0 has_anchor=0 [[ -f "$keyfile" ]] && has_local_key=1 + clan_ssh_key_exists "$host" "$repo_root" && has_clan_key=1 grep -qE "^ - &${host} age1" "$sops_yaml" && has_anchor=1 - if [[ "$has_local_key" -eq 0 && "$has_anchor" -eq 1 ]]; then + if [[ "$has_local_key" -eq 0 && "$has_clan_key" -eq 0 && "$has_anchor" -eq 1 ]]; then echo "SKIP ${host}: .sops.yaml already has an &${host} anchor, but" - echo " host-keys/${host}_ssh_host_ed25519_key is missing locally." + echo " neither host-keys/${host}_ssh_host_ed25519_key nor" + echo " vars/per-machine/${host}/openssh/ exist locally." echo " Not generating a replacement -- it wouldn't match whatever's" echo " already registered (and possibly deployed). Remove the" echo " &${host} line from .sops.yaml first if you really want a" @@ -160,21 +164,26 @@ queue_host_sync() { return 1 fi - if [[ "$has_local_key" -eq 0 ]]; then + if [[ "$has_local_key" -eq 0 && "$has_clan_key" -eq 0 ]]; then if [[ "$dry_run" -eq 1 ]]; then - echo "[dry-run] ${host}: would generate host key" + echo "[dry-run] ${host}: would generate host key via clan vars" else - echo "==> ${host}: generating host key" - generate_host_ed25519_key "$host" "$keyfile" + echo "==> ${host}: generating host key via clan vars" + clan_generate_ssh_key "$host" "$repo_root" + has_clan_key=1 fi + elif [[ "$has_clan_key" -eq 1 ]]; then + echo "==> ${host}: clan-managed SSH host key already present" else - echo "==> ${host}: host key already present" + echo "==> ${host}: host key already present (host-keys/)" fi if [[ "$has_anchor" -eq 0 ]]; then local age_pub if [[ "$dry_run" -eq 1 ]]; then age_pub="dry-run-placeholder-not-a-real-key" + elif [[ "$has_clan_key" -eq 1 ]]; then + age_pub="$(ssh_pubkey_to_age "$(clan_ssh_pubkey_path "$host" "$repo_root")")" else age_pub="$(ssh_pubkey_to_age "${keyfile}.pub")" fi -- 2.54.0 From 055577ee9134426a436ee4f3ba851c0d5f9ae58a Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Sat, 25 Jul 2026 19:07:30 +1000 Subject: [PATCH 4/4] fix(lxc): fix activation ordering and add boot-time sops reinstall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs prevented nixos-rebuild switch from working on lxc-* hosts after first boot, both confirmed live on a deployed lxc-tor-relay container: 1. Ordering bug: preserveSshHostKey had no explicit deps, so the topological sort placed it at position 7 — after etc at position 5. By the time it tried to save the SSH key, etc had already removed it as "obsolete" (absent from the current generation's environment.etc when built without NIXOS_HOST_KEYS_DIR). Consolidate all four system.activationScripts entries into one block and add etc = { deps = ["preserveSshHostKey"]; } and setupSecrets = { deps = ["restoreSshHostKey"]; } to enforce the correct save→etc→restore→sops chain. 2. No boot-time secrets: /run/secrets is a tmpfs cleared on every reboot, and sops-nix does NOT generate a boot-time service in this configuration (confirmed live: no sops-nix.service in systemctl list-unit-files). Add nixos-lxc-sops-reinstall.service, modelled after sops-nix's own service placement (wantedBy/before sysinit.target, DefaultDependencies=false), so secrets are reinstalled before basic.target on every non-first boot. ConditionPathExists skips it on first boot; nixos-lxc-first-boot-activate handles that case. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01B2EJ4qTsM5KUqhS5c3GAwx --- modules/platforms/lxc.nix | 95 ++++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 25 deletions(-) diff --git a/modules/platforms/lxc.nix b/modules/platforms/lxc.nix index 7d6cee4..0188cef 100644 --- a/modules/platforms/lxc.nix +++ b/modules/platforms/lxc.nix @@ -114,30 +114,39 @@ in # /etc/ssh/ssh_host_ed25519_key; deletion cascades into every sops secret # failing with "Error getting data key: 0 successful groups required, got 0". # - # Fix: two activation scripts that bracket the etc step. - # preserveSshHostKey — no deps, runs before etc — saves the live key to - # /run (tmpfs) before etc can delete it. - # restoreSshHostKey — deps=[etc], runs after etc — reinstalls the key via - # `install` (atomic, sets mode) if etc removed it. - # The resulting file is not registered in environment.etc - # for either the previous or current generation, so - # subsequent rebuilds leave it alone permanently. - system.activationScripts.preserveSshHostKey = '' - if [ -f /etc/ssh/ssh_host_ed25519_key ]; then - cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp - cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp - fi - ''; - - system.activationScripts.restoreSshHostKey = { - deps = [ "etc" ]; - text = '' - if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then - install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key - install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub + # Fix: activation scripts that bracket the etc step, with explicit deps + # to enforce the correct ordering. Without deps the topological sort places + # preserveSshHostKey AFTER etc (confirmed live on a deployed lxc-tor-relay: + # position 7 vs etc's position 5) -- the key is already gone by the time it + # tries to save it. The etc/setupSecrets entries ADD to existing deps + # (types.listOf concatenates across module definitions). + system.activationScripts = { + # Saves the live key to /run before etc can delete it. + preserveSshHostKey = '' + if [ -f /etc/ssh/ssh_host_ed25519_key ]; then + cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp + cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp fi - rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp ''; + + # Reinstalls the key after etc runs if it was removed as "obsolete". + # The resulting file is not registered in environment.etc for either + # generation, so subsequent rebuilds leave it alone permanently. + restoreSshHostKey = { + deps = [ "etc" ]; + text = '' + if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then + install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key + install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub + fi + rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp + ''; + }; + + # Force etc to wait until the key is saved, and sops to wait until the + # key is restored. Without these the topological sort breaks the chain. + etc = { deps = [ "preserveSshHostKey" ]; }; + setupSecrets = { deps = [ "restoreSshHostKey" ]; }; }; # virtualisation/proxmox-lxc.nix (imported above) registers the Nix @@ -146,9 +155,12 @@ in # sops-nix's "for users" secrets (password hashes -- installed by the # activation script itself, not a systemd service, since they need to # exist *before* user creation) nor the user-creation step that - # consumes them ever run on a real lxc-* boot. Regular secrets - # (nix-serve's key, beszel's token, etc.) work anyway because sops-nix - # provides its own systemd service for those. + # consumes them ever run on a real lxc-* boot. In this config sops-nix + # does NOT generate its own boot-time service (confirmed live: no + # sops-nix.service in systemctl list-unit-files on a deployed + # lxc-tor-relay container); /run/secrets is a tmpfs cleared on every + # reboot, so secrets must be reinstalled on each non-first boot by + # nixos-lxc-sops-reinstall (below). # # A systemd service, not boot.postBootCommands: tried that first (it's # a genuine, generally-invoked hook -- nixos/modules/system/boot/stage-2-init.sh, @@ -199,4 +211,37 @@ in touch /var/lib/nixos-lxc-first-boot-activated ''; }; + + # Reinstalls sops secrets on every non-first boot. /run/secrets is a + # tmpfs that is cleared on each reboot; without this service, secrets + # are permanently absent after the first boot and every service that + # reads from /run/secrets fails on start. + # + # wantedBy/before sysinit.target + DefaultDependencies=false mirrors how + # the sops-nix module places its own service when it generates one. This + # ensures secrets exist before basic.target (and thus before any user + # service) starts. DefaultDependencies=false is required to avoid a + # circular ordering: without it, systemd would add After=sysinit.target + # to a service that is itself part of sysinit.target. + # + # ConditionPathExists=... skips this service on the genuine first boot + # (the marker doesn't exist yet); nixos-lxc-first-boot-activate handles + # that case. On every subsequent boot the condition passes and secrets + # are reinstalled before user services start. + systemd.services.nixos-lxc-sops-reinstall = { + description = "Reinstall sops secrets on each non-first boot (LXC, /run is tmpfs)"; + wantedBy = [ "sysinit.target" ]; + before = [ "sysinit.target" ]; + unitConfig = { + DefaultDependencies = false; + ConditionPathExists = "/var/lib/nixos-lxc-first-boot-activated"; + }; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + /run/current-system/bin/switch-to-configuration test + ''; + }; } -- 2.54.0