Merge pull request 'feat(provision): Phase 0-2 + fix — clan-core SSH host keys, activation ordering, boot-time sops' (#56) from worktree-phase0-provision-ordering-fix into main

Merge feat(provision): Phase 0-2 + fix — clan-core SSH host keys, activation ordering, boot-time sops

flake.lock

@@ -1,5 +1,62 @@
 {
   "nodes": {
+    "clan-core": {
+      "inputs": {
+        "data-mesher": "data-mesher",
+        "disko": [
+          "disko"
+        ],
+        "flake-parts": "flake-parts",
+        "nix-darwin": "nix-darwin",
+        "nix-select": "nix-select",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "sops-nix": [
+          "sops-nix"
+        ],
+        "systems": "systems",
+        "treefmt-nix": "treefmt-nix"
+      },
+      "locked": {
+        "lastModified": 1783497933,
+        "narHash": "sha256-TxmwEews6URFPqOWEHNychtXbFDgLZjbOfEXtvtOm6U=",
+        "rev": "3dc0221ca09033599fe98055e9bbc81bdf32732a",
+        "type": "tarball",
+        "url": "https://git.clan.lol/api/v1/repos/clan/clan-core/archive/3dc0221ca09033599fe98055e9bbc81bdf32732a.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz"
+      }
+    },
+    "data-mesher": {
+      "inputs": {
+        "flake-parts": [
+          "clan-core",
+          "flake-parts"
+        ],
+        "nixpkgs": [
+          "clan-core",
+          "nixpkgs"
+        ],
+        "treefmt-nix": [
+          "clan-core",
+          "treefmt-nix"
+        ]
+      },
+      "locked": {
+        "lastModified": 1778718524,
+        "narHash": "sha256-pXLoI6Ax0EnUK6r34UM1vibVC7CfTu6j72R2692ZzPs=",
+        "rev": "12c552ad547d87254f33f33bddd1a2cdbeac754d",
+        "type": "tarball",
+        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/12c552ad547d87254f33f33bddd1a2cdbeac754d.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://git.clan.lol/clan/data-mesher/archive/main.tar.gz"
+      }
+    },
     "disko": {
       "inputs": {
         "nixpkgs": [
@@ -51,9 +108,30 @@
         "type": "github"
       }
     },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "clan-core",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1778716662,
+        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "inputs": {
-        "systems": "systems"
+        "systems": "systems_2"
       },
       "locked": {
         "lastModified": 1694529238,
@@ -109,6 +187,40 @@
         "type": "github"
       }
     },
+    "nix-darwin": {
+      "inputs": {
+        "nixpkgs": [
+          "clan-core",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1779036909,
+        "narHash": "sha256-zXcwYQGCT6pzinK+1dBB2ekTVtfxGZAapb3Evdcu4fY=",
+        "owner": "nix-darwin",
+        "repo": "nix-darwin",
+        "rev": "56c666e108467d87d13508936aade6d567f2a501",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-darwin",
+        "repo": "nix-darwin",
+        "type": "github"
+      }
+    },
+    "nix-select": {
+      "locked": {
+        "lastModified": 1763303120,
+        "narHash": "sha256-yxcNOha7Cfv2nhVpz9ZXSNKk0R7wt4AiBklJ8D24rVg=",
+        "rev": "3d1e3860bef36857a01a2ddecba7cdb0a14c35a9",
+        "type": "tarball",
+        "url": "https://git.clan.lol/api/v1/repos/clan/nix-select/archive/3d1e3860bef36857a01a2ddecba7cdb0a14c35a9.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://git.clan.lol/clan/nix-select/archive/main.tar.gz"
+      }
+    },
     "nixos-conf-editor": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -163,6 +275,7 @@
     },
     "root": {
       "inputs": {
+        "clan-core": "clan-core",
         "disko": "disko",
         "home-manager": "home-manager",
         "nixos-conf-editor": "nixos-conf-editor",
@@ -214,6 +327,22 @@
       }
     },
     "systems": {
+      "locked": {
+        "lastModified": 1774449309,
+        "narHash": "sha256-brhZ8DmuGtzkCYHJg4HEd602amKm89Y9ytsFZ5uWD1w=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "c29398b59d2048c4ab79345812849c9bd15e9150",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "ref": "future-26.11",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
       "locked": {
         "lastModified": 1681028828,
         "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
@@ -227,6 +356,27 @@
         "repo": "default",
         "type": "github"
       }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "clan-core",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1780220602,
+        "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "db947814a175b7ca6ded66e21383d938df01c227",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -16,6 +16,19 @@
       url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+    clan-core = {
+      url = "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz";
+      # Deduplicate modules: clan-core bundles its own disko and sops-nix
+      # (both imported by nixosModules.clanCore). Without follows, we'd get
+      # two different versions of each, and disko's _module.args.diskoLib
+      # unique option would conflict. With follows, clan-core uses the same
+      # store paths as us, so NixOS deduplicates the imports.
+      inputs = {
+        nixpkgs.follows = "nixpkgs";
+        disko.follows = "disko";
+        sops-nix.follows = "sops-nix";
+      };
+    };
   };
 
   outputs = { self, nixpkgs, nixos-conf-editor, home-manager, sops-nix, ... } @ inputs:
@@ -41,6 +54,22 @@
           modules = [
             inputs.disko.nixosModules.disko
             sops-nix.nixosModules.sops
+            inputs.clan-core.nixosModules.clanCore
+            {
+              # Required clan settings. directory is the flake root (where
+              # vars/ and sops/ directories live); machine.name is the flake
+              # target name (matches what clan vars generate uses as the key
+              # under vars/per-machine/). enableRecommendedDefaults = false
+              # is mandatory: without it, clan unconditionally enables
+              # networking.useNetworkd, adds packages, and tweaks nix settings
+              # -- none of which belong here.
+              clan.core = {
+                settings.directory = self;
+                settings.machine.name = flakeTarget;
+                enableRecommendedDefaults = false;
+              };
+            }
+            ./modules/clan/ssh-host-key.nix
             ./modules/common/configuration.nix
             ./modules/platforms/${platform}.nix
             ./modules/build-types/${buildType}.nix

modules/clan/ssh-host-key.nix

@@ -0,0 +1,30 @@
+{ pkgs, ... }: {
+  # Defines the SSH host key as a clan vars generator so that:
+  #   - `clan vars generate <target>` creates and encrypts the key pair
+  #   - The private key lives at vars/per-machine/<target>/openssh/ssh_host_ed25519_key/secret
+  #     (sops binary-encrypted, admin-key-only; decrypted by the build script)
+  #   - The public key lives at vars/per-machine/<target>/openssh/ssh_host_ed25519_key.pub/value
+  #     (plaintext; used by sync-host-keys.sh to derive the sops age fingerprint)
+  #
+  # neededFor = "activation" means clan's deployment tool would upload this
+  # before running nixos-rebuild/nixos-install (for VM/baremetal via
+  # nixos-anywhere). For lxc-* hosts, the build script bakes it into the
+  # tarball directly via NIXOS_HOST_KEYS_DIR -- the neededFor value here
+  # simply ensures it is NOT mapped to sops.secrets (which would try to
+  # decrypt it at runtime as a regular service secret, which is wrong: the
+  # SSH host key reaches the container via the tarball, not sops).
+  clan.core.vars.generators.openssh = {
+    files."ssh_host_ed25519_key" = {
+      secret = true;
+      neededFor = "activation";
+    };
+    files."ssh_host_ed25519_key.pub" = {
+      secret = false;
+      neededFor = "activation";
+    };
+    runtimeInputs = [ pkgs.openssh ];
+    script = ''
+      ssh-keygen -t ed25519 -N "" -C "" -f "$out/ssh_host_ed25519_key"
+    '';
+  };
+}

modules/platforms/lxc.nix

@@ -114,30 +114,39 @@ in
   # /etc/ssh/ssh_host_ed25519_key; deletion cascades into every sops secret
   # failing with "Error getting data key: 0 successful groups required, got 0".
   #
-  # Fix: two activation scripts that bracket the etc step.
-  #   preserveSshHostKey — no deps, runs before etc — saves the live key to
-  #                        /run (tmpfs) before etc can delete it.
-  #   restoreSshHostKey  — deps=[etc], runs after etc — reinstalls the key via
-  #                        `install` (atomic, sets mode) if etc removed it.
-  #                        The resulting file is not registered in environment.etc
-  #                        for either the previous or current generation, so
-  #                        subsequent rebuilds leave it alone permanently.
-  system.activationScripts.preserveSshHostKey = ''
-    if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
-      cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
-      cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
-    fi
-  '';
-
-  system.activationScripts.restoreSshHostKey = {
-    deps = [ "etc" ];
-    text = ''
-      if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
-        install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
-        install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
+  # Fix: activation scripts that bracket the etc step, with explicit deps
+  # to enforce the correct ordering. Without deps the topological sort places
+  # preserveSshHostKey AFTER etc (confirmed live on a deployed lxc-tor-relay:
+  # position 7 vs etc's position 5) -- the key is already gone by the time it
+  # tries to save it. The etc/setupSecrets entries ADD to existing deps
+  # (types.listOf concatenates across module definitions).
+  system.activationScripts = {
+    # Saves the live key to /run before etc can delete it.
+    preserveSshHostKey = ''
+      if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
+        cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
+        cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
       fi
-      rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
     '';
+
+    # Reinstalls the key after etc runs if it was removed as "obsolete".
+    # The resulting file is not registered in environment.etc for either
+    # generation, so subsequent rebuilds leave it alone permanently.
+    restoreSshHostKey = {
+      deps = [ "etc" ];
+      text = ''
+        if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
+          install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
+          install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
+        fi
+        rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
+      '';
+    };
+
+    # Force etc to wait until the key is saved, and sops to wait until the
+    # key is restored. Without these the topological sort breaks the chain.
+    etc = { deps = [ "preserveSshHostKey" ]; };
+    setupSecrets = { deps = [ "restoreSshHostKey" ]; };
   };
 
   # virtualisation/proxmox-lxc.nix (imported above) registers the Nix
@@ -146,9 +155,12 @@ in
   # sops-nix's "for users" secrets (password hashes -- installed by the
   # activation script itself, not a systemd service, since they need to
   # exist *before* user creation) nor the user-creation step that
-  # consumes them ever run on a real lxc-* boot. Regular secrets
-  # (nix-serve's key, beszel's token, etc.) work anyway because sops-nix
-  # provides its own systemd service for those.
+  # consumes them ever run on a real lxc-* boot. In this config sops-nix
+  # does NOT generate its own boot-time service (confirmed live: no
+  # sops-nix.service in systemctl list-unit-files on a deployed
+  # lxc-tor-relay container); /run/secrets is a tmpfs cleared on every
+  # reboot, so secrets must be reinstalled on each non-first boot by
+  # nixos-lxc-sops-reinstall (below).
   #
   # A systemd service, not boot.postBootCommands: tried that first (it's
   # a genuine, generally-invoked hook -- nixos/modules/system/boot/stage-2-init.sh,
@@ -199,4 +211,37 @@ in
       touch /var/lib/nixos-lxc-first-boot-activated
     '';
   };
+
+  # Reinstalls sops secrets on every non-first boot. /run/secrets is a
+  # tmpfs that is cleared on each reboot; without this service, secrets
+  # are permanently absent after the first boot and every service that
+  # reads from /run/secrets fails on start.
+  #
+  # wantedBy/before sysinit.target + DefaultDependencies=false mirrors how
+  # the sops-nix module places its own service when it generates one. This
+  # ensures secrets exist before basic.target (and thus before any user
+  # service) starts. DefaultDependencies=false is required to avoid a
+  # circular ordering: without it, systemd would add After=sysinit.target
+  # to a service that is itself part of sysinit.target.
+  #
+  # ConditionPathExists=... skips this service on the genuine first boot
+  # (the marker doesn't exist yet); nixos-lxc-first-boot-activate handles
+  # that case. On every subsequent boot the condition passes and secrets
+  # are reinstalled before user services start.
+  systemd.services.nixos-lxc-sops-reinstall = {
+    description = "Reinstall sops secrets on each non-first boot (LXC, /run is tmpfs)";
+    wantedBy = [ "sysinit.target" ];
+    before = [ "sysinit.target" ];
+    unitConfig = {
+      DefaultDependencies = false;
+      ConditionPathExists = "/var/lib/nixos-lxc-first-boot-activated";
+    };
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    script = ''
+      /run/current-system/bin/switch-to-configuration test
+    '';
+  };
 }

scripts/lib/clan-vars.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# Clan vars helpers: manage SSH host keys stored as clan vars (sops-encrypted
+# binary files under vars/per-machine/<target>/openssh/) instead of the
+# gitignored host-keys/ directory.
+#
+# Layout (per clan's convention):
+#   vars/per-machine/<target>/openssh/ssh_host_ed25519_key/secret  -- sops binary (admin-encrypted)
+#   vars/per-machine/<target>/openssh/ssh_host_ed25519_key.pub/value  -- plaintext SSH pubkey
+#
+# Sourced by create-proxmox-resource.sh and sync-host-keys.sh.
+# Depends on sops-age.sh and ssh-host-keys.sh being sourced first (for
+# sops_yaml_admin_pubkey, ssh_pubkey_to_age, and NIX_OPTS).
+
+if ! declare -p NIX_OPTS >/dev/null 2>&1; then
+  declare -a NIX_OPTS=()
+fi
+
+# clan_ssh_key_exists <target> <repo_root>
+# Returns 0 if clan vars hold a SSH host key for <target>, 1 otherwise.
+clan_ssh_key_exists() {
+  local target="$1" repo_root="$2"
+  [[ -f "${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key/secret" ]]
+}
+
+# clan_ssh_pubkey_path <target> <repo_root>
+# Prints the path to the plaintext SSH public key value file.
+clan_ssh_pubkey_path() {
+  local target="$1" repo_root="$2"
+  echo "${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key.pub/value"
+}
+
+# clan_decrypt_ssh_key <target> <repo_root> <dest_dir>
+# Decrypts the sops-encrypted SSH host private key for <target> into <dest_dir>,
+# naming it <target>_ssh_host_ed25519_key (to match NIXOS_HOST_KEYS_DIR
+# conventions that lxc.nix and the disko build already expect). Also copies
+# the plaintext public key. The caller is responsible for protecting and
+# cleaning up <dest_dir>.
+clan_decrypt_ssh_key() {
+  local target="$1" repo_root="$2" dest_dir="$3"
+  local secret="${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key/secret"
+  local pubval="${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key.pub/value"
+  local dest_priv="${dest_dir}/${target}_ssh_host_ed25519_key"
+  local dest_pub="${dest_dir}/${target}_ssh_host_ed25519_key.pub"
+
+  nix-shell "${NIX_OPTS[@]}" -p sops --run \
+    "sops -d --output-type binary '${secret}'" > "$dest_priv"
+  chmod 0600 "$dest_priv"
+  cp "$pubval" "$dest_pub"
+}
+
+# clan_generate_ssh_key <target> <repo_root>
+# Generates a new SSH host key pair and stores it in clan vars format:
+#   - private key: sops binary-encrypted for the admin age key
+#   - public key: plaintext value file
+# Idempotent: if the secret already exists, prints a note and returns 0.
+# Requires sops_yaml_admin_pubkey (from sops-age.sh) to be available.
+clan_generate_ssh_key() {
+  local target="$1" repo_root="$2"
+  local var_base="${repo_root}/vars/per-machine/${target}/openssh"
+  local secret_dir="${var_base}/ssh_host_ed25519_key"
+  local pubval_dir="${var_base}/ssh_host_ed25519_key.pub"
+
+  if [[ -f "${secret_dir}/secret" ]]; then
+    echo "Clan SSH host key for ${target} already exists -- skipping generation."
+    return 0
+  fi
+
+  # Resolve admin age public key from .sops.yaml
+  local admin_pubkey
+  admin_pubkey="$(sops_yaml_admin_pubkey "${repo_root}/.sops.yaml")"
+  if [[ -z "$admin_pubkey" ]]; then
+    echo "ERROR: Could not find &admin age key in ${repo_root}/.sops.yaml" >&2
+    return 1
+  fi
+
+  # Generate the SSH key pair in a secure temp directory
+  local tmpdir
+  tmpdir="$(mktemp -d)"
+  local priv_tmp="${tmpdir}/ssh_host_ed25519_key"
+
+  # shellcheck disable=SC2064
+  trap "rm -rf '${tmpdir}'" RETURN
+
+  nix-shell "${NIX_OPTS[@]}" -p openssh --run \
+    "ssh-keygen -t ed25519 -N '' -C '${target}' -f '${priv_tmp}'" >/dev/null
+
+  # Create a minimal sops config that uses only the admin age key -- this
+  # prevents sops from merging in ALL recipients from .sops.yaml (which
+  # would unnecessarily encrypt for every host's key, not just admin).
+  local sops_cfg="${tmpdir}/sops-config.json"
+  printf '{"creation_rules":[{"key_groups":[{"age":["%s"]}]}]}\n' \
+    "$admin_pubkey" > "$sops_cfg"
+
+  # Encrypt the private key in sops binary format (admin-only recipient)
+  mkdir -p "$secret_dir" "$pubval_dir"
+  nix-shell "${NIX_OPTS[@]}" -p sops --run \
+    "sops -e --config '${sops_cfg}' --input-type binary '${priv_tmp}'" \
+    > "${secret_dir}/secret"
+
+  # Store the public key as a plaintext value file
+  cp "${priv_tmp}.pub" "${pubval_dir}/value"
+
+  echo "Generated and stored clan SSH host key for ${target}."
+  echo "  Private key: ${secret_dir}/secret (sops binary, admin-key encrypted)"
+  echo "  Public key:  ${pubval_dir}/value"
+}

scripts/proxmox/create-proxmox-resource.sh

@@ -59,6 +59,10 @@ source "${repo_root}/scripts/env.sh"
 source "${repo_root}/scripts/lib/nix-eval.sh"
 # shellcheck source=../lib/confirm.sh
 source "${repo_root}/scripts/lib/confirm.sh"
+# shellcheck source=../lib/sops-age.sh
+source "${repo_root}/scripts/lib/sops-age.sh"
+# shellcheck source=../lib/clan-vars.sh
+source "${repo_root}/scripts/lib/clan-vars.sh"
 
 sync_keys="${repo_root}/scripts/secrets/sync-host-keys.sh"
 
@@ -485,12 +489,37 @@ echo "Target: ${flake_target} (host=${host}, type=${type}) -> Proxmox resource '
 nix_extra_opts
 
 # --- make sure this target has a registered host key --------------------
+# sync-host-keys.sh is idempotent and generates the key (via clan vars) if
+# no key exists yet -- the old inline prepare-host-key.sh call is gone.
 echo
 echo "==> Ensuring host key exists and is registered..."
 sync_args=("$flake_target")
 [[ "$dry_run" -eq 1 ]] && sync_args+=(--dry-run)
 bash "$sync_keys" "${sync_args[@]}"
 
+# If sync-host-keys.sh changed .sops.yaml or secrets/, those changes must be
+# committed and pushed before the remote `git pull` below picks them up --
+# the PVE node builds from whatever HEAD is checked out there, not the local
+# working tree. Detect uncommitted changes and block until the operator
+# confirms they've pushed, so the build never runs against a stale flake.
+if [[ "$dry_run" -eq 0 ]]; then
+  _sops_dirty="$(git -C "$repo_root" status --porcelain -- .sops.yaml secrets/ 2>/dev/null || true)"
+  if [[ -n "$_sops_dirty" ]]; then
+    echo
+    echo "==> COMMIT + PUSH REQUIRED before the remote build can succeed:"
+    echo "    sync-host-keys.sh modified .sops.yaml / secrets/ to register the"
+    echo "    new host's sops recipient. The PVE node builds from the git-tracked"
+    echo "    flake, so these changes must be committed and pushed first -- otherwise"
+    echo "    the image build will succeed but the host cannot decrypt secrets on"
+    echo "    first boot (its age key isn't in the encrypted secrets files yet)."
+    echo
+    git -C "$repo_root" status --short -- .sops.yaml secrets/ || true
+    echo
+    read -rp "    Commit and push those changes, then press Enter to continue (Ctrl-C to abort): "
+  fi
+  unset _sops_dirty
+fi
+
 # --- VMID: pick one, and refuse to touch anything that already exists ---
 echo
 if [[ -z "$vmid" ]]; then
@@ -613,21 +642,37 @@ ensure_remote_repo() {
   fi
 }
 
-# --- sync locally-managed host-keys/ to the node ---------------------------
-# Gitignored (see .gitignore), so `git pull` above never carries it -- both
-# build paths need it present as NIXOS_HOST_KEYS_DIR / --pre-format-files
-# input on the node itself now that the build runs there. scp (not rsync,
-# not already a dependency anywhere else in this repo) mirrors how this
-# script already transfers the --image case below.
+# --- sync host key to the node ---------------------------------------------
+# Clan-managed keys live in vars/per-machine/ (committed, sops-encrypted),
+# so they arrive on the node via `git pull`. But the build scripts expect a
+# plaintext key file in host-keys/ (NIXOS_HOST_KEYS_DIR for LXC, or
+# --pre-format-files for VM). For clan keys, decrypt locally and scp just the
+# two files for this target; for legacy host-keys/ entries, scp the whole dir.
 sync_remote_host_keys() {
   echo
-  echo "==> Syncing host-keys/ to ${node}..."
+  echo "==> Syncing host key for ${flake_target} to ${node}..."
   if [[ "$dry_run" -eq 1 ]]; then
-    echo "[dry-run] would copy ${repo_root}/host-keys/ to ${ssh_target}:${remote_repo_dir}/host-keys/"
+    if clan_ssh_key_exists "$flake_target" "$repo_root"; then
+      echo "[dry-run] would decrypt clan SSH key for ${flake_target} and copy to ${ssh_target}:${remote_repo_dir}/host-keys/"
+    else
+      echo "[dry-run] would copy ${repo_root}/host-keys/ to ${ssh_target}:${remote_repo_dir}/host-keys/"
+    fi
     return
   fi
   ssh "$ssh_target" "mkdir -p '${remote_repo_dir}/host-keys'"
-  scp -pr "${repo_root}/host-keys/." "${ssh_target}:${remote_repo_dir}/host-keys/"
+  if clan_ssh_key_exists "$flake_target" "$repo_root"; then
+    local tmpdir
+    tmpdir="$(mktemp -d)"
+    # shellcheck disable=SC2064
+    trap "rm -rf '${tmpdir}'" RETURN
+    echo "  Decrypting clan SSH key for ${flake_target}..."
+    clan_decrypt_ssh_key "$flake_target" "$repo_root" "$tmpdir"
+    scp -p "${tmpdir}/${flake_target}_ssh_host_ed25519_key" \
+           "${tmpdir}/${flake_target}_ssh_host_ed25519_key.pub" \
+           "${ssh_target}:${remote_repo_dir}/host-keys/"
+  else
+    scp -pr "${repo_root}/host-keys/." "${ssh_target}:${remote_repo_dir}/host-keys/"
+  fi
 }
 
 # --- build (or reuse an image already on the node) ------------------------
@@ -696,6 +741,12 @@ cd "$repo_dir"
 # right after a successful install.
 . scripts/lib/nix-bootstrap.sh
 ensure_nix_profile
+if [[ ! -f "host-keys/${target}_ssh_host_ed25519_key" ]]; then
+  echo "ERROR: host-keys/${target}_ssh_host_ed25519_key not found in ${repo_dir}." >&2
+  echo "Generate the key locally (scripts/secrets/sync-host-keys.sh ${target})" >&2
+  echo "and ensure it was synced here before starting the build." >&2
+  exit 1
+fi
 NIXOS_HOST_KEYS_DIR="$(pwd)/host-keys" nix build --impure \
   --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \
   ".#nixosConfigurations.${target}.config.system.build.tarball" \
@@ -736,12 +787,18 @@ declare -a NIX_OPTS=()
 cd "$repo_dir"
 . scripts/lib/nix-bootstrap.sh
 ensure_nix_profile
+if [[ ! -f "host-keys/${target}_ssh_host_ed25519_key" ]]; then
+  echo "ERROR: host-keys/${target}_ssh_host_ed25519_key not found in ${repo_dir}." >&2
+  echo "Generate the key locally (scripts/secrets/sync-host-keys.sh ${target})" >&2
+  echo "and ensure it was synced here before starting the build." >&2
+  exit 1
+fi
 nix build --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \
   ".#nixosConfigurations.${target}.config.system.build.diskoImagesScript" \
   --out-link "result-${target}"
 $sudo_pfx "./result-${target}" \
-  --pre-format-files "host-keys/${target}_ssh_host_ed25519_key" /etc/ssh/ssh_host_ed25519_key \
-  --pre-format-files "host-keys/${target}_ssh_host_ed25519_key.pub" /etc/ssh/ssh_host_ed25519_key.pub \
+  --pre-format-files "$(pwd)/host-keys/${target}_ssh_host_ed25519_key" /etc/ssh/ssh_host_ed25519_key \
+  --pre-format-files "$(pwd)/host-keys/${target}_ssh_host_ed25519_key.pub" /etc/ssh/ssh_host_ed25519_key.pub \
   --build-memory 2048
 built="$(find . -maxdepth 1 -name '*.raw' -newer "result-${target}" | head -1)"
 if [[ -z "$built" ]]; then

scripts/secrets/prepare-host-key.sh

@@ -41,8 +41,9 @@ mkdir -p "$keydir"
 keyfile="${keydir}/${hostname}_ssh_host_ed25519_key"
 
 if [[ -f "$keyfile" ]]; then
-  echo "ERROR: $keyfile already exists. Remove it first if you want to regenerate." >&2
-  exit 1
+  echo "Key already exists: ${keyfile}"
+  echo "Reusing the existing key. Remove it first if you want to regenerate."
+  exit 0
 fi
 
 nix_extra_opts

scripts/secrets/sync-host-keys.sh

@@ -39,6 +39,8 @@ source "${repo_root}/scripts/lib/ssh-host-keys.sh"
 source "${repo_root}/scripts/lib/sops-age.sh"
 # shellcheck source=../lib/confirm.sh
 source "${repo_root}/scripts/lib/confirm.sh"
+# shellcheck source=../lib/clan-vars.sh
+source "${repo_root}/scripts/lib/clan-vars.sh"
 
 mkdir -p "$keydir"
 
@@ -146,13 +148,15 @@ dry_run=0
 queue_host_sync() {
   local host="$1"
   local keyfile="${keydir}/${host}_ssh_host_ed25519_key"
-  local has_local_key=0 has_anchor=0
+  local has_local_key=0 has_clan_key=0 has_anchor=0
   [[ -f "$keyfile" ]] && has_local_key=1
+  clan_ssh_key_exists "$host" "$repo_root" && has_clan_key=1
   grep -qE "^  - &${host} age1" "$sops_yaml" && has_anchor=1
 
-  if [[ "$has_local_key" -eq 0 && "$has_anchor" -eq 1 ]]; then
+  if [[ "$has_local_key" -eq 0 && "$has_clan_key" -eq 0 && "$has_anchor" -eq 1 ]]; then
     echo "SKIP ${host}: .sops.yaml already has an &${host} anchor, but"
-    echo "     host-keys/${host}_ssh_host_ed25519_key is missing locally."
+    echo "     neither host-keys/${host}_ssh_host_ed25519_key nor"
+    echo "     vars/per-machine/${host}/openssh/ exist locally."
     echo "     Not generating a replacement -- it wouldn't match whatever's"
     echo "     already registered (and possibly deployed). Remove the"
     echo "     &${host} line from .sops.yaml first if you really want a"
@@ -160,21 +164,26 @@ queue_host_sync() {
     return 1
   fi
 
-  if [[ "$has_local_key" -eq 0 ]]; then
+  if [[ "$has_local_key" -eq 0 && "$has_clan_key" -eq 0 ]]; then
     if [[ "$dry_run" -eq 1 ]]; then
-      echo "[dry-run] ${host}: would generate host key"
+      echo "[dry-run] ${host}: would generate host key via clan vars"
     else
-      echo "==> ${host}: generating host key"
-      generate_host_ed25519_key "$host" "$keyfile"
+      echo "==> ${host}: generating host key via clan vars"
+      clan_generate_ssh_key "$host" "$repo_root"
+      has_clan_key=1
     fi
+  elif [[ "$has_clan_key" -eq 1 ]]; then
+    echo "==> ${host}: clan-managed SSH host key already present"
   else
-    echo "==> ${host}: host key already present"
+    echo "==> ${host}: host key already present (host-keys/)"
   fi
 
   if [[ "$has_anchor" -eq 0 ]]; then
     local age_pub
     if [[ "$dry_run" -eq 1 ]]; then
       age_pub="dry-run-placeholder-not-a-real-key"
+    elif [[ "$has_clan_key" -eq 1 ]]; then
+      age_pub="$(ssh_pubkey_to_age "$(clan_ssh_pubkey_path "$host" "$repo_root")")"
     else
       age_pub="$(ssh_pubkey_to_age "${keyfile}.pub")"
     fi