feat(provision): Phase 2 — migrate SSH host keys to clan vars

Replaces the gitignored host-keys/ directory with clan vars as the
authoritative storage for SSH host keys. Keys are now generated as
sops-binary-encrypted clan var files (admin-key only) and checked into
vars/per-machine/<target>/openssh/, eliminating the plaintext private
key that previously had to live outside the repo.

Changes:
- modules/clan/ssh-host-key.nix: clan vars generator for the ed25519
  SSH host key pair (neededFor="activation" — not mapped to sops.secrets,
  delivered via tarball baking for LXC or --pre-format-files for VMs)
- flake.nix: add clanCore module + required settings to every mkTarget;
  deduplicate bundled disko/sops-nix via follows; all 27 hosts eval clean
- flake.lock: updated to reflect the new follows constraints
- scripts/lib/clan-vars.sh: new helper library with
  clan_ssh_key_exists / clan_ssh_pubkey_path / clan_decrypt_ssh_key /
  clan_generate_ssh_key for use by the provisioning and sync scripts
- scripts/secrets/sync-host-keys.sh: queue_host_sync() now checks clan
  vars first; generates via clan_generate_ssh_key if no key exists;
  derives age fingerprint from clan pub key for .sops.yaml registration
- scripts/proxmox/create-proxmox-resource.sh: key management simplified
  (sync-host-keys.sh now generates the key if missing, so the inline
  prepare-host-key.sh call is gone); sync_remote_host_keys() decrypts
  the clan key into a temp dir and scps just the two files to the node
  when a clan key exists, falling back to the old host-keys/ scp for
  any remaining legacy entries

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B2EJ4qTsM5KUqhS5c3GAwx

flake.lock

@@ -3,14 +3,18 @@
     "clan-core": {
       "inputs": {
         "data-mesher": "data-mesher",
-        "disko": "disko",
+        "disko": [
+          "disko"
+        ],
         "flake-parts": "flake-parts",
         "nix-darwin": "nix-darwin",
         "nix-select": "nix-select",
         "nixpkgs": [
           "nixpkgs"
         ],
-        "sops-nix": "sops-nix",
+        "sops-nix": [
+          "sops-nix"
+        ],
         "systems": "systems",
         "treefmt-nix": "treefmt-nix"
       },
@@ -54,27 +58,6 @@
       }
     },
     "disko": {
-      "inputs": {
-        "nixpkgs": [
-          "clan-core",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1780290312,
-        "narHash": "sha256-eTAlX0CwgB84Ts3GaBd944A3DRXVMzgA0EqroZBISUo=",
-        "owner": "nix-community",
-        "repo": "disko",
-        "rev": "115e5211780054d8a890b41f0b7734cafad54dfe",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "disko",
-        "type": "github"
-      }
-    },
-    "disko_2": {
       "inputs": {
         "nixpkgs": [
           "nixpkgs"
@@ -293,11 +276,11 @@
     "root": {
       "inputs": {
         "clan-core": "clan-core",
-        "disko": "disko_2",
+        "disko": "disko",
         "home-manager": "home-manager",
         "nixos-conf-editor": "nixos-conf-editor",
         "nixpkgs": "nixpkgs_2",
-        "sops-nix": "sops-nix_2"
+        "sops-nix": "sops-nix"
       }
     },
     "snowfall-lib": {
@@ -324,27 +307,6 @@
       }
     },
     "sops-nix": {
-      "inputs": {
-        "nixpkgs": [
-          "clan-core",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1780547341,
-        "narHash": "sha256-Gq8KNx5A7hBB3uGJaj6eQfLDIz5YdLu92gqBcvHvoUo=",
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "rev": "9ed65852b6257fbeae4355bc24ecfea307ca759a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "type": "github"
-      }
-    },
-    "sops-nix_2": {
       "inputs": {
         "nixpkgs": [
           "nixpkgs"

flake.nix

@@ -18,7 +18,16 @@
     };
     clan-core = {
       url = "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz";
-      inputs.nixpkgs.follows = "nixpkgs";
+      # Deduplicate modules: clan-core bundles its own disko and sops-nix
+      # (both imported by nixosModules.clanCore). Without follows, we'd get
+      # two different versions of each, and disko's _module.args.diskoLib
+      # unique option would conflict. With follows, clan-core uses the same
+      # store paths as us, so NixOS deduplicates the imports.
+      inputs = {
+        nixpkgs.follows = "nixpkgs";
+        disko.follows = "disko";
+        sops-nix.follows = "sops-nix";
+      };
     };
   };
 
@@ -45,6 +54,22 @@
           modules = [
             inputs.disko.nixosModules.disko
             sops-nix.nixosModules.sops
+            inputs.clan-core.nixosModules.clanCore
+            {
+              # Required clan settings. directory is the flake root (where
+              # vars/ and sops/ directories live); machine.name is the flake
+              # target name (matches what clan vars generate uses as the key
+              # under vars/per-machine/). enableRecommendedDefaults = false
+              # is mandatory: without it, clan unconditionally enables
+              # networking.useNetworkd, adds packages, and tweaks nix settings
+              # -- none of which belong here.
+              clan.core = {
+                settings.directory = self;
+                settings.machine.name = flakeTarget;
+                enableRecommendedDefaults = false;
+              };
+            }
+            ./modules/clan/ssh-host-key.nix
             ./modules/common/configuration.nix
             ./modules/platforms/${platform}.nix
             ./modules/build-types/${buildType}.nix

modules/clan/ssh-host-key.nix

@@ -0,0 +1,30 @@
+{ pkgs, ... }: {
+  # Defines the SSH host key as a clan vars generator so that:
+  #   - `clan vars generate <target>` creates and encrypts the key pair
+  #   - The private key lives at vars/per-machine/<target>/openssh/ssh_host_ed25519_key/secret
+  #     (sops binary-encrypted, admin-key-only; decrypted by the build script)
+  #   - The public key lives at vars/per-machine/<target>/openssh/ssh_host_ed25519_key.pub/value
+  #     (plaintext; used by sync-host-keys.sh to derive the sops age fingerprint)
+  #
+  # neededFor = "activation" means clan's deployment tool would upload this
+  # before running nixos-rebuild/nixos-install (for VM/baremetal via
+  # nixos-anywhere). For lxc-* hosts, the build script bakes it into the
+  # tarball directly via NIXOS_HOST_KEYS_DIR -- the neededFor value here
+  # simply ensures it is NOT mapped to sops.secrets (which would try to
+  # decrypt it at runtime as a regular service secret, which is wrong: the
+  # SSH host key reaches the container via the tarball, not sops).
+  clan.core.vars.generators.openssh = {
+    files."ssh_host_ed25519_key" = {
+      secret = true;
+      neededFor = "activation";
+    };
+    files."ssh_host_ed25519_key.pub" = {
+      secret = false;
+      neededFor = "activation";
+    };
+    runtimeInputs = [ pkgs.openssh ];
+    script = ''
+      ssh-keygen -t ed25519 -N "" -C "" -f "$out/ssh_host_ed25519_key"
+    '';
+  };
+}

scripts/lib/clan-vars.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# Clan vars helpers: manage SSH host keys stored as clan vars (sops-encrypted
+# binary files under vars/per-machine/<target>/openssh/) instead of the
+# gitignored host-keys/ directory.
+#
+# Layout (per clan's convention):
+#   vars/per-machine/<target>/openssh/ssh_host_ed25519_key/secret  -- sops binary (admin-encrypted)
+#   vars/per-machine/<target>/openssh/ssh_host_ed25519_key.pub/value  -- plaintext SSH pubkey
+#
+# Sourced by create-proxmox-resource.sh and sync-host-keys.sh.
+# Depends on sops-age.sh and ssh-host-keys.sh being sourced first (for
+# sops_yaml_admin_pubkey, ssh_pubkey_to_age, and NIX_OPTS).
+
+if ! declare -p NIX_OPTS >/dev/null 2>&1; then
+  declare -a NIX_OPTS=()
+fi
+
+# clan_ssh_key_exists <target> <repo_root>
+# Returns 0 if clan vars hold a SSH host key for <target>, 1 otherwise.
+clan_ssh_key_exists() {
+  local target="$1" repo_root="$2"
+  [[ -f "${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key/secret" ]]
+}
+
+# clan_ssh_pubkey_path <target> <repo_root>
+# Prints the path to the plaintext SSH public key value file.
+clan_ssh_pubkey_path() {
+  local target="$1" repo_root="$2"
+  echo "${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key.pub/value"
+}
+
+# clan_decrypt_ssh_key <target> <repo_root> <dest_dir>
+# Decrypts the sops-encrypted SSH host private key for <target> into <dest_dir>,
+# naming it <target>_ssh_host_ed25519_key (to match NIXOS_HOST_KEYS_DIR
+# conventions that lxc.nix and the disko build already expect). Also copies
+# the plaintext public key. The caller is responsible for protecting and
+# cleaning up <dest_dir>.
+clan_decrypt_ssh_key() {
+  local target="$1" repo_root="$2" dest_dir="$3"
+  local secret="${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key/secret"
+  local pubval="${repo_root}/vars/per-machine/${target}/openssh/ssh_host_ed25519_key.pub/value"
+  local dest_priv="${dest_dir}/${target}_ssh_host_ed25519_key"
+  local dest_pub="${dest_dir}/${target}_ssh_host_ed25519_key.pub"
+
+  nix-shell "${NIX_OPTS[@]}" -p sops --run \
+    "sops -d --output-type binary '${secret}'" > "$dest_priv"
+  chmod 0600 "$dest_priv"
+  cp "$pubval" "$dest_pub"
+}
+
+# clan_generate_ssh_key <target> <repo_root>
+# Generates a new SSH host key pair and stores it in clan vars format:
+#   - private key: sops binary-encrypted for the admin age key
+#   - public key: plaintext value file
+# Idempotent: if the secret already exists, prints a note and returns 0.
+# Requires sops_yaml_admin_pubkey (from sops-age.sh) to be available.
+clan_generate_ssh_key() {
+  local target="$1" repo_root="$2"
+  local var_base="${repo_root}/vars/per-machine/${target}/openssh"
+  local secret_dir="${var_base}/ssh_host_ed25519_key"
+  local pubval_dir="${var_base}/ssh_host_ed25519_key.pub"
+
+  if [[ -f "${secret_dir}/secret" ]]; then
+    echo "Clan SSH host key for ${target} already exists -- skipping generation."
+    return 0
+  fi
+
+  # Resolve admin age public key from .sops.yaml
+  local admin_pubkey
+  admin_pubkey="$(sops_yaml_admin_pubkey "${repo_root}/.sops.yaml")"
+  if [[ -z "$admin_pubkey" ]]; then
+    echo "ERROR: Could not find &admin age key in ${repo_root}/.sops.yaml" >&2
+    return 1
+  fi
+
+  # Generate the SSH key pair in a secure temp directory
+  local tmpdir
+  tmpdir="$(mktemp -d)"
+  local priv_tmp="${tmpdir}/ssh_host_ed25519_key"
+
+  # shellcheck disable=SC2064
+  trap "rm -rf '${tmpdir}'" RETURN
+
+  nix-shell "${NIX_OPTS[@]}" -p openssh --run \
+    "ssh-keygen -t ed25519 -N '' -C '${target}' -f '${priv_tmp}'" >/dev/null
+
+  # Create a minimal sops config that uses only the admin age key -- this
+  # prevents sops from merging in ALL recipients from .sops.yaml (which
+  # would unnecessarily encrypt for every host's key, not just admin).
+  local sops_cfg="${tmpdir}/sops-config.json"
+  printf '{"creation_rules":[{"key_groups":[{"age":["%s"]}]}]}\n' \
+    "$admin_pubkey" > "$sops_cfg"
+
+  # Encrypt the private key in sops binary format (admin-only recipient)
+  mkdir -p "$secret_dir" "$pubval_dir"
+  nix-shell "${NIX_OPTS[@]}" -p sops --run \
+    "sops -e --config '${sops_cfg}' --input-type binary '${priv_tmp}'" \
+    > "${secret_dir}/secret"
+
+  # Store the public key as a plaintext value file
+  cp "${priv_tmp}.pub" "${pubval_dir}/value"
+
+  echo "Generated and stored clan SSH host key for ${target}."
+  echo "  Private key: ${secret_dir}/secret (sops binary, admin-key encrypted)"
+  echo "  Public key:  ${pubval_dir}/value"
+}

scripts/proxmox/create-proxmox-resource.sh

@@ -59,6 +59,10 @@ source "${repo_root}/scripts/env.sh"
 source "${repo_root}/scripts/lib/nix-eval.sh"
 # shellcheck source=../lib/confirm.sh
 source "${repo_root}/scripts/lib/confirm.sh"
+# shellcheck source=../lib/sops-age.sh
+source "${repo_root}/scripts/lib/sops-age.sh"
+# shellcheck source=../lib/clan-vars.sh
+source "${repo_root}/scripts/lib/clan-vars.sh"
 
 sync_keys="${repo_root}/scripts/secrets/sync-host-keys.sh"
 
@@ -485,19 +489,10 @@ echo "Target: ${flake_target} (host=${host}, type=${type}) -> Proxmox resource '
 nix_extra_opts
 
 # --- make sure this target has a registered host key --------------------
+# sync-host-keys.sh is idempotent and generates the key (via clan vars) if
+# no key exists yet -- the old inline prepare-host-key.sh call is gone.
 echo
 echo "==> Ensuring host key exists and is registered..."
-_host_keyfile="${repo_root}/host-keys/${flake_target}_ssh_host_ed25519_key"
-if [[ ! -f "$_host_keyfile" ]]; then
-  echo "  No host key found for ${flake_target}; generating one via prepare-host-key.sh..."
-  _prepare_host_key="${repo_root}/scripts/secrets/prepare-host-key.sh"
-  if [[ "$dry_run" -eq 1 ]]; then
-    echo "[dry-run] would run: bash ${_prepare_host_key} ${flake_target}"
-  else
-    bash "$_prepare_host_key" "$flake_target"
-  fi
-fi
-unset _host_keyfile _prepare_host_key
 sync_args=("$flake_target")
 [[ "$dry_run" -eq 1 ]] && sync_args+=(--dry-run)
 bash "$sync_keys" "${sync_args[@]}"
@@ -647,21 +642,37 @@ ensure_remote_repo() {
   fi
 }
 
-# --- sync locally-managed host-keys/ to the node ---------------------------
+# --- sync host key to the node ---------------------------------------------
-# Gitignored (see .gitignore), so `git pull` above never carries it -- both
+# Clan-managed keys live in vars/per-machine/ (committed, sops-encrypted),
-# build paths need it present as NIXOS_HOST_KEYS_DIR / --pre-format-files
+# so they arrive on the node via `git pull`. But the build scripts expect a
-# input on the node itself now that the build runs there. scp (not rsync,
+# plaintext key file in host-keys/ (NIXOS_HOST_KEYS_DIR for LXC, or
-# not already a dependency anywhere else in this repo) mirrors how this
+# --pre-format-files for VM). For clan keys, decrypt locally and scp just the
-# script already transfers the --image case below.
+# two files for this target; for legacy host-keys/ entries, scp the whole dir.
 sync_remote_host_keys() {
   echo
-  echo "==> Syncing host-keys/ to ${node}..."
+  echo "==> Syncing host key for ${flake_target} to ${node}..."
   if [[ "$dry_run" -eq 1 ]]; then
+    if clan_ssh_key_exists "$flake_target" "$repo_root"; then
+      echo "[dry-run] would decrypt clan SSH key for ${flake_target} and copy to ${ssh_target}:${remote_repo_dir}/host-keys/"
+    else
       echo "[dry-run] would copy ${repo_root}/host-keys/ to ${ssh_target}:${remote_repo_dir}/host-keys/"
+    fi
     return
   fi
   ssh "$ssh_target" "mkdir -p '${remote_repo_dir}/host-keys'"
+  if clan_ssh_key_exists "$flake_target" "$repo_root"; then
+    local tmpdir
+    tmpdir="$(mktemp -d)"
+    # shellcheck disable=SC2064
+    trap "rm -rf '${tmpdir}'" RETURN
+    echo "  Decrypting clan SSH key for ${flake_target}..."
+    clan_decrypt_ssh_key "$flake_target" "$repo_root" "$tmpdir"
+    scp -p "${tmpdir}/${flake_target}_ssh_host_ed25519_key" \
+           "${tmpdir}/${flake_target}_ssh_host_ed25519_key.pub" \
+           "${ssh_target}:${remote_repo_dir}/host-keys/"
+  else
     scp -pr "${repo_root}/host-keys/." "${ssh_target}:${remote_repo_dir}/host-keys/"
+  fi
 }
 
 # --- build (or reuse an image already on the node) ------------------------

scripts/secrets/sync-host-keys.sh

@@ -39,6 +39,8 @@ source "${repo_root}/scripts/lib/ssh-host-keys.sh"
 source "${repo_root}/scripts/lib/sops-age.sh"
 # shellcheck source=../lib/confirm.sh
 source "${repo_root}/scripts/lib/confirm.sh"
+# shellcheck source=../lib/clan-vars.sh
+source "${repo_root}/scripts/lib/clan-vars.sh"
 
 mkdir -p "$keydir"
 
@@ -146,13 +148,15 @@ dry_run=0
 queue_host_sync() {
   local host="$1"
   local keyfile="${keydir}/${host}_ssh_host_ed25519_key"
-  local has_local_key=0 has_anchor=0
+  local has_local_key=0 has_clan_key=0 has_anchor=0
   [[ -f "$keyfile" ]] && has_local_key=1
+  clan_ssh_key_exists "$host" "$repo_root" && has_clan_key=1
   grep -qE "^  - &${host} age1" "$sops_yaml" && has_anchor=1
 
-  if [[ "$has_local_key" -eq 0 && "$has_anchor" -eq 1 ]]; then
+  if [[ "$has_local_key" -eq 0 && "$has_clan_key" -eq 0 && "$has_anchor" -eq 1 ]]; then
     echo "SKIP ${host}: .sops.yaml already has an &${host} anchor, but"
-    echo "     host-keys/${host}_ssh_host_ed25519_key is missing locally."
+    echo "     neither host-keys/${host}_ssh_host_ed25519_key nor"
+    echo "     vars/per-machine/${host}/openssh/ exist locally."
     echo "     Not generating a replacement -- it wouldn't match whatever's"
     echo "     already registered (and possibly deployed). Remove the"
     echo "     &${host} line from .sops.yaml first if you really want a"
@@ -160,21 +164,26 @@ queue_host_sync() {
     return 1
   fi
 
-  if [[ "$has_local_key" -eq 0 ]]; then
+  if [[ "$has_local_key" -eq 0 && "$has_clan_key" -eq 0 ]]; then
     if [[ "$dry_run" -eq 1 ]]; then
-      echo "[dry-run] ${host}: would generate host key"
+      echo "[dry-run] ${host}: would generate host key via clan vars"
     else
-      echo "==> ${host}: generating host key"
+      echo "==> ${host}: generating host key via clan vars"
-      generate_host_ed25519_key "$host" "$keyfile"
+      clan_generate_ssh_key "$host" "$repo_root"
+      has_clan_key=1
     fi
+  elif [[ "$has_clan_key" -eq 1 ]]; then
+    echo "==> ${host}: clan-managed SSH host key already present"
   else
-    echo "==> ${host}: host key already present"
+    echo "==> ${host}: host key already present (host-keys/)"
   fi
 
   if [[ "$has_anchor" -eq 0 ]]; then
     local age_pub
     if [[ "$dry_run" -eq 1 ]]; then
       age_pub="dry-run-placeholder-not-a-real-key"
+    elif [[ "$has_clan_key" -eq 1 ]]; then
+      age_pub="$(ssh_pubkey_to_age "$(clan_ssh_pubkey_path "$host" "$repo_root")")"
     else
       age_pub="$(ssh_pubkey_to_age "${keyfile}.pub")"
     fi