fix(proxmox): embed SSH host key via NIXOS_HOST_KEYS_DIR so sops can decrypt on first boot

--pre-format-files placed the key on the QEMU builder VM's rootfs, not the
target disk. nixos-install chroots into the target and runs sshd-keygen, which
found no key in the chroot and generated a fresh (unregistered) one. sops then
could not decrypt on first boot because the key didn't match .sops.yaml, leaving
both root and nixos with '!' in /etc/shadow even after mutableUsers = false was
set (hashedPasswordFile pointed to paths sops never wrote).

Fix modules/platforms/proxmox.nix to embed the clan SSH host key in
environment.etc via NIXOS_HOST_KEYS_DIR at eval time -- the same pattern
lxc.nix uses. nixos-install's own activation places the key on the target disk,
sshd-keygen finds it already present and skips generation, and sops decrypts
correctly on first boot. Includes the same preserveSshHostKey/restoreSshHostKey
activation scripts as lxc.nix so subsequent nixos-rebuild switch calls (without
NIXOS_HOST_KEYS_DIR) don't remove the key as "obsolete" from environment.etc.

Update create-proxmox-resource.sh: switch VM builds from
  ./result-<target> --pre-format-files ... --build-memory 2048
to
  NIXOS_HOST_KEYS_DIR=$(pwd)/host-keys nix build --impure ... diskoImagesScript
  ./result-<target> --build-memory 2048
matching the LXC build path.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011uRcikkTp3D5VbXj2DwNpQ

modules/platforms/proxmox.nix

@@ -1,9 +1,81 @@
-{ ... }:
+{ lib, flakeTarget, ... }:
 
+let
+  # Bakes this exact flake target's pre-generated SSH host key straight
+  # into /etc/ssh/ -- mirrors lxc.nix's builtins.getEnv pattern (impure
+  # and empty under normal `nix build`/`nix eval`, so this is a no-op
+  # unless explicitly opted into with NIXOS_HOST_KEYS_DIR=... --impure).
+  #
+  # Unlike --pre-format-files (which places files on the QEMU builder VM's
+  # rootfs, not the target disk), embedding via environment.etc here means
+  # nixos-install's own activation step installs the key onto the target
+  # disk. sshd-keygen then finds it already present and skips generation,
+  # so the disk image boots with the clan-registered key and sops can
+  # decrypt on first boot.
+  #
+  # Without this, nixos-install's sshd-keygen activation generates a fresh
+  # key (unregistered in .sops.yaml), sops decryption fails permanently,
+  # and password hashes are never applied -- confirmed live: passwords
+  # stayed '!' even with mutableUsers = false because hashedPasswordFile
+  # pointed to a path that sops never wrote.
+  hostKeysDirStr = builtins.getEnv "NIXOS_HOST_KEYS_DIR";
+  hasHostKeysDir = hostKeysDirStr != "" && builtins.pathExists hostKeysDirStr;
+  hostKeysDir = /. + hostKeysDirStr;
+
+  privKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key";
+  pubKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key.pub";
+  hasKeyForThisTarget =
+    hasHostKeysDir
+    && builtins.pathExists privKeyFile
+    && builtins.pathExists pubKeyFile;
+in
 {
   imports = [
     ../hardware-configuration/vm/proxmox.nix
     ../boot/efi.nix
     ../disko/proxmox.nix
   ];
+
+  environment.etc = lib.mkIf hasKeyForThisTarget {
+    "ssh/ssh_host_ed25519_key" = {
+      source = privKeyFile;
+      mode = "0600";
+    };
+    "ssh/ssh_host_ed25519_key.pub" = {
+      source = pubKeyFile;
+      mode = "0644";
+    };
+  };
+
+  # NixOS's etc activation removes any /etc file that was in the previous
+  # generation's environment.etc but is absent from the current one. Since
+  # the SSH key is only in environment.etc during the --impure build (when
+  # NIXOS_HOST_KEYS_DIR is set), normal rebuilds would remove it as
+  # "obsolete". These scripts mirror lxc.nix's approach: save the live key
+  # before etc runs, restore it after. Without the explicit deps, the
+  # topological sort places preserveSshHostKey after etc (confirmed live on
+  # lxc-tor-relay: position 7 vs etc's position 5), so the key is gone
+  # before it can be saved.
+  system.activationScripts = {
+    preserveSshHostKey = ''
+      if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
+        cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
+        cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
+      fi
+    '';
+
+    restoreSshHostKey = {
+      deps = [ "etc" ];
+      text = ''
+        if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
+          install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
+          install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
+        fi
+        rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
+      '';
+    };
+
+    etc = { deps = [ "preserveSshHostKey" ]; };
+    setupSecrets = { deps = [ "restoreSshHostKey" ]; };
+  };
 }

scripts/proxmox/create-proxmox-resource.sh

@@ -778,12 +778,9 @@ REMOTE_SCRIPT
     fi
   else
     if [[ "$dry_run" -eq 1 ]]; then
-      echo "[dry-run] would build on ${node}: nix build --no-use-registries --no-accept-flake-config${nix_opts_display} \\"
+      echo "[dry-run] would build on ${node}: NIXOS_HOST_KEYS_DIR=\$(pwd)/host-keys nix build --impure --no-use-registries --no-accept-flake-config${nix_opts_display} \\"
       echo "[dry-run]   .#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript"
-      echo "[dry-run] would run:   ${sudo_display}./result-${flake_target} \\"
-      echo "[dry-run]                --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key \\"
-      echo "[dry-run]                --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key.pub /etc/ssh/ssh_host_ed25519_key.pub \\"
-      echo "[dry-run]                --build-memory 2048"
+      echo "[dry-run] would run:   ${sudo_display}./result-${flake_target} --build-memory 2048"
       echo "[dry-run] image will be at ${vm_built_raw} (imported from there; no mv to /var/lib/vz/import/)"
       local_image="<built-image>.raw"
     else
@@ -791,6 +788,14 @@ REMOTE_SCRIPT
       # See the LXC branch above for why this is one %q-quoted command
       # string rather than separate ssh argv elements.
       # $7 = image_name (hostname, the diskoImagesScript's own output filename).
+      #
+      # NIXOS_HOST_KEYS_DIR + --impure: modules/platforms/proxmox.nix reads
+      # this env var at eval time (like lxc.nix) to embed the clan SSH host
+      # key in environment.etc. nixos-install's own activation then places the
+      # key on the target disk, so sshd-keygen finds it already present and
+      # skips generation. --pre-format-files put the key on the QEMU builder
+      # VM's rootfs (not the target disk), so sshd-keygen regenerated a fresh
+      # key -- one not registered in .sops.yaml -- and sops could never decrypt.
       printf -v remote_cmd 'bash -s -- %q %q %q %q %q %q %q' \
         "$remote_repo_dir" "$flake_target" "$remote_dir" "$remote_filename" "$NIX_EXTRA_OPTS" "$sudo_prefix" "$host"
       ssh "$ssh_target" "$remote_cmd" <<'REMOTE_SCRIPT'
@@ -807,16 +812,18 @@ if [[ ! -f "host-keys/${target}_ssh_host_ed25519_key" ]]; then
   echo "and ensure it was synced here before starting the build." >&2
   exit 1
 fi
-nix build --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \
+# Build diskoImagesScript with NIXOS_HOST_KEYS_DIR so proxmox.nix embeds the
+# clan SSH key in environment.etc (same as lxc.nix). This causes nixos-install
+# to place the key on the target disk, so sshd-keygen finds it and skips
+# generation -- the disk image boots with the registered key, sops decrypts.
+NIXOS_HOST_KEYS_DIR="$(pwd)/host-keys" nix build --impure \
+  --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \
   ".#nixosConfigurations.${target}.config.system.build.diskoImagesScript" \
   --out-link "result-${target}"
 # Remove any stale .raw from a previous failed build so the post-build check
 # below is unambiguous (diskoImagesScript writes to CWD as ${image_name}.raw).
 $sudo_pfx rm -f "${image_name}.raw" 2>/dev/null || true
-$sudo_pfx "./result-${target}" \
-  --pre-format-files "$(pwd)/host-keys/${target}_ssh_host_ed25519_key" /etc/ssh/ssh_host_ed25519_key \
-  --pre-format-files "$(pwd)/host-keys/${target}_ssh_host_ed25519_key.pub" /etc/ssh/ssh_host_ed25519_key.pub \
-  --build-memory 2048
+$sudo_pfx "./result-${target}" --build-memory 2048
 if [[ ! -f "${image_name}.raw" ]]; then
   echo "ERROR: ${image_name}.raw not found in ${repo_dir} after build -- disko/QEMU may have failed." >&2
   exit 1