fix(ha): pre-seed SSH host key in disko image; fix DRBD init race

Root cause of recurring sops failures on new VM boots: disko builds raw
disk images, and create-proxmox-resource.sh only syncs the clan-var SSH
host key to the Proxmox node (for proxmox.nix to bake into the image) when
it actually builds — reusing a cached image skips sync_remote_host_keys, so
destroy+recreate reuses a stale image with the wrong or randomly-generated
key baked in.  On first boot the VM gets a different key than what .sops.yaml
was encrypted for, and sops fails permanently.

Fix 1 — deploy.sh Phase 3: always pass --force-rebuild so every VM creation
rebuilds the disko image fresh with the current clan-var key baked in via
NIXOS_HOST_KEYS_DIR (proxmox.nix already reads this under --impure).

Fix 2 — deploy.sh Phase 5.5: after VMs boot, scan their actual ed25519 host
keys and, if they drift from clan vars, update the clan var pub-key files,
rewrite the .sops.yaml age anchors, and re-encrypt all affected sops files.
Defence-in-depth: normally a no-op after Fix 1, but catches any residual
mismatch (e.g. --skip-create-vms reuse of an older image).

Fix 3 — cluster-init.sh: add crm_standby -v on for both nodes before DRBD
metadata init.  Without this, Pacemaker's OCF DRBD agent races: it sees
drbdadm down as a failure and immediately calls drbdadm up again, leaving
/dev/sdb busy when create-md / write-dev-uuid runs (drbdmeta exits 20 with
"stdin not a TTY, not waiting for confirmation").  Standby suppresses
resource scheduling during init; crm_standby -v off restores it after DRBD
is up on both nodes.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scripts/ha/cluster-init.sh

@@ -135,14 +135,33 @@ for i in $(seq 1 30); do
 done
 
 # ── 2. DRBD initialisation ────────────────────────────────────────────────
-# Down DRBD first on both nodes before (re-)initialising metadata.
+# Put both nodes in Pacemaker standby before touching DRBD metadata.
-# This ensures /dev/sdb is not held open by the kernel module, which would
+# Without this, the OCF DRBD agent races: it sees drbdadm-down as a failure
-# trigger a drbdmeta TTY-confirmation prompt ("stdin not a TTY, not waiting
+# and immediately calls drbdadm-up again, leaving /dev/sdb busy when
-# for confirmation") during the write-dev-uuid step even when --force is set.
+# create-md / write-dev-uuid runs.  On a fresh cluster with no resources
-log "Detaching DRBD on $NODE1 (idempotent pre-init clean-up)..."
+# configured this is a no-op; on a re-run it stops the race.
+log "Setting both nodes to Pacemaker standby for DRBD metadata init..."
+crm_standby -N "$NODE1" -v on 2>/dev/null || true
+crm_standby -N "$NODE2" -v on 2>/dev/null || true
+
+# Wait for Pacemaker to actually stop DRBD (if it was managing it).
+log "Waiting for DRBD to stop under Pacemaker control..."
+for i in $(seq 1 30); do
+  n1_role=$(drbdadm role ha-data 2>/dev/null || echo "Unconfigured")
+  n2_role=$(n2_ssh "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "Unconfigured")
+  if [[ "$n1_role" == "Unconfigured" ]] && [[ "$n2_role" == "Unconfigured" ]]; then
+    log "DRBD stopped on both nodes"
+    break
+  fi
+  [[ $i -eq 30 ]] && warn "DRBD still active after 60s standby — forcing down anyway"
+  sleep 2
+done
+
+log "Detaching DRBD on $NODE1 (belt-and-suspenders after standby)..."
 drbdadm down ha-data 2>/dev/null || true
-log "Detaching DRBD on $NODE2 (idempotent pre-init clean-up)..."
+log "Detaching DRBD on $NODE2..."
 n2_ssh "drbdadm down ha-data 2>/dev/null || true"
+sleep 2
 
 log "Initialising DRBD metadata on $NODE1..."
 if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then
@@ -157,6 +176,10 @@ log "Bringing up DRBD on both nodes..."
 drbdadm up ha-data 2>/dev/null || true
 n2_ssh "drbdadm up ha-data" 2>/dev/null || true
 
+log "Clearing Pacemaker standby — DRBD is up, letting Pacemaker resume..."
+crm_standby -N "$NODE1" -v off 2>/dev/null || true
+crm_standby -N "$NODE2" -v off 2>/dev/null || true
+
 log "Forcing $NODE1 to DRBD Primary for initial sync..."
 drbdadm primary ha-data --force
 

scripts/ha/deploy.sh

@@ -32,6 +32,7 @@
 #   --skip-create-vms        Skip VM creation (VMs already exist)
 #   --skip-add-hardware      Skip net1/scsi1 attachment (already attached)
 #   --skip-boot-wait         Skip boot/SSH wait (VMs already running)
+#   --skip-refresh-sops-keys Skip scanning running VMs for fresh SSH host keys
 #   --skip-cluster-init      Skip cluster formation (cluster already configured)
 #   --skip-tests             Skip acceptance tests
 #   --force-rebuild          Pass --force-rebuild to create-proxmox-resource.sh
@@ -70,6 +71,7 @@ SKIP_SYNC_KEYS=false
 SKIP_CREATE_VMS=false
 SKIP_ADD_HARDWARE=false
 SKIP_BOOT_WAIT=false
+SKIP_REFRESH_SOPS_KEYS=false
 SKIP_CLUSTER_INIT=false
 SKIP_TESTS=false
 FORCE_REBUILD=false
@@ -109,6 +111,7 @@ while [[ $# -gt 0 ]]; do
     --skip-create-vms)    SKIP_CREATE_VMS=true; shift ;;
     --skip-add-hardware)  SKIP_ADD_HARDWARE=true; shift ;;
     --skip-boot-wait)         SKIP_BOOT_WAIT=true; shift ;;
+    --skip-refresh-sops-keys) SKIP_REFRESH_SOPS_KEYS=true; shift ;;
     --skip-cluster-init)      SKIP_CLUSTER_INIT=true; shift ;;
     --skip-tests)         SKIP_TESTS=true;     shift ;;
     --force-rebuild)      FORCE_REBUILD=true;  shift ;;
@@ -275,14 +278,16 @@ fi
 if ! $SKIP_CREATE_VMS; then
   log "Phase 3: Building and creating VMs on ${NODE}"
 
-  REBUILD_FLAG=""
-  $FORCE_REBUILD && REBUILD_FLAG="--force-rebuild"
-
   CREATE="${REPO_ROOT}/scripts/proxmox/create-proxmox-resource.sh"
 
   for spec in "${VMID1}:ha-server-1:proxmox-ha-server-1" "${VMID2}:ha-server-2:proxmox-ha-server-2"; do
     IFS=: read -r vmid host_name flake_target <<< "$spec"
     log "Creating ${flake_target} (VMID ${vmid}) on ${NODE}..."
+    # Always --force-rebuild: create-proxmox-resource.sh only calls
+    # sync_remote_host_keys (which populates host-keys/ for proxmox.nix to
+    # bake the clan-var SSH key into the disko image) when it actually builds.
+    # Reusing a cached image skips that step, so destroy+recreate would reuse
+    # an image with a stale/random key baked in → sops fails on first boot.
     run bash "$CREATE" \
       --type vm \
       --host "$host_name" \
@@ -291,7 +296,7 @@ if ! $SKIP_CREATE_VMS; then
       --storage "$STORAGE" \
       --memory "$MEMORY_MB" \
       --cores "$CORES" \
-      ${REBUILD_FLAG}
+      --force-rebuild
   done
 fi
 
@@ -325,6 +330,90 @@ if ! $SKIP_BOOT_WAIT; then
   sleep 10
 fi
 
+# ── Phase 5.5: Refresh sops host-key registrations ───────────────────────────
+#
+# Disko builds raw disk images; each new VM boots with a freshly-generated SSH
+# host key rather than the one pre-seeded in clan vars.  This phase scans the
+# actual running VMs, and if their ed25519 host keys differ from what clan vars
+# record: updates the clan var pub-key files, rewrites the .sops.yaml age-key
+# anchors, and re-encrypts all affected sops files so the nodes can decrypt
+# secrets on the next nixos-rebuild.  Safe no-op when keys haven't changed.
+
+if ! $SKIP_REFRESH_SOPS_KEYS; then
+  if $DRY_RUN; then
+    logn "[dry-run] Would scan VM host keys and refresh .sops.yaml / secrets if needed"
+  else
+    log "Phase 5.5: Refreshing sops host-key registrations (disko key drift fix)"
+    SOPS_UPDATED=false
+
+    for spec in \
+      "${NODE1_IP}:proxmox-ha-server-1:${NODE1_HOST}" \
+      "${NODE2_IP}:proxmox-ha-server-2:${NODE2_HOST}"; do
+      IFS=: read -r node_ip flake_target host_name <<< "$spec"
+      CLAN_PUB="${REPO_ROOT}/vars/per-machine/${flake_target}/openssh/ssh_host_ed25519_key.pub/value"
+
+      logn "Scanning ed25519 host key from ${host_name} (${node_ip})..."
+      RAW=$(ssh-keyscan -t ed25519 "${node_ip}" 2>/dev/null | grep -v "^#") || true
+      if [[ -z "$RAW" ]]; then
+        logn "WARNING: no ed25519 key returned by ssh-keyscan for ${node_ip} — skipping"
+        continue
+      fi
+      # ssh-keyscan returns:  <ip>  ssh-ed25519  <b64key>
+      SCANNED_TYPE=$(awk '{print $2}' <<< "$RAW")
+      SCANNED_KEY=$(awk  '{print $3}' <<< "$RAW")
+      SCANNED_PUBKEY="${SCANNED_TYPE} ${SCANNED_KEY} ${host_name}"
+
+      CURRENT=$(tr -d '\n' < "$CLAN_PUB" 2>/dev/null || true)
+      if [[ "$SCANNED_PUBKEY" == "$CURRENT" ]]; then
+        logn "${host_name}: clan var matches running key — no update needed"
+        continue
+      fi
+
+      logn "${host_name}: key drift detected — updating clan var"
+      logn "  old: ${CURRENT}"
+      logn "  new: ${SCANNED_PUBKEY}"
+      echo "$SCANNED_PUBKEY" > "$CLAN_PUB"
+      SOPS_UPDATED=true
+
+      # Rewrite the .sops.yaml anchor for this host with the new age key.
+      ANCHOR="${flake_target}"   # e.g. proxmox-ha-server-1
+      NEW_AGE=$(echo "$SCANNED_PUBKEY" | \
+        nix run --quiet --no-warn-dirty nixpkgs#ssh-to-age 2>/dev/null)
+      if [[ -z "$NEW_AGE" ]]; then
+        err "ssh-to-age produced no output for ${host_name} — check nixpkgs#ssh-to-age"
+      fi
+      logn "  new age key: ${NEW_AGE}"
+      sed -i "/&${ANCHOR} /s| age[a-z0-9]*$| ${NEW_AGE}|" "${REPO_ROOT}/.sops.yaml"
+    done
+
+    if $SOPS_UPDATED; then
+      logn "Running sops updatekeys on affected secrets..."
+      SOPS="nix run --quiet --no-warn-dirty nixpkgs#sops --"
+      (cd "${REPO_ROOT}" && \
+        $SOPS updatekeys -y secrets/common.yaml && \
+        $SOPS updatekeys -y secrets/ha-server-1.yaml && \
+        $SOPS updatekeys -y secrets/ha-server-2.yaml && \
+        $SOPS updatekeys -y secrets/ha-server-1.keytab && \
+        $SOPS updatekeys -y secrets/ha-server-2.keytab)
+      # Note: ha-corosync-authkey is re-generated and re-encrypted by cluster-init below.
+
+      logn "Committing refreshed host keys and re-encrypted secrets..."
+      (cd "${REPO_ROOT}" && \
+        git add \
+          vars/per-machine/proxmox-ha-server-1/openssh/ssh_host_ed25519_key.pub/value \
+          vars/per-machine/proxmox-ha-server-2/openssh/ssh_host_ed25519_key.pub/value \
+          .sops.yaml \
+          secrets/common.yaml \
+          secrets/ha-server-1.yaml \
+          secrets/ha-server-2.yaml \
+          secrets/ha-server-1.keytab \
+          secrets/ha-server-2.keytab && \
+        git commit -m "secrets(ha): refresh sops host-key registrations for new VM instances" || true)
+      logn "Sops keys refreshed and committed."
+    fi
+  fi
+fi
+
 # ── Phase 6: Cluster init ─────────────────────────────────────────────────────
 
 if ! $SKIP_CLUSTER_INIT; then