From 684351b89b958366dea84e8b4f006387abe2fe2b Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Tue, 28 Jul 2026 22:51:29 +1000 Subject: [PATCH] fix(ha): pre-seed SSH host key in disko image; fix DRBD init race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of recurring sops failures on new VM boots: disko builds raw disk images, and create-proxmox-resource.sh only syncs the clan-var SSH host key to the Proxmox node (for proxmox.nix to bake into the image) when it actually builds — reusing a cached image skips sync_remote_host_keys, so destroy+recreate reuses a stale image with the wrong or randomly-generated key baked in. On first boot the VM gets a different key than what .sops.yaml was encrypted for, and sops fails permanently. Fix 1 — deploy.sh Phase 3: always pass --force-rebuild so every VM creation rebuilds the disko image fresh with the current clan-var key baked in via NIXOS_HOST_KEYS_DIR (proxmox.nix already reads this under --impure). Fix 2 — deploy.sh Phase 5.5: after VMs boot, scan their actual ed25519 host keys and, if they drift from clan vars, update the clan var pub-key files, rewrite the .sops.yaml age anchors, and re-encrypt all affected sops files. Defence-in-depth: normally a no-op after Fix 1, but catches any residual mismatch (e.g. --skip-create-vms reuse of an older image). Fix 3 — cluster-init.sh: add crm_standby -v on for both nodes before DRBD metadata init. Without this, Pacemaker's OCF DRBD agent races: it sees drbdadm down as a failure and immediately calls drbdadm up again, leaving /dev/sdb busy when create-md / write-dev-uuid runs (drbdmeta exits 20 with "stdin not a TTY, not waiting for confirmation"). Standby suppresses resource scheduling during init; crm_standby -v off restores it after DRBD is up on both nodes. Co-Authored-By: Claude Sonnet 4.6 --- scripts/ha/cluster-init.sh | 35 ++++++++++--- scripts/ha/deploy.sh | 101 ++++++++++++++++++++++++++++++++++--- 2 files changed, 124 insertions(+), 12 deletions(-) diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh index dd76e69..7bcd798 100755 --- a/scripts/ha/cluster-init.sh +++ b/scripts/ha/cluster-init.sh @@ -135,14 +135,33 @@ for i in $(seq 1 30); do done # ── 2. DRBD initialisation ──────────────────────────────────────────────── -# Down DRBD first on both nodes before (re-)initialising metadata. -# This ensures /dev/sdb is not held open by the kernel module, which would -# trigger a drbdmeta TTY-confirmation prompt ("stdin not a TTY, not waiting -# for confirmation") during the write-dev-uuid step even when --force is set. -log "Detaching DRBD on $NODE1 (idempotent pre-init clean-up)..." +# Put both nodes in Pacemaker standby before touching DRBD metadata. +# Without this, the OCF DRBD agent races: it sees drbdadm-down as a failure +# and immediately calls drbdadm-up again, leaving /dev/sdb busy when +# create-md / write-dev-uuid runs. On a fresh cluster with no resources +# configured this is a no-op; on a re-run it stops the race. +log "Setting both nodes to Pacemaker standby for DRBD metadata init..." +crm_standby -N "$NODE1" -v on 2>/dev/null || true +crm_standby -N "$NODE2" -v on 2>/dev/null || true + +# Wait for Pacemaker to actually stop DRBD (if it was managing it). +log "Waiting for DRBD to stop under Pacemaker control..." +for i in $(seq 1 30); do + n1_role=$(drbdadm role ha-data 2>/dev/null || echo "Unconfigured") + n2_role=$(n2_ssh "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "Unconfigured") + if [[ "$n1_role" == "Unconfigured" ]] && [[ "$n2_role" == "Unconfigured" ]]; then + log "DRBD stopped on both nodes" + break + fi + [[ $i -eq 30 ]] && warn "DRBD still active after 60s standby — forcing down anyway" + sleep 2 +done + +log "Detaching DRBD on $NODE1 (belt-and-suspenders after standby)..." drbdadm down ha-data 2>/dev/null || true -log "Detaching DRBD on $NODE2 (idempotent pre-init clean-up)..." +log "Detaching DRBD on $NODE2..." n2_ssh "drbdadm down ha-data 2>/dev/null || true" +sleep 2 log "Initialising DRBD metadata on $NODE1..." if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then @@ -157,6 +176,10 @@ log "Bringing up DRBD on both nodes..." drbdadm up ha-data 2>/dev/null || true n2_ssh "drbdadm up ha-data" 2>/dev/null || true +log "Clearing Pacemaker standby — DRBD is up, letting Pacemaker resume..." +crm_standby -N "$NODE1" -v off 2>/dev/null || true +crm_standby -N "$NODE2" -v off 2>/dev/null || true + log "Forcing $NODE1 to DRBD Primary for initial sync..." drbdadm primary ha-data --force diff --git a/scripts/ha/deploy.sh b/scripts/ha/deploy.sh index 2d35064..6f1f9e3 100755 --- a/scripts/ha/deploy.sh +++ b/scripts/ha/deploy.sh @@ -32,6 +32,7 @@ # --skip-create-vms Skip VM creation (VMs already exist) # --skip-add-hardware Skip net1/scsi1 attachment (already attached) # --skip-boot-wait Skip boot/SSH wait (VMs already running) +# --skip-refresh-sops-keys Skip scanning running VMs for fresh SSH host keys # --skip-cluster-init Skip cluster formation (cluster already configured) # --skip-tests Skip acceptance tests # --force-rebuild Pass --force-rebuild to create-proxmox-resource.sh @@ -70,6 +71,7 @@ SKIP_SYNC_KEYS=false SKIP_CREATE_VMS=false SKIP_ADD_HARDWARE=false SKIP_BOOT_WAIT=false +SKIP_REFRESH_SOPS_KEYS=false SKIP_CLUSTER_INIT=false SKIP_TESTS=false FORCE_REBUILD=false @@ -108,8 +110,9 @@ while [[ $# -gt 0 ]]; do --skip-sync-keys) SKIP_SYNC_KEYS=true; shift ;; --skip-create-vms) SKIP_CREATE_VMS=true; shift ;; --skip-add-hardware) SKIP_ADD_HARDWARE=true; shift ;; - --skip-boot-wait) SKIP_BOOT_WAIT=true; shift ;; - --skip-cluster-init) SKIP_CLUSTER_INIT=true; shift ;; + --skip-boot-wait) SKIP_BOOT_WAIT=true; shift ;; + --skip-refresh-sops-keys) SKIP_REFRESH_SOPS_KEYS=true; shift ;; + --skip-cluster-init) SKIP_CLUSTER_INIT=true; shift ;; --skip-tests) SKIP_TESTS=true; shift ;; --force-rebuild) FORCE_REBUILD=true; shift ;; --destroy) DESTROY=true; shift ;; @@ -275,14 +278,16 @@ fi if ! $SKIP_CREATE_VMS; then log "Phase 3: Building and creating VMs on ${NODE}" - REBUILD_FLAG="" - $FORCE_REBUILD && REBUILD_FLAG="--force-rebuild" - CREATE="${REPO_ROOT}/scripts/proxmox/create-proxmox-resource.sh" for spec in "${VMID1}:ha-server-1:proxmox-ha-server-1" "${VMID2}:ha-server-2:proxmox-ha-server-2"; do IFS=: read -r vmid host_name flake_target <<< "$spec" log "Creating ${flake_target} (VMID ${vmid}) on ${NODE}..." + # Always --force-rebuild: create-proxmox-resource.sh only calls + # sync_remote_host_keys (which populates host-keys/ for proxmox.nix to + # bake the clan-var SSH key into the disko image) when it actually builds. + # Reusing a cached image skips that step, so destroy+recreate would reuse + # an image with a stale/random key baked in → sops fails on first boot. run bash "$CREATE" \ --type vm \ --host "$host_name" \ @@ -291,7 +296,7 @@ if ! $SKIP_CREATE_VMS; then --storage "$STORAGE" \ --memory "$MEMORY_MB" \ --cores "$CORES" \ - ${REBUILD_FLAG} + --force-rebuild done fi @@ -325,6 +330,90 @@ if ! $SKIP_BOOT_WAIT; then sleep 10 fi +# ── Phase 5.5: Refresh sops host-key registrations ─────────────────────────── +# +# Disko builds raw disk images; each new VM boots with a freshly-generated SSH +# host key rather than the one pre-seeded in clan vars. This phase scans the +# actual running VMs, and if their ed25519 host keys differ from what clan vars +# record: updates the clan var pub-key files, rewrites the .sops.yaml age-key +# anchors, and re-encrypts all affected sops files so the nodes can decrypt +# secrets on the next nixos-rebuild. Safe no-op when keys haven't changed. + +if ! $SKIP_REFRESH_SOPS_KEYS; then + if $DRY_RUN; then + logn "[dry-run] Would scan VM host keys and refresh .sops.yaml / secrets if needed" + else + log "Phase 5.5: Refreshing sops host-key registrations (disko key drift fix)" + SOPS_UPDATED=false + + for spec in \ + "${NODE1_IP}:proxmox-ha-server-1:${NODE1_HOST}" \ + "${NODE2_IP}:proxmox-ha-server-2:${NODE2_HOST}"; do + IFS=: read -r node_ip flake_target host_name <<< "$spec" + CLAN_PUB="${REPO_ROOT}/vars/per-machine/${flake_target}/openssh/ssh_host_ed25519_key.pub/value" + + logn "Scanning ed25519 host key from ${host_name} (${node_ip})..." + RAW=$(ssh-keyscan -t ed25519 "${node_ip}" 2>/dev/null | grep -v "^#") || true + if [[ -z "$RAW" ]]; then + logn "WARNING: no ed25519 key returned by ssh-keyscan for ${node_ip} — skipping" + continue + fi + # ssh-keyscan returns: ssh-ed25519 + SCANNED_TYPE=$(awk '{print $2}' <<< "$RAW") + SCANNED_KEY=$(awk '{print $3}' <<< "$RAW") + SCANNED_PUBKEY="${SCANNED_TYPE} ${SCANNED_KEY} ${host_name}" + + CURRENT=$(tr -d '\n' < "$CLAN_PUB" 2>/dev/null || true) + if [[ "$SCANNED_PUBKEY" == "$CURRENT" ]]; then + logn "${host_name}: clan var matches running key — no update needed" + continue + fi + + logn "${host_name}: key drift detected — updating clan var" + logn " old: ${CURRENT}" + logn " new: ${SCANNED_PUBKEY}" + echo "$SCANNED_PUBKEY" > "$CLAN_PUB" + SOPS_UPDATED=true + + # Rewrite the .sops.yaml anchor for this host with the new age key. + ANCHOR="${flake_target}" # e.g. proxmox-ha-server-1 + NEW_AGE=$(echo "$SCANNED_PUBKEY" | \ + nix run --quiet --no-warn-dirty nixpkgs#ssh-to-age 2>/dev/null) + if [[ -z "$NEW_AGE" ]]; then + err "ssh-to-age produced no output for ${host_name} — check nixpkgs#ssh-to-age" + fi + logn " new age key: ${NEW_AGE}" + sed -i "/&${ANCHOR} /s| age[a-z0-9]*$| ${NEW_AGE}|" "${REPO_ROOT}/.sops.yaml" + done + + if $SOPS_UPDATED; then + logn "Running sops updatekeys on affected secrets..." + SOPS="nix run --quiet --no-warn-dirty nixpkgs#sops --" + (cd "${REPO_ROOT}" && \ + $SOPS updatekeys -y secrets/common.yaml && \ + $SOPS updatekeys -y secrets/ha-server-1.yaml && \ + $SOPS updatekeys -y secrets/ha-server-2.yaml && \ + $SOPS updatekeys -y secrets/ha-server-1.keytab && \ + $SOPS updatekeys -y secrets/ha-server-2.keytab) + # Note: ha-corosync-authkey is re-generated and re-encrypted by cluster-init below. + + logn "Committing refreshed host keys and re-encrypted secrets..." + (cd "${REPO_ROOT}" && \ + git add \ + vars/per-machine/proxmox-ha-server-1/openssh/ssh_host_ed25519_key.pub/value \ + vars/per-machine/proxmox-ha-server-2/openssh/ssh_host_ed25519_key.pub/value \ + .sops.yaml \ + secrets/common.yaml \ + secrets/ha-server-1.yaml \ + secrets/ha-server-2.yaml \ + secrets/ha-server-1.keytab \ + secrets/ha-server-2.keytab && \ + git commit -m "secrets(ha): refresh sops host-key registrations for new VM instances" || true) + logn "Sops keys refreshed and committed." + fi + fi +fi + # ── Phase 6: Cluster init ───────────────────────────────────────────────────── if ! $SKIP_CLUSTER_INIT; then