fix(ha): fix deploy script bugs and correct cluster configuration

- Add root SSH key + passwordless sudo to ha-server nodes (needed for
  deploy script to run cluster-init.sh via SSH as root on node1)
- Fix cluster-init.sh: correct default IPs (228/227/229 per variables.nix),
  use \${VAR:-default} for all config so deploy.sh can override via env
- Fix acceptance-tests.sh: same IP corrections, add -i flag to SSH calls,
  use \${VAR:-default} pattern
- Fix deploy.sh dry-run bugs: pve_check() always runs SSH for read-only
  probes so bridge existence check is accurate; wait_for_ssh skips in
  dry-run instead of timing out
- Fix cluster-init invocation: upload script via scp and run via SSH as
  root on node1 (was incorrectly trying to run the script locally)
- Fix acceptance-tests invocation: pass IP env vars from deploy.sh

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8

modules/ha/cluster-config.nix

@@ -19,6 +19,14 @@
 #   cluster-enable-stonith.sh once the fence key is deployed.
 { lib, vars, ... }:
 {
+  # Root SSH access with the admin key — needed for the deploy script to upload
+  # and run cluster-init.sh as root on node1, and for node1→node2 SSH during init.
+  users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey ];
+
+  # Passwordless sudo for wheel — operator SSHes as nixos and uses sudo for
+  # cluster management commands (drbdadm, crm*, pcs, etc.)
+  security.sudo.wheelNeedsPassword = lib.mkForce false;
+
   services.drbd = {
     enable = true;
     config = ''

scripts/ha/acceptance-tests.sh

@@ -7,13 +7,14 @@
 set -euo pipefail
 
 # ── Configuration ─────────────────────────────────────────────────────────
-NODE1="ha-server-1"
+# All values override-able via environment variables; defaults match variables.nix.
-NODE2="ha-server-2"
+NODE1="${NODE1:-ha-server-1}"
-NODE1_IP="192.168.2.200"   # vars.haServer1Ip
+NODE2="${NODE2:-ha-server-2}"
-NODE2_IP="192.168.2.201"   # vars.haServer2Ip
+NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
-VIP="192.168.2.202"        # vars.haServerVip
+NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
-XFS_MOUNT="/srv/ha-data"   # vars.haStorageRoot
+VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
-ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage"  # vars.haIscsiIqn
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
+ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}"  # vars.haIscsiIqn
 # ──────────────────────────────────────────────────────────────────────────
 
 PASS=0
@@ -23,8 +24,8 @@ RESULTS=()
 pass() { echo "  PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); }
 fail() { echo "  FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); }
 
-n1() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; }
+n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; }
-n2() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; }
+n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; }
 
 echo "════════════════════════════════════════════════════"
 echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"

scripts/ha/cluster-init.sh

@@ -21,22 +21,21 @@
 set -euo pipefail
 
 # ── Configuration ─────────────────────────────────────────────────────────
-# These must match variables.nix haServer* values and the Proxmox VMID
+# All values override-able via environment variables; defaults match variables.nix.
-# assignments.  Update before running.
+NODE1="${NODE1:-ha-server-1}"
-NODE1="ha-server-1"
+NODE2="${NODE2:-ha-server-2}"
-NODE2="ha-server-2"
+NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
-NODE1_IP="192.168.2.200"   # vars.haServer1Ip
+NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
-NODE2_IP="192.168.2.201"   # vars.haServer2Ip
+VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
-VIP="192.168.2.202"        # vars.haServerVip
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
-XFS_MOUNT="/srv/ha-data"   # vars.haStorageRoot
+ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}"  # vars.haIscsiIqn
-ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage"  # vars.haIscsiIqn
 ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
 ISCSI_LUN_SIZE="10G"
 DRBD_DEVICE="/dev/drbd0"
-VMID_NODE1=""              # FILL IN: Proxmox VMID for ha-server-1
+VMID_NODE1="${VMID_NODE1:-}"            # set by deploy.sh; needed for STONITH
-VMID_NODE2=""              # FILL IN: Proxmox VMID for ha-server-2
+VMID_NODE2="${VMID_NODE2:-}"
-PVE_HOST="pve1.sweet.home"
+PVE_HOST="${PVE_HOST:-pve1.sweet.home}"
-PVE_USER="wayne"
+PVE_USER="${PVE_USER:-wayne}"
 
 # NFS dataset subdirectories to create under XFS_MOUNT.
 # Must mirror vars.nfsShares subpath values in variables.nix.

scripts/ha/deploy.sh

@@ -142,6 +142,11 @@ pve() {
   fi
 }
 
+pve_check() {
+  # Run a read-only probe on the Proxmox node — always executes even in dry-run.
+  ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
+}
+
 n1() {
   # Run a command on ha-server-1 via SSH.
   ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null
@@ -153,7 +158,12 @@ n2() {
 }
 
 wait_for_ssh() {
-  local ip="$1" label="$2" deadline=$(( $(date +%s) + 300 ))
+  local ip="$1" label="$2"
+  if $DRY_RUN; then
+    logn "[dry-run] Skipping SSH wait for ${label} (${ip})"
+    return 0
+  fi
+  local deadline=$(( $(date +%s) + 300 ))
   log "Waiting for SSH on ${label} (${ip}) — up to 5 min..."
   while [[ $(date +%s) -lt $deadline ]]; do
     if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=3 \
@@ -192,7 +202,7 @@ fi
 
 if ! $SKIP_ENSURE_BRIDGE; then
   log "Phase 1: Ensuring storage bridge ${STORAGE_BRIDGE} on ${NODE}"
-  if pve "ip link show ${STORAGE_BRIDGE}" &>/dev/null; then
+  if pve_check "test -d /sys/class/net/${STORAGE_BRIDGE}" &>/dev/null; then
     logn "${STORAGE_BRIDGE} already exists — skipping."
   else
     logn "Creating isolated internal bridge ${STORAGE_BRIDGE} (no upstream port, ${STORAGE_CIDR})"
@@ -287,27 +297,31 @@ if ! $SKIP_CLUSTER_INIT; then
   log "Phase 6: Initialising HA cluster"
 
   CLUSTER_INIT="${REPO_ROOT}/scripts/ha/cluster-init.sh"
-  if [[ ! -x "$CLUSTER_INIT" ]]; then
+  [[ -x "$CLUSTER_INIT" ]] || chmod +x "$CLUSTER_INIT"
-    chmod +x "$CLUSTER_INIT"
-  fi
 
-  run bash "$CLUSTER_INIT" \
+  if $DRY_RUN; then
-    NODE1="$NODE1_HOST" NODE2="$NODE2_HOST" \
+    logn "[dry-run] Would scp cluster-init.sh to root@${NODE1_IP} and run it"
-    NODE1_IP="$NODE1_IP" NODE2_IP="$NODE2_IP" \
+  else
-    VIP="192.168.2.229" \
+    logn "Uploading cluster-init.sh to ${NODE1_HOST}..."
-    XFS_MOUNT="/srv/ha-data" \
+    scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
-    ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" \
+      "$CLUSTER_INIT" "root@${NODE1_IP}:/tmp/cluster-init.sh"
-    VMID_NODE1="$VMID1" VMID_NODE2="$VMID2"
+
+    logn "Running cluster-init.sh on ${NODE1_HOST}..."
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "root@${NODE1_IP}" \
+      "NODE1=${NODE1_HOST} NODE2=${NODE2_HOST} \
+       NODE1_IP=${NODE1_IP} NODE2_IP=${NODE2_IP} \
+       VIP=192.168.2.229 XFS_MOUNT=/srv/ha-data \
+       ISCSI_IQN=iqn.2026-01.home.sweet:ha-storage \
+       VMID_NODE1=${VMID1} VMID_NODE2=${VMID2} \
+       bash /tmp/cluster-init.sh"
 
     # Encrypt the corosync authkey generated by cluster-init and commit it.
     log "  Encrypting corosync authkey into secrets/ha-corosync-authkey..."
-  if ! $DRY_RUN; then
     AUTHKEY_TMP="${REPO_ROOT}/secrets/ha-corosync-authkey.tmp"
     n1 "cat /etc/corosync/authkey" > "$AUTHKEY_TMP"
     if [[ ! -s "$AUTHKEY_TMP" ]]; then
       err "corosync authkey on node1 is empty — cluster-init may have failed."
     fi
-    # sops-encrypt in-place; creation rule matches secrets/ha-corosync-authkey
     mv "$AUTHKEY_TMP" "${REPO_ROOT}/secrets/ha-corosync-authkey"
     (cd "${REPO_ROOT}" && nix run nixpkgs#sops -- -e --input-type binary -i secrets/ha-corosync-authkey)
     logn "Authkey encrypted. Committing..."
@@ -321,7 +335,14 @@ fi
 
 if ! $SKIP_TESTS; then
   log "Phase 7: Running acceptance tests (T1–T7)"
-  run bash "${REPO_ROOT}/scripts/ha/acceptance-tests.sh"
+  if $DRY_RUN; then
+    logn "[dry-run] Would run acceptance-tests.sh against ${NODE1_HOST}/${NODE2_HOST}"
+  else
+    NODE1="$NODE1_HOST" NODE2="$NODE2_HOST" \
+    NODE1_IP="$NODE1_IP" NODE2_IP="$NODE2_IP" \
+    VIP="192.168.2.229" \
+    bash "${REPO_ROOT}/scripts/ha/acceptance-tests.sh"
+  fi
 fi
 
 log "Deploy complete."