fix(ha/deploy): use nixos+sudo instead of root SSH; temp key for inter-node comms

Root SSH was failing because only the RSA admin key was authorized but the
local dev box only has an ed25519 key. Fix:

- cluster-config.nix: add ed25519 keys to root (same set as nixos user) so
  future deployments work without the temp-key workaround
- deploy.sh/acceptance-tests.sh: SSH as nixos user with sudo instead of root@
- cluster-init.sh: HA_USER/HA_KEY env vars + n2_ssh()/n2_scp() helpers so
  inter-node SSH works regardless of whether root-to-root is available
- deploy.sh Phase 6: generate temp keypair, authorize on node2, place on node1
  for root to use during cluster-init, clean up afterward

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8

modules/ha/cluster-config.nix

@@ -19,9 +19,12 @@
 #   cluster-enable-stonith.sh once the fence key is deployed.
 { lib, vars, ... }:
 {
-  # Root SSH access with the admin key — needed for the deploy script to upload
+  # Root SSH access — same key set as nixos user so all admin keys can reach root.
-  # and run cluster-init.sh as root on node1, and for node1→node2 SSH during init.
+  users.users.root.openssh.authorizedKeys.keys = [
-  users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey ];
+    vars.adminSshKey
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
+  ];
 
   # Passwordless sudo for wheel — operator SSHes as nixos and uses sudo for
   # cluster management commands (drbdadm, crm*, pcs, etc.)

scripts/ha/acceptance-tests.sh

@@ -24,8 +24,9 @@ RESULTS=()
 pass() { echo "  PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); }
 fail() { echo "  FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); }
 
-n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; }
+HA_USER="nixos"
-n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; }
+n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
 
 echo "════════════════════════════════════════════════════"
 echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"

scripts/ha/cluster-init.sh

@@ -36,6 +36,12 @@ VMID_NODE1="${VMID_NODE1:-}"            # set by deploy.sh; needed for STONITH
 VMID_NODE2="${VMID_NODE2:-}"
 PVE_HOST="${PVE_HOST:-pve1.sweet.home}"
 PVE_USER="${PVE_USER:-wayne}"
+# Inter-node SSH: HA_USER is the user to SSH as on NODE2; HA_KEY is the private
+# key to use.  Default is root-to-root (no key arg).  deploy.sh sets HA_USER=nixos
+# and HA_KEY=/tmp/cluster-init-key so the script works even when root-to-root SSH
+# is not available.
+HA_USER="${HA_USER:-root}"
+HA_KEY="${HA_KEY:-}"
 
 # NFS dataset subdirectories to create under XFS_MOUNT.
 # Must mirror vars.nfsShares subpath values in variables.nix.
@@ -58,6 +64,24 @@ warn() { echo "[cluster-init] WARNING: $*" >&2; }
 [[ $(id -u) -eq 0 ]] || die "must run as root"
 [[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1"
 
+# Inter-node SSH/SCP helpers — abstract over root-to-root vs nixos+sudo.
+_SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10"
+[[ -n "$HA_KEY" ]] && _SSH_OPTS="-i $HA_KEY $_SSH_OPTS"
+if [[ "$HA_USER" == "root" ]]; then
+  n2_ssh() { ssh $_SSH_OPTS "root@${NODE2_IP}" "$@"; }
+  n2_scp() { scp $_SSH_OPTS "$1" "root@${NODE2_IP}:$2"; }
+else
+  # Non-root user with passwordless sudo; wrap each command with sudo.
+  n2_ssh() { ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@"; }
+  n2_scp() {
+    # SCP to a tmp path, then sudo-move to the real destination as the remote user.
+    local src="$1" dst="$2"
+    local tmp="/tmp/_cluster_init_scp_$$"
+    scp $_SSH_OPTS "$src" "${HA_USER}@${NODE2_IP}:${tmp}"
+    ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo mv "${tmp}" "${dst}"
+  }
+fi
+
 # ── 0. Corosync authkey ───────────────────────────────────────────────────
 AUTHKEY="/etc/corosync/authkey"
 mkdir -p /etc/corosync
@@ -67,13 +91,13 @@ if [[ ! -f "$AUTHKEY" ]]; then
   chmod 0400 "$AUTHKEY"
 fi
 log "Distributing authkey to $NODE2..."
-ssh "root@${NODE2_IP}" "mkdir -p /etc/corosync"
+n2_ssh "mkdir -p /etc/corosync"
-scp -q "$AUTHKEY" "root@${NODE2_IP}:${AUTHKEY}"
+n2_scp "$AUTHKEY" "$AUTHKEY"
-ssh "root@${NODE2_IP}" "chmod 0400 '${AUTHKEY}'"
+n2_ssh "chmod 0400 '${AUTHKEY}'"
 
 log "Restarting corosync on both nodes..."
 systemctl restart corosync
-ssh "root@${NODE2_IP}" "systemctl restart corosync"
+n2_ssh "systemctl restart corosync"
 sleep 3
 
 # ── 1. Corosync quorum ────────────────────────────────────────────────────
@@ -104,15 +128,11 @@ if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskl
 fi
 
 log "Initialising DRBD metadata on $NODE2..."
-ssh "root@${NODE2_IP}" "
+n2_ssh "bash -c 'if ! drbdadm dstate ha-data 2>/dev/null | grep -q UpToDate.Inconsistent.Diskless; then drbdadm create-md ha-data --force; fi'"
-  if ! drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent\|Diskless'; then
-    drbdadm create-md ha-data --force
-  fi
-"
 
 log "Bringing up DRBD on both nodes..."
 drbdadm up ha-data 2>/dev/null || true
-ssh "root@${NODE2_IP}" "drbdadm up ha-data 2>/dev/null" || true
+n2_ssh "drbdadm up ha-data" 2>/dev/null || true
 
 log "Forcing $NODE1 to DRBD Primary for initial sync..."
 drbdadm primary ha-data --force
@@ -163,7 +183,7 @@ saveconfig /etc/target/saveconfig.json
 EOF
 
 log "Distributing iSCSI saveconfig to $NODE2..."
-scp -q /etc/target/saveconfig.json "root@${NODE2_IP}:/etc/target/saveconfig.json"
+n2_scp /etc/target/saveconfig.json /etc/target/saveconfig.json
 
 log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..."
 umount "${XFS_MOUNT}"

scripts/ha/deploy.sh

@@ -147,14 +147,16 @@ pve_check() {
   ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
 }
 
+HA_USER="nixos"
+
 n1() {
-  # Run a command on ha-server-1 via SSH.
+  # Run a command on ha-server-1 via SSH as nixos with sudo.
-  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null
+  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null
 }
 
 n2() {
-  # Run a command on ha-server-2 via SSH.
+  # Run a command on ha-server-2 via SSH as nixos with sudo.
-  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null
+  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null
 }
 
 wait_for_ssh() {
@@ -167,7 +169,7 @@ wait_for_ssh() {
   log "Waiting for SSH on ${label} (${ip}) — up to 5 min..."
   while [[ $(date +%s) -lt $deadline ]]; do
     if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=3 \
-        -o BatchMode=yes "root@${ip}" true 2>/dev/null; then
+        -o BatchMode=yes "${HA_USER}@${ip}" true 2>/dev/null; then
       logn "${label} is up."
       return 0
     fi
@@ -332,25 +334,53 @@ if ! $SKIP_CLUSTER_INIT; then
   [[ -x "$CLUSTER_INIT" ]] || chmod +x "$CLUSTER_INIT"
 
   if $DRY_RUN; then
-    logn "[dry-run] Would scp cluster-init.sh to root@${NODE1_IP} and run it"
+    logn "[dry-run] Would generate temp key, authorise on ${NODE2_HOST}, scp cluster-init.sh to ${NODE1_HOST}, and run it as root via sudo"
   else
+    # Generate a temp keypair so cluster-init.sh can SSH node1→node2 as ${HA_USER}.
+    # Root on node1 has no keys; a temp key bridging node1→node2 nixos solves this.
+    TEMP_KEY="${REPO_ROOT}/.tmp-cluster-init-key"
+    TEMP_KEY_PUB="${TEMP_KEY}.pub"
+    rm -f "$TEMP_KEY" "$TEMP_KEY_PUB"
+    ssh-keygen -t ed25519 -f "$TEMP_KEY" -N "" -C "cluster-init-temp-$(date +%s)" -q
+    TEMP_PUBKEY=$(cat "$TEMP_KEY_PUB")
+
+    logn "Authorising temp key on ${NODE2_HOST} for ${HA_USER}..."
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE2_IP}" \
+      "mkdir -p ~/.ssh && chmod 700 ~/.ssh && echo '${TEMP_PUBKEY}' >> ~/.ssh/authorized_keys"
+
+    logn "Placing temp key on ${NODE1_HOST} for root..."
+    scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
+      "$TEMP_KEY" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init-key"
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo mkdir -p /root/.ssh && sudo cp /tmp/cluster-init-key /root/.ssh/cluster-init-key && \
+       sudo chmod 600 /root/.ssh/cluster-init-key && rm -f /tmp/cluster-init-key"
+
     logn "Uploading cluster-init.sh to ${NODE1_HOST}..."
     scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
-      "$CLUSTER_INIT" "root@${NODE1_IP}:/tmp/cluster-init.sh"
+      "$CLUSTER_INIT" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init.sh"
 
     logn "Running cluster-init.sh on ${NODE1_HOST}..."
-    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "root@${NODE1_IP}" \
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
-      "NODE1=${NODE1_HOST} NODE2=${NODE2_HOST} \
+      "sudo env NODE1=${NODE1_HOST} NODE2=${NODE2_HOST} \
        NODE1_IP=${NODE1_IP} NODE2_IP=${NODE2_IP} \
        VIP=192.168.2.229 XFS_MOUNT=/srv/ha-data \
        ISCSI_IQN=iqn.2026-01.home.sweet:ha-storage \
        VMID_NODE1=${VMID1} VMID_NODE2=${VMID2} \
+       HA_USER=${HA_USER} HA_KEY=/root/.ssh/cluster-init-key \
        bash /tmp/cluster-init.sh"
 
+    logn "Cleaning up temp key from both nodes..."
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE2_IP}" \
+      "sed -i '/cluster-init-temp/d' ~/.ssh/authorized_keys" 2>/dev/null || true
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo rm -f /root/.ssh/cluster-init-key" 2>/dev/null || true
+    rm -f "$TEMP_KEY" "$TEMP_KEY_PUB"
+
     # Encrypt the corosync authkey generated by cluster-init and commit it.
     log "  Encrypting corosync authkey into secrets/ha-corosync-authkey..."
     AUTHKEY_TMP="${REPO_ROOT}/secrets/ha-corosync-authkey.tmp"
-    n1 "cat /etc/corosync/authkey" > "$AUTHKEY_TMP"
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo cat /etc/corosync/authkey" > "$AUTHKEY_TMP"
     if [[ ! -s "$AUTHKEY_TMP" ]]; then
       err "corosync authkey on node1 is empty — cluster-init may have failed."
     fi