fix(ha/deploy): use nixos+sudo instead of root SSH; temp key for inter-node comms

Root SSH was failing because only the RSA admin key was authorized but the
local dev box only has an ed25519 key. Fix:

- cluster-config.nix: add ed25519 keys to root (same set as nixos user) so
  future deployments work without the temp-key workaround
- deploy.sh/acceptance-tests.sh: SSH as nixos user with sudo instead of root@
- cluster-init.sh: HA_USER/HA_KEY env vars + n2_ssh()/n2_scp() helpers so
  inter-node SSH works regardless of whether root-to-root is available
- deploy.sh Phase 6: generate temp keypair, authorize on node2, place on node1
  for root to use during cluster-init, clean up afterward

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8

scripts/ha/cluster-init.sh

@@ -36,6 +36,12 @@ VMID_NODE1="${VMID_NODE1:-}"            # set by deploy.sh; needed for STONITH
 VMID_NODE2="${VMID_NODE2:-}"
 PVE_HOST="${PVE_HOST:-pve1.sweet.home}"
 PVE_USER="${PVE_USER:-wayne}"
+# Inter-node SSH: HA_USER is the user to SSH as on NODE2; HA_KEY is the private
+# key to use.  Default is root-to-root (no key arg).  deploy.sh sets HA_USER=nixos
+# and HA_KEY=/tmp/cluster-init-key so the script works even when root-to-root SSH
+# is not available.
+HA_USER="${HA_USER:-root}"
+HA_KEY="${HA_KEY:-}"
 
 # NFS dataset subdirectories to create under XFS_MOUNT.
 # Must mirror vars.nfsShares subpath values in variables.nix.
@@ -58,6 +64,24 @@ warn() { echo "[cluster-init] WARNING: $*" >&2; }
 [[ $(id -u) -eq 0 ]] || die "must run as root"
 [[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1"
 
+# Inter-node SSH/SCP helpers — abstract over root-to-root vs nixos+sudo.
+_SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10"
+[[ -n "$HA_KEY" ]] && _SSH_OPTS="-i $HA_KEY $_SSH_OPTS"
+if [[ "$HA_USER" == "root" ]]; then
+  n2_ssh() { ssh $_SSH_OPTS "root@${NODE2_IP}" "$@"; }
+  n2_scp() { scp $_SSH_OPTS "$1" "root@${NODE2_IP}:$2"; }
+else
+  # Non-root user with passwordless sudo; wrap each command with sudo.
+  n2_ssh() { ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@"; }
+  n2_scp() {
+    # SCP to a tmp path, then sudo-move to the real destination as the remote user.
+    local src="$1" dst="$2"
+    local tmp="/tmp/_cluster_init_scp_$$"
+    scp $_SSH_OPTS "$src" "${HA_USER}@${NODE2_IP}:${tmp}"
+    ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo mv "${tmp}" "${dst}"
+  }
+fi
+
 # ── 0. Corosync authkey ───────────────────────────────────────────────────
 AUTHKEY="/etc/corosync/authkey"
 mkdir -p /etc/corosync
@@ -67,13 +91,13 @@ if [[ ! -f "$AUTHKEY" ]]; then
   chmod 0400 "$AUTHKEY"
 fi
 log "Distributing authkey to $NODE2..."
-ssh "root@${NODE2_IP}" "mkdir -p /etc/corosync"
-scp -q "$AUTHKEY" "root@${NODE2_IP}:${AUTHKEY}"
-ssh "root@${NODE2_IP}" "chmod 0400 '${AUTHKEY}'"
+n2_ssh "mkdir -p /etc/corosync"
+n2_scp "$AUTHKEY" "$AUTHKEY"
+n2_ssh "chmod 0400 '${AUTHKEY}'"
 
 log "Restarting corosync on both nodes..."
 systemctl restart corosync
-ssh "root@${NODE2_IP}" "systemctl restart corosync"
+n2_ssh "systemctl restart corosync"
 sleep 3
 
 # ── 1. Corosync quorum ────────────────────────────────────────────────────
@@ -104,15 +128,11 @@ if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskl
 fi
 
 log "Initialising DRBD metadata on $NODE2..."
-ssh "root@${NODE2_IP}" "
-  if ! drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent\|Diskless'; then
-    drbdadm create-md ha-data --force
-  fi
-"
+n2_ssh "bash -c 'if ! drbdadm dstate ha-data 2>/dev/null | grep -q UpToDate.Inconsistent.Diskless; then drbdadm create-md ha-data --force; fi'"
 
 log "Bringing up DRBD on both nodes..."
 drbdadm up ha-data 2>/dev/null || true
-ssh "root@${NODE2_IP}" "drbdadm up ha-data 2>/dev/null" || true
+n2_ssh "drbdadm up ha-data" 2>/dev/null || true
 
 log "Forcing $NODE1 to DRBD Primary for initial sync..."
 drbdadm primary ha-data --force
@@ -163,7 +183,7 @@ saveconfig /etc/target/saveconfig.json
 EOF
 
 log "Distributing iSCSI saveconfig to $NODE2..."
-scp -q /etc/target/saveconfig.json "root@${NODE2_IP}:/etc/target/saveconfig.json"
+n2_scp /etc/target/saveconfig.json /etc/target/saveconfig.json
 
 log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..."
 umount "${XFS_MOUNT}"