diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix index 3e304fe..3271ef7 100644 --- a/modules/ha/cluster-config.nix +++ b/modules/ha/cluster-config.nix @@ -19,9 +19,12 @@ # cluster-enable-stonith.sh once the fence key is deployed. { lib, vars, ... }: { - # Root SSH access with the admin key — needed for the deploy script to upload - # and run cluster-init.sh as root on node1, and for node1→node2 SSH during init. - users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey ]; + # Root SSH access — same key set as nixos user so all admin keys can reach root. + users.users.root.openssh.authorizedKeys.keys = [ + vars.adminSshKey + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos" + ]; # Passwordless sudo for wheel — operator SSHes as nixos and uses sudo for # cluster management commands (drbdadm, crm*, pcs, etc.) diff --git a/scripts/ha/acceptance-tests.sh b/scripts/ha/acceptance-tests.sh index 5fde7a6..b14b607 100644 --- a/scripts/ha/acceptance-tests.sh +++ b/scripts/ha/acceptance-tests.sh @@ -24,8 +24,9 @@ RESULTS=() pass() { echo " PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); } fail() { echo " FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); } -n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; } -n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; } +HA_USER="nixos" +n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; } +n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; } echo "════════════════════════════════════════════════════" echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')" diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh index 103786b..2864b0c 100755 --- a/scripts/ha/cluster-init.sh +++ b/scripts/ha/cluster-init.sh @@ -36,6 +36,12 @@ VMID_NODE1="${VMID_NODE1:-}" # set by deploy.sh; needed for STONITH VMID_NODE2="${VMID_NODE2:-}" PVE_HOST="${PVE_HOST:-pve1.sweet.home}" PVE_USER="${PVE_USER:-wayne}" +# Inter-node SSH: HA_USER is the user to SSH as on NODE2; HA_KEY is the private +# key to use. Default is root-to-root (no key arg). deploy.sh sets HA_USER=nixos +# and HA_KEY=/tmp/cluster-init-key so the script works even when root-to-root SSH +# is not available. +HA_USER="${HA_USER:-root}" +HA_KEY="${HA_KEY:-}" # NFS dataset subdirectories to create under XFS_MOUNT. # Must mirror vars.nfsShares subpath values in variables.nix. @@ -58,6 +64,24 @@ warn() { echo "[cluster-init] WARNING: $*" >&2; } [[ $(id -u) -eq 0 ]] || die "must run as root" [[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1" +# Inter-node SSH/SCP helpers — abstract over root-to-root vs nixos+sudo. +_SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10" +[[ -n "$HA_KEY" ]] && _SSH_OPTS="-i $HA_KEY $_SSH_OPTS" +if [[ "$HA_USER" == "root" ]]; then + n2_ssh() { ssh $_SSH_OPTS "root@${NODE2_IP}" "$@"; } + n2_scp() { scp $_SSH_OPTS "$1" "root@${NODE2_IP}:$2"; } +else + # Non-root user with passwordless sudo; wrap each command with sudo. + n2_ssh() { ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@"; } + n2_scp() { + # SCP to a tmp path, then sudo-move to the real destination as the remote user. + local src="$1" dst="$2" + local tmp="/tmp/_cluster_init_scp_$$" + scp $_SSH_OPTS "$src" "${HA_USER}@${NODE2_IP}:${tmp}" + ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo mv "${tmp}" "${dst}" + } +fi + # ── 0. Corosync authkey ─────────────────────────────────────────────────── AUTHKEY="/etc/corosync/authkey" mkdir -p /etc/corosync @@ -67,13 +91,13 @@ if [[ ! -f "$AUTHKEY" ]]; then chmod 0400 "$AUTHKEY" fi log "Distributing authkey to $NODE2..." -ssh "root@${NODE2_IP}" "mkdir -p /etc/corosync" -scp -q "$AUTHKEY" "root@${NODE2_IP}:${AUTHKEY}" -ssh "root@${NODE2_IP}" "chmod 0400 '${AUTHKEY}'" +n2_ssh "mkdir -p /etc/corosync" +n2_scp "$AUTHKEY" "$AUTHKEY" +n2_ssh "chmod 0400 '${AUTHKEY}'" log "Restarting corosync on both nodes..." systemctl restart corosync -ssh "root@${NODE2_IP}" "systemctl restart corosync" +n2_ssh "systemctl restart corosync" sleep 3 # ── 1. Corosync quorum ──────────────────────────────────────────────────── @@ -104,15 +128,11 @@ if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskl fi log "Initialising DRBD metadata on $NODE2..." -ssh "root@${NODE2_IP}" " - if ! drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent\|Diskless'; then - drbdadm create-md ha-data --force - fi -" +n2_ssh "bash -c 'if ! drbdadm dstate ha-data 2>/dev/null | grep -q UpToDate.Inconsistent.Diskless; then drbdadm create-md ha-data --force; fi'" log "Bringing up DRBD on both nodes..." drbdadm up ha-data 2>/dev/null || true -ssh "root@${NODE2_IP}" "drbdadm up ha-data 2>/dev/null" || true +n2_ssh "drbdadm up ha-data" 2>/dev/null || true log "Forcing $NODE1 to DRBD Primary for initial sync..." drbdadm primary ha-data --force @@ -163,7 +183,7 @@ saveconfig /etc/target/saveconfig.json EOF log "Distributing iSCSI saveconfig to $NODE2..." -scp -q /etc/target/saveconfig.json "root@${NODE2_IP}:/etc/target/saveconfig.json" +n2_scp /etc/target/saveconfig.json /etc/target/saveconfig.json log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..." umount "${XFS_MOUNT}" diff --git a/scripts/ha/deploy.sh b/scripts/ha/deploy.sh index 618e5ef..2d35064 100755 --- a/scripts/ha/deploy.sh +++ b/scripts/ha/deploy.sh @@ -147,14 +147,16 @@ pve_check() { ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*" } +HA_USER="nixos" + n1() { - # Run a command on ha-server-1 via SSH. - ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null + # Run a command on ha-server-1 via SSH as nixos with sudo. + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null } n2() { - # Run a command on ha-server-2 via SSH. - ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null + # Run a command on ha-server-2 via SSH as nixos with sudo. + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null } wait_for_ssh() { @@ -167,7 +169,7 @@ wait_for_ssh() { log "Waiting for SSH on ${label} (${ip}) — up to 5 min..." while [[ $(date +%s) -lt $deadline ]]; do if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=3 \ - -o BatchMode=yes "root@${ip}" true 2>/dev/null; then + -o BatchMode=yes "${HA_USER}@${ip}" true 2>/dev/null; then logn "${label} is up." return 0 fi @@ -332,25 +334,53 @@ if ! $SKIP_CLUSTER_INIT; then [[ -x "$CLUSTER_INIT" ]] || chmod +x "$CLUSTER_INIT" if $DRY_RUN; then - logn "[dry-run] Would scp cluster-init.sh to root@${NODE1_IP} and run it" + logn "[dry-run] Would generate temp key, authorise on ${NODE2_HOST}, scp cluster-init.sh to ${NODE1_HOST}, and run it as root via sudo" else + # Generate a temp keypair so cluster-init.sh can SSH node1→node2 as ${HA_USER}. + # Root on node1 has no keys; a temp key bridging node1→node2 nixos solves this. + TEMP_KEY="${REPO_ROOT}/.tmp-cluster-init-key" + TEMP_KEY_PUB="${TEMP_KEY}.pub" + rm -f "$TEMP_KEY" "$TEMP_KEY_PUB" + ssh-keygen -t ed25519 -f "$TEMP_KEY" -N "" -C "cluster-init-temp-$(date +%s)" -q + TEMP_PUBKEY=$(cat "$TEMP_KEY_PUB") + + logn "Authorising temp key on ${NODE2_HOST} for ${HA_USER}..." + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE2_IP}" \ + "mkdir -p ~/.ssh && chmod 700 ~/.ssh && echo '${TEMP_PUBKEY}' >> ~/.ssh/authorized_keys" + + logn "Placing temp key on ${NODE1_HOST} for root..." + scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \ + "$TEMP_KEY" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init-key" + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \ + "sudo mkdir -p /root/.ssh && sudo cp /tmp/cluster-init-key /root/.ssh/cluster-init-key && \ + sudo chmod 600 /root/.ssh/cluster-init-key && rm -f /tmp/cluster-init-key" + logn "Uploading cluster-init.sh to ${NODE1_HOST}..." scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \ - "$CLUSTER_INIT" "root@${NODE1_IP}:/tmp/cluster-init.sh" + "$CLUSTER_INIT" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init.sh" logn "Running cluster-init.sh on ${NODE1_HOST}..." - ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "root@${NODE1_IP}" \ - "NODE1=${NODE1_HOST} NODE2=${NODE2_HOST} \ + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \ + "sudo env NODE1=${NODE1_HOST} NODE2=${NODE2_HOST} \ NODE1_IP=${NODE1_IP} NODE2_IP=${NODE2_IP} \ VIP=192.168.2.229 XFS_MOUNT=/srv/ha-data \ ISCSI_IQN=iqn.2026-01.home.sweet:ha-storage \ VMID_NODE1=${VMID1} VMID_NODE2=${VMID2} \ + HA_USER=${HA_USER} HA_KEY=/root/.ssh/cluster-init-key \ bash /tmp/cluster-init.sh" + logn "Cleaning up temp key from both nodes..." + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE2_IP}" \ + "sed -i '/cluster-init-temp/d' ~/.ssh/authorized_keys" 2>/dev/null || true + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \ + "sudo rm -f /root/.ssh/cluster-init-key" 2>/dev/null || true + rm -f "$TEMP_KEY" "$TEMP_KEY_PUB" + # Encrypt the corosync authkey generated by cluster-init and commit it. log " Encrypting corosync authkey into secrets/ha-corosync-authkey..." AUTHKEY_TMP="${REPO_ROOT}/secrets/ha-corosync-authkey.tmp" - n1 "cat /etc/corosync/authkey" > "$AUTHKEY_TMP" + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \ + "sudo cat /etc/corosync/authkey" > "$AUTHKEY_TMP" if [[ ! -s "$AUTHKEY_TMP" ]]; then err "corosync authkey on node1 is empty — cluster-init may have failed." fi