diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix index 5b55776..3e304fe 100644 --- a/modules/ha/cluster-config.nix +++ b/modules/ha/cluster-config.nix @@ -19,6 +19,14 @@ # cluster-enable-stonith.sh once the fence key is deployed. { lib, vars, ... }: { + # Root SSH access with the admin key — needed for the deploy script to upload + # and run cluster-init.sh as root on node1, and for node1→node2 SSH during init. + users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey ]; + + # Passwordless sudo for wheel — operator SSHes as nixos and uses sudo for + # cluster management commands (drbdadm, crm*, pcs, etc.) + security.sudo.wheelNeedsPassword = lib.mkForce false; + services.drbd = { enable = true; config = '' diff --git a/scripts/ha/acceptance-tests.sh b/scripts/ha/acceptance-tests.sh index ab3081c..5fde7a6 100644 --- a/scripts/ha/acceptance-tests.sh +++ b/scripts/ha/acceptance-tests.sh @@ -7,13 +7,14 @@ set -euo pipefail # ── Configuration ───────────────────────────────────────────────────────── -NODE1="ha-server-1" -NODE2="ha-server-2" -NODE1_IP="192.168.2.200" # vars.haServer1Ip -NODE2_IP="192.168.2.201" # vars.haServer2Ip -VIP="192.168.2.202" # vars.haServerVip -XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot -ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn +# All values override-able via environment variables; defaults match variables.nix. +NODE1="${NODE1:-ha-server-1}" +NODE2="${NODE2:-ha-server-2}" +NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip +NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip +VIP="${VIP:-192.168.2.229}" # vars.haServerVip +XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot +ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}" # vars.haIscsiIqn # ────────────────────────────────────────────────────────────────────────── PASS=0 @@ -23,8 +24,8 @@ RESULTS=() pass() { echo " PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); } fail() { echo " FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); } -n1() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; } -n2() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; } +n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; } +n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; } echo "════════════════════════════════════════════════════" echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')" diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh old mode 100644 new mode 100755 index 1aac090..103786b --- a/scripts/ha/cluster-init.sh +++ b/scripts/ha/cluster-init.sh @@ -21,22 +21,21 @@ set -euo pipefail # ── Configuration ───────────────────────────────────────────────────────── -# These must match variables.nix haServer* values and the Proxmox VMID -# assignments. Update before running. -NODE1="ha-server-1" -NODE2="ha-server-2" -NODE1_IP="192.168.2.200" # vars.haServer1Ip -NODE2_IP="192.168.2.201" # vars.haServer2Ip -VIP="192.168.2.202" # vars.haServerVip -XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot -ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn +# All values override-able via environment variables; defaults match variables.nix. +NODE1="${NODE1:-ha-server-1}" +NODE2="${NODE2:-ha-server-2}" +NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip +NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip +VIP="${VIP:-192.168.2.229}" # vars.haServerVip +XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot +ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}" # vars.haIscsiIqn ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img" ISCSI_LUN_SIZE="10G" DRBD_DEVICE="/dev/drbd0" -VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1 -VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2 -PVE_HOST="pve1.sweet.home" -PVE_USER="wayne" +VMID_NODE1="${VMID_NODE1:-}" # set by deploy.sh; needed for STONITH +VMID_NODE2="${VMID_NODE2:-}" +PVE_HOST="${PVE_HOST:-pve1.sweet.home}" +PVE_USER="${PVE_USER:-wayne}" # NFS dataset subdirectories to create under XFS_MOUNT. # Must mirror vars.nfsShares subpath values in variables.nix. diff --git a/scripts/ha/deploy.sh b/scripts/ha/deploy.sh index 5cd2587..0c9b78b 100755 --- a/scripts/ha/deploy.sh +++ b/scripts/ha/deploy.sh @@ -142,6 +142,11 @@ pve() { fi } +pve_check() { + # Run a read-only probe on the Proxmox node — always executes even in dry-run. + ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*" +} + n1() { # Run a command on ha-server-1 via SSH. ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null @@ -153,7 +158,12 @@ n2() { } wait_for_ssh() { - local ip="$1" label="$2" deadline=$(( $(date +%s) + 300 )) + local ip="$1" label="$2" + if $DRY_RUN; then + logn "[dry-run] Skipping SSH wait for ${label} (${ip})" + return 0 + fi + local deadline=$(( $(date +%s) + 300 )) log "Waiting for SSH on ${label} (${ip}) — up to 5 min..." while [[ $(date +%s) -lt $deadline ]]; do if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=3 \ @@ -192,7 +202,7 @@ fi if ! $SKIP_ENSURE_BRIDGE; then log "Phase 1: Ensuring storage bridge ${STORAGE_BRIDGE} on ${NODE}" - if pve "ip link show ${STORAGE_BRIDGE}" &>/dev/null; then + if pve_check "test -d /sys/class/net/${STORAGE_BRIDGE}" &>/dev/null; then logn "${STORAGE_BRIDGE} already exists — skipping." else logn "Creating isolated internal bridge ${STORAGE_BRIDGE} (no upstream port, ${STORAGE_CIDR})" @@ -287,27 +297,31 @@ if ! $SKIP_CLUSTER_INIT; then log "Phase 6: Initialising HA cluster" CLUSTER_INIT="${REPO_ROOT}/scripts/ha/cluster-init.sh" - if [[ ! -x "$CLUSTER_INIT" ]]; then - chmod +x "$CLUSTER_INIT" - fi + [[ -x "$CLUSTER_INIT" ]] || chmod +x "$CLUSTER_INIT" - run bash "$CLUSTER_INIT" \ - NODE1="$NODE1_HOST" NODE2="$NODE2_HOST" \ - NODE1_IP="$NODE1_IP" NODE2_IP="$NODE2_IP" \ - VIP="192.168.2.229" \ - XFS_MOUNT="/srv/ha-data" \ - ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" \ - VMID_NODE1="$VMID1" VMID_NODE2="$VMID2" + if $DRY_RUN; then + logn "[dry-run] Would scp cluster-init.sh to root@${NODE1_IP} and run it" + else + logn "Uploading cluster-init.sh to ${NODE1_HOST}..." + scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \ + "$CLUSTER_INIT" "root@${NODE1_IP}:/tmp/cluster-init.sh" - # Encrypt the corosync authkey generated by cluster-init and commit it. - log " Encrypting corosync authkey into secrets/ha-corosync-authkey..." - if ! $DRY_RUN; then + logn "Running cluster-init.sh on ${NODE1_HOST}..." + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "root@${NODE1_IP}" \ + "NODE1=${NODE1_HOST} NODE2=${NODE2_HOST} \ + NODE1_IP=${NODE1_IP} NODE2_IP=${NODE2_IP} \ + VIP=192.168.2.229 XFS_MOUNT=/srv/ha-data \ + ISCSI_IQN=iqn.2026-01.home.sweet:ha-storage \ + VMID_NODE1=${VMID1} VMID_NODE2=${VMID2} \ + bash /tmp/cluster-init.sh" + + # Encrypt the corosync authkey generated by cluster-init and commit it. + log " Encrypting corosync authkey into secrets/ha-corosync-authkey..." AUTHKEY_TMP="${REPO_ROOT}/secrets/ha-corosync-authkey.tmp" n1 "cat /etc/corosync/authkey" > "$AUTHKEY_TMP" if [[ ! -s "$AUTHKEY_TMP" ]]; then err "corosync authkey on node1 is empty — cluster-init may have failed." fi - # sops-encrypt in-place; creation rule matches secrets/ha-corosync-authkey mv "$AUTHKEY_TMP" "${REPO_ROOT}/secrets/ha-corosync-authkey" (cd "${REPO_ROOT}" && nix run nixpkgs#sops -- -e --input-type binary -i secrets/ha-corosync-authkey) logn "Authkey encrypted. Committing..." @@ -321,7 +335,14 @@ fi if ! $SKIP_TESTS; then log "Phase 7: Running acceptance tests (T1–T7)" - run bash "${REPO_ROOT}/scripts/ha/acceptance-tests.sh" + if $DRY_RUN; then + logn "[dry-run] Would run acceptance-tests.sh against ${NODE1_HOST}/${NODE2_HOST}" + else + NODE1="$NODE1_HOST" NODE2="$NODE2_HOST" \ + NODE1_IP="$NODE1_IP" NODE2_IP="$NODE2_IP" \ + VIP="192.168.2.229" \ + bash "${REPO_ROOT}/scripts/ha/acceptance-tests.sh" + fi fi log "Deploy complete."