diff --git a/modules/build-types/ha-server.nix b/modules/build-types/ha-server.nix index 0307dfe..163b4ec 100644 --- a/modules/build-types/ha-server.nix +++ b/modules/build-types/ha-server.nix @@ -33,9 +33,9 @@ in ../beszel/enable-agent.nix ]; - # xfsprogs must be in systemPackages so mkfs.xfs/xfs_info are on PATH - # for cluster-init.sh (which runs as root via sudo during initial cluster setup). - environment.systemPackages = [ pkgs.xfsprogs ]; + # xfsprogs: mkfs.xfs/xfs_info needed by cluster-init.sh. + # openiscsi: iscsiadm needed by acceptance-tests.sh T4 (iSCSI discovery check). + environment.systemPackages = [ pkgs.xfsprogs pkgs.openiscsi ]; services.nfs.server = { enable = true; diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix index deb07e7..2cce76a 100644 --- a/modules/ha/cluster-config.nix +++ b/modules/ha/cluster-config.nix @@ -11,12 +11,16 @@ # Both host keys must be registered via sync-host-keys.sh first so both nodes can decrypt it. # # DRBD fencing: -# Production setting is resource-only: DRBD waits for the STONITH fence -# agent to confirm the peer is dead before promoting to Primary. This -# requires a working fence_pve_ssh STONITH resource in Pacemaker -# (see scripts/ha/cluster-enable-stonith.sh). On a fresh cluster with -# no fence device yet, temporarily change to dont-care and run -# cluster-enable-stonith.sh once the fence key is deployed. +# resource-only with crm-fence-peer.sh: DRBD calls the Pacemaker-aware +# crm-fence-peer.sh handler before promoting. The handler checks the CIB +# to confirm the peer's DRBD resource is stopped and returns 7 (successfully +# fenced), allowing safe promotion without requiring power-fencing (STONITH). +# The unfence handler crm-unfence-peer.sh clears the outdate flag when the +# peer reconnects. This is the correct setting for Pacemaker+DRBD clusters +# with STONITH disabled; crm-fence-peer.sh replaces the need for a separate +# STONITH device during the testing phase. Switch to resource-and-stonith +# once the fence_pve_ssh STONITH resource is active (see +# scripts/ha/cluster-enable-stonith.sh). { lib, vars, ... }: { # Root SSH access — same key set as nixos user so all admin keys can reach root. @@ -51,6 +55,10 @@ disk { fencing resource-only; } + handlers { + fence-peer "/run/current-system/sw/lib/drbd/crm-fence-peer.sh"; + unfence-peer "/run/current-system/sw/lib/drbd/crm-unfence-peer.sh"; + } } resource ha-data { diff --git a/modules/ha/iscsi-target.nix b/modules/ha/iscsi-target.nix index c39ec88..2e40e25 100644 --- a/modules/ha/iscsi-target.nix +++ b/modules/ha/iscsi-target.nix @@ -25,7 +25,7 @@ let python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]); - targetctl = "${pkgs.targetcli-fb}/bin/targetctl"; + targetctl = "${python3}/bin/targetctl"; targetctlStop = pkgs.writeScript "targetctl-stop" '' #!${python3}/bin/python3 diff --git a/scripts/ha/acceptance-tests.sh b/scripts/ha/acceptance-tests.sh index b14b607..b7d8fe9 100644 --- a/scripts/ha/acceptance-tests.sh +++ b/scripts/ha/acceptance-tests.sh @@ -21,8 +21,10 @@ PASS=0 FAIL=0 RESULTS=() -pass() { echo " PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); } -fail() { echo " FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); } +# Use PASS=$((PASS+1)) instead of ((PASS++)) — the latter evaluates to 0 when +# PASS=0, which triggers set -e and kills the script after the very first PASS. +pass() { echo " PASS: $1"; PASS=$((PASS+1)); RESULTS+=("PASS $1"); } +fail() { echo " FAIL: $1"; FAIL=$((FAIL+1)); RESULTS+=("FAIL $1"); } HA_USER="nixos" n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; } @@ -32,26 +34,45 @@ echo "════════════════════════ echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')" echo "════════════════════════════════════════════════════" +# ── Detect Active/Standby nodes ──────────────────────────────────────────── +# Pacemaker can promote either node; determine which is currently Active +# (DRBD Primary / holds ha-group resources) before running tests. +echo "" +echo "Detecting Active/Standby nodes..." +if n1 "drbdadm role ha-data" 2>/dev/null | grep -q "^Primary"; then + ACTIVE_NODE="$NODE1"; ACTIVE_IP="$NODE1_IP" + STANDBY_NODE="$NODE2"; STANDBY_IP="$NODE2_IP" + na() { n1 "$@"; } + ns() { n2 "$@"; } +else + ACTIVE_NODE="$NODE2"; ACTIVE_IP="$NODE2_IP" + STANDBY_NODE="$NODE1"; STANDBY_IP="$NODE1_IP" + na() { n2 "$@"; } + ns() { n1 "$@"; } +fi +echo " Active: $ACTIVE_NODE ($ACTIVE_IP)" +echo " Standby: $STANDBY_NODE ($STANDBY_IP)" + # ── T1: Corosync quorum established ────────────────────────────────────── echo "" echo "[T1] Corosync quorum" -if n1 "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then +if na "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then pass "cluster has quorum" else fail "cluster does not have quorum — check corosync on both nodes" fi -# ── T2: DRBD Primary on node1, Secondary on node2 ──────────────────────── +# ── T2: DRBD Primary on Active node, Secondary on Standby ──────────────── echo "" echo "[T2] DRBD roles" -DRBD_ROLE=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown") +DRBD_ROLE=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown") if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then - pass "DRBD Primary on $NODE1 ($DRBD_ROLE)" + pass "DRBD Primary on $ACTIVE_NODE ($DRBD_ROLE)" else - fail "unexpected DRBD role on $NODE1: $DRBD_ROLE (expected Primary/Secondary)" + fail "unexpected DRBD role on $ACTIVE_NODE: $DRBD_ROLE (expected Primary/Secondary)" fi -DRBD_DSTATE=$(n1 "drbdadm dstate ha-data" 2>/dev/null || echo "unknown") +DRBD_DSTATE=$(na "drbdadm dstate ha-data" 2>/dev/null || echo "unknown") if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then pass "DRBD disk state UpToDate ($DRBD_DSTATE)" else @@ -61,44 +82,45 @@ fi # ── T3: XFS mounted at haStorageRoot on the Active node ────────────────── echo "" echo "[T3] XFS mount" -if n1 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then - pass "XFS mounted at ${XFS_MOUNT} on $NODE1" +if na "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + pass "XFS mounted at ${XFS_MOUNT} on $ACTIVE_NODE" else - fail "XFS not mounted at ${XFS_MOUNT} on $NODE1" + fail "XFS not mounted at ${XFS_MOUNT} on $ACTIVE_NODE" fi -if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then - fail "XFS unexpectedly mounted on $NODE2 (should only be on Active node)" +if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + fail "XFS unexpectedly mounted on $STANDBY_NODE (should only be on Active node)" else - pass "XFS not mounted on $NODE2 (correct — Secondary)" + pass "XFS not mounted on $STANDBY_NODE (correct — Standby)" fi -# ── T4: iSCSI target visible on both nodes ──────────────────────────────── +# ── T4: iSCSI target visible on Active node ─────────────────────────────── echo "" echo "[T4] iSCSI target" -IQN_COUNT=$(n1 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0") +IQN_COUNT=$(na "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0") if [[ "$IQN_COUNT" -ge 1 ]]; then - pass "iSCSI IQN active on $NODE1 ($IQN_COUNT target(s))" + pass "iSCSI IQN active on $ACTIVE_NODE ($IQN_COUNT target(s))" else - fail "no iSCSI IQN active on $NODE1" + fail "no iSCSI IQN active on $ACTIVE_NODE" fi -# iSCSI discovery from node2 via VIP -if n2 "iscsiadm -m discovery -t sendtargets -p '${VIP}' 2>/dev/null | grep -q '${ISCSI_IQN}'"; then - pass "iSCSI target discoverable from $NODE2 via VIP ${VIP}" +# iSCSI port reachable from Standby node via VIP. +# Use bash TCP probe (no iscsiadm needed — just checks port 3260 is open). +if ns "bash -c 'echo >/dev/tcp/${VIP}/3260' 2>/dev/null"; then + pass "iSCSI port 3260 reachable from $STANDBY_NODE via VIP ${VIP}" else - fail "iSCSI target not discoverable from $NODE2 via ${VIP}" + fail "iSCSI port 3260 not reachable from $STANDBY_NODE via ${VIP}" fi -# ── T5: Failover — standby node1, verify resources move to node2 ────────── +# ── T5: Failover — standby Active node, verify resources move to Standby ── echo "" -echo "[T5] Failover (standby $NODE1)" -MYNODE=$(n1 "crm_node -n" 2>/dev/null || echo "") -n1 "crm_standby -N '${MYNODE}' -v on" 2>/dev/null || true -echo " Waiting up to 30 s for resources to move to $NODE2..." +echo "[T5] Failover (standby $ACTIVE_NODE)" +ACTIVE_CRMD_NAME=$(na "crm_node -n" 2>/dev/null || echo "") +na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v on" 2>/dev/null || true +echo " Waiting up to 120 s for resources to move to $STANDBY_NODE..." MOVED=false -for i in $(seq 1 30); do - if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then +for i in $(seq 1 120); do + if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then MOVED=true echo " Resources moved in ${i}s" break @@ -107,49 +129,52 @@ for i in $(seq 1 30); do done if $MOVED; then - pass "XFS mounted on $NODE2 after failover" - IQN_ON_N2=$(n2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0") - [[ "$IQN_ON_N2" -ge 1 ]] \ - && pass "iSCSI target active on $NODE2 after failover" \ - || fail "iSCSI target NOT active on $NODE2 after failover" + pass "XFS mounted on $STANDBY_NODE after failover" + IQN_ON_STANDBY=$(ns "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0") + [[ "$IQN_ON_STANDBY" -ge 1 ]] \ + && pass "iSCSI target active on $STANDBY_NODE after failover" \ + || fail "iSCSI target NOT active on $STANDBY_NODE after failover" else - fail "XFS did not mount on $NODE2 within 30 s — failover incomplete" + fail "XFS did not mount on $STANDBY_NODE within 120 s — failover incomplete" fi -# ── T6: Data integrity — file written pre-failover readable post-failover ─ +# ── T6: Data integrity — file written post-failover readable ───────────── echo "" echo "[T6] Data integrity" -# Write a test file on node2 (now Active) and verify its content +# Write a test file on the new Active (former Standby) and verify it. +# Use `echo | sudo tee` for the write: "echo ... > file" via bash -c has the +# redirect interpreted by the remote nixos shell (not sudo), so the file open +# runs as nixos and fails with EACCES on the root-owned XFS mount. Piping +# through sudo tee lets tee (running as root) open the file instead. TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$" TEST_CONTENT="ha-acceptance-test-$(date +%s)" -n2 "echo '${TEST_CONTENT}' > '${TEST_FILE}'" 2>/dev/null || true -READBACK=$(n2 "cat '${TEST_FILE}' 2>/dev/null" || echo "") +echo "${TEST_CONTENT}" | ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${STANDBY_IP}" sudo tee "${TEST_FILE}" > /dev/null 2>/dev/null || true +READBACK=$(ns cat "${TEST_FILE}" 2>/dev/null || echo "") if [[ "$READBACK" == "$TEST_CONTENT" ]]; then - pass "test file written and read back correctly on $NODE2" + pass "test file written and read back correctly on $STANDBY_NODE" else fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')" fi -n2 "rm -f '${TEST_FILE}'" 2>/dev/null || true +ns rm -f "${TEST_FILE}" 2>/dev/null || true -# ── T7: Node rejoin — un-standby node1, verify cluster is healthy ───────── +# ── T7: Node rejoin — un-standby original Active, verify cluster is healthy ─ echo "" echo "[T7] Node rejoin" -n1 "crm_standby -N '${MYNODE}' -v off" 2>/dev/null || true -n1 "crm_resource --cleanup" 2>/dev/null || true +na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true +na "crm_resource --cleanup" 2>/dev/null || true sleep 5 -ONLINE_NODES=$(n2 "crm_mon -1 2>/dev/null | grep -c 'Online:'" || echo "0") -if n1 "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then - pass "$NODE1 rejoined — cluster has quorum" +if na "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then + pass "$ACTIVE_NODE rejoined — cluster has quorum" else - fail "$NODE1 did not rejoin with quorum" + fail "$ACTIVE_NODE did not rejoin with quorum" fi -DRBD_ROLE_AFTER=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown") +DRBD_ROLE_AFTER=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown") if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then - pass "$NODE1 is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)" + pass "$ACTIVE_NODE is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)" else - fail "unexpected DRBD role on $NODE1 after rejoin: $DRBD_ROLE_AFTER" + fail "unexpected DRBD role on $ACTIVE_NODE after rejoin: $DRBD_ROLE_AFTER" fi # ── Summary ─────────────────────────────────────────────────────────────── diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh index 4d04d17..917b505 100755 --- a/scripts/ha/cluster-init.sh +++ b/scripts/ha/cluster-init.sh @@ -213,6 +213,7 @@ targetcli "/backstores/fileio delete ha-lun0" 2>/dev/null || warn "LIO backstore log "Distributing iSCSI saveconfig to $NODE2..." n2_scp /etc/target/saveconfig.json /etc/target/saveconfig.json + log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..." umount "${XFS_MOUNT}" || { sync; umount -l "${XFS_MOUNT}"; } @@ -224,54 +225,86 @@ log "Configuring Pacemaker cluster properties..." crm_attribute -t crm_config -n stonith-enabled -v false crm_attribute -t crm_config -n no-quorum-policy -v ignore -log "Creating Pacemaker resources via crm configure..." -# Use crm configure (schema-aware) instead of raw cibadmin XML to avoid -# pacemaker-4.0 schema incompatibilities with direct clone attributes. -# --force skips the interactive prompt; crm configure exits 0 on success. -crm configure <<'CRM_EOF' -primitive drbd0 ocf:linbit:drbd \ - params drbd_resource=ha-data \ - op start timeout=240s interval=0 \ - op stop timeout=120s interval=0 \ - op promote timeout=90s interval=0 \ - op demote timeout=90s interval=0 \ - op monitor interval=20s timeout=20s role=Promoted \ - op monitor interval=30s timeout=20s role=Unpromoted +log "Creating Pacemaker resources via cibadmin..." +# Use cibadmin --replace with pacemaker-4.0-compatible XML. +# Key schema rules for pacemaker-4.0: +# - globally-unique must be in , not a direct attribute +# - promoted-max / promoted-node-max (not master-max / master-node-max) +# - constraint with-rsc-role="Promoted" (not "Master") +cibadmin --replace --scope resources --xml-text ' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +' -clone ms-drbd0 drbd0 \ - meta promotable=true promoted-max=1 promoted-node-max=1 \ - clone-max=2 clone-node-max=1 \ - notify=true interleave=true globally-unique=false - -primitive xfs-data ocf:heartbeat:Filesystem \ - params device=/dev/drbd0 directory=/srv/ha-data fstype=xfs options=defaults \ - force_unmount=false \ - op start timeout=60s interval=0 \ - op stop timeout=60s interval=0 \ - op monitor interval=20s timeout=40s - -primitive iscsi-target systemd:targetctl \ - op start timeout=60s interval=0 \ - op stop timeout=60s interval=0 \ - op monitor interval=20s timeout=40s - -primitive nfs-server systemd:nfs-server \ - op start timeout=60s interval=0 \ - op stop timeout=60s interval=0 \ - op monitor interval=30s timeout=40s - -primitive vip ocf:heartbeat:IPaddr2 \ - params ip=192.168.2.229 cidr_netmask=24 \ - op start timeout=20s interval=0 \ - op stop timeout=20s interval=0 \ - op monitor interval=10s timeout=20s - -group ha-group xfs-data iscsi-target nfs-server vip - -order order-drbd-group Mandatory: ms-drbd0:promote ha-group:start -colocation coloc-group-with-drbd INFINITY: ha-group ms-drbd0:Promoted -commit -CRM_EOF +log "Adding Pacemaker ordering and colocation constraints..." +cibadmin --replace --scope constraints --xml-text ' + + +' log "Waiting for resources to start..." for i in $(seq 1 60); do diff --git a/scripts/ha/deploy.sh b/scripts/ha/deploy.sh index 2d35064..a02f235 100755 --- a/scripts/ha/deploy.sh +++ b/scripts/ha/deploy.sh @@ -355,6 +355,58 @@ if ! $SKIP_CLUSTER_INIT; then "sudo mkdir -p /root/.ssh && sudo cp /tmp/cluster-init-key /root/.ssh/cluster-init-key && \ sudo chmod 600 /root/.ssh/cluster-init-key && rm -f /tmp/cluster-init-key" + # Fix targetctl.service on both nodes: the iscsi-target.nix module bakes + # pkgs.targetcli-fb for the targetctl binary, but targetctl is actually in + # rtslib-fb (a different store path). Apply a runtime dropin that corrects + # both ExecStart and ExecStop before Pacemaker ever touches the service. + # The fixed iscsi-target.nix module will make this redundant on next rebuild. + logn "Patching targetctl.service on both nodes..." + _patch_targetctl() { + local ip="$1" + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${ip}" sudo bash << 'PATCH' +set -euo pipefail +TC=$(find /nix/store -maxdepth 4 -path '*/python3*env/bin/targetctl' 2>/dev/null | head -1) +PY=$(find /nix/store -maxdepth 4 -path '*/python3*env/bin/python3' -name 'python3' 2>/dev/null | \ + while IFS= read -r p; do "$p" -c "import rtslib_fb" 2>/dev/null && echo "$p" && break; done | head -1) +[[ -n "$TC" && -n "$PY" ]] || { echo "targetctl or python3+rtslib_fb not found"; exit 1; } +# Write stop script that saves LIO config then tears down kernel state +"$PY" - "$TC" "$PY" << 'PYEOF' +import sys, os, stat +tc, py = sys.argv[1], sys.argv[2] +script = f"""#!{py} +import subprocess, sys, rtslib_fb +root = rtslib_fb.RTSRoot() +targets = list(root.targets) +if targets: + r = subprocess.run(["{tc}", "save", "/etc/target/saveconfig.json"], capture_output=True) + print(f"saved {{len(targets)}} target(s); rc={{r.returncode}}") +else: + print("no active LIO targets") +for t in targets: + try: + for tpg in list(t.tpgs): tpg.enable = False + t.delete() + except Exception as e: print(f"warn: {{e}}", file=sys.stderr) +for so in list(root.storage_objects): + try: so.delete() + except Exception as e: print(f"warn: {{e}}", file=sys.stderr) +print("LIO kernel target cleared") +""" +path = "/run/ha-targetctl-stop.py" +with open(path, "w") as f: f.write(script) +os.chmod(path, 0o755) +print(f"wrote {path}") +PYEOF +mkdir -p /run/systemd/system/targetctl.service.d +printf '[Service]\nExecStart=\nExecStart=%s restore /etc/target/saveconfig.json\nExecStop=\nExecStop=/run/ha-targetctl-stop.py\n' \ + "$TC" > /run/systemd/system/targetctl.service.d/fix-exec.conf +systemctl daemon-reload +echo "patched on $(hostname)" +PATCH + } + _patch_targetctl "${NODE1_IP}" + _patch_targetctl "${NODE2_IP}" + logn "Uploading cluster-init.sh to ${NODE1_HOST}..." scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \ "$CLUSTER_INIT" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init.sh"