Archived
beatzaplentyandClaude Sonnet 4.6
724d9a45af
All 7 acceptance tests pass on live NixOS 25.11 VMs (VMIDs 200/201 on
pve1). Failover completes in ~5 s with data integrity verified.
modules/ha/pacemaker-stack.nix — fixes four NixOS-specific breakages:
- systemd StateDirectory resets /var/lib/pacemaker to root:root; removed
and replaced with ExecStartPre to create/chown dirs as hacluster
- HA_SBIN_DIR points to a non-existent Nix store path; overridden to
/run/current-system/sw/bin so crm_master resolves correctly
- OCF agents need an explicit broad PATH (iproute2, util-linux, xfsprogs,
drbd, bash, etc.) — NixOS services have no implicit PATH
- FUSER=true bypasses the psmisc fuser check_binary call in the
Filesystem OCF agent (psmisc not installed on minimal hosts)
modules/ha/iscsi-target.nix — LIO iSCSI target via targetctl with a
Python/rtslib_fb ExecStop that explicitly clears the kernel LIO state
(not just saves JSON), so the XFS backing store's file descriptor is
released before umount — preventing EBUSY stop timeouts on failover.
Includes an empty-config guard so the secondary node never overwrites
the primary's saveconfig.json with an empty one.
test-lab/ha/common.nix — updated to import both modules, use fencing
dont-care (no STONITH in test lab), omit LVM handlers (non-existent on
NixOS paths), and merge repeated services/networking attr sets to satisfy
statix W20. test-lab/ha/acceptance-tests.sh — final v4 with crm_standby
fix (pacemaker 3.x API).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
172 lines
6.7 KiB
Bash
Executable File
172 lines
6.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# HA File Server Acceptance Tests T1-T7
|
|
# Failover (T5) uses pacemaker standby mode to gracefully move resources,
|
|
# simulating what STONITH + node-restart does in production.
|
|
# Note: A production cluster requires real STONITH (fence agent for the hypervisor).
|
|
set -euo pipefail
|
|
|
|
NODE2="root@192.168.2.201"
|
|
VIP="192.168.2.202"
|
|
IQN="iqn.2026-01.local.ha-test:storage"
|
|
MOUNT="/srv/ha-data"
|
|
PACEMAKERD="/nix/store/3v9sb74cg2qmpcyzb4h6fq0z8bvp5gw1-pacemaker-3.0.1/sbin/pacemakerd"
|
|
MYNODE=$(hostname)
|
|
|
|
SSH="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15"
|
|
PASS=0; FAIL=0
|
|
|
|
pass() { echo "[PASS] $1"; ((PASS++)) || true; }
|
|
fail() { echo "[FAIL] $1"; ((FAIL++)) || true; }
|
|
info() { echo "[INFO] $1"; }
|
|
|
|
echo "=== HA File Server Acceptance Tests ==="
|
|
echo "Node: $MYNODE Date: $(date)"
|
|
echo ""
|
|
|
|
# T1: Corosync 2-node cluster with quorum
|
|
echo "--- T1: Corosync cluster quorum ---"
|
|
QUORATE=$(corosync-quorumtool -s 2>/dev/null | grep 'Quorate:' | awk '{print $2}')
|
|
NODE_COUNT=$(corosync-quorumtool -s 2>/dev/null | grep '^Nodes:' | awk '{print $2}')
|
|
if [ "$QUORATE" = "Yes" ] && [ "$NODE_COUNT" = "2" ]; then
|
|
pass "T1: Corosync quorate with 2 nodes"
|
|
else
|
|
fail "T1: quorate=$QUORATE nodes=$NODE_COUNT"
|
|
fi
|
|
|
|
# T2: DRBD both UpToDate, replication Established
|
|
echo "--- T2: DRBD replication healthy ---"
|
|
DRBD_STATUS=$(drbdadm status ha-data 2>/dev/null)
|
|
N1_ROLE=$(echo "$DRBD_STATUS" | grep '^ha-data role:' | awk -F: '{print $2}')
|
|
N1_DISK=$(echo "$DRBD_STATUS" | grep -oP 'disk:\K\S+' | head -1)
|
|
N2_DISK=$(echo "$DRBD_STATUS" | grep -oP 'peer-disk:\K\S+' | head -1)
|
|
REPL=$(echo "$DRBD_STATUS" | grep -oP 'replication:\K\S+' | head -1)
|
|
info "DRBD: role=$N1_ROLE local_disk=$N1_DISK peer_disk=$N2_DISK replication=$REPL"
|
|
if [ "$N1_DISK" = "UpToDate" ] && [ "$N2_DISK" = "UpToDate" ] && [ "$REPL" = "Established" ]; then
|
|
pass "T2: DRBD both UpToDate, Established (role=$N1_ROLE)"
|
|
else
|
|
fail "T2: DRBD issue: local=$N1_DISK peer=$N2_DISK replication=$REPL"
|
|
fi
|
|
|
|
# T3: XFS mounted on primary
|
|
echo "--- T3: XFS mount on primary ---"
|
|
if mountpoint -q $MOUNT && df -t xfs $MOUNT &>/dev/null; then
|
|
FSINFO=$(df -h $MOUNT | tail -1)
|
|
pass "T3: XFS mounted at $MOUNT: $FSINFO"
|
|
else
|
|
fail "T3: XFS not mounted at $MOUNT"
|
|
fi
|
|
|
|
# T4: iSCSI active on primary, VIP responds on port 3260
|
|
echo "--- T4: iSCSI target active ---"
|
|
ACTIVE_IQN=$(ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep iqn | head -1)
|
|
if [ "$ACTIVE_IQN" = "$IQN" ] && nc -w3 $VIP 3260 < /dev/null 2>/dev/null; then
|
|
pass "T4: iSCSI $IQN active, port 3260 open on VIP $VIP"
|
|
elif [ "$ACTIVE_IQN" = "$IQN" ]; then
|
|
fail "T4: iSCSI IQN active but port 3260 not reachable on VIP"
|
|
else
|
|
fail "T4: iSCSI not active (got '$ACTIVE_IQN')"
|
|
fi
|
|
|
|
# Pre-T5: write test file for data integrity check
|
|
echo "--- Pre-T5: writing test data ---"
|
|
TESTFILE="$MOUNT/failover-test.txt"
|
|
TESTDATA="FAILOVER_INTEGRITY_$(date +%s)"
|
|
echo "$TESTDATA" > "$TESTFILE"
|
|
sync
|
|
info "Wrote: $TESTFILE (data: $TESTDATA)"
|
|
|
|
# T5: Failover — put this node into pacemaker standby, forcing resource migration
|
|
# (In production this is triggered by real STONITH; standby simulates the result.)
|
|
echo "--- T5: Failover (pacemaker standby + node isolation) ---"
|
|
info "Putting $MYNODE into standby mode (triggers resource migration to node2)..."
|
|
crm_standby -N "$MYNODE" -v on 2>&1 || true
|
|
|
|
FAILOVER_OK=false
|
|
info "Waiting up to 90s for node2 failover..."
|
|
for i in $(seq 1 18); do
|
|
sleep 5
|
|
STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "UNREACHABLE")
|
|
if echo "$STATUS" | grep -q "Started ha-test-node2"; then
|
|
FAILOVER_OK=true
|
|
info "Failover complete at $((i*5))s"
|
|
echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted'
|
|
break
|
|
fi
|
|
done
|
|
|
|
if $FAILOVER_OK; then
|
|
N2_DRBD=$($SSH $NODE2 "drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print \$2}'" 2>/dev/null || echo "unknown")
|
|
N2_MOUNT=$($SSH $NODE2 "mountpoint -q $MOUNT && echo 'mounted' || echo 'not-mounted'" 2>/dev/null || echo "unknown")
|
|
N2_ISCSI=$($SSH $NODE2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" 2>/dev/null || echo "0")
|
|
N2_VIP=$($SSH $NODE2 "ip addr show | grep -c '$VIP'" 2>/dev/null || echo "0")
|
|
info "Node2: DRBD=$N2_DRBD mount=$N2_MOUNT iSCSI_IQNs=$N2_ISCSI VIP=$N2_VIP"
|
|
|
|
FAILS=0
|
|
[ "$N2_DRBD" = "Primary" ] || { info "FAIL: DRBD not Primary on node2"; ((FAILS++)) || true; }
|
|
[ "$N2_MOUNT" = "mounted" ] || { info "FAIL: XFS not mounted on node2"; ((FAILS++)) || true; }
|
|
[ "$N2_ISCSI" -ge "1" ] 2>/dev/null || { info "FAIL: iSCSI not active on node2"; ((FAILS++)) || true; }
|
|
[ "$N2_VIP" -ge "1" ] 2>/dev/null || { info "FAIL: VIP not on node2"; ((FAILS++)) || true; }
|
|
|
|
if [ $FAILS -eq 0 ]; then
|
|
pass "T5: Failover complete — DRBD Primary, XFS, iSCSI, VIP all on node2"
|
|
else
|
|
fail "T5: Partial failover ($FAILS sub-checks failed)"
|
|
fi
|
|
else
|
|
fail "T5: No failover detected within 90s"
|
|
fi
|
|
|
|
# T7: Data integrity — test file readable on node2 after failover
|
|
echo "--- T7: Data integrity after failover ---"
|
|
if $SSH $NODE2 "grep -q '$TESTDATA' $TESTFILE 2>/dev/null"; then
|
|
pass "T7: Test data intact on node2 after failover"
|
|
else
|
|
ACTUAL=$($SSH $NODE2 "cat $TESTFILE 2>/dev/null || echo FILE_MISSING" 2>/dev/null || echo "SSH_FAIL")
|
|
fail "T7: Data integrity check failed (expected '$TESTDATA', got '$ACTUAL')"
|
|
fi
|
|
|
|
# T6: Node rejoin — take node out of standby
|
|
echo "--- T6: Node rejoin ---"
|
|
info "Taking $MYNODE out of standby..."
|
|
crm_standby -N "$MYNODE" -v off 2>&1 || true
|
|
|
|
REJOIN_OK=false
|
|
for i in $(seq 1 12); do
|
|
sleep 5
|
|
# Check if this node is back online (no longer standby)
|
|
STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "")
|
|
if echo "$STATUS" | grep -q "Online:.*$MYNODE"; then
|
|
REJOIN_OK=true
|
|
info "Rejoined at $((i*5))s"
|
|
echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted'
|
|
break
|
|
fi
|
|
done
|
|
|
|
if $REJOIN_OK; then
|
|
sleep 5
|
|
N1_DRBD=$(drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print $2}')
|
|
N1_DISK=$(drbdadm status ha-data 2>/dev/null | grep -oP 'disk:\K\S+' | head -1)
|
|
info "Node1 DRBD after rejoin: role=$N1_DRBD disk=$N1_DISK"
|
|
pass "T6: Node rejoined cluster (DRBD role=$N1_DRBD, disk=$N1_DISK)"
|
|
else
|
|
fail "T6: Node did not rejoin within 60s"
|
|
fi
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo "RESULTS: $PASS passed, $FAIL failed"
|
|
echo "========================================"
|
|
echo ""
|
|
echo "NOTES:"
|
|
echo " T5 uses pacemaker standby to simulate failover (production needs STONITH"
|
|
echo " fence agent, e.g. fence_pve_ssh, to crash the VM — same requirement on Debian)"
|
|
echo ""
|
|
if [ $FAIL -eq 0 ]; then
|
|
echo "VERDICT: ALL TESTS PASSED → Deliverable A (NixOS modules)"
|
|
exit 0
|
|
else
|
|
echo "VERDICT: $FAIL TEST(S) FAILED → review above"
|
|
exit 1
|
|
fi
|