Archived
beatzaplentyandClaude Sonnet 4.6
71a6c4738c
Check NixOS configurations / eval-hosts (push) Successful in 10m23s
crm_mon 2.x formats the Promoted line as " * Promoted: [ node ]" — the asterisk bullet means the previous grep -E '^\s*(Promoted|Masters):' never matched, so active-node detection silently returned empty in health.sh, failover.sh, and acceptance-tests.sh. Fix: pipe through grep -v Unpromoted first, then grep -E '(Promoted|Masters):' without anchoring to start-of-line. Also: remove the SSH_OPTS=-i ~/.ssh/... variable pattern in health.sh and failover.sh; tilde is not expanded inside double-quoted strings, so $SSH_OPTS was passing a literal ~ to SSH. Inline the key path in each function definition instead (same as acceptance-tests.sh already did). Also: drop the 2>/dev/null embedded in the crm_mon argument string — the outer 2>/dev/null on the n1/n2 call already suppresses SSH stderr; the embedded one was harmless but noisy to reason about. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
232 lines
10 KiB
Bash
Executable File
232 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# acceptance-tests.sh — HA cluster acceptance tests (T1–T7)
|
||
#
|
||
# Run from a host with SSH access to both HA nodes (or from node1 itself).
|
||
# All 7 tests must pass before considering the cluster production-ready.
|
||
# Test values below must match variables.nix haServer* values.
|
||
set -euo pipefail
|
||
|
||
# ── Configuration ─────────────────────────────────────────────────────────
|
||
# All values override-able via environment variables; defaults match variables.nix.
|
||
NODE1="${NODE1:-ha-server-1}"
|
||
NODE2="${NODE2:-ha-server-2}"
|
||
NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip
|
||
NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip
|
||
VIP="${VIP:-192.168.2.229}" # vars.haServerVip
|
||
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot
|
||
ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}" # vars.haIscsiIqn
|
||
# ──────────────────────────────────────────────────────────────────────────
|
||
|
||
PASS=0
|
||
FAIL=0
|
||
RESULTS=()
|
||
|
||
# Use PASS=$((PASS+1)) instead of ((PASS++)) — the latter evaluates to 0 when
|
||
# PASS=0, which triggers set -e and kills the script after the very first PASS.
|
||
pass() { echo " PASS: $1"; PASS=$((PASS+1)); RESULTS+=("PASS $1"); }
|
||
fail() { echo " FAIL: $1"; FAIL=$((FAIL+1)); RESULTS+=("FAIL $1"); }
|
||
|
||
HA_USER="nixos"
|
||
n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
|
||
n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
|
||
|
||
echo "════════════════════════════════════════════════════"
|
||
echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo "════════════════════════════════════════════════════"
|
||
|
||
# ── Pre-flight: DRBD sync must be complete ────────────────────────────────
|
||
# Tests that check disk state, XFS mount, and iSCSI will fail or give false
|
||
# results while the initial full sync is in progress. Block until done.
|
||
echo ""
|
||
echo "Pre-flight: verifying DRBD sync is complete..."
|
||
DRBD_PREFLIGHT=$(n1 "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
|
||
if ! echo "$DRBD_PREFLIGHT" | grep -q "^UpToDate/UpToDate$"; then
|
||
echo ""
|
||
echo " ERROR: DRBD initial sync not complete."
|
||
echo " Current dstate on $NODE1: $DRBD_PREFLIGHT"
|
||
echo ""
|
||
echo " Monitor progress:"
|
||
echo " ssh nixos@$NODE1_IP 'sudo watch -n3 cat /proc/drbd'"
|
||
echo ""
|
||
echo " Re-run this script once dstate shows UpToDate/UpToDate."
|
||
exit 1
|
||
fi
|
||
echo " dstate: $DRBD_PREFLIGHT — ready."
|
||
|
||
# ── Detect Active/Standby nodes ────────────────────────────────────────────
|
||
# Use crm_mon to detect which node holds the Promoted (Primary) DRBD resource.
|
||
# Pacemaker is authoritative; DRBD role can briefly read as Secondary while
|
||
# Pacemaker is mid-transition, giving a false Active/Standby swap.
|
||
# Wait up to 90 s for Pacemaker to settle before giving up.
|
||
echo ""
|
||
echo "Detecting Active/Standby nodes (waiting for Pacemaker to settle)..."
|
||
ACTIVE_NODE=""
|
||
for i in $(seq 1 30); do
|
||
# crm_mon -1 output contains "Promoted: [ <node> ]" for the DRBD master.
|
||
CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || n2 "crm_mon -1" 2>/dev/null || true)
|
||
# crm_mon 2.x formats Promoted lines as " * Promoted: [ node ]" — the * bullet
|
||
# means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
|
||
ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | grep -E '(Promoted|Masters):' | grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
|
||
[[ -n "$ACTIVE_NODE" ]] && break
|
||
sleep 3
|
||
done
|
||
|
||
if [[ -z "$ACTIVE_NODE" ]]; then
|
||
echo " WARNING: could not determine Active node from crm_mon after 90 s — defaulting to $NODE1"
|
||
ACTIVE_NODE="$NODE1"
|
||
fi
|
||
|
||
if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
|
||
ACTIVE_IP="$NODE1_IP"
|
||
STANDBY_NODE="$NODE2"; STANDBY_IP="$NODE2_IP"
|
||
na() { n1 "$@"; }
|
||
ns() { n2 "$@"; }
|
||
else
|
||
ACTIVE_IP="$NODE2_IP"
|
||
STANDBY_NODE="$NODE1"; STANDBY_IP="$NODE1_IP"
|
||
na() { n2 "$@"; }
|
||
ns() { n1 "$@"; }
|
||
fi
|
||
echo " Active: $ACTIVE_NODE ($ACTIVE_IP)"
|
||
echo " Standby: $STANDBY_NODE ($STANDBY_IP)"
|
||
|
||
# ── T1: Corosync quorum established ──────────────────────────────────────
|
||
echo ""
|
||
echo "[T1] Corosync quorum"
|
||
if na "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
|
||
pass "cluster has quorum"
|
||
else
|
||
fail "cluster does not have quorum — check corosync on both nodes"
|
||
fi
|
||
|
||
# ── T2: DRBD Primary on Active node, Secondary on Standby ────────────────
|
||
echo ""
|
||
echo "[T2] DRBD roles"
|
||
DRBD_ROLE=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
|
||
if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then
|
||
pass "DRBD Primary on $ACTIVE_NODE ($DRBD_ROLE)"
|
||
else
|
||
fail "unexpected DRBD role on $ACTIVE_NODE: $DRBD_ROLE (expected Primary/Secondary)"
|
||
fi
|
||
|
||
DRBD_DSTATE=$(na "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
|
||
if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then
|
||
pass "DRBD disk state UpToDate ($DRBD_DSTATE)"
|
||
else
|
||
fail "DRBD disk not UpToDate: $DRBD_DSTATE"
|
||
fi
|
||
|
||
# ── T3: XFS mounted at haStorageRoot on the Active node ──────────────────
|
||
echo ""
|
||
echo "[T3] XFS mount"
|
||
if na "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||
pass "XFS mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
|
||
else
|
||
fail "XFS not mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
|
||
fi
|
||
|
||
if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||
fail "XFS unexpectedly mounted on $STANDBY_NODE (should only be on Active node)"
|
||
else
|
||
pass "XFS not mounted on $STANDBY_NODE (correct — Standby)"
|
||
fi
|
||
|
||
# ── T4: iSCSI target visible on Active node ───────────────────────────────
|
||
echo ""
|
||
echo "[T4] iSCSI target"
|
||
IQN_COUNT=$(na "bash -c 'ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn || true'" 2>/dev/null || echo "0")
|
||
if [[ "$IQN_COUNT" -ge 1 ]]; then
|
||
pass "iSCSI IQN active on $ACTIVE_NODE ($IQN_COUNT target(s))"
|
||
else
|
||
fail "no iSCSI IQN active on $ACTIVE_NODE"
|
||
fi
|
||
|
||
# iSCSI port reachable from Standby node via VIP.
|
||
# Use bash TCP probe (no iscsiadm needed — just checks port 3260 is open).
|
||
if ns "bash -c 'echo >/dev/tcp/${VIP}/3260' 2>/dev/null"; then
|
||
pass "iSCSI port 3260 reachable from $STANDBY_NODE via VIP ${VIP}"
|
||
else
|
||
fail "iSCSI port 3260 not reachable from $STANDBY_NODE via ${VIP}"
|
||
fi
|
||
|
||
# ── T5: Failover — standby Active node, verify resources move to Standby ──
|
||
echo ""
|
||
echo "[T5] Failover (standby $ACTIVE_NODE)"
|
||
ACTIVE_CRMD_NAME=$(na "crm_node -n" 2>/dev/null || echo "")
|
||
na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v on" 2>/dev/null || true
|
||
echo " Waiting up to 120 s for resources to move to $STANDBY_NODE..."
|
||
MOVED=false
|
||
for i in $(seq 1 120); do
|
||
if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||
MOVED=true
|
||
echo " Resources moved in ${i}s"
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
|
||
if $MOVED; then
|
||
pass "XFS mounted on $STANDBY_NODE after failover"
|
||
IQN_ON_STANDBY=$(ns "bash -c 'ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn || true'" 2>/dev/null || echo "0")
|
||
[[ "$IQN_ON_STANDBY" -ge 1 ]] \
|
||
&& pass "iSCSI target active on $STANDBY_NODE after failover" \
|
||
|| fail "iSCSI target NOT active on $STANDBY_NODE after failover"
|
||
else
|
||
fail "XFS did not mount on $STANDBY_NODE within 120 s — failover incomplete"
|
||
fi
|
||
|
||
# ── T6: Data integrity — file written post-failover readable ─────────────
|
||
echo ""
|
||
echo "[T6] Data integrity"
|
||
# Write a test file on the new Active (former Standby) and verify it.
|
||
# Use `echo | sudo tee` for the write: "echo ... > file" via bash -c has the
|
||
# redirect interpreted by the remote nixos shell (not sudo), so the file open
|
||
# runs as nixos and fails with EACCES on the root-owned XFS mount. Piping
|
||
# through sudo tee lets tee (running as root) open the file instead.
|
||
TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$"
|
||
TEST_CONTENT="ha-acceptance-test-$(date +%s)"
|
||
echo "${TEST_CONTENT}" | ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${STANDBY_IP}" sudo tee "${TEST_FILE}" > /dev/null 2>/dev/null || true
|
||
READBACK=$(ns cat "${TEST_FILE}" 2>/dev/null || echo "")
|
||
if [[ "$READBACK" == "$TEST_CONTENT" ]]; then
|
||
pass "test file written and read back correctly on $STANDBY_NODE"
|
||
else
|
||
fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')"
|
||
fi
|
||
ns rm -f "${TEST_FILE}" 2>/dev/null || true
|
||
|
||
# ── T7: Node rejoin — un-standby original Active, verify cluster is healthy ─
|
||
echo ""
|
||
echo "[T7] Node rejoin"
|
||
na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
|
||
na "crm_resource --cleanup" 2>/dev/null || true
|
||
sleep 5
|
||
|
||
if na "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
|
||
pass "$ACTIVE_NODE rejoined — cluster has quorum"
|
||
else
|
||
fail "$ACTIVE_NODE did not rejoin with quorum"
|
||
fi
|
||
|
||
DRBD_ROLE_AFTER=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
|
||
if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then
|
||
pass "$ACTIVE_NODE is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
|
||
else
|
||
fail "unexpected DRBD role on $ACTIVE_NODE after rejoin: $DRBD_ROLE_AFTER"
|
||
fi
|
||
|
||
# ── Summary ───────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "════════════════════════════════════════════════════"
|
||
echo " Results: ${PASS} PASS, ${FAIL} FAIL"
|
||
echo "════════════════════════════════════════════════════"
|
||
for r in "${RESULTS[@]}"; do echo " $r"; done
|
||
echo ""
|
||
|
||
if [[ "$FAIL" -eq 0 ]]; then
|
||
echo "ALL PASS — cluster is production-ready."
|
||
exit 0
|
||
else
|
||
echo "SOME TESTS FAILED — investigate before deploying."
|
||
exit 1
|
||
fi
|