Archived
beatzaplentyandClaude Sonnet 4.6
d740064a35
Check NixOS configurations / eval-hosts (pull_request) Successful in 10m37s
Deploy/init fixes: - iscsi-target.nix: targetctl binary is in rtslib-fb (python3 env), not targetcli-fb — fixes ExecStart and ExecStop for the targetctl.service - deploy.sh: _patch_targetctl() applies runtime dropin to both nodes before cluster-init so Pacemaker can manage the iSCSI target from first start - cluster-init.sh: replace crm configure heredoc with cibadmin --replace XML (pacemaker-4.0 schema: globally-unique in meta_attributes, promoted-max/ promoted-node-max, Promoted role in constraints); force_unmount=true on xfs-data; DRBD promote timeout 240s - cluster-config.nix: add crm-fence-peer.sh/crm-unfence-peer.sh handlers; update fencing comment to reflect resource-only + Pacemaker-aware handler replacing STONITH during testing phase - ha-server.nix: add openiscsi to systemPackages for T4 iscsiadm availability Acceptance test fixes: - acceptance-tests.sh: fix ((PASS++)) set -e bug → PASS=$((PASS+1)); detect Active/Standby dynamically via drbdadm role (Pacemaker can promote either node); T4 bash TCP probe instead of iscsiadm; T5 timeout 120s; T6 echo|sudo tee for root-owned XFS write (bash -c redirect runs as nixos not sudo — permission denied); use ns cat / ns rm for root-owned reads Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
195 lines
8.7 KiB
Bash
195 lines
8.7 KiB
Bash
#!/usr/bin/env bash
|
||
# acceptance-tests.sh — HA cluster acceptance tests (T1–T7)
|
||
#
|
||
# Run from a host with SSH access to both HA nodes (or from node1 itself).
|
||
# All 7 tests must pass before considering the cluster production-ready.
|
||
# Test values below must match variables.nix haServer* values.
|
||
set -euo pipefail
|
||
|
||
# ── Configuration ─────────────────────────────────────────────────────────
|
||
# All values override-able via environment variables; defaults match variables.nix.
|
||
NODE1="${NODE1:-ha-server-1}"
|
||
NODE2="${NODE2:-ha-server-2}"
|
||
NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip
|
||
NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip
|
||
VIP="${VIP:-192.168.2.229}" # vars.haServerVip
|
||
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot
|
||
ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}" # vars.haIscsiIqn
|
||
# ──────────────────────────────────────────────────────────────────────────
|
||
|
||
PASS=0
|
||
FAIL=0
|
||
RESULTS=()
|
||
|
||
# Use PASS=$((PASS+1)) instead of ((PASS++)) — the latter evaluates to 0 when
|
||
# PASS=0, which triggers set -e and kills the script after the very first PASS.
|
||
pass() { echo " PASS: $1"; PASS=$((PASS+1)); RESULTS+=("PASS $1"); }
|
||
fail() { echo " FAIL: $1"; FAIL=$((FAIL+1)); RESULTS+=("FAIL $1"); }
|
||
|
||
HA_USER="nixos"
|
||
n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
|
||
n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
|
||
|
||
echo "════════════════════════════════════════════════════"
|
||
echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo "════════════════════════════════════════════════════"
|
||
|
||
# ── Detect Active/Standby nodes ────────────────────────────────────────────
|
||
# Pacemaker can promote either node; determine which is currently Active
|
||
# (DRBD Primary / holds ha-group resources) before running tests.
|
||
echo ""
|
||
echo "Detecting Active/Standby nodes..."
|
||
if n1 "drbdadm role ha-data" 2>/dev/null | grep -q "^Primary"; then
|
||
ACTIVE_NODE="$NODE1"; ACTIVE_IP="$NODE1_IP"
|
||
STANDBY_NODE="$NODE2"; STANDBY_IP="$NODE2_IP"
|
||
na() { n1 "$@"; }
|
||
ns() { n2 "$@"; }
|
||
else
|
||
ACTIVE_NODE="$NODE2"; ACTIVE_IP="$NODE2_IP"
|
||
STANDBY_NODE="$NODE1"; STANDBY_IP="$NODE1_IP"
|
||
na() { n2 "$@"; }
|
||
ns() { n1 "$@"; }
|
||
fi
|
||
echo " Active: $ACTIVE_NODE ($ACTIVE_IP)"
|
||
echo " Standby: $STANDBY_NODE ($STANDBY_IP)"
|
||
|
||
# ── T1: Corosync quorum established ──────────────────────────────────────
|
||
echo ""
|
||
echo "[T1] Corosync quorum"
|
||
if na "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
|
||
pass "cluster has quorum"
|
||
else
|
||
fail "cluster does not have quorum — check corosync on both nodes"
|
||
fi
|
||
|
||
# ── T2: DRBD Primary on Active node, Secondary on Standby ────────────────
|
||
echo ""
|
||
echo "[T2] DRBD roles"
|
||
DRBD_ROLE=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
|
||
if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then
|
||
pass "DRBD Primary on $ACTIVE_NODE ($DRBD_ROLE)"
|
||
else
|
||
fail "unexpected DRBD role on $ACTIVE_NODE: $DRBD_ROLE (expected Primary/Secondary)"
|
||
fi
|
||
|
||
DRBD_DSTATE=$(na "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
|
||
if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then
|
||
pass "DRBD disk state UpToDate ($DRBD_DSTATE)"
|
||
else
|
||
fail "DRBD disk not UpToDate: $DRBD_DSTATE"
|
||
fi
|
||
|
||
# ── T3: XFS mounted at haStorageRoot on the Active node ──────────────────
|
||
echo ""
|
||
echo "[T3] XFS mount"
|
||
if na "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||
pass "XFS mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
|
||
else
|
||
fail "XFS not mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
|
||
fi
|
||
|
||
if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||
fail "XFS unexpectedly mounted on $STANDBY_NODE (should only be on Active node)"
|
||
else
|
||
pass "XFS not mounted on $STANDBY_NODE (correct — Standby)"
|
||
fi
|
||
|
||
# ── T4: iSCSI target visible on Active node ───────────────────────────────
|
||
echo ""
|
||
echo "[T4] iSCSI target"
|
||
IQN_COUNT=$(na "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
|
||
if [[ "$IQN_COUNT" -ge 1 ]]; then
|
||
pass "iSCSI IQN active on $ACTIVE_NODE ($IQN_COUNT target(s))"
|
||
else
|
||
fail "no iSCSI IQN active on $ACTIVE_NODE"
|
||
fi
|
||
|
||
# iSCSI port reachable from Standby node via VIP.
|
||
# Use bash TCP probe (no iscsiadm needed — just checks port 3260 is open).
|
||
if ns "bash -c 'echo >/dev/tcp/${VIP}/3260' 2>/dev/null"; then
|
||
pass "iSCSI port 3260 reachable from $STANDBY_NODE via VIP ${VIP}"
|
||
else
|
||
fail "iSCSI port 3260 not reachable from $STANDBY_NODE via ${VIP}"
|
||
fi
|
||
|
||
# ── T5: Failover — standby Active node, verify resources move to Standby ──
|
||
echo ""
|
||
echo "[T5] Failover (standby $ACTIVE_NODE)"
|
||
ACTIVE_CRMD_NAME=$(na "crm_node -n" 2>/dev/null || echo "")
|
||
na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v on" 2>/dev/null || true
|
||
echo " Waiting up to 120 s for resources to move to $STANDBY_NODE..."
|
||
MOVED=false
|
||
for i in $(seq 1 120); do
|
||
if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||
MOVED=true
|
||
echo " Resources moved in ${i}s"
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
|
||
if $MOVED; then
|
||
pass "XFS mounted on $STANDBY_NODE after failover"
|
||
IQN_ON_STANDBY=$(ns "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
|
||
[[ "$IQN_ON_STANDBY" -ge 1 ]] \
|
||
&& pass "iSCSI target active on $STANDBY_NODE after failover" \
|
||
|| fail "iSCSI target NOT active on $STANDBY_NODE after failover"
|
||
else
|
||
fail "XFS did not mount on $STANDBY_NODE within 120 s — failover incomplete"
|
||
fi
|
||
|
||
# ── T6: Data integrity — file written post-failover readable ─────────────
|
||
echo ""
|
||
echo "[T6] Data integrity"
|
||
# Write a test file on the new Active (former Standby) and verify it.
|
||
# Use `echo | sudo tee` for the write: "echo ... > file" via bash -c has the
|
||
# redirect interpreted by the remote nixos shell (not sudo), so the file open
|
||
# runs as nixos and fails with EACCES on the root-owned XFS mount. Piping
|
||
# through sudo tee lets tee (running as root) open the file instead.
|
||
TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$"
|
||
TEST_CONTENT="ha-acceptance-test-$(date +%s)"
|
||
echo "${TEST_CONTENT}" | ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${STANDBY_IP}" sudo tee "${TEST_FILE}" > /dev/null 2>/dev/null || true
|
||
READBACK=$(ns cat "${TEST_FILE}" 2>/dev/null || echo "")
|
||
if [[ "$READBACK" == "$TEST_CONTENT" ]]; then
|
||
pass "test file written and read back correctly on $STANDBY_NODE"
|
||
else
|
||
fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')"
|
||
fi
|
||
ns rm -f "${TEST_FILE}" 2>/dev/null || true
|
||
|
||
# ── T7: Node rejoin — un-standby original Active, verify cluster is healthy ─
|
||
echo ""
|
||
echo "[T7] Node rejoin"
|
||
na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
|
||
na "crm_resource --cleanup" 2>/dev/null || true
|
||
sleep 5
|
||
|
||
if na "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
|
||
pass "$ACTIVE_NODE rejoined — cluster has quorum"
|
||
else
|
||
fail "$ACTIVE_NODE did not rejoin with quorum"
|
||
fi
|
||
|
||
DRBD_ROLE_AFTER=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
|
||
if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then
|
||
pass "$ACTIVE_NODE is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
|
||
else
|
||
fail "unexpected DRBD role on $ACTIVE_NODE after rejoin: $DRBD_ROLE_AFTER"
|
||
fi
|
||
|
||
# ── Summary ───────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "════════════════════════════════════════════════════"
|
||
echo " Results: ${PASS} PASS, ${FAIL} FAIL"
|
||
echo "════════════════════════════════════════════════════"
|
||
for r in "${RESULTS[@]}"; do echo " $r"; done
|
||
echo ""
|
||
|
||
if [[ "$FAIL" -eq 0 ]]; then
|
||
echo "ALL PASS — cluster is production-ready."
|
||
exit 0
|
||
else
|
||
echo "SOME TESTS FAILED — investigate before deploying."
|
||
exit 1
|
||
fi
|