Archived
beatzaplentyandClaude Sonnet 4.6
acebbdbe26
Root SSH was failing because only the RSA admin key was authorized but the local dev box only has an ed25519 key. Fix: - cluster-config.nix: add ed25519 keys to root (same set as nixos user) so future deployments work without the temp-key workaround - deploy.sh/acceptance-tests.sh: SSH as nixos user with sudo instead of root@ - cluster-init.sh: HA_USER/HA_KEY env vars + n2_ssh()/n2_scp() helpers so inter-node SSH works regardless of whether root-to-root is available - deploy.sh Phase 6: generate temp keypair, authorize on node2, place on node1 for root to use during cluster-init, clean up afterward Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
170 lines
7.2 KiB
Bash
170 lines
7.2 KiB
Bash
#!/usr/bin/env bash
|
||
# acceptance-tests.sh — HA cluster acceptance tests (T1–T7)
|
||
#
|
||
# Run from a host with SSH access to both HA nodes (or from node1 itself).
|
||
# All 7 tests must pass before considering the cluster production-ready.
|
||
# Test values below must match variables.nix haServer* values.
|
||
set -euo pipefail
|
||
|
||
# ── Configuration ─────────────────────────────────────────────────────────
|
||
# All values override-able via environment variables; defaults match variables.nix.
|
||
NODE1="${NODE1:-ha-server-1}"
|
||
NODE2="${NODE2:-ha-server-2}"
|
||
NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip
|
||
NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip
|
||
VIP="${VIP:-192.168.2.229}" # vars.haServerVip
|
||
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot
|
||
ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}" # vars.haIscsiIqn
|
||
# ──────────────────────────────────────────────────────────────────────────
|
||
|
||
PASS=0
|
||
FAIL=0
|
||
RESULTS=()
|
||
|
||
pass() { echo " PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); }
|
||
fail() { echo " FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); }
|
||
|
||
HA_USER="nixos"
|
||
n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
|
||
n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
|
||
|
||
echo "════════════════════════════════════════════════════"
|
||
echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo "════════════════════════════════════════════════════"
|
||
|
||
# ── T1: Corosync quorum established ──────────────────────────────────────
|
||
echo ""
|
||
echo "[T1] Corosync quorum"
|
||
if n1 "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
|
||
pass "cluster has quorum"
|
||
else
|
||
fail "cluster does not have quorum — check corosync on both nodes"
|
||
fi
|
||
|
||
# ── T2: DRBD Primary on node1, Secondary on node2 ────────────────────────
|
||
echo ""
|
||
echo "[T2] DRBD roles"
|
||
DRBD_ROLE=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
|
||
if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then
|
||
pass "DRBD Primary on $NODE1 ($DRBD_ROLE)"
|
||
else
|
||
fail "unexpected DRBD role on $NODE1: $DRBD_ROLE (expected Primary/Secondary)"
|
||
fi
|
||
|
||
DRBD_DSTATE=$(n1 "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
|
||
if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then
|
||
pass "DRBD disk state UpToDate ($DRBD_DSTATE)"
|
||
else
|
||
fail "DRBD disk not UpToDate: $DRBD_DSTATE"
|
||
fi
|
||
|
||
# ── T3: XFS mounted at haStorageRoot on the Active node ──────────────────
|
||
echo ""
|
||
echo "[T3] XFS mount"
|
||
if n1 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||
pass "XFS mounted at ${XFS_MOUNT} on $NODE1"
|
||
else
|
||
fail "XFS not mounted at ${XFS_MOUNT} on $NODE1"
|
||
fi
|
||
|
||
if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||
fail "XFS unexpectedly mounted on $NODE2 (should only be on Active node)"
|
||
else
|
||
pass "XFS not mounted on $NODE2 (correct — Secondary)"
|
||
fi
|
||
|
||
# ── T4: iSCSI target visible on both nodes ────────────────────────────────
|
||
echo ""
|
||
echo "[T4] iSCSI target"
|
||
IQN_COUNT=$(n1 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
|
||
if [[ "$IQN_COUNT" -ge 1 ]]; then
|
||
pass "iSCSI IQN active on $NODE1 ($IQN_COUNT target(s))"
|
||
else
|
||
fail "no iSCSI IQN active on $NODE1"
|
||
fi
|
||
|
||
# iSCSI discovery from node2 via VIP
|
||
if n2 "iscsiadm -m discovery -t sendtargets -p '${VIP}' 2>/dev/null | grep -q '${ISCSI_IQN}'"; then
|
||
pass "iSCSI target discoverable from $NODE2 via VIP ${VIP}"
|
||
else
|
||
fail "iSCSI target not discoverable from $NODE2 via ${VIP}"
|
||
fi
|
||
|
||
# ── T5: Failover — standby node1, verify resources move to node2 ──────────
|
||
echo ""
|
||
echo "[T5] Failover (standby $NODE1)"
|
||
MYNODE=$(n1 "crm_node -n" 2>/dev/null || echo "")
|
||
n1 "crm_standby -N '${MYNODE}' -v on" 2>/dev/null || true
|
||
echo " Waiting up to 30 s for resources to move to $NODE2..."
|
||
MOVED=false
|
||
for i in $(seq 1 30); do
|
||
if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||
MOVED=true
|
||
echo " Resources moved in ${i}s"
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
|
||
if $MOVED; then
|
||
pass "XFS mounted on $NODE2 after failover"
|
||
IQN_ON_N2=$(n2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
|
||
[[ "$IQN_ON_N2" -ge 1 ]] \
|
||
&& pass "iSCSI target active on $NODE2 after failover" \
|
||
|| fail "iSCSI target NOT active on $NODE2 after failover"
|
||
else
|
||
fail "XFS did not mount on $NODE2 within 30 s — failover incomplete"
|
||
fi
|
||
|
||
# ── T6: Data integrity — file written pre-failover readable post-failover ─
|
||
echo ""
|
||
echo "[T6] Data integrity"
|
||
# Write a test file on node2 (now Active) and verify its content
|
||
TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$"
|
||
TEST_CONTENT="ha-acceptance-test-$(date +%s)"
|
||
n2 "echo '${TEST_CONTENT}' > '${TEST_FILE}'" 2>/dev/null || true
|
||
READBACK=$(n2 "cat '${TEST_FILE}' 2>/dev/null" || echo "")
|
||
if [[ "$READBACK" == "$TEST_CONTENT" ]]; then
|
||
pass "test file written and read back correctly on $NODE2"
|
||
else
|
||
fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')"
|
||
fi
|
||
n2 "rm -f '${TEST_FILE}'" 2>/dev/null || true
|
||
|
||
# ── T7: Node rejoin — un-standby node1, verify cluster is healthy ─────────
|
||
echo ""
|
||
echo "[T7] Node rejoin"
|
||
n1 "crm_standby -N '${MYNODE}' -v off" 2>/dev/null || true
|
||
n1 "crm_resource --cleanup" 2>/dev/null || true
|
||
sleep 5
|
||
|
||
ONLINE_NODES=$(n2 "crm_mon -1 2>/dev/null | grep -c 'Online:'" || echo "0")
|
||
if n1 "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
|
||
pass "$NODE1 rejoined — cluster has quorum"
|
||
else
|
||
fail "$NODE1 did not rejoin with quorum"
|
||
fi
|
||
|
||
DRBD_ROLE_AFTER=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
|
||
if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then
|
||
pass "$NODE1 is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
|
||
else
|
||
fail "unexpected DRBD role on $NODE1 after rejoin: $DRBD_ROLE_AFTER"
|
||
fi
|
||
|
||
# ── Summary ───────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "════════════════════════════════════════════════════"
|
||
echo " Results: ${PASS} PASS, ${FAIL} FAIL"
|
||
echo "════════════════════════════════════════════════════"
|
||
for r in "${RESULTS[@]}"; do echo " $r"; done
|
||
echo ""
|
||
|
||
if [[ "$FAIL" -eq 0 ]]; then
|
||
echo "ALL PASS — cluster is production-ready."
|
||
exit 0
|
||
else
|
||
echo "SOME TESTS FAILED — investigate before deploying."
|
||
exit 1
|
||
fi
|