nixos/scripts/ha/acceptance-tests.sh

#!/usr/bin/env bash
# acceptance-tests.sh — HA cluster acceptance tests (T1–T7)
#
# Run from a host with SSH access to both HA nodes (or from node1 itself).
# All 7 tests must pass before considering the cluster production-ready.
# Test values below must match variables.nix haServer* values.
set -euo pipefail

# ── Configuration ─────────────────────────────────────────────────────────
# All values override-able via environment variables; defaults match variables.nix.
NODE1="${NODE1:-ha-server-1}"
NODE2="${NODE2:-ha-server-2}"
NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}"  # vars.haIscsiIqn
# ──────────────────────────────────────────────────────────────────────────

PASS=0
FAIL=0
RESULTS=()

# Use PASS=$((PASS+1)) instead of ((PASS++)) — the latter evaluates to 0 when
# PASS=0, which triggers set -e and kills the script after the very first PASS.
pass() { echo "  PASS: $1"; PASS=$((PASS+1)); RESULTS+=("PASS $1"); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL+1)); RESULTS+=("FAIL $1"); }

HA_USER="nixos"
n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }

echo "════════════════════════════════════════════════════"
echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
echo "════════════════════════════════════════════════════"

# ── Detect Active/Standby nodes ────────────────────────────────────────────
# Pacemaker can promote either node; determine which is currently Active
# (DRBD Primary / holds ha-group resources) before running tests.
echo ""
echo "Detecting Active/Standby nodes..."
if n1 "drbdadm role ha-data" 2>/dev/null | grep -q "^Primary"; then
  ACTIVE_NODE="$NODE1"; ACTIVE_IP="$NODE1_IP"
  STANDBY_NODE="$NODE2"; STANDBY_IP="$NODE2_IP"
  na() { n1 "$@"; }
  ns() { n2 "$@"; }
else
  ACTIVE_NODE="$NODE2"; ACTIVE_IP="$NODE2_IP"
  STANDBY_NODE="$NODE1"; STANDBY_IP="$NODE1_IP"
  na() { n2 "$@"; }
  ns() { n1 "$@"; }
fi
echo "  Active:  $ACTIVE_NODE ($ACTIVE_IP)"
echo "  Standby: $STANDBY_NODE ($STANDBY_IP)"

# ── T1: Corosync quorum established ──────────────────────────────────────
echo ""
echo "[T1] Corosync quorum"
if na "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
  pass "cluster has quorum"
else
  fail "cluster does not have quorum — check corosync on both nodes"
fi

# ── T2: DRBD Primary on Active node, Secondary on Standby ────────────────
echo ""
echo "[T2] DRBD roles"
DRBD_ROLE=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then
  pass "DRBD Primary on $ACTIVE_NODE ($DRBD_ROLE)"
else
  fail "unexpected DRBD role on $ACTIVE_NODE: $DRBD_ROLE (expected Primary/Secondary)"
fi

DRBD_DSTATE=$(na "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then
  pass "DRBD disk state UpToDate ($DRBD_DSTATE)"
else
  fail "DRBD disk not UpToDate: $DRBD_DSTATE"
fi

# ── T3: XFS mounted at haStorageRoot on the Active node ──────────────────
echo ""
echo "[T3] XFS mount"
if na "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
  pass "XFS mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
else
  fail "XFS not mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
fi

if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
  fail "XFS unexpectedly mounted on $STANDBY_NODE (should only be on Active node)"
else
  pass "XFS not mounted on $STANDBY_NODE (correct — Standby)"
fi

# ── T4: iSCSI target visible on Active node ───────────────────────────────
echo ""
echo "[T4] iSCSI target"
IQN_COUNT=$(na "bash -c 'ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn || true'" 2>/dev/null || echo "0")
if [[ "$IQN_COUNT" -ge 1 ]]; then
  pass "iSCSI IQN active on $ACTIVE_NODE ($IQN_COUNT target(s))"
else
  fail "no iSCSI IQN active on $ACTIVE_NODE"
fi

# iSCSI port reachable from Standby node via VIP.
# Use bash TCP probe (no iscsiadm needed — just checks port 3260 is open).
if ns "bash -c 'echo >/dev/tcp/${VIP}/3260' 2>/dev/null"; then
  pass "iSCSI port 3260 reachable from $STANDBY_NODE via VIP ${VIP}"
else
  fail "iSCSI port 3260 not reachable from $STANDBY_NODE via ${VIP}"
fi

# ── T5: Failover — standby Active node, verify resources move to Standby ──
echo ""
echo "[T5] Failover (standby $ACTIVE_NODE)"
ACTIVE_CRMD_NAME=$(na "crm_node -n" 2>/dev/null || echo "")
na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v on" 2>/dev/null || true
echo "  Waiting up to 120 s for resources to move to $STANDBY_NODE..."
MOVED=false
for i in $(seq 1 120); do
  if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
    MOVED=true
    echo "  Resources moved in ${i}s"
    break
  fi
  sleep 1
done

if $MOVED; then
  pass "XFS mounted on $STANDBY_NODE after failover"
  IQN_ON_STANDBY=$(ns "bash -c 'ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn || true'" 2>/dev/null || echo "0")
  [[ "$IQN_ON_STANDBY" -ge 1 ]] \
    && pass "iSCSI target active on $STANDBY_NODE after failover" \
    || fail "iSCSI target NOT active on $STANDBY_NODE after failover"
else
  fail "XFS did not mount on $STANDBY_NODE within 120 s — failover incomplete"
fi

# ── T6: Data integrity — file written post-failover readable ─────────────
echo ""
echo "[T6] Data integrity"
# Write a test file on the new Active (former Standby) and verify it.
# Use `echo | sudo tee` for the write: "echo ... > file" via bash -c has the
# redirect interpreted by the remote nixos shell (not sudo), so the file open
# runs as nixos and fails with EACCES on the root-owned XFS mount.  Piping
# through sudo tee lets tee (running as root) open the file instead.
TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$"
TEST_CONTENT="ha-acceptance-test-$(date +%s)"
echo "${TEST_CONTENT}" | ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${STANDBY_IP}" sudo tee "${TEST_FILE}" > /dev/null 2>/dev/null || true
READBACK=$(ns cat "${TEST_FILE}" 2>/dev/null || echo "")
if [[ "$READBACK" == "$TEST_CONTENT" ]]; then
  pass "test file written and read back correctly on $STANDBY_NODE"
else
  fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')"
fi
ns rm -f "${TEST_FILE}" 2>/dev/null || true

# ── T7: Node rejoin — un-standby original Active, verify cluster is healthy ─
echo ""
echo "[T7] Node rejoin"
na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
na "crm_resource --cleanup" 2>/dev/null || true
sleep 5

if na "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
  pass "$ACTIVE_NODE rejoined — cluster has quorum"
else
  fail "$ACTIVE_NODE did not rejoin with quorum"
fi

DRBD_ROLE_AFTER=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then
  pass "$ACTIVE_NODE is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
else
  fail "unexpected DRBD role on $ACTIVE_NODE after rejoin: $DRBD_ROLE_AFTER"
fi

# ── Summary ───────────────────────────────────────────────────────────────
echo ""
echo "════════════════════════════════════════════════════"
echo " Results: ${PASS} PASS, ${FAIL} FAIL"
echo "════════════════════════════════════════════════════"
for r in "${RESULTS[@]}"; do echo "  $r"; done
echo ""

if [[ "$FAIL" -eq 0 ]]; then
  echo "ALL PASS — cluster is production-ready."
  exit 0
else
  echo "SOME TESTS FAILED — investigate before deploying."
  exit 1
fi