#!/usr/bin/env bash
# HA File Server Acceptance Tests T1-T7
# Failover (T5) uses pacemaker standby mode to gracefully move resources,
# simulating what STONITH + node-restart does in production.
# Note: A production cluster requires real STONITH (fence agent for the hypervisor).
set -euo pipefail

NODE2="root@192.168.2.201"
VIP="192.168.2.202"
IQN="iqn.2026-01.local.ha-test:storage"
MOUNT="/srv/ha-data"
PACEMAKERD="/nix/store/3v9sb74cg2qmpcyzb4h6fq0z8bvp5gw1-pacemaker-3.0.1/sbin/pacemakerd"
MYNODE=$(hostname)

SSH="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15"
PASS=0; FAIL=0

pass() { echo "[PASS] $1"; ((PASS++)) || true; }
fail() { echo "[FAIL] $1"; ((FAIL++)) || true; }
info() { echo "[INFO] $1"; }

echo "=== HA File Server Acceptance Tests ==="
echo "Node: $MYNODE  Date: $(date)"
echo ""

# T1: Corosync 2-node cluster with quorum
echo "--- T1: Corosync cluster quorum ---"
QUORATE=$(corosync-quorumtool -s 2>/dev/null | grep 'Quorate:' | awk '{print $2}')
NODE_COUNT=$(corosync-quorumtool -s 2>/dev/null | grep '^Nodes:' | awk '{print $2}')
if [ "$QUORATE" = "Yes" ] && [ "$NODE_COUNT" = "2" ]; then
  pass "T1: Corosync quorate with 2 nodes"
else
  fail "T1: quorate=$QUORATE nodes=$NODE_COUNT"
fi

# T2: DRBD both UpToDate, replication Established
echo "--- T2: DRBD replication healthy ---"
DRBD_STATUS=$(drbdadm status ha-data 2>/dev/null)
N1_ROLE=$(echo "$DRBD_STATUS" | grep '^ha-data role:' | awk -F: '{print $2}')
N1_DISK=$(echo "$DRBD_STATUS" | grep -oP 'disk:\K\S+' | head -1)
N2_DISK=$(echo "$DRBD_STATUS" | grep -oP 'peer-disk:\K\S+' | head -1)
REPL=$(echo "$DRBD_STATUS" | grep -oP 'replication:\K\S+' | head -1)
info "DRBD: role=$N1_ROLE local_disk=$N1_DISK peer_disk=$N2_DISK replication=$REPL"
if [ "$N1_DISK" = "UpToDate" ] && [ "$N2_DISK" = "UpToDate" ] && [ "$REPL" = "Established" ]; then
  pass "T2: DRBD both UpToDate, Established (role=$N1_ROLE)"
else
  fail "T2: DRBD issue: local=$N1_DISK peer=$N2_DISK replication=$REPL"
fi

# T3: XFS mounted on primary
echo "--- T3: XFS mount on primary ---"
if mountpoint -q $MOUNT && df -t xfs $MOUNT &>/dev/null; then
  FSINFO=$(df -h $MOUNT | tail -1)
  pass "T3: XFS mounted at $MOUNT: $FSINFO"
else
  fail "T3: XFS not mounted at $MOUNT"
fi

# T4: iSCSI active on primary, VIP responds on port 3260
echo "--- T4: iSCSI target active ---"
ACTIVE_IQN=$(ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep iqn | head -1)
if [ "$ACTIVE_IQN" = "$IQN" ] && nc -w3 $VIP 3260 < /dev/null 2>/dev/null; then
  pass "T4: iSCSI $IQN active, port 3260 open on VIP $VIP"
elif [ "$ACTIVE_IQN" = "$IQN" ]; then
  fail "T4: iSCSI IQN active but port 3260 not reachable on VIP"
else
  fail "T4: iSCSI not active (got '$ACTIVE_IQN')"
fi

# Pre-T5: write test file for data integrity check
echo "--- Pre-T5: writing test data ---"
TESTFILE="$MOUNT/failover-test.txt"
TESTDATA="FAILOVER_INTEGRITY_$(date +%s)"
echo "$TESTDATA" > "$TESTFILE"
sync
info "Wrote: $TESTFILE (data: $TESTDATA)"

# T5: Failover — put this node into pacemaker standby, forcing resource migration
# (In production this is triggered by real STONITH; standby simulates the result.)
echo "--- T5: Failover (pacemaker standby + node isolation) ---"
info "Putting $MYNODE into standby mode (triggers resource migration to node2)..."
crm_standby -N "$MYNODE" -v on 2>&1 || true

FAILOVER_OK=false
info "Waiting up to 90s for node2 failover..."
for i in $(seq 1 18); do
  sleep 5
  STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "UNREACHABLE")
  if echo "$STATUS" | grep -q "Started ha-test-node2"; then
    FAILOVER_OK=true
    info "Failover complete at $((i*5))s"
    echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted'
    break
  fi
done

if $FAILOVER_OK; then
  N2_DRBD=$($SSH $NODE2 "drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print \$2}'" 2>/dev/null || echo "unknown")
  N2_MOUNT=$($SSH $NODE2 "mountpoint -q $MOUNT && echo 'mounted' || echo 'not-mounted'" 2>/dev/null || echo "unknown")
  N2_ISCSI=$($SSH $NODE2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" 2>/dev/null || echo "0")
  N2_VIP=$($SSH $NODE2 "ip addr show | grep -c '$VIP'" 2>/dev/null || echo "0")
  info "Node2: DRBD=$N2_DRBD mount=$N2_MOUNT iSCSI_IQNs=$N2_ISCSI VIP=$N2_VIP"
  
  FAILS=0
  [ "$N2_DRBD" = "Primary" ]  || { info "FAIL: DRBD not Primary on node2"; ((FAILS++)) || true; }
  [ "$N2_MOUNT" = "mounted" ] || { info "FAIL: XFS not mounted on node2"; ((FAILS++)) || true; }
  [ "$N2_ISCSI" -ge "1" ] 2>/dev/null || { info "FAIL: iSCSI not active on node2"; ((FAILS++)) || true; }
  [ "$N2_VIP" -ge "1" ] 2>/dev/null || { info "FAIL: VIP not on node2"; ((FAILS++)) || true; }
  
  if [ $FAILS -eq 0 ]; then
    pass "T5: Failover complete — DRBD Primary, XFS, iSCSI, VIP all on node2"
  else
    fail "T5: Partial failover ($FAILS sub-checks failed)"
  fi
else
  fail "T5: No failover detected within 90s"
fi

# T7: Data integrity — test file readable on node2 after failover
echo "--- T7: Data integrity after failover ---"
if $SSH $NODE2 "grep -q '$TESTDATA' $TESTFILE 2>/dev/null"; then
  pass "T7: Test data intact on node2 after failover"
else
  ACTUAL=$($SSH $NODE2 "cat $TESTFILE 2>/dev/null || echo FILE_MISSING" 2>/dev/null || echo "SSH_FAIL")
  fail "T7: Data integrity check failed (expected '$TESTDATA', got '$ACTUAL')"
fi

# T6: Node rejoin — take node out of standby
echo "--- T6: Node rejoin ---"
info "Taking $MYNODE out of standby..."
crm_standby -N "$MYNODE" -v off 2>&1 || true

REJOIN_OK=false
for i in $(seq 1 12); do
  sleep 5
  # Check if this node is back online (no longer standby)
  STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "")
  if echo "$STATUS" | grep -q "Online:.*$MYNODE"; then
    REJOIN_OK=true
    info "Rejoined at $((i*5))s"
    echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted'
    break
  fi
done

if $REJOIN_OK; then
  sleep 5
  N1_DRBD=$(drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print $2}')
  N1_DISK=$(drbdadm status ha-data 2>/dev/null | grep -oP 'disk:\K\S+' | head -1)
  info "Node1 DRBD after rejoin: role=$N1_DRBD disk=$N1_DISK"
  pass "T6: Node rejoined cluster (DRBD role=$N1_DRBD, disk=$N1_DISK)"
else
  fail "T6: Node did not rejoin within 60s"
fi

echo ""
echo "========================================"
echo "RESULTS: $PASS passed, $FAIL failed"
echo "========================================"
echo ""
echo "NOTES:"
echo "  T5 uses pacemaker standby to simulate failover (production needs STONITH"
echo "  fence agent, e.g. fence_pve_ssh, to crash the VM — same requirement on Debian)"
echo ""
if [ $FAIL -eq 0 ]; then
  echo "VERDICT: ALL TESTS PASSED → Deliverable A (NixOS modules)"
  exit 0
else
  echo "VERDICT: $FAIL TEST(S) FAILED → review above"
  exit 1
fi