From e7531b276e3d4f4d0795d3708530aac632ddc49f Mon Sep 17 00:00:00 2001
From: beatzaplenty <wayne.bennett@live.com>
Date: Wed, 29 Jul 2026 00:24:47 +1000
Subject: [PATCH] fix(ha): use crm_mon for Active node detection and add
 cleanup before VIP wait

acceptance-tests.sh: replace drbdadm-role-based Active node detection with
crm_mon-based detection. DRBD role briefly reads Secondary during Pacemaker
transitions, causing T2/T3/T4 to target the wrong node. crm_mon Promoted:/
Masters: lines are authoritative; wait up to 90 s for Pacemaker to settle
before falling back to NODE1.

cluster-init.sh: add crm_resource --cleanup before the VIP wait loop so
stale migration-threshold failure counters from previous cluster-init runs
are cleared before Pacemaker decides resource placement.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 scripts/ha/acceptance-tests.sh | 29 +++++++++++++++++++++++------
 scripts/ha/cluster-init.sh     |  3 +++
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/scripts/ha/acceptance-tests.sh b/scripts/ha/acceptance-tests.sh
index 376edc2..6b10d3c 100755
--- a/scripts/ha/acceptance-tests.sh
+++ b/scripts/ha/acceptance-tests.sh
@@ -35,17 +35,34 @@ echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
 echo "════════════════════════════════════════════════════"
 
 # ── Detect Active/Standby nodes ────────────────────────────────────────────
-# Pacemaker can promote either node; determine which is currently Active
-# (DRBD Primary / holds ha-group resources) before running tests.
+# Use crm_mon to detect which node holds the Promoted (Primary) DRBD resource.
+# Pacemaker is authoritative; DRBD role can briefly read as Secondary while
+# Pacemaker is mid-transition, giving a false Active/Standby swap.
+# Wait up to 90 s for Pacemaker to settle before giving up.
 echo ""
-echo "Detecting Active/Standby nodes..."
-if n1 "drbdadm role ha-data" 2>/dev/null | grep -q "^Primary"; then
-  ACTIVE_NODE="$NODE1"; ACTIVE_IP="$NODE1_IP"
+echo "Detecting Active/Standby nodes (waiting for Pacemaker to settle)..."
+ACTIVE_NODE=""
+for i in $(seq 1 30); do
+  # crm_mon -1 output contains "Promoted: [ <node> ]" for the DRBD master.
+  CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
+  # Match "Promoted:" (Pacemaker 2.x) or "Masters:" (Pacemaker 1.x), not "Unpromoted:"/"Slaves:"
+  ACTIVE_NODE=$(echo "$CRM_OUT" | grep -E '^\s*(Promoted|Masters):' | grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+  [[ -n "$ACTIVE_NODE" ]] && break
+  sleep 3
+done
+
+if [[ -z "$ACTIVE_NODE" ]]; then
+  echo "  WARNING: could not determine Active node from crm_mon after 90 s — defaulting to $NODE1"
+  ACTIVE_NODE="$NODE1"
+fi
+
+if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
+  ACTIVE_IP="$NODE1_IP"
   STANDBY_NODE="$NODE2"; STANDBY_IP="$NODE2_IP"
   na() { n1 "$@"; }
   ns() { n2 "$@"; }
 else
-  ACTIVE_NODE="$NODE2"; ACTIVE_IP="$NODE2_IP"
+  ACTIVE_IP="$NODE2_IP"
   STANDBY_NODE="$NODE1"; STANDBY_IP="$NODE1_IP"
   na() { n2 "$@"; }
   ns() { n1 "$@"; }
diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh
index 9495052..cb0130d 100755
--- a/scripts/ha/cluster-init.sh
+++ b/scripts/ha/cluster-init.sh
@@ -399,6 +399,9 @@ cibadmin --replace --scope constraints --xml-text '<constraints>
   <rsc_colocation id="coloc-group-with-drbd" score="INFINITY" rsc="ha-group" with-rsc="ms-drbd0" with-rsc-role="Promoted"/>
 </constraints>'
 
+log "Clearing stale Pacemaker failure history..."
+crm_resource --cleanup 2>/dev/null || true
+
 log "Waiting for resources to start..."
 for i in $(seq 1 60); do
   if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then