fix(ha): fix crm_mon active-node detection in all three scripts

crm_mon 2.x formats the Promoted line as "    * Promoted: [ node ]" — the
asterisk bullet means the previous grep -E '^\s*(Promoted|Masters):' never
matched, so active-node detection silently returned empty in health.sh,
failover.sh, and acceptance-tests.sh.

Fix: pipe through grep -v Unpromoted first, then grep -E '(Promoted|Masters):'
without anchoring to start-of-line.

Also: remove the SSH_OPTS=-i ~/.ssh/... variable pattern in health.sh and
failover.sh; tilde is not expanded inside double-quoted strings, so $SSH_OPTS
was passing a literal ~ to SSH.  Inline the key path in each function
definition instead (same as acceptance-tests.sh already did).

Also: drop the 2>/dev/null embedded in the crm_mon argument string — the
outer 2>/dev/null on the n1/n2 call already suppresses SSH stderr; the
embedded one was harmless but noisy to reason about.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scripts/ha/acceptance-tests.sh

@@ -63,9 +63,10 @@ echo "Detecting Active/Standby nodes (waiting for Pacemaker to settle)..."
 ACTIVE_NODE=""
 for i in $(seq 1 30); do
   # crm_mon -1 output contains "Promoted: [ <node> ]" for the DRBD master.
-  CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
-  # Match "Promoted:" (Pacemaker 2.x) or "Masters:" (Pacemaker 1.x), not "Unpromoted:"/"Slaves:"
-  ACTIVE_NODE=$(echo "$CRM_OUT" | grep -E '^\s*(Promoted|Masters):' | grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || n2 "crm_mon -1" 2>/dev/null || true)
+  # crm_mon 2.x formats Promoted lines as "    * Promoted: [ node ]" — the * bullet
+  # means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
+  ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | grep -E '(Promoted|Masters):' | grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
   [[ -n "$ACTIVE_NODE" ]] && break
   sleep 3
 done

scripts/ha/failover.sh

@@ -24,10 +24,8 @@ XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"
 HA_USER="${HA_USER:-nixos}"
 # ──────────────────────────────────────────────────────────────────────────
 
-SSH_OPTS="-i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5"
-
-n1() { ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
-n2() { ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
+n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
 
 # ── Argument parsing ───────────────────────────────────────────────────────
 TARGET_NODE=""
@@ -66,16 +64,19 @@ echo ""
 echo "Detecting active node..."
 
 CRM_OUT=""
-if ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then
-  CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
+if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
 fi
 if [[ -z "$CRM_OUT" ]]; then
-  if ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then
-    CRM_OUT=$(n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
+  if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then
+    CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
   fi
 fi
 
-ACTIVE_NODE=$(echo "$CRM_OUT" | grep -E '^\s*(Promoted|Masters):' | \
+# crm_mon 2.x formats Promoted lines as "    * Promoted: [ node ]" — the * bullet
+# means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
+ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
   grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
 
 if [[ -z "$ACTIVE_NODE" ]]; then
@@ -223,12 +224,13 @@ echo ""
 
 CRM_OUT_AFTER=""
 if ! $DRY_RUN; then
-  CRM_OUT_AFTER=$(ns "crm_mon -1 2>/dev/null" 2>/dev/null || na "crm_mon -1 2>/dev/null" 2>/dev/null || true)
+  CRM_OUT_AFTER=$(ns "crm_mon -1" 2>/dev/null || na "crm_mon -1" 2>/dev/null || true)
 else
   CRM_OUT_AFTER="$CRM_OUT"
 fi
 
-NEW_ACTIVE=$(echo "$CRM_OUT_AFTER" | grep -E '^\s*(Promoted|Masters):' | \
+NEW_ACTIVE=$(echo "$CRM_OUT_AFTER" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
   grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
 
 if [[ -n "$NEW_ACTIVE" ]]; then

scripts/ha/health.sh

@@ -16,18 +16,15 @@ XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
 HA_USER="${HA_USER:-nixos}"
 # ──────────────────────────────────────────────────────────────────────────
 
-SSH_OPTS="-i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5"
-
-# Returns "OK" or "UNREACHABLE" and sets global REACHABLE_{1,2}.
 REACHABLE_1=false
 REACHABLE_2=false
 
-n1()  { ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
-n2()  { ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
+n1()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
 
 probe_node() {
   local ip=$1
-  ssh $SSH_OPTS "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
+  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
 }
 
 section() { echo ""; echo "── $* ──"; }
@@ -49,13 +46,18 @@ if ! $REACHABLE_1 && ! $REACHABLE_2; then
 fi
 
 # ── Detect active node ─────────────────────────────────────────────────────
+# crm_mon 2.x formats the Promoted line as "    * Promoted: [ node ]" — the
+# bullet * means ^\s*(Promoted|Masters): never matches.  Filter out Unpromoted
+# first, then match anywhere on the line.
 ACTIVE_NODE=""
+CRM_OUT=""
 if $REACHABLE_1; then
-  CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
 elif $REACHABLE_2; then
-  CRM_OUT=$(n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
+  CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
 fi
-ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -E '^\s*(Promoted|Masters):' | \
+ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
   grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
 
 STANDBY_NODE=""
@@ -115,37 +117,17 @@ drbd_info_from() {
 $REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo "  $NODE1  [OFFLINE]"
 $REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo "  $NODE2  [OFFLINE]"
 
-# ── Pacemaker resources ────────────────────────────────────────────────────
-section "Pacemaker Resources"
+# ── Pacemaker (full crm_mon output) ───────────────────────────────────────
+section "Pacemaker"
 if [[ -n "${CRM_OUT:-}" ]]; then
-  # Print the resource lines: everything from "Full List of Resources" onward,
-  # or fall back to printing any line with a resource state keyword.
-  RESOURCES=$(echo "$CRM_OUT" | awk '/Full List of Resources/,0' | tail -n +2 || true)
-  if [[ -z "$RESOURCES" ]]; then
-    # Older crm_mon: print lines containing Started/Stopped/Promoted/Unpromoted/FAILED
-    RESOURCES=$(echo "$CRM_OUT" | grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED|Master|Slave' || true)
-  fi
-  if [[ -n "$RESOURCES" ]]; then
-    echo "$RESOURCES" | sed 's/^/  /'
+  echo "$CRM_OUT" | sed 's/^/  /'
 else
-    echo "  (no resource output parsed — run: crm_mon -1)"
+  echo "  crm_mon returned no output — trying again without suppression:"
+  if $REACHABLE_1; then
+    n1 "crm_mon -1" || true
+  elif $REACHABLE_2; then
+    n2 "crm_mon -1" || true
   fi
-else
-  echo "  crm_mon output unavailable"
-fi
-
-# ── Failure history ────────────────────────────────────────────────────────
-section "Failure History"
-if [[ -n "${CRM_OUT:-}" ]]; then
-  FAILURES=$(echo "$CRM_OUT" | awk '/Failed Resource Actions/,/^$/' | tail -n +2 | grep -v '^$' || true)
-  if [[ -n "$FAILURES" ]]; then
-    echo "$FAILURES" | sed 's/^/  /'
-    echo "  (clear with: crm_resource --cleanup)"
-  else
-    echo "  none"
-  fi
-else
-  echo "  unavailable"
 fi
 
 # ── XFS mount ─────────────────────────────────────────────────────────────