beatzaplenty/nixos
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ha): fix crm_mon active-node detection in all three scripts
Check NixOS configurations / eval-hosts (push) Successful in 10m23s
Details

Browse Source 
crm_mon 2.x formats the Promoted line as "    * Promoted: [ node ]" — the
asterisk bullet means the previous grep -E '^\s*(Promoted|Masters):' never
matched, so active-node detection silently returned empty in health.sh,
failover.sh, and acceptance-tests.sh.

Fix: pipe through grep -v Unpromoted first, then grep -E '(Promoted|Masters):'
without anchoring to start-of-line.

Also: remove the SSH_OPTS=-i ~/.ssh/... variable pattern in health.sh and
failover.sh; tilde is not expanded inside double-quoted strings, so $SSH_OPTS
was passing a literal ~ to SSH.  Inline the key path in each function
definition instead (same as acceptance-tests.sh already did).

Also: drop the 2>/dev/null embedded in the crm_mon argument string — the
outer 2>/dev/null on the n1/n2 call already suppresses SSH stderr; the
embedded one was harmless but noisy to reason about.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
beatzaplenty
2026-07-29 02:36:22 +10:00
co-authored by Claude Sonnet 4.6
parent 793a2b5924
commit 71a6c4738c
3 changed files with 36 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ha/health.sh
+19 -37
View File
@@ -16,18 +16,15 @@ XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot
HA_USER="${HA_USER:-nixos}"
# ──────────────────────────────────────────────────────────────────────────
SSH_OPTS="-i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5"
# Returns "OK" or "UNREACHABLE" and sets global REACHABLE_{1,2}.
REACHABLE_1=false
REACHABLE_2=false
n1() { ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
n2() { ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
probe_node() {
local ip=$1
ssh $SSH_OPTS "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
}
section() { echo ""; echo "── $* ──"; }
@@ -49,13 +46,18 @@ if ! $REACHABLE_1 && ! $REACHABLE_2; then
fi
# ── Detect active node ─────────────────────────────────────────────────────
# crm_mon 2.x formats the Promoted line as " * Promoted: [ node ]" — the
# bullet * means ^\s*(Promoted|Masters): never matches. Filter out Unpromoted
# first, then match anywhere on the line.
ACTIVE_NODE=""
CRM_OUT=""
if $REACHABLE_1; then
CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
elif $REACHABLE_2; then
CRM_OUT=$(n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
fi
ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -E '^\s*(Promoted|Masters):' | \
ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -v 'Unpromoted\|Unmanaged' | \
grep -E '(Promoted|Masters):' | \
grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
STANDBY_NODE=""
@@ -115,37 +117,17 @@ drbd_info_from() {
$REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo " $NODE1 [OFFLINE]"
$REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo " $NODE2 [OFFLINE]"
# ── Pacemaker resources ────────────────────────────────────────────────────
section "Pacemaker Resources"
# ── Pacemaker (full crm_mon output) ───────────────────────────────────────
section "Pacemaker"
if [[ -n "${CRM_OUT:-}" ]]; then
# Print the resource lines: everything from "Full List of Resources" onward,
# or fall back to printing any line with a resource state keyword.
RESOURCES=$(echo "$CRM_OUT" | awk '/Full List of Resources/,0' | tail -n +2 || true)
if [[ -z "$RESOURCES" ]]; then
# Older crm_mon: print lines containing Started/Stopped/Promoted/Unpromoted/FAILED
RESOURCES=$(echo "$CRM_OUT" | grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED|Master|Slave' || true)
fi
if [[ -n "$RESOURCES" ]]; then
echo "$RESOURCES" | sed 's/^/ /'
else
echo " (no resource output parsed — run: crm_mon -1)"
fi
echo "$CRM_OUT" | sed 's/^/ /'
else
echo " crm_mon output unavailable"
fi
# ── Failure history ────────────────────────────────────────────────────────
section "Failure History"
if [[ -n "${CRM_OUT:-}" ]]; then
FAILURES=$(echo "$CRM_OUT" | awk '/Failed Resource Actions/,/^$/' | tail -n +2 | grep -v '^$' || true)
if [[ -n "$FAILURES" ]]; then
echo "$FAILURES" | sed 's/^/ /'
echo " (clear with: crm_resource --cleanup)"
else
echo " none"
echo " crm_mon returned no output — trying again without suppression:"
if $REACHABLE_1; then
n1 "crm_mon -1" || true
elif $REACHABLE_2; then
n2 "crm_mon -1" || true
fi
else
echo " unavailable"
fi
# ── XFS mount ─────────────────────────────────────────────────────────────