beatzaplenty/nixos
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ha): fix crm_mon active-node detection in all three scripts
Check NixOS configurations / eval-hosts (push) Successful in 10m23s
Details

Browse Source 
crm_mon 2.x formats the Promoted line as "    * Promoted: [ node ]" — the
asterisk bullet means the previous grep -E '^\s*(Promoted|Masters):' never
matched, so active-node detection silently returned empty in health.sh,
failover.sh, and acceptance-tests.sh.

Fix: pipe through grep -v Unpromoted first, then grep -E '(Promoted|Masters):'
without anchoring to start-of-line.

Also: remove the SSH_OPTS=-i ~/.ssh/... variable pattern in health.sh and
failover.sh; tilde is not expanded inside double-quoted strings, so $SSH_OPTS
was passing a literal ~ to SSH.  Inline the key path in each function
definition instead (same as acceptance-tests.sh already did).

Also: drop the 2>/dev/null embedded in the crm_mon argument string — the
outer 2>/dev/null on the n1/n2 call already suppresses SSH stderr; the
embedded one was harmless but noisy to reason about.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
beatzaplenty
2026-07-29 02:36:22 +10:00
co-authored by Claude Sonnet 4.6
parent 793a2b5924
commit 71a6c4738c
3 changed files with 36 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ha/acceptance-tests.sh
+4 -3
View File
@@ -63,9 +63,10 @@ echo "Detecting Active/Standby nodes (waiting for Pacemaker to settle)..."
ACTIVE_NODE="" ACTIVE_NODE=""
for i in $(seq 1 30); do for i in $(seq 1 30); do
# crm_mon -1 output contains "Promoted: [ <node> ]" for the DRBD master. # crm_mon -1 output contains "Promoted: [ <node> ]" for the DRBD master.
CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true) CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || n2 "crm_mon -1" 2>/dev/null || true)
# Match "Promoted:" (Pacemaker 2.x) or "Masters:" (Pacemaker 1.x), not "Unpromoted:"/"Slaves:" # crm_mon 2.x formats Promoted lines as " * Promoted: [ node ]" — the * bullet
ACTIVE_NODE=$(echo "$CRM_OUT" | grep -E '^\s*(Promoted|Masters):' | grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true) # means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | grep -E '(Promoted|Masters):' | grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
[[ -n "$ACTIVE_NODE" ]] && break [[ -n "$ACTIVE_NODE" ]] && break
sleep 3 sleep 3
done done
scripts/ha/failover.sh
+13 -11
View File
@@ -24,10 +24,8 @@ XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"
HA_USER="${HA_USER:-nixos}" HA_USER="${HA_USER:-nixos}"
# ────────────────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────────────────
SSH_OPTS="-i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5" n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
n1() { ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
n2() { ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
# ── Argument parsing ─────────────────────────────────────────────────────── # ── Argument parsing ───────────────────────────────────────────────────────
TARGET_NODE="" TARGET_NODE=""
@@ -66,16 +64,19 @@ echo ""
echo "Detecting active node..." echo "Detecting active node..."
CRM_OUT="" CRM_OUT=""
if ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then
CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || true) CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
fi fi
if [[ -z "$CRM_OUT" ]]; then if [[ -z "$CRM_OUT" ]]; then
if ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then
CRM_OUT=$(n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true) CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
fi fi
fi fi
ACTIVE_NODE=$(echo "$CRM_OUT" | grep -E '^\s*(Promoted|Masters):' | \ # crm_mon 2.x formats Promoted lines as " * Promoted: [ node ]" — the * bullet
# means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | \
grep -E '(Promoted|Masters):' | \
grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true) grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
if [[ -z "$ACTIVE_NODE" ]]; then if [[ -z "$ACTIVE_NODE" ]]; then
@@ -223,12 +224,13 @@ echo ""
CRM_OUT_AFTER="" CRM_OUT_AFTER=""
if ! $DRY_RUN; then if ! $DRY_RUN; then
CRM_OUT_AFTER=$(ns "crm_mon -1 2>/dev/null" 2>/dev/null || na "crm_mon -1 2>/dev/null" 2>/dev/null || true) CRM_OUT_AFTER=$(ns "crm_mon -1" 2>/dev/null || na "crm_mon -1" 2>/dev/null || true)
else else
CRM_OUT_AFTER="$CRM_OUT" CRM_OUT_AFTER="$CRM_OUT"
fi fi
NEW_ACTIVE=$(echo "$CRM_OUT_AFTER" | grep -E '^\s*(Promoted|Masters):' | \ NEW_ACTIVE=$(echo "$CRM_OUT_AFTER" | grep -v 'Unpromoted\|Unmanaged' | \
grep -E '(Promoted|Masters):' | \
grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true) grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
if [[ -n "$NEW_ACTIVE" ]]; then if [[ -n "$NEW_ACTIVE" ]]; then
scripts/ha/health.sh
+19 -37
View File
@@ -16,18 +16,15 @@ XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot
HA_USER="${HA_USER:-nixos}" HA_USER="${HA_USER:-nixos}"
# ────────────────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────────────────
SSH_OPTS="-i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5"
# Returns "OK" or "UNREACHABLE" and sets global REACHABLE_{1,2}.
REACHABLE_1=false REACHABLE_1=false
REACHABLE_2=false REACHABLE_2=false
n1() { ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; } n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
n2() { ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; } n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
probe_node() { probe_node() {
local ip=$1 local ip=$1
ssh $SSH_OPTS "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE" ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
} }
section() { echo ""; echo "── $* ──"; } section() { echo ""; echo "── $* ──"; }
@@ -49,13 +46,18 @@ if ! $REACHABLE_1 && ! $REACHABLE_2; then
fi fi
# ── Detect active node ───────────────────────────────────────────────────── # ── Detect active node ─────────────────────────────────────────────────────
# crm_mon 2.x formats the Promoted line as " * Promoted: [ node ]" — the
# bullet * means ^\s*(Promoted|Masters): never matches. Filter out Unpromoted
# first, then match anywhere on the line.
ACTIVE_NODE="" ACTIVE_NODE=""
CRM_OUT=""
if $REACHABLE_1; then if $REACHABLE_1; then
CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || true) CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
elif $REACHABLE_2; then elif $REACHABLE_2; then
CRM_OUT=$(n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true) CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
fi fi
ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -E '^\s*(Promoted|Masters):' | \ ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -v 'Unpromoted\|Unmanaged' | \
grep -E '(Promoted|Masters):' | \
grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true) grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
STANDBY_NODE="" STANDBY_NODE=""
@@ -115,37 +117,17 @@ drbd_info_from() {
$REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo " $NODE1 [OFFLINE]" $REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo " $NODE1 [OFFLINE]"
$REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo " $NODE2 [OFFLINE]" $REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo " $NODE2 [OFFLINE]"
# ── Pacemaker resources ──────────────────────────────────────────────────── # ── Pacemaker (full crm_mon output) ───────────────────────────────────────
section "Pacemaker Resources" section "Pacemaker"
if [[ -n "${CRM_OUT:-}" ]]; then if [[ -n "${CRM_OUT:-}" ]]; then
# Print the resource lines: everything from "Full List of Resources" onward, echo "$CRM_OUT" | sed 's/^/ /'
# or fall back to printing any line with a resource state keyword.
RESOURCES=$(echo "$CRM_OUT" | awk '/Full List of Resources/,0' | tail -n +2 || true)
if [[ -z "$RESOURCES" ]]; then
# Older crm_mon: print lines containing Started/Stopped/Promoted/Unpromoted/FAILED
RESOURCES=$(echo "$CRM_OUT" | grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED|Master|Slave' || true)
fi
if [[ -n "$RESOURCES" ]]; then
echo "$RESOURCES" | sed 's/^/ /'
else
echo " (no resource output parsed — run: crm_mon -1)"
fi
else else
echo " crm_mon output unavailable" echo " crm_mon returned no output — trying again without suppression:"
fi if $REACHABLE_1; then
n1 "crm_mon -1" || true
# ── Failure history ──────────────────────────────────────────────────────── elif $REACHABLE_2; then
section "Failure History" n2 "crm_mon -1" || true
if [[ -n "${CRM_OUT:-}" ]]; then
FAILURES=$(echo "$CRM_OUT" | awk '/Failed Resource Actions/,/^$/' | tail -n +2 | grep -v '^$' || true)
if [[ -n "$FAILURES" ]]; then
echo "$FAILURES" | sed 's/^/ /'
echo " (clear with: crm_resource --cleanup)"
else
echo " none"
fi fi
else
echo " unavailable"
fi fi
# ── XFS mount ───────────────────────────────────────────────────────────── # ── XFS mount ─────────────────────────────────────────────────────────────