diff --git a/scripts/ha/acceptance-tests.sh b/scripts/ha/acceptance-tests.sh index c405038..4b7a3bb 100755 --- a/scripts/ha/acceptance-tests.sh +++ b/scripts/ha/acceptance-tests.sh @@ -63,9 +63,10 @@ echo "Detecting Active/Standby nodes (waiting for Pacemaker to settle)..." ACTIVE_NODE="" for i in $(seq 1 30); do # crm_mon -1 output contains "Promoted: [ ]" for the DRBD master. - CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true) - # Match "Promoted:" (Pacemaker 2.x) or "Masters:" (Pacemaker 1.x), not "Unpromoted:"/"Slaves:" - ACTIVE_NODE=$(echo "$CRM_OUT" | grep -E '^\s*(Promoted|Masters):' | grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true) + CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || n2 "crm_mon -1" 2>/dev/null || true) + # crm_mon 2.x formats Promoted lines as " * Promoted: [ node ]" — the * bullet + # means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead. + ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | grep -E '(Promoted|Masters):' | grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true) [[ -n "$ACTIVE_NODE" ]] && break sleep 3 done diff --git a/scripts/ha/failover.sh b/scripts/ha/failover.sh index c4c2a22..0706d34 100755 --- a/scripts/ha/failover.sh +++ b/scripts/ha/failover.sh @@ -24,10 +24,8 @@ XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" HA_USER="${HA_USER:-nixos}" # ────────────────────────────────────────────────────────────────────────── -SSH_OPTS="-i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5" - -n1() { ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; } -n2() { ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; } +n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; } +n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; } # ── Argument parsing ─────────────────────────────────────────────────────── TARGET_NODE="" @@ -66,16 +64,19 @@ echo "" echo "Detecting active node..." CRM_OUT="" -if ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then - CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || true) +if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then + CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true) fi if [[ -z "$CRM_OUT" ]]; then - if ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then - CRM_OUT=$(n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true) + if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then + CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true) fi fi -ACTIVE_NODE=$(echo "$CRM_OUT" | grep -E '^\s*(Promoted|Masters):' | \ +# crm_mon 2.x formats Promoted lines as " * Promoted: [ node ]" — the * bullet +# means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead. +ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | \ + grep -E '(Promoted|Masters):' | \ grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true) if [[ -z "$ACTIVE_NODE" ]]; then @@ -223,12 +224,13 @@ echo "" CRM_OUT_AFTER="" if ! $DRY_RUN; then - CRM_OUT_AFTER=$(ns "crm_mon -1 2>/dev/null" 2>/dev/null || na "crm_mon -1 2>/dev/null" 2>/dev/null || true) + CRM_OUT_AFTER=$(ns "crm_mon -1" 2>/dev/null || na "crm_mon -1" 2>/dev/null || true) else CRM_OUT_AFTER="$CRM_OUT" fi -NEW_ACTIVE=$(echo "$CRM_OUT_AFTER" | grep -E '^\s*(Promoted|Masters):' | \ +NEW_ACTIVE=$(echo "$CRM_OUT_AFTER" | grep -v 'Unpromoted\|Unmanaged' | \ + grep -E '(Promoted|Masters):' | \ grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true) if [[ -n "$NEW_ACTIVE" ]]; then diff --git a/scripts/ha/health.sh b/scripts/ha/health.sh index 15fc8b4..5bdc33e 100755 --- a/scripts/ha/health.sh +++ b/scripts/ha/health.sh @@ -16,18 +16,15 @@ XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot HA_USER="${HA_USER:-nixos}" # ────────────────────────────────────────────────────────────────────────── -SSH_OPTS="-i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5" - -# Returns "OK" or "UNREACHABLE" and sets global REACHABLE_{1,2}. REACHABLE_1=false REACHABLE_2=false -n1() { ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; } -n2() { ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; } +n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; } +n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; } probe_node() { local ip=$1 - ssh $SSH_OPTS "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE" + ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE" } section() { echo ""; echo "── $* ──"; } @@ -49,13 +46,18 @@ if ! $REACHABLE_1 && ! $REACHABLE_2; then fi # ── Detect active node ───────────────────────────────────────────────────── +# crm_mon 2.x formats the Promoted line as " * Promoted: [ node ]" — the +# bullet * means ^\s*(Promoted|Masters): never matches. Filter out Unpromoted +# first, then match anywhere on the line. ACTIVE_NODE="" +CRM_OUT="" if $REACHABLE_1; then - CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || true) + CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true) elif $REACHABLE_2; then - CRM_OUT=$(n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true) + CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true) fi -ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -E '^\s*(Promoted|Masters):' | \ +ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -v 'Unpromoted\|Unmanaged' | \ + grep -E '(Promoted|Masters):' | \ grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true) STANDBY_NODE="" @@ -115,37 +117,17 @@ drbd_info_from() { $REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo " $NODE1 [OFFLINE]" $REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo " $NODE2 [OFFLINE]" -# ── Pacemaker resources ──────────────────────────────────────────────────── -section "Pacemaker Resources" +# ── Pacemaker (full crm_mon output) ─────────────────────────────────────── +section "Pacemaker" if [[ -n "${CRM_OUT:-}" ]]; then - # Print the resource lines: everything from "Full List of Resources" onward, - # or fall back to printing any line with a resource state keyword. - RESOURCES=$(echo "$CRM_OUT" | awk '/Full List of Resources/,0' | tail -n +2 || true) - if [[ -z "$RESOURCES" ]]; then - # Older crm_mon: print lines containing Started/Stopped/Promoted/Unpromoted/FAILED - RESOURCES=$(echo "$CRM_OUT" | grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED|Master|Slave' || true) - fi - if [[ -n "$RESOURCES" ]]; then - echo "$RESOURCES" | sed 's/^/ /' - else - echo " (no resource output parsed — run: crm_mon -1)" - fi + echo "$CRM_OUT" | sed 's/^/ /' else - echo " crm_mon output unavailable" -fi - -# ── Failure history ──────────────────────────────────────────────────────── -section "Failure History" -if [[ -n "${CRM_OUT:-}" ]]; then - FAILURES=$(echo "$CRM_OUT" | awk '/Failed Resource Actions/,/^$/' | tail -n +2 | grep -v '^$' || true) - if [[ -n "$FAILURES" ]]; then - echo "$FAILURES" | sed 's/^/ /' - echo " (clear with: crm_resource --cleanup)" - else - echo " none" + echo " crm_mon returned no output — trying again without suppression:" + if $REACHABLE_1; then + n1 "crm_mon -1" || true + elif $REACHABLE_2; then + n2 "crm_mon -1" || true fi -else - echo " unavailable" fi # ── XFS mount ─────────────────────────────────────────────────────────────