Archived
beatzaplentyandClaude Sonnet 4.6
71a6c4738c
Check NixOS configurations / eval-hosts (push) Successful in 10m23s
crm_mon 2.x formats the Promoted line as " * Promoted: [ node ]" — the asterisk bullet means the previous grep -E '^\s*(Promoted|Masters):' never matched, so active-node detection silently returned empty in health.sh, failover.sh, and acceptance-tests.sh. Fix: pipe through grep -v Unpromoted first, then grep -E '(Promoted|Masters):' without anchoring to start-of-line. Also: remove the SSH_OPTS=-i ~/.ssh/... variable pattern in health.sh and failover.sh; tilde is not expanded inside double-quoted strings, so $SSH_OPTS was passing a literal ~ to SSH. Inline the key path in each function definition instead (same as acceptance-tests.sh already did). Also: drop the 2>/dev/null embedded in the crm_mon argument string — the outer 2>/dev/null on the n1/n2 call already suppresses SSH stderr; the embedded one was harmless but noisy to reason about. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
181 lines
7.9 KiB
Bash
Executable File
181 lines
7.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# health.sh — HA cluster health snapshot (read-only, non-destructive)
|
|
#
|
|
# Prints a compact status panel across both nodes: SSH reachability, quorum,
|
|
# DRBD state, Pacemaker resources, and service ports via the VIP.
|
|
# Run from any host with SSH access to the HA nodes.
|
|
set -euo pipefail
|
|
|
|
# ── Configuration ─────────────────────────────────────────────────────────
|
|
NODE1="${NODE1:-ha-server-1}"
|
|
NODE2="${NODE2:-ha-server-2}"
|
|
NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip
|
|
NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip
|
|
VIP="${VIP:-192.168.2.229}" # vars.haServerVip
|
|
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot
|
|
HA_USER="${HA_USER:-nixos}"
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
|
|
REACHABLE_1=false
|
|
REACHABLE_2=false
|
|
|
|
n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
|
|
n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
|
|
|
|
probe_node() {
|
|
local ip=$1
|
|
ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
|
|
}
|
|
|
|
section() { echo ""; echo "── $* ──"; }
|
|
|
|
echo "════════════════════════════════════════════════════"
|
|
echo " HA Cluster Health — $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo "════════════════════════════════════════════════════"
|
|
|
|
# ── Node reachability ──────────────────────────────────────────────────────
|
|
section "Nodes"
|
|
N1_STATUS=$(probe_node "$NODE1_IP")
|
|
N2_STATUS=$(probe_node "$NODE2_IP")
|
|
[[ "$N1_STATUS" == "ONLINE" ]] && REACHABLE_1=true
|
|
[[ "$N2_STATUS" == "ONLINE" ]] && REACHABLE_2=true
|
|
|
|
if ! $REACHABLE_1 && ! $REACHABLE_2; then
|
|
echo " ERROR: both nodes unreachable — cannot continue."
|
|
exit 1
|
|
fi
|
|
|
|
# ── Detect active node ─────────────────────────────────────────────────────
|
|
# crm_mon 2.x formats the Promoted line as " * Promoted: [ node ]" — the
|
|
# bullet * means ^\s*(Promoted|Masters): never matches. Filter out Unpromoted
|
|
# first, then match anywhere on the line.
|
|
ACTIVE_NODE=""
|
|
CRM_OUT=""
|
|
if $REACHABLE_1; then
|
|
CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
|
|
elif $REACHABLE_2; then
|
|
CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
|
|
fi
|
|
ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -v 'Unpromoted\|Unmanaged' | \
|
|
grep -E '(Promoted|Masters):' | \
|
|
grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
|
|
|
|
STANDBY_NODE=""
|
|
if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
|
|
STANDBY_NODE="$NODE2"
|
|
elif [[ "$ACTIVE_NODE" == "$NODE2" ]]; then
|
|
STANDBY_NODE="$NODE1"
|
|
fi
|
|
|
|
n1_tag=""; n2_tag=""
|
|
[[ "$ACTIVE_NODE" == "$NODE1" ]] && n1_tag=" [ACTIVE]" || n1_tag=" [STANDBY]"
|
|
[[ "$ACTIVE_NODE" == "$NODE2" ]] && n2_tag=" [ACTIVE]" || n2_tag=" [STANDBY]"
|
|
[[ -z "$ACTIVE_NODE" ]] && { n1_tag=""; n2_tag=""; }
|
|
|
|
printf " %-14s [%s]%s\n" "$NODE1" "$N1_STATUS" "$n1_tag"
|
|
printf " %-14s [%s]%s\n" "$NODE2" "$N2_STATUS" "$n2_tag"
|
|
|
|
if [[ -z "$ACTIVE_NODE" ]]; then
|
|
echo ""
|
|
echo " WARNING: could not determine active node from crm_mon."
|
|
echo " Pacemaker may still be settling, or both nodes may be in standby."
|
|
fi
|
|
|
|
# ── Quorum ─────────────────────────────────────────────────────────────────
|
|
section "Quorum"
|
|
if $REACHABLE_1; then
|
|
QUORUM=$(n1 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
|
|
elif $REACHABLE_2; then
|
|
QUORUM=$(n2 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
|
|
fi
|
|
|
|
if [[ -z "${QUORUM:-}" ]]; then
|
|
echo " corosync-quorumtool: unavailable"
|
|
else
|
|
QUORATE=$(echo "$QUORUM" | grep "Quorate:" | awk '{print $2}' || echo "?")
|
|
VOTES=$(echo "$QUORUM" | grep "Total votes:" | awk '{print $3}' || echo "?")
|
|
NEEDED=$(echo "$QUORUM" | grep "Quorum votes:" | awk '{print $3}' || echo "?")
|
|
echo " Quorate: $QUORATE Votes: $VOTES / Expected: $NEEDED"
|
|
fi
|
|
|
|
# ── DRBD ───────────────────────────────────────────────────────────────────
|
|
section "DRBD (ha-data)"
|
|
|
|
drbd_info_from() {
|
|
local node=$1 run=$2
|
|
local role dstate cs pct
|
|
role=$($run "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
|
|
dstate=$($run "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
|
|
cs=$($run "grep -oE 'cs:[A-Za-z]+' /proc/drbd 2>/dev/null | head -1 | sed 's/cs://'" 2>/dev/null || echo "unknown")
|
|
pct=$($run "grep -oE \"sync'ed:[[:space:]]+[0-9.]+\" /proc/drbd 2>/dev/null | grep -oE '[0-9.]+$' | head -1" 2>/dev/null || echo "")
|
|
printf " %-14s role: %-22s dstate: %-25s cs: %s" \
|
|
"$node" "${role:-unknown}" "${dstate:-unknown}" "${cs:-unknown}"
|
|
[[ -n "$pct" ]] && printf " syncing: %s%%" "$pct"
|
|
echo ""
|
|
}
|
|
|
|
$REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo " $NODE1 [OFFLINE]"
|
|
$REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo " $NODE2 [OFFLINE]"
|
|
|
|
# ── Pacemaker (full crm_mon output) ───────────────────────────────────────
|
|
section "Pacemaker"
|
|
if [[ -n "${CRM_OUT:-}" ]]; then
|
|
echo "$CRM_OUT" | sed 's/^/ /'
|
|
else
|
|
echo " crm_mon returned no output — trying again without suppression:"
|
|
if $REACHABLE_1; then
|
|
n1 "crm_mon -1" || true
|
|
elif $REACHABLE_2; then
|
|
n2 "crm_mon -1" || true
|
|
fi
|
|
fi
|
|
|
|
# ── XFS mount ─────────────────────────────────────────────────────────────
|
|
section "XFS Mount ($XFS_MOUNT)"
|
|
check_mount() {
|
|
local node=$1 run=$2
|
|
local status
|
|
if $run "mountpoint -q '$XFS_MOUNT' 2>/dev/null" 2>/dev/null; then
|
|
local usage
|
|
usage=$($run "df -h '$XFS_MOUNT' 2>/dev/null | tail -1 | awk '{print \$3\"/\"\$2\" used (\"\$5\")\";}'" 2>/dev/null || echo "")
|
|
status="mounted"
|
|
[[ -n "$usage" ]] && status="mounted $usage"
|
|
else
|
|
status="not mounted"
|
|
fi
|
|
printf " %-14s %s\n" "$node" "$status"
|
|
}
|
|
|
|
$REACHABLE_1 && check_mount "$NODE1" n1 || echo " $NODE1 [OFFLINE]"
|
|
$REACHABLE_2 && check_mount "$NODE2" n2 || echo " $NODE2 [OFFLINE]"
|
|
|
|
# ── Service ports via VIP ──────────────────────────────────────────────────
|
|
section "Services via VIP ($VIP)"
|
|
|
|
check_port() {
|
|
local name=$1 port=$2
|
|
if bash -c "echo >/dev/tcp/${VIP}/${port}" 2>/dev/null; then
|
|
printf " %-10s port %-5s OK\n" "$name" "$port"
|
|
else
|
|
printf " %-10s port %-5s UNREACHABLE\n" "$name" "$port"
|
|
fi
|
|
}
|
|
|
|
if ping -c1 -W2 "$VIP" >/dev/null 2>&1; then
|
|
echo " Ping OK"
|
|
else
|
|
echo " Ping UNREACHABLE"
|
|
fi
|
|
check_port "NFS" 2049
|
|
check_port "iSCSI" 3260
|
|
|
|
echo ""
|
|
echo "════════════════════════════════════════════════════"
|
|
if [[ -n "$ACTIVE_NODE" ]]; then
|
|
echo " Active: $ACTIVE_NODE Standby: $STANDBY_NODE"
|
|
else
|
|
echo " Active: unknown (Pacemaker not settled)"
|
|
fi
|
|
echo "════════════════════════════════════════════════════"
|
|
echo ""
|