Archived
beatzaplentyandClaude Sonnet 4.6
e9a4913069
Check NixOS configurations / eval-hosts (push) Successful in 10m43s
Storage VIP (192.168.20.229) is on vmbr2 — an internal-only bridge with no physical uplink, unreachable from the LAN. Test it via SSH to the active node (which has VLAN 20 connectivity) instead of directly. LAN VIP (192.168.2.229) continues to be tested directly from wherever the script runs. Also adds VIP_LAN variable to config section. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J8djTWdXVzXZc99iujU6T2
202 lines
8.9 KiB
Bash
Executable File
202 lines
8.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# health.sh — HA cluster health snapshot (read-only, non-destructive)
|
|
#
|
|
# Prints a compact status panel across both nodes: SSH reachability, quorum,
|
|
# DRBD state, Pacemaker resources, and service ports via the VIP.
|
|
# Run from any host with SSH access to the HA nodes.
|
|
set -euo pipefail
|
|
|
|
# ── Configuration ─────────────────────────────────────────────────────────
|
|
NODE1="${NODE1:-ha-server-1}"
|
|
NODE2="${NODE2:-ha-server-2}"
|
|
NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip
|
|
NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip
|
|
VIP="${VIP:-192.168.20.229}" # vars.haServerVip (storage-client, VLAN 20 — internal only)
|
|
VIP_LAN="${VIP_LAN:-192.168.2.229}" # vars.haServerLanVip (LAN, VLAN 2 — reachable from workstation)
|
|
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot
|
|
HA_USER="${HA_USER:-nixos}"
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
|
|
REACHABLE_1=false
|
|
REACHABLE_2=false
|
|
|
|
n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
|
|
n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
|
|
|
|
probe_node() {
|
|
local ip=$1
|
|
ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
|
|
}
|
|
|
|
section() { echo ""; echo "── $* ──"; }
|
|
|
|
echo "════════════════════════════════════════════════════"
|
|
echo " HA Cluster Health — $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo "════════════════════════════════════════════════════"
|
|
|
|
# ── Node reachability ──────────────────────────────────────────────────────
|
|
section "Nodes"
|
|
N1_STATUS=$(probe_node "$NODE1_IP")
|
|
N2_STATUS=$(probe_node "$NODE2_IP")
|
|
[[ "$N1_STATUS" == "ONLINE" ]] && REACHABLE_1=true
|
|
[[ "$N2_STATUS" == "ONLINE" ]] && REACHABLE_2=true
|
|
|
|
if ! $REACHABLE_1 && ! $REACHABLE_2; then
|
|
echo " ERROR: both nodes unreachable — cannot continue."
|
|
exit 1
|
|
fi
|
|
|
|
# ── Detect active node ─────────────────────────────────────────────────────
|
|
# crm_mon 2.x formats the Promoted line as " * Promoted: [ node ]" — the
|
|
# bullet * means ^\s*(Promoted|Masters): never matches. Filter out Unpromoted
|
|
# first, then match anywhere on the line.
|
|
ACTIVE_NODE=""
|
|
CRM_OUT=""
|
|
if $REACHABLE_1; then
|
|
CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
|
|
elif $REACHABLE_2; then
|
|
CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
|
|
fi
|
|
ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -v 'Unpromoted\|Unmanaged' | \
|
|
grep -E '(Promoted|Masters):' | \
|
|
grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
|
|
|
|
STANDBY_NODE=""
|
|
if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
|
|
STANDBY_NODE="$NODE2"
|
|
elif [[ "$ACTIVE_NODE" == "$NODE2" ]]; then
|
|
STANDBY_NODE="$NODE1"
|
|
fi
|
|
|
|
n1_tag=""; n2_tag=""
|
|
[[ "$ACTIVE_NODE" == "$NODE1" ]] && n1_tag=" [ACTIVE]" || n1_tag=" [STANDBY]"
|
|
[[ "$ACTIVE_NODE" == "$NODE2" ]] && n2_tag=" [ACTIVE]" || n2_tag=" [STANDBY]"
|
|
[[ -z "$ACTIVE_NODE" ]] && { n1_tag=""; n2_tag=""; }
|
|
|
|
printf " %-14s [%s]%s\n" "$NODE1" "$N1_STATUS" "$n1_tag"
|
|
printf " %-14s [%s]%s\n" "$NODE2" "$N2_STATUS" "$n2_tag"
|
|
|
|
if [[ -z "$ACTIVE_NODE" ]]; then
|
|
echo ""
|
|
echo " WARNING: could not determine active node from crm_mon."
|
|
echo " Pacemaker may still be settling, or both nodes may be in standby."
|
|
fi
|
|
|
|
# ── Quorum ─────────────────────────────────────────────────────────────────
|
|
section "Quorum"
|
|
if $REACHABLE_1; then
|
|
QUORUM=$(n1 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
|
|
elif $REACHABLE_2; then
|
|
QUORUM=$(n2 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
|
|
fi
|
|
|
|
if [[ -z "${QUORUM:-}" ]]; then
|
|
echo " corosync-quorumtool: unavailable"
|
|
else
|
|
QUORATE=$(echo "$QUORUM" | grep "Quorate:" | awk '{print $2}' || echo "?")
|
|
VOTES=$(echo "$QUORUM" | grep "Total votes:" | awk '{print $3}' || echo "?")
|
|
NEEDED=$(echo "$QUORUM" | grep "Quorum votes:" | awk '{print $3}' || echo "?")
|
|
echo " Quorate: $QUORATE Votes: $VOTES / Expected: $NEEDED"
|
|
fi
|
|
|
|
# ── DRBD ───────────────────────────────────────────────────────────────────
|
|
section "DRBD (ha-data)"
|
|
|
|
drbd_info_from() {
|
|
local node=$1 run=$2
|
|
local role dstate cs pct
|
|
role=$($run "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
|
|
dstate=$($run "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
|
|
cs=$($run "grep -oE 'cs:[A-Za-z]+' /proc/drbd 2>/dev/null | head -1 | sed 's/cs://'" 2>/dev/null || echo "unknown")
|
|
pct=$($run "grep -oE \"sync'ed:[[:space:]]+[0-9.]+\" /proc/drbd 2>/dev/null | grep -oE '[0-9.]+$' | head -1" 2>/dev/null || echo "")
|
|
printf " %-14s role: %-22s dstate: %-25s cs: %s" \
|
|
"$node" "${role:-unknown}" "${dstate:-unknown}" "${cs:-unknown}"
|
|
[[ -n "$pct" ]] && printf " syncing: %s%%" "$pct"
|
|
echo ""
|
|
}
|
|
|
|
$REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo " $NODE1 [OFFLINE]"
|
|
$REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo " $NODE2 [OFFLINE]"
|
|
|
|
# ── Pacemaker (full crm_mon output) ───────────────────────────────────────
|
|
section "Pacemaker"
|
|
if [[ -n "${CRM_OUT:-}" ]]; then
|
|
echo "$CRM_OUT" | sed 's/^/ /'
|
|
else
|
|
echo " crm_mon returned no output — trying again without suppression:"
|
|
if $REACHABLE_1; then
|
|
n1 "crm_mon -1" || true
|
|
elif $REACHABLE_2; then
|
|
n2 "crm_mon -1" || true
|
|
fi
|
|
fi
|
|
|
|
# ── XFS mount ─────────────────────────────────────────────────────────────
|
|
section "XFS Mount ($XFS_MOUNT)"
|
|
check_mount() {
|
|
local node=$1 run=$2
|
|
local status
|
|
if $run "mountpoint -q '$XFS_MOUNT' 2>/dev/null" 2>/dev/null; then
|
|
local usage
|
|
usage=$($run "df -h '$XFS_MOUNT' 2>/dev/null | tail -1 | awk '{print \$3\"/\"\$2\" used (\"\$5\")\";}'" 2>/dev/null || echo "")
|
|
status="mounted"
|
|
[[ -n "$usage" ]] && status="mounted $usage"
|
|
else
|
|
status="not mounted"
|
|
fi
|
|
printf " %-14s %s\n" "$node" "$status"
|
|
}
|
|
|
|
$REACHABLE_1 && check_mount "$NODE1" n1 || echo " $NODE1 [OFFLINE]"
|
|
$REACHABLE_2 && check_mount "$NODE2" n2 || echo " $NODE2 [OFFLINE]"
|
|
|
|
# ── LAN VIP (NFS) — reachable from workstation ────────────────────────────────
|
|
section "LAN VIP ($VIP_LAN) — NFS"
|
|
if ping -c1 -W2 "$VIP_LAN" >/dev/null 2>&1; then
|
|
echo " Ping OK"
|
|
else
|
|
echo " Ping UNREACHABLE"
|
|
fi
|
|
if bash -c "echo >/dev/tcp/${VIP_LAN}/2049" 2>/dev/null; then
|
|
printf " %-10s port %-5s OK\n" "NFS" "2049"
|
|
else
|
|
printf " %-10s port %-5s UNREACHABLE\n" "NFS" "2049"
|
|
fi
|
|
|
|
# ── Storage VIP (iSCSI) — VLAN 20 internal bridge, tested via active node ─────
|
|
section "Storage VIP ($VIP) — iSCSI (via ${ACTIVE_NODE:-unknown})"
|
|
|
|
run_active_raw() {
|
|
local active_ip=""
|
|
[[ "$ACTIVE_NODE" == "$NODE1" ]] && active_ip="$NODE1_IP"
|
|
[[ "$ACTIVE_NODE" == "$NODE2" ]] && active_ip="$NODE2_IP"
|
|
[[ -z "$active_ip" ]] && return 1
|
|
ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
|
|
"${HA_USER}@${active_ip}" "$@" 2>/dev/null
|
|
}
|
|
|
|
if [[ -z "$ACTIVE_NODE" ]]; then
|
|
echo " Cannot determine active node — skipping"
|
|
else
|
|
if run_active_raw "ping -c1 -W2 '$VIP' >/dev/null 2>&1"; then
|
|
echo " Ping OK"
|
|
else
|
|
echo " Ping UNREACHABLE"
|
|
fi
|
|
if run_active_raw "bash -c 'echo >/dev/tcp/${VIP}/3260' 2>/dev/null"; then
|
|
printf " %-10s port %-5s OK\n" "iSCSI" "3260"
|
|
else
|
|
printf " %-10s port %-5s UNREACHABLE\n" "iSCSI" "3260"
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
echo "════════════════════════════════════════════════════"
|
|
if [[ -n "$ACTIVE_NODE" ]]; then
|
|
echo " Active: $ACTIVE_NODE Standby: $STANDBY_NODE"
|
|
else
|
|
echo " Active: unknown (Pacemaker not settled)"
|
|
fi
|
|
echo "════════════════════════════════════════════════════"
|
|
echo ""
|