Archived
feat(ha): add failover.sh and health.sh operator scripts
Check NixOS configurations / eval-hosts (push) Successful in 10m23s
Check NixOS configurations / eval-hosts (push) Successful in 10m23s
failover.sh: graceful active→standby resource migration with pre-checks (DRBD UpToDate/UpToDate, quorum), spinner wait for XFS mount on target, rollback on timeout, --to/--force/--dry-run/--timeout flags. health.sh: read-only status panel — node reachability, quorum, DRBD role/dstate/cs per node, Pacemaker resources, failure history, XFS mount usage, and service port checks (NFS 2049, iSCSI 3260) via VIP. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
Executable
+254
@@ -0,0 +1,254 @@
|
||||
#!/usr/bin/env bash
|
||||
# failover.sh — graceful HA cluster failover
|
||||
#
|
||||
# Detects which node is active and moves all resources to the other node by
|
||||
# putting the active node into Pacemaker standby. Waits for the XFS mount to
|
||||
# appear on the target before returning.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/ha/failover.sh [--to node1|node2] [--force] [--timeout <s>] [--dry-run]
|
||||
#
|
||||
# --to node1|node2 target node (default: the node that is NOT currently active)
|
||||
# --force skip the interactive confirmation prompt
|
||||
# --timeout <s> seconds to wait for resources to move (default: 120)
|
||||
# --dry-run show what would be done without changing anything
|
||||
set -euo pipefail
|
||||
|
||||
# ── Configuration ─────────────────────────────────────────────────────────
|
||||
NODE1="${NODE1:-ha-server-1}"
|
||||
NODE2="${NODE2:-ha-server-2}"
|
||||
NODE1_IP="${NODE1_IP:-192.168.2.228}"
|
||||
NODE2_IP="${NODE2_IP:-192.168.2.227}"
|
||||
VIP="${VIP:-192.168.2.229}"
|
||||
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"
|
||||
HA_USER="${HA_USER:-nixos}"
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
SSH_OPTS="-i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5"
|
||||
|
||||
n1() { ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
|
||||
n2() { ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
|
||||
|
||||
# ── Argument parsing ───────────────────────────────────────────────────────
|
||||
TARGET_NODE=""
|
||||
FORCE=false
|
||||
DRY_RUN=false
|
||||
TIMEOUT=120
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--to)
|
||||
shift
|
||||
case "${1:-}" in
|
||||
node1|ha-server-1) TARGET_NODE="$NODE1" ;;
|
||||
node2|ha-server-2) TARGET_NODE="$NODE2" ;;
|
||||
*) echo "ERROR: --to must be node1, node2, ha-server-1, or ha-server-2"; exit 1 ;;
|
||||
esac
|
||||
;;
|
||||
--force) FORCE=true ;;
|
||||
--dry-run) DRY_RUN=true ;;
|
||||
--timeout) shift; TIMEOUT="${1:?--timeout requires a value}" ;;
|
||||
*) echo "Unknown argument: $1"; echo "Usage: $0 [--to node1|node2] [--force] [--timeout <s>] [--dry-run]"; exit 1 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
DRY_PREFIX=""
|
||||
$DRY_RUN && DRY_PREFIX="[dry-run] "
|
||||
|
||||
echo "════════════════════════════════════════════════════"
|
||||
echo " HA Cluster Failover — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
$DRY_RUN && echo " MODE: dry-run — no changes will be made"
|
||||
echo "════════════════════════════════════════════════════"
|
||||
|
||||
# ── Detect active node ─────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "Detecting active node..."
|
||||
|
||||
CRM_OUT=""
|
||||
if ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then
|
||||
CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
|
||||
fi
|
||||
if [[ -z "$CRM_OUT" ]]; then
|
||||
if ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then
|
||||
CRM_OUT=$(n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
|
||||
fi
|
||||
fi
|
||||
|
||||
ACTIVE_NODE=$(echo "$CRM_OUT" | grep -E '^\s*(Promoted|Masters):' | \
|
||||
grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
|
||||
|
||||
if [[ -z "$ACTIVE_NODE" ]]; then
|
||||
echo ""
|
||||
echo "ERROR: could not determine active node from crm_mon."
|
||||
echo " Is Pacemaker still settling? Try running scripts/ha/health.sh first."
|
||||
echo " If Pacemaker is down on both nodes, manual recovery is required."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
|
||||
ACTIVE_IP="$NODE1_IP"
|
||||
STANDBY_NODE="$NODE2"
|
||||
STANDBY_IP="$NODE2_IP"
|
||||
na() { n1 "$@"; }
|
||||
ns() { n2 "$@"; }
|
||||
else
|
||||
ACTIVE_IP="$NODE2_IP"
|
||||
STANDBY_NODE="$NODE1"
|
||||
STANDBY_IP="$NODE1_IP"
|
||||
na() { n2 "$@"; }
|
||||
ns() { n1 "$@"; }
|
||||
fi
|
||||
|
||||
echo " Active: $ACTIVE_NODE ($ACTIVE_IP)"
|
||||
echo " Standby: $STANDBY_NODE ($STANDBY_IP)"
|
||||
|
||||
# ── Validate target ────────────────────────────────────────────────────────
|
||||
if [[ -n "$TARGET_NODE" ]]; then
|
||||
if [[ "$TARGET_NODE" == "$ACTIVE_NODE" ]]; then
|
||||
echo ""
|
||||
echo "ERROR: $TARGET_NODE is already the active node — nothing to do."
|
||||
exit 1
|
||||
fi
|
||||
echo " Target: $TARGET_NODE (as requested)"
|
||||
else
|
||||
echo " Target: $STANDBY_NODE (auto — the other node)"
|
||||
fi
|
||||
|
||||
# ── Pre-checks ─────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "Pre-checks..."
|
||||
|
||||
DRBD_DSTATE=$(na "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
|
||||
if ! echo "$DRBD_DSTATE" | grep -q "UpToDate/UpToDate"; then
|
||||
echo ""
|
||||
echo " WARNING: DRBD dstate is '$DRBD_DSTATE' (not UpToDate/UpToDate)."
|
||||
echo " Failing over with a partially-synced disk risks split-brain."
|
||||
if ! $FORCE; then
|
||||
echo " Use --force to proceed anyway (not recommended)."
|
||||
exit 1
|
||||
fi
|
||||
echo " --force specified — proceeding despite non-ideal DRBD state."
|
||||
else
|
||||
echo " DRBD dstate: $DRBD_DSTATE — OK"
|
||||
fi
|
||||
|
||||
QUORUM_OK=$(na "corosync-quorumtool -s 2>/dev/null | grep -c 'Quorate:.*Yes'" 2>/dev/null || echo "0")
|
||||
if [[ "$QUORUM_OK" -lt 1 ]]; then
|
||||
echo " ERROR: cluster does not have quorum — failover would be unsafe."
|
||||
exit 1
|
||||
fi
|
||||
echo " Quorum: OK"
|
||||
|
||||
# ── Confirm ────────────────────────────────────────────────────────────────
|
||||
if ! $FORCE && ! $DRY_RUN; then
|
||||
echo ""
|
||||
echo " This will move all resources from $ACTIVE_NODE → $STANDBY_NODE."
|
||||
echo " VIP and services will be unreachable for ~10–30 seconds."
|
||||
printf " Proceed? [y/N] "
|
||||
read -r ANSWER
|
||||
[[ "${ANSWER,,}" == "y" || "${ANSWER,,}" == "yes" ]] || { echo "Aborted."; exit 0; }
|
||||
fi
|
||||
|
||||
# ── Capture active node's crm_node name ───────────────────────────────────
|
||||
# crm_node -n returns the node name as registered in Pacemaker (may differ
|
||||
# from hostname if Pacemaker was configured with explicit node names).
|
||||
ACTIVE_CRMD_NAME=$(na "crm_node -n 2>/dev/null" 2>/dev/null || echo "$ACTIVE_NODE")
|
||||
|
||||
# ── Perform failover ───────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "${DRY_PREFIX}Putting $ACTIVE_NODE into standby (resources will migrate to $STANDBY_NODE)..."
|
||||
if ! $DRY_RUN; then
|
||||
na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v on" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# ── Wait for resources to move ─────────────────────────────────────────────
|
||||
echo "${DRY_PREFIX}Waiting up to ${TIMEOUT}s for XFS to mount on $STANDBY_NODE..."
|
||||
MOVED=false
|
||||
SPIN_CHARS=('|' '/' '-' '\')
|
||||
SPIN_I=0
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo " [dry-run] would wait for mountpoint $XFS_MOUNT on $STANDBY_NODE"
|
||||
MOVED=true
|
||||
else
|
||||
for i in $(seq 1 "$TIMEOUT"); do
|
||||
if ns "mountpoint -q '${XFS_MOUNT}' 2>/dev/null" 2>/dev/null; then
|
||||
printf "\r%-80s\r" ""
|
||||
echo " Resources moved in ${i}s"
|
||||
MOVED=true
|
||||
break
|
||||
fi
|
||||
SPIN_I=$(( SPIN_I + 1 ))
|
||||
SC="${SPIN_CHARS[$((SPIN_I % 4))]}"
|
||||
printf "\r [%s] waiting... (%ds) " "$SC" "$i"
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
if ! $MOVED; then
|
||||
echo ""
|
||||
echo "ERROR: XFS did not mount on $STANDBY_NODE within ${TIMEOUT}s."
|
||||
echo ""
|
||||
echo " Current resource state:"
|
||||
na "crm_mon -1 2>/dev/null" 2>/dev/null | grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED' | sed 's/^/ /' || true
|
||||
echo ""
|
||||
echo " Clearing standby to restore $ACTIVE_NODE (undo the failover attempt)..."
|
||||
na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
|
||||
na "crm_resource --cleanup" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Clear failure history ──────────────────────────────────────────────────
|
||||
echo "${DRY_PREFIX}Clearing Pacemaker failure history..."
|
||||
if ! $DRY_RUN; then
|
||||
ns "crm_resource --cleanup 2>/dev/null" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# ── Re-enable original active node as standby ─────────────────────────────
|
||||
echo "${DRY_PREFIX}Re-enabling $ACTIVE_NODE (now standby — will not claim resources)..."
|
||||
if ! $DRY_RUN; then
|
||||
na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# ── Wait briefly for DRBD resync to begin ─────────────────────────────────
|
||||
if ! $DRY_RUN; then
|
||||
sleep 5
|
||||
fi
|
||||
|
||||
# ── Final state ────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "Failover complete. Final state:"
|
||||
echo ""
|
||||
|
||||
CRM_OUT_AFTER=""
|
||||
if ! $DRY_RUN; then
|
||||
CRM_OUT_AFTER=$(ns "crm_mon -1 2>/dev/null" 2>/dev/null || na "crm_mon -1 2>/dev/null" 2>/dev/null || true)
|
||||
else
|
||||
CRM_OUT_AFTER="$CRM_OUT"
|
||||
fi
|
||||
|
||||
NEW_ACTIVE=$(echo "$CRM_OUT_AFTER" | grep -E '^\s*(Promoted|Masters):' | \
|
||||
grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
|
||||
|
||||
if [[ -n "$NEW_ACTIVE" ]]; then
|
||||
if [[ "$NEW_ACTIVE" == "$ACTIVE_NODE" ]]; then
|
||||
echo " WARNING: $ACTIVE_NODE is still showing as active in crm_mon."
|
||||
echo " Pacemaker may still be settling — check again in a few seconds."
|
||||
else
|
||||
echo " Active: $NEW_ACTIVE"
|
||||
echo " Standby: $ACTIVE_NODE"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
RESOURCES_AFTER=$(echo "$CRM_OUT_AFTER" | awk '/Full List of Resources/,0' | tail -n +2 || true)
|
||||
[[ -z "$RESOURCES_AFTER" ]] && RESOURCES_AFTER=$(echo "$CRM_OUT_AFTER" | \
|
||||
grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED|Master|Slave' || true)
|
||||
[[ -n "$RESOURCES_AFTER" ]] && echo "$RESOURCES_AFTER" | sed 's/^/ /'
|
||||
|
||||
echo ""
|
||||
echo " (DRBD resync of $ACTIVE_NODE may take a moment; monitor with:"
|
||||
echo " ssh nixos@${ACTIVE_IP} 'sudo watch -n3 cat /proc/drbd')"
|
||||
echo ""
|
||||
echo "════════════════════════════════════════════════════"
|
||||
Executable
+198
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env bash
|
||||
# health.sh — HA cluster health snapshot (read-only, non-destructive)
|
||||
#
|
||||
# Prints a compact status panel across both nodes: SSH reachability, quorum,
|
||||
# DRBD state, Pacemaker resources, and service ports via the VIP.
|
||||
# Run from any host with SSH access to the HA nodes.
|
||||
set -euo pipefail
|
||||
|
||||
# ── Configuration ─────────────────────────────────────────────────────────
|
||||
NODE1="${NODE1:-ha-server-1}"
|
||||
NODE2="${NODE2:-ha-server-2}"
|
||||
NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip
|
||||
NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip
|
||||
VIP="${VIP:-192.168.2.229}" # vars.haServerVip
|
||||
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot
|
||||
HA_USER="${HA_USER:-nixos}"
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
SSH_OPTS="-i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5"
|
||||
|
||||
# Returns "OK" or "UNREACHABLE" and sets global REACHABLE_{1,2}.
|
||||
REACHABLE_1=false
|
||||
REACHABLE_2=false
|
||||
|
||||
n1() { ssh $SSH_OPTS "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
|
||||
n2() { ssh $SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
|
||||
|
||||
probe_node() {
|
||||
local ip=$1
|
||||
ssh $SSH_OPTS "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
|
||||
}
|
||||
|
||||
section() { echo ""; echo "── $* ──"; }
|
||||
|
||||
echo "════════════════════════════════════════════════════"
|
||||
echo " HA Cluster Health — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "════════════════════════════════════════════════════"
|
||||
|
||||
# ── Node reachability ──────────────────────────────────────────────────────
|
||||
section "Nodes"
|
||||
N1_STATUS=$(probe_node "$NODE1_IP")
|
||||
N2_STATUS=$(probe_node "$NODE2_IP")
|
||||
[[ "$N1_STATUS" == "ONLINE" ]] && REACHABLE_1=true
|
||||
[[ "$N2_STATUS" == "ONLINE" ]] && REACHABLE_2=true
|
||||
|
||||
if ! $REACHABLE_1 && ! $REACHABLE_2; then
|
||||
echo " ERROR: both nodes unreachable — cannot continue."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Detect active node ─────────────────────────────────────────────────────
|
||||
ACTIVE_NODE=""
|
||||
if $REACHABLE_1; then
|
||||
CRM_OUT=$(n1 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
|
||||
elif $REACHABLE_2; then
|
||||
CRM_OUT=$(n2 "crm_mon -1 2>/dev/null" 2>/dev/null || true)
|
||||
fi
|
||||
ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -E '^\s*(Promoted|Masters):' | \
|
||||
grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
|
||||
|
||||
STANDBY_NODE=""
|
||||
if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
|
||||
STANDBY_NODE="$NODE2"
|
||||
elif [[ "$ACTIVE_NODE" == "$NODE2" ]]; then
|
||||
STANDBY_NODE="$NODE1"
|
||||
fi
|
||||
|
||||
n1_tag=""; n2_tag=""
|
||||
[[ "$ACTIVE_NODE" == "$NODE1" ]] && n1_tag=" [ACTIVE]" || n1_tag=" [STANDBY]"
|
||||
[[ "$ACTIVE_NODE" == "$NODE2" ]] && n2_tag=" [ACTIVE]" || n2_tag=" [STANDBY]"
|
||||
[[ -z "$ACTIVE_NODE" ]] && { n1_tag=""; n2_tag=""; }
|
||||
|
||||
printf " %-14s [%s]%s\n" "$NODE1" "$N1_STATUS" "$n1_tag"
|
||||
printf " %-14s [%s]%s\n" "$NODE2" "$N2_STATUS" "$n2_tag"
|
||||
|
||||
if [[ -z "$ACTIVE_NODE" ]]; then
|
||||
echo ""
|
||||
echo " WARNING: could not determine active node from crm_mon."
|
||||
echo " Pacemaker may still be settling, or both nodes may be in standby."
|
||||
fi
|
||||
|
||||
# ── Quorum ─────────────────────────────────────────────────────────────────
|
||||
section "Quorum"
|
||||
if $REACHABLE_1; then
|
||||
QUORUM=$(n1 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
|
||||
elif $REACHABLE_2; then
|
||||
QUORUM=$(n2 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
|
||||
fi
|
||||
|
||||
if [[ -z "${QUORUM:-}" ]]; then
|
||||
echo " corosync-quorumtool: unavailable"
|
||||
else
|
||||
QUORATE=$(echo "$QUORUM" | grep "Quorate:" | awk '{print $2}' || echo "?")
|
||||
VOTES=$(echo "$QUORUM" | grep "Total votes:" | awk '{print $3}' || echo "?")
|
||||
NEEDED=$(echo "$QUORUM" | grep "Quorum votes:" | awk '{print $3}' || echo "?")
|
||||
echo " Quorate: $QUORATE Votes: $VOTES / Expected: $NEEDED"
|
||||
fi
|
||||
|
||||
# ── DRBD ───────────────────────────────────────────────────────────────────
|
||||
section "DRBD (ha-data)"
|
||||
|
||||
drbd_info_from() {
|
||||
local node=$1 run=$2
|
||||
local role dstate cs pct
|
||||
role=$($run "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
|
||||
dstate=$($run "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
|
||||
cs=$($run "grep -oE 'cs:[A-Za-z]+' /proc/drbd 2>/dev/null | head -1 | sed 's/cs://'" 2>/dev/null || echo "unknown")
|
||||
pct=$($run "grep -oE \"sync'ed:[[:space:]]+[0-9.]+\" /proc/drbd 2>/dev/null | grep -oE '[0-9.]+$' | head -1" 2>/dev/null || echo "")
|
||||
printf " %-14s role: %-22s dstate: %-25s cs: %s" \
|
||||
"$node" "${role:-unknown}" "${dstate:-unknown}" "${cs:-unknown}"
|
||||
[[ -n "$pct" ]] && printf " syncing: %s%%" "$pct"
|
||||
echo ""
|
||||
}
|
||||
|
||||
$REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo " $NODE1 [OFFLINE]"
|
||||
$REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo " $NODE2 [OFFLINE]"
|
||||
|
||||
# ── Pacemaker resources ────────────────────────────────────────────────────
|
||||
section "Pacemaker Resources"
|
||||
if [[ -n "${CRM_OUT:-}" ]]; then
|
||||
# Print the resource lines: everything from "Full List of Resources" onward,
|
||||
# or fall back to printing any line with a resource state keyword.
|
||||
RESOURCES=$(echo "$CRM_OUT" | awk '/Full List of Resources/,0' | tail -n +2 || true)
|
||||
if [[ -z "$RESOURCES" ]]; then
|
||||
# Older crm_mon: print lines containing Started/Stopped/Promoted/Unpromoted/FAILED
|
||||
RESOURCES=$(echo "$CRM_OUT" | grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED|Master|Slave' || true)
|
||||
fi
|
||||
if [[ -n "$RESOURCES" ]]; then
|
||||
echo "$RESOURCES" | sed 's/^/ /'
|
||||
else
|
||||
echo " (no resource output parsed — run: crm_mon -1)"
|
||||
fi
|
||||
else
|
||||
echo " crm_mon output unavailable"
|
||||
fi
|
||||
|
||||
# ── Failure history ────────────────────────────────────────────────────────
|
||||
section "Failure History"
|
||||
if [[ -n "${CRM_OUT:-}" ]]; then
|
||||
FAILURES=$(echo "$CRM_OUT" | awk '/Failed Resource Actions/,/^$/' | tail -n +2 | grep -v '^$' || true)
|
||||
if [[ -n "$FAILURES" ]]; then
|
||||
echo "$FAILURES" | sed 's/^/ /'
|
||||
echo " (clear with: crm_resource --cleanup)"
|
||||
else
|
||||
echo " none"
|
||||
fi
|
||||
else
|
||||
echo " unavailable"
|
||||
fi
|
||||
|
||||
# ── XFS mount ─────────────────────────────────────────────────────────────
|
||||
section "XFS Mount ($XFS_MOUNT)"
|
||||
check_mount() {
|
||||
local node=$1 run=$2
|
||||
local status
|
||||
if $run "mountpoint -q '$XFS_MOUNT' 2>/dev/null" 2>/dev/null; then
|
||||
local usage
|
||||
usage=$($run "df -h '$XFS_MOUNT' 2>/dev/null | tail -1 | awk '{print \$3\"/\"\$2\" used (\"\$5\")\";}'" 2>/dev/null || echo "")
|
||||
status="mounted"
|
||||
[[ -n "$usage" ]] && status="mounted $usage"
|
||||
else
|
||||
status="not mounted"
|
||||
fi
|
||||
printf " %-14s %s\n" "$node" "$status"
|
||||
}
|
||||
|
||||
$REACHABLE_1 && check_mount "$NODE1" n1 || echo " $NODE1 [OFFLINE]"
|
||||
$REACHABLE_2 && check_mount "$NODE2" n2 || echo " $NODE2 [OFFLINE]"
|
||||
|
||||
# ── Service ports via VIP ──────────────────────────────────────────────────
|
||||
section "Services via VIP ($VIP)"
|
||||
|
||||
check_port() {
|
||||
local name=$1 port=$2
|
||||
if bash -c "echo >/dev/tcp/${VIP}/${port}" 2>/dev/null; then
|
||||
printf " %-10s port %-5s OK\n" "$name" "$port"
|
||||
else
|
||||
printf " %-10s port %-5s UNREACHABLE\n" "$name" "$port"
|
||||
fi
|
||||
}
|
||||
|
||||
if ping -c1 -W2 "$VIP" >/dev/null 2>&1; then
|
||||
echo " Ping OK"
|
||||
else
|
||||
echo " Ping UNREACHABLE"
|
||||
fi
|
||||
check_port "NFS" 2049
|
||||
check_port "iSCSI" 3260
|
||||
|
||||
echo ""
|
||||
echo "════════════════════════════════════════════════════"
|
||||
if [[ -n "$ACTIVE_NODE" ]]; then
|
||||
echo " Active: $ACTIVE_NODE Standby: $STANDBY_NODE"
|
||||
else
|
||||
echo " Active: unknown (Pacemaker not settled)"
|
||||
fi
|
||||
echo "════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user