#!/usr/bin/env bash
# health.sh — HA cluster health snapshot (read-only, non-destructive)
#
# Prints a compact status panel across both nodes: SSH reachability, quorum,
# DRBD state, Pacemaker resources, and service ports via the VIP.
# Run from any host with SSH access to the HA nodes.
set -euo pipefail

# ── Configuration ─────────────────────────────────────────────────────────
NODE1="${NODE1:-ha-server-1}"
NODE2="${NODE2:-ha-server-2}"
NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
HA_USER="${HA_USER:-nixos}"
# ──────────────────────────────────────────────────────────────────────────

REACHABLE_1=false
REACHABLE_2=false

n1()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
n2()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }

probe_node() {
  local ip=$1
  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
}

section() { echo ""; echo "── $* ──"; }

echo "════════════════════════════════════════════════════"
echo " HA Cluster Health — $(date '+%Y-%m-%d %H:%M:%S')"
echo "════════════════════════════════════════════════════"

# ── Node reachability ──────────────────────────────────────────────────────
section "Nodes"
N1_STATUS=$(probe_node "$NODE1_IP")
N2_STATUS=$(probe_node "$NODE2_IP")
[[ "$N1_STATUS" == "ONLINE" ]] && REACHABLE_1=true
[[ "$N2_STATUS" == "ONLINE" ]] && REACHABLE_2=true

if ! $REACHABLE_1 && ! $REACHABLE_2; then
  echo "  ERROR: both nodes unreachable — cannot continue."
  exit 1
fi

# ── Detect active node ─────────────────────────────────────────────────────
# crm_mon 2.x formats the Promoted line as "    * Promoted: [ node ]" — the
# bullet * means ^\s*(Promoted|Masters): never matches.  Filter out Unpromoted
# first, then match anywhere on the line.
ACTIVE_NODE=""
CRM_OUT=""
if $REACHABLE_1; then
  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
elif $REACHABLE_2; then
  CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
fi
ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -v 'Unpromoted\|Unmanaged' | \
  grep -E '(Promoted|Masters):' | \
  grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)

STANDBY_NODE=""
if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
  STANDBY_NODE="$NODE2"
elif [[ "$ACTIVE_NODE" == "$NODE2" ]]; then
  STANDBY_NODE="$NODE1"
fi

n1_tag=""; n2_tag=""
[[ "$ACTIVE_NODE" == "$NODE1" ]] && n1_tag=" [ACTIVE]"  || n1_tag=" [STANDBY]"
[[ "$ACTIVE_NODE" == "$NODE2" ]] && n2_tag=" [ACTIVE]"  || n2_tag=" [STANDBY]"
[[ -z "$ACTIVE_NODE" ]] && { n1_tag=""; n2_tag=""; }

printf "  %-14s  [%s]%s\n" "$NODE1" "$N1_STATUS" "$n1_tag"
printf "  %-14s  [%s]%s\n" "$NODE2" "$N2_STATUS" "$n2_tag"

if [[ -z "$ACTIVE_NODE" ]]; then
  echo ""
  echo "  WARNING: could not determine active node from crm_mon."
  echo "  Pacemaker may still be settling, or both nodes may be in standby."
fi

# ── Quorum ─────────────────────────────────────────────────────────────────
section "Quorum"
if $REACHABLE_1; then
  QUORUM=$(n1 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
elif $REACHABLE_2; then
  QUORUM=$(n2 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
fi

if [[ -z "${QUORUM:-}" ]]; then
  echo "  corosync-quorumtool: unavailable"
else
  QUORATE=$(echo "$QUORUM" | grep "Quorate:" | awk '{print $2}' || echo "?")
  VOTES=$(echo "$QUORUM" | grep "Total votes:" | awk '{print $3}' || echo "?")
  NEEDED=$(echo "$QUORUM" | grep "Quorum votes:" | awk '{print $3}' || echo "?")
  echo "  Quorate: $QUORATE   Votes: $VOTES / Expected: $NEEDED"
fi

# ── DRBD ───────────────────────────────────────────────────────────────────
section "DRBD (ha-data)"

drbd_info_from() {
  local node=$1 run=$2
  local role dstate cs pct
  role=$($run "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
  dstate=$($run "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
  cs=$($run "grep -oE 'cs:[A-Za-z]+' /proc/drbd 2>/dev/null | head -1 | sed 's/cs://'" 2>/dev/null || echo "unknown")
  pct=$($run "grep -oE \"sync'ed:[[:space:]]+[0-9.]+\" /proc/drbd 2>/dev/null | grep -oE '[0-9.]+$' | head -1" 2>/dev/null || echo "")
  printf "  %-14s  role: %-22s  dstate: %-25s  cs: %s" \
    "$node" "${role:-unknown}" "${dstate:-unknown}" "${cs:-unknown}"
  [[ -n "$pct" ]] && printf "  syncing: %s%%" "$pct"
  echo ""
}

$REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo "  $NODE1  [OFFLINE]"
$REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo "  $NODE2  [OFFLINE]"

# ── Pacemaker (full crm_mon output) ───────────────────────────────────────
section "Pacemaker"
if [[ -n "${CRM_OUT:-}" ]]; then
  echo "$CRM_OUT" | sed 's/^/  /'
else
  echo "  crm_mon returned no output — trying again without suppression:"
  if $REACHABLE_1; then
    n1 "crm_mon -1" || true
  elif $REACHABLE_2; then
    n2 "crm_mon -1" || true
  fi
fi

# ── XFS mount ─────────────────────────────────────────────────────────────
section "XFS Mount ($XFS_MOUNT)"
check_mount() {
  local node=$1 run=$2
  local status
  if $run "mountpoint -q '$XFS_MOUNT' 2>/dev/null" 2>/dev/null; then
    local usage
    usage=$($run "df -h '$XFS_MOUNT' 2>/dev/null | tail -1 | awk '{print \$3\"/\"\$2\" used (\"\$5\")\";}'" 2>/dev/null || echo "")
    status="mounted"
    [[ -n "$usage" ]] && status="mounted  $usage"
  else
    status="not mounted"
  fi
  printf "  %-14s  %s\n" "$node" "$status"
}

$REACHABLE_1 && check_mount "$NODE1" n1 || echo "  $NODE1  [OFFLINE]"
$REACHABLE_2 && check_mount "$NODE2" n2 || echo "  $NODE2  [OFFLINE]"

# ── Service ports via VIP ──────────────────────────────────────────────────
section "Services via VIP ($VIP)"

check_port() {
  local name=$1 port=$2
  if bash -c "echo >/dev/tcp/${VIP}/${port}" 2>/dev/null; then
    printf "  %-10s  port %-5s  OK\n" "$name" "$port"
  else
    printf "  %-10s  port %-5s  UNREACHABLE\n" "$name" "$port"
  fi
}

if ping -c1 -W2 "$VIP" >/dev/null 2>&1; then
  echo "  Ping       OK"
else
  echo "  Ping       UNREACHABLE"
fi
check_port "NFS"    2049
check_port "iSCSI"  3260

echo ""
echo "════════════════════════════════════════════════════"
if [[ -n "$ACTIVE_NODE" ]]; then
  echo " Active: $ACTIVE_NODE   Standby: $STANDBY_NODE"
else
  echo " Active: unknown (Pacemaker not settled)"
fi
echo "════════════════════════════════════════════════════"
echo ""