Archived
beatzaplentyandClaude Sonnet 4.6
539bdf9833
Check NixOS configurations / eval-hosts (pull_request) Successful in 10m21s
The by-id lookup is read-only so it's safe to run in dry-run mode. Previously it was gated behind `if ! $DRY_RUN`, which always triggered the sdb fallback warning in dry-run — making it look like the device path wasn't reliable when the symlink actually exists on both servers. Now the lookup always runs and the script errors out with a clear message if the by-id symlink is genuinely missing, instead of silently falling back to a guessed /dev/sd* name. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
275 lines
12 KiB
Bash
Executable File
275 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# resize-data-disk.sh — online resize of the HA cluster data disk
|
|
#
|
|
# Three-phase process (all online-safe, no downtime required):
|
|
# 1. Proxmox: grow scsi1 on both HA VMs (qm resize)
|
|
# 2. Guest: rescan block device on both nodes so the kernel sees the new size
|
|
# 3. DRBD + XFS: drbdadm resize, then xfs_growfs — active node only
|
|
#
|
|
# Usage:
|
|
# scripts/ha/resize-data-disk.sh --size +20G [--force] [--dry-run]
|
|
#
|
|
# --size +NNg amount to grow scsi1 by, e.g. +20G, +50G (required)
|
|
# XFS and DRBD cannot shrink; only positive deltas accepted
|
|
# --force skip the interactive confirmation prompt
|
|
# --dry-run show what would be done without changing anything
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=../env.sh
|
|
source "${SCRIPT_DIR}/../env.sh"
|
|
|
|
# ── Configuration ─────────────────────────────────────────────────────────
|
|
NODE1="${NODE1:-ha-server-1}"
|
|
NODE2="${NODE2:-ha-server-2}"
|
|
NODE1_IP="${NODE1_IP:-192.168.2.228}"
|
|
NODE2_IP="${NODE2_IP:-192.168.2.227}"
|
|
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"
|
|
DRBD_RESOURCE="${DRBD_RESOURCE:-ha-data}"
|
|
DATA_DISK_SLOT="${DATA_DISK_SLOT:-scsi1}" # Proxmox disk name (scsi1 = data disk)
|
|
HA_USER="${HA_USER:-nixos}"
|
|
PVE_HOST="${PVE_HOST:-${PVE1_HOST}}"
|
|
PVE_SSH_USER="${PVE_SSH_USER:-${PROXMOX_SSH_USER}}"
|
|
PVE_SUDO=""
|
|
[[ "$PVE_SSH_USER" != "root" ]] && PVE_SUDO="sudo"
|
|
# By-id symlink for the data disk; basename resolves to the raw block device.
|
|
# matches variables.nix's haServerDrbdDisk.
|
|
DATA_DISK_BYID="${DATA_DISK_BYID:-scsi-0QEMU_QEMU_HARDDISK_drive-${DATA_DISK_SLOT}}"
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
|
|
pve() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=10 \
|
|
"${PVE_SSH_USER}@${PVE_HOST}" "$@"; }
|
|
n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
|
|
"${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
|
|
n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
|
|
"${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
|
|
|
|
# ── Argument parsing ───────────────────────────────────────────────────────
|
|
SIZE=""
|
|
FORCE=false
|
|
DRY_RUN=false
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--size) shift; SIZE="${1:?--size requires a value (e.g. +20G)}" ;;
|
|
--force) FORCE=true ;;
|
|
--dry-run) DRY_RUN=true ;;
|
|
*) echo "Unknown argument: $1"
|
|
echo "Usage: $0 --size +NNg [--force] [--dry-run]"
|
|
exit 1 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
if [[ -z "$SIZE" ]]; then
|
|
echo "ERROR: --size is required (e.g. --size +20G)"
|
|
echo "Usage: $0 --size +NNg [--force] [--dry-run]"
|
|
exit 1
|
|
fi
|
|
|
|
# Only positive deltas — qm resize, DRBD, and XFS all refuse to shrink.
|
|
if [[ ! "$SIZE" =~ ^\+[0-9]+(G|M|T)$ ]]; then
|
|
echo "ERROR: --size must be a positive delta like +20G, +50G, +500M, +2T"
|
|
echo " (qm resize, drbdadm resize, and xfs_growfs can only grow, not shrink)"
|
|
exit 1
|
|
fi
|
|
|
|
DRY_PREFIX=""
|
|
$DRY_RUN && DRY_PREFIX="[dry-run] "
|
|
|
|
echo "════════════════════════════════════════════════════"
|
|
echo " HA Data Disk Resize — $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo " Size delta: $SIZE • Proxmox: $PVE_HOST"
|
|
$DRY_RUN && echo " MODE: dry-run — no changes will be made"
|
|
echo "════════════════════════════════════════════════════"
|
|
|
|
# ── Detect active node ─────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "Detecting active node..."
|
|
|
|
CRM_OUT=""
|
|
if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
|
|
"${HA_USER}@${NODE1_IP}" true 2>/dev/null; then
|
|
CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
|
|
fi
|
|
if [[ -z "$CRM_OUT" ]]; then
|
|
if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
|
|
"${HA_USER}@${NODE2_IP}" true 2>/dev/null; then
|
|
CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
|
|
fi
|
|
fi
|
|
|
|
# crm_mon 2.x formats Promoted lines as " * Promoted: [ node ]" (bullet *),
|
|
# so ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
|
|
ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | \
|
|
grep -E '(Promoted|Masters):' | \
|
|
grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
|
|
|
|
if [[ -z "$ACTIVE_NODE" ]]; then
|
|
echo "ERROR: could not determine active node from crm_mon."
|
|
echo " Is Pacemaker still settling? Try running scripts/ha/health.sh first."
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
|
|
ACTIVE_IP="$NODE1_IP"
|
|
na() { n1 "$@"; }
|
|
else
|
|
ACTIVE_IP="$NODE2_IP"
|
|
na() { n2 "$@"; }
|
|
fi
|
|
|
|
echo " Active node: $ACTIVE_NODE ($ACTIVE_IP)"
|
|
|
|
# ── Pre-check DRBD state ───────────────────────────────────────────────────
|
|
echo ""
|
|
echo "Pre-checks..."
|
|
|
|
DRBD_DSTATE=$(na "drbdadm dstate ${DRBD_RESOURCE} 2>/dev/null" 2>/dev/null || echo "unknown")
|
|
if ! echo "$DRBD_DSTATE" | grep -q "UpToDate/UpToDate"; then
|
|
echo " WARNING: DRBD dstate is '$DRBD_DSTATE' (expected UpToDate/UpToDate)."
|
|
echo " Resizing with a partially-synced disk may cause issues."
|
|
if ! $FORCE; then
|
|
echo " Use --force to proceed anyway."
|
|
exit 1
|
|
fi
|
|
echo " --force specified — proceeding despite non-ideal DRBD state."
|
|
else
|
|
echo " DRBD dstate: $DRBD_DSTATE — OK"
|
|
fi
|
|
|
|
# ── Find VMIDs on Proxmox ─────────────────────────────────────────────────
|
|
echo ""
|
|
echo "Looking up VM IDs on ${PVE_HOST}..."
|
|
|
|
QM_LIST=$(pve "$PVE_SUDO qm list 2>/dev/null" || true)
|
|
VMID1=$(echo "$QM_LIST" | awk -v name="$NODE1" '$0 ~ name {print $1}' | head -1)
|
|
VMID2=$(echo "$QM_LIST" | awk -v name="$NODE2" '$0 ~ name {print $1}' | head -1)
|
|
|
|
if [[ -z "$VMID1" ]]; then
|
|
echo " ERROR: could not find VMID for $NODE1 on $PVE_HOST"
|
|
echo " qm list output:"
|
|
echo "$QM_LIST" | sed 's/^/ /'
|
|
exit 1
|
|
fi
|
|
if [[ -z "$VMID2" ]]; then
|
|
echo " ERROR: could not find VMID for $NODE2 on $PVE_HOST"
|
|
echo " qm list output:"
|
|
echo "$QM_LIST" | sed 's/^/ /'
|
|
exit 1
|
|
fi
|
|
|
|
echo " $NODE1: VMID $VMID1"
|
|
echo " $NODE2: VMID $VMID2"
|
|
|
|
# ── Confirm ────────────────────────────────────────────────────────────────
|
|
if ! $FORCE && ! $DRY_RUN; then
|
|
echo ""
|
|
echo " Plan:"
|
|
echo " Phase 1 — qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE} (on $PVE_HOST)"
|
|
echo " qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE} (on $PVE_HOST)"
|
|
echo " Phase 2 — block device rescan on $NODE1 and $NODE2"
|
|
echo " Phase 3 — drbdadm resize + xfs_growfs on $ACTIVE_NODE"
|
|
echo " No downtime required (all operations are online-safe)."
|
|
printf " Proceed? [y/N] "
|
|
read -r ANSWER
|
|
[[ "${ANSWER,,}" == "y" || "${ANSWER,,}" == "yes" ]] || { echo "Aborted."; exit 0; }
|
|
fi
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# Phase 1 — Resize both VM data disks in Proxmox
|
|
# ═══════════════════════════════════════════════════════════════
|
|
echo ""
|
|
echo "── Phase 1 — Proxmox disk resize (${DATA_DISK_SLOT} ${SIZE} on both VMs) ──"
|
|
|
|
echo " ${DRY_PREFIX}qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE} ($NODE1 on ${PVE_HOST})"
|
|
if ! $DRY_RUN; then
|
|
pve "$PVE_SUDO qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}"
|
|
fi
|
|
|
|
echo " ${DRY_PREFIX}qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE} ($NODE2 on ${PVE_HOST})"
|
|
if ! $DRY_RUN; then
|
|
pve "$PVE_SUDO qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}"
|
|
fi
|
|
|
|
echo " Phase 1 done."
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# Phase 2 — Rescan block device on both guest nodes
|
|
# ═══════════════════════════════════════════════════════════════
|
|
echo ""
|
|
echo "── Phase 2 — Block device rescan (both nodes) ──"
|
|
|
|
rescan_node() {
|
|
local node_name=$1 run_fn=$2
|
|
|
|
# Resolve block device name from the stable by-id symlink on the guest.
|
|
# Read-only lookup — safe to run even in dry-run so we show the real device.
|
|
local blk_dev=""
|
|
blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
|
|
if [[ -z "$blk_dev" ]]; then
|
|
echo " ERROR: /dev/disk/by-id/${DATA_DISK_BYID} not found on $node_name" >&2
|
|
echo " Check DATA_DISK_BYID or DATA_DISK_SLOT configuration." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo " ${DRY_PREFIX}Rescanning /dev/${blk_dev} on ${node_name}..."
|
|
if ! $DRY_RUN; then
|
|
$run_fn "bash -c 'echo 1 > /sys/block/${blk_dev}/device/rescan'" 2>/dev/null
|
|
local new_size
|
|
new_size=$($run_fn "lsblk -nd -o SIZE /dev/${blk_dev} 2>/dev/null" 2>/dev/null || echo "unknown")
|
|
echo " /dev/${blk_dev} on $node_name now reports: $new_size"
|
|
fi
|
|
}
|
|
|
|
rescan_node "$NODE1" n1
|
|
rescan_node "$NODE2" n2
|
|
|
|
echo " Phase 2 done."
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# Phase 3 — Grow DRBD metadata, then XFS (active node only)
|
|
# ═══════════════════════════════════════════════════════════════
|
|
echo ""
|
|
echo "── Phase 3 — DRBD resize + XFS grow (on active node: $ACTIVE_NODE) ──"
|
|
|
|
echo " ${DRY_PREFIX}drbdadm resize ${DRBD_RESOURCE}"
|
|
if ! $DRY_RUN; then
|
|
na "drbdadm resize ${DRBD_RESOURCE}"
|
|
fi
|
|
|
|
echo " ${DRY_PREFIX}xfs_growfs ${XFS_MOUNT}"
|
|
if ! $DRY_RUN; then
|
|
na "xfs_growfs ${XFS_MOUNT}"
|
|
fi
|
|
|
|
echo " Phase 3 done."
|
|
|
|
# ── Verify ────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "── Verify ──"
|
|
|
|
if ! $DRY_RUN; then
|
|
DF_OUT=$(na "df -h '${XFS_MOUNT}' 2>/dev/null" 2>/dev/null || echo "")
|
|
if [[ -n "$DF_OUT" ]]; then
|
|
echo " ${XFS_MOUNT}:"
|
|
echo "$DF_OUT" | sed 's/^/ /'
|
|
fi
|
|
|
|
DRBD_DSTATE_AFTER=$(na "drbdadm dstate ${DRBD_RESOURCE} 2>/dev/null" 2>/dev/null || echo "unknown")
|
|
echo " DRBD dstate: $DRBD_DSTATE_AFTER"
|
|
if ! echo "$DRBD_DSTATE_AFTER" | grep -q "UpToDate/UpToDate"; then
|
|
echo " NOTE: DRBD is resyncing — normal immediately after resize."
|
|
echo " Monitor: ssh nixos@${ACTIVE_IP} 'sudo watch -n3 cat /proc/drbd'"
|
|
fi
|
|
else
|
|
echo " [dry-run] would verify df -h ${XFS_MOUNT} and drbdadm dstate on $ACTIVE_NODE"
|
|
fi
|
|
|
|
echo ""
|
|
echo "════════════════════════════════════════════════════"
|
|
echo " Resize complete."
|
|
echo " Active node: $ACTIVE_NODE"
|
|
echo "════════════════════════════════════════════════════"
|
|
echo ""
|