From e18b605706a79cbbf56f52e4c34f677abaeb666d Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Wed, 29 Jul 2026 09:59:39 +1000 Subject: [PATCH] feat(ha): add resize-data-disk.sh for online data disk expansion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three-phase online resize: Proxmox qm resize → guest block device rescan → drbdadm resize + xfs_growfs. No downtime required. Detects active node via crm_mon, pre-checks DRBD UpToDate/UpToDate, auto-discovers VMIDs from qm list, and resolves the block device name from the stable scsi-0QEMU... by-id symlink with a slot-index fallback. Co-Authored-By: Claude Sonnet 4.6 --- scripts/ha/resize-data-disk.sh | 276 +++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100755 scripts/ha/resize-data-disk.sh diff --git a/scripts/ha/resize-data-disk.sh b/scripts/ha/resize-data-disk.sh new file mode 100755 index 0000000..3bb01f0 --- /dev/null +++ b/scripts/ha/resize-data-disk.sh @@ -0,0 +1,276 @@ +#!/usr/bin/env bash +# resize-data-disk.sh — online resize of the HA cluster data disk +# +# Three-phase process (all online-safe, no downtime required): +# 1. Proxmox: grow scsi1 on both HA VMs (qm resize) +# 2. Guest: rescan block device on both nodes so the kernel sees the new size +# 3. DRBD + XFS: drbdadm resize, then xfs_growfs — active node only +# +# Usage: +# scripts/ha/resize-data-disk.sh --size +20G [--force] [--dry-run] +# +# --size +NNg amount to grow scsi1 by, e.g. +20G, +50G (required) +# XFS and DRBD cannot shrink; only positive deltas accepted +# --force skip the interactive confirmation prompt +# --dry-run show what would be done without changing anything +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../env.sh +source "${SCRIPT_DIR}/../env.sh" + +# ── Configuration ───────────────────────────────────────────────────────── +NODE1="${NODE1:-ha-server-1}" +NODE2="${NODE2:-ha-server-2}" +NODE1_IP="${NODE1_IP:-192.168.2.228}" +NODE2_IP="${NODE2_IP:-192.168.2.227}" +XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" +DRBD_RESOURCE="${DRBD_RESOURCE:-ha-data}" +DATA_DISK_SLOT="${DATA_DISK_SLOT:-scsi1}" # Proxmox disk name (scsi1 = data disk) +HA_USER="${HA_USER:-nixos}" +PVE_HOST="${PVE_HOST:-${PVE1_HOST}}" +PVE_SSH_USER="${PVE_SSH_USER:-${PROXMOX_SSH_USER}}" +# By-id symlink for the data disk; basename resolves to the raw block device. +# matches variables.nix's haServerDrbdDisk. +DATA_DISK_BYID="${DATA_DISK_BYID:-scsi-0QEMU_QEMU_HARDDISK_drive-${DATA_DISK_SLOT}}" +# ────────────────────────────────────────────────────────────────────────── + +pve() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=10 \ + "${PVE_SSH_USER}@${PVE_HOST}" "$@"; } +n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ + "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; } +n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ + "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; } + +# ── Argument parsing ─────────────────────────────────────────────────────── +SIZE="" +FORCE=false +DRY_RUN=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --size) shift; SIZE="${1:?--size requires a value (e.g. +20G)}" ;; + --force) FORCE=true ;; + --dry-run) DRY_RUN=true ;; + *) echo "Unknown argument: $1" + echo "Usage: $0 --size +NNg [--force] [--dry-run]" + exit 1 ;; + esac + shift +done + +if [[ -z "$SIZE" ]]; then + echo "ERROR: --size is required (e.g. --size +20G)" + echo "Usage: $0 --size +NNg [--force] [--dry-run]" + exit 1 +fi + +# Only positive deltas — qm resize, DRBD, and XFS all refuse to shrink. +if [[ ! "$SIZE" =~ ^\+[0-9]+(G|M|T)$ ]]; then + echo "ERROR: --size must be a positive delta like +20G, +50G, +500M, +2T" + echo " (qm resize, drbdadm resize, and xfs_growfs can only grow, not shrink)" + exit 1 +fi + +DRY_PREFIX="" +$DRY_RUN && DRY_PREFIX="[dry-run] " + +echo "════════════════════════════════════════════════════" +echo " HA Data Disk Resize — $(date '+%Y-%m-%d %H:%M:%S')" +echo " Size delta: $SIZE • Proxmox: $PVE_HOST" +$DRY_RUN && echo " MODE: dry-run — no changes will be made" +echo "════════════════════════════════════════════════════" + +# ── Detect active node ───────────────────────────────────────────────────── +echo "" +echo "Detecting active node..." + +CRM_OUT="" +if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ + "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then + CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true) +fi +if [[ -z "$CRM_OUT" ]]; then + if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ + "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then + CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true) + fi +fi + +# crm_mon 2.x formats Promoted lines as " * Promoted: [ node ]" (bullet *), +# so ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead. +ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | \ + grep -E '(Promoted|Masters):' | \ + grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true) + +if [[ -z "$ACTIVE_NODE" ]]; then + echo "ERROR: could not determine active node from crm_mon." + echo " Is Pacemaker still settling? Try running scripts/ha/health.sh first." + exit 1 +fi + +if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then + ACTIVE_IP="$NODE1_IP" + na() { n1 "$@"; } +else + ACTIVE_IP="$NODE2_IP" + na() { n2 "$@"; } +fi + +echo " Active node: $ACTIVE_NODE ($ACTIVE_IP)" + +# ── Pre-check DRBD state ─────────────────────────────────────────────────── +echo "" +echo "Pre-checks..." + +DRBD_DSTATE=$(na "drbdadm dstate ${DRBD_RESOURCE} 2>/dev/null" 2>/dev/null || echo "unknown") +if ! echo "$DRBD_DSTATE" | grep -q "UpToDate/UpToDate"; then + echo " WARNING: DRBD dstate is '$DRBD_DSTATE' (expected UpToDate/UpToDate)." + echo " Resizing with a partially-synced disk may cause issues." + if ! $FORCE; then + echo " Use --force to proceed anyway." + exit 1 + fi + echo " --force specified — proceeding despite non-ideal DRBD state." +else + echo " DRBD dstate: $DRBD_DSTATE — OK" +fi + +# ── Find VMIDs on Proxmox ───────────────────────────────────────────────── +echo "" +echo "Looking up VM IDs on ${PVE_HOST}..." + +QM_LIST=$(pve "qm list 2>/dev/null" || true) +VMID1=$(echo "$QM_LIST" | awk -v name="$NODE1" '$0 ~ name {print $1}' | head -1) +VMID2=$(echo "$QM_LIST" | awk -v name="$NODE2" '$0 ~ name {print $1}' | head -1) + +if [[ -z "$VMID1" ]]; then + echo " ERROR: could not find VMID for $NODE1 on $PVE_HOST" + echo " qm list output:" + echo "$QM_LIST" | sed 's/^/ /' + exit 1 +fi +if [[ -z "$VMID2" ]]; then + echo " ERROR: could not find VMID for $NODE2 on $PVE_HOST" + echo " qm list output:" + echo "$QM_LIST" | sed 's/^/ /' + exit 1 +fi + +echo " $NODE1: VMID $VMID1" +echo " $NODE2: VMID $VMID2" + +# ── Confirm ──────────────────────────────────────────────────────────────── +if ! $FORCE && ! $DRY_RUN; then + echo "" + echo " Plan:" + echo " Phase 1 — qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE} (on $PVE_HOST)" + echo " qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE} (on $PVE_HOST)" + echo " Phase 2 — block device rescan on $NODE1 and $NODE2" + echo " Phase 3 — drbdadm resize + xfs_growfs on $ACTIVE_NODE" + echo " No downtime required (all operations are online-safe)." + printf " Proceed? [y/N] " + read -r ANSWER + [[ "${ANSWER,,}" == "y" || "${ANSWER,,}" == "yes" ]] || { echo "Aborted."; exit 0; } +fi + +# ═══════════════════════════════════════════════════════════════ +# Phase 1 — Resize both VM data disks in Proxmox +# ═══════════════════════════════════════════════════════════════ +echo "" +echo "── Phase 1 — Proxmox disk resize (${DATA_DISK_SLOT} ${SIZE} on both VMs) ──" + +echo " ${DRY_PREFIX}qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE} ($NODE1 on ${PVE_HOST})" +if ! $DRY_RUN; then + pve "qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}" +fi + +echo " ${DRY_PREFIX}qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE} ($NODE2 on ${PVE_HOST})" +if ! $DRY_RUN; then + pve "qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}" +fi + +echo " Phase 1 done." + +# ═══════════════════════════════════════════════════════════════ +# Phase 2 — Rescan block device on both guest nodes +# ═══════════════════════════════════════════════════════════════ +echo "" +echo "── Phase 2 — Block device rescan (both nodes) ──" + +rescan_node() { + local node_name=$1 run_fn=$2 + + # Resolve block device name from the stable by-id symlink on the guest. + local blk_dev="" + if ! $DRY_RUN; then + blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true) + fi + if [[ -z "$blk_dev" ]]; then + # Fall back: scsi1 → index 1 → sdb, scsi2 → sdc, etc. + local slot_idx + slot_idx=$(echo "$DATA_DISK_SLOT" | grep -oE '[0-9]+$' || echo "1") + blk_dev=$(printf "sd%s" "$(echo "abcdefghij" | cut -c$((slot_idx + 1)))") + echo " WARNING: could not resolve /dev/disk/by-id/${DATA_DISK_BYID} on $node_name;" \ + "falling back to /dev/${blk_dev}" >&2 + fi + + echo " ${DRY_PREFIX}Rescanning /dev/${blk_dev} on ${node_name}..." + if ! $DRY_RUN; then + $run_fn "bash -c 'echo 1 > /sys/block/${blk_dev}/device/rescan'" 2>/dev/null + local new_size + new_size=$($run_fn "lsblk -nd -o SIZE /dev/${blk_dev} 2>/dev/null" 2>/dev/null || echo "unknown") + echo " /dev/${blk_dev} on $node_name now reports: $new_size" + fi +} + +rescan_node "$NODE1" n1 +rescan_node "$NODE2" n2 + +echo " Phase 2 done." + +# ═══════════════════════════════════════════════════════════════ +# Phase 3 — Grow DRBD metadata, then XFS (active node only) +# ═══════════════════════════════════════════════════════════════ +echo "" +echo "── Phase 3 — DRBD resize + XFS grow (on active node: $ACTIVE_NODE) ──" + +echo " ${DRY_PREFIX}drbdadm resize ${DRBD_RESOURCE}" +if ! $DRY_RUN; then + na "drbdadm resize ${DRBD_RESOURCE}" +fi + +echo " ${DRY_PREFIX}xfs_growfs ${XFS_MOUNT}" +if ! $DRY_RUN; then + na "xfs_growfs ${XFS_MOUNT}" +fi + +echo " Phase 3 done." + +# ── Verify ──────────────────────────────────────────────────────────────── +echo "" +echo "── Verify ──" + +if ! $DRY_RUN; then + DF_OUT=$(na "df -h '${XFS_MOUNT}' 2>/dev/null" 2>/dev/null || echo "") + if [[ -n "$DF_OUT" ]]; then + echo " ${XFS_MOUNT}:" + echo "$DF_OUT" | sed 's/^/ /' + fi + + DRBD_DSTATE_AFTER=$(na "drbdadm dstate ${DRBD_RESOURCE} 2>/dev/null" 2>/dev/null || echo "unknown") + echo " DRBD dstate: $DRBD_DSTATE_AFTER" + if ! echo "$DRBD_DSTATE_AFTER" | grep -q "UpToDate/UpToDate"; then + echo " NOTE: DRBD is resyncing — normal immediately after resize." + echo " Monitor: ssh nixos@${ACTIVE_IP} 'sudo watch -n3 cat /proc/drbd'" + fi +else + echo " [dry-run] would verify df -h ${XFS_MOUNT} and drbdadm dstate on $ACTIVE_NODE" +fi + +echo "" +echo "════════════════════════════════════════════════════" +echo " Resize complete." +echo " Active node: $ACTIVE_NODE" +echo "════════════════════════════════════════════════════" +echo ""