From ec90753a09246bbeba350c790a0fa581e1b78e40 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Tue, 28 Jul 2026 23:57:07 +1000 Subject: [PATCH] fix(ha): per-node DRBD disk assignment (NODE2 data disk is /dev/sda not /dev/sdb) Proxmox VM disk ordering differs between the two HA nodes: ha-server-1: sda=OS (50G), sdb=DRBD data (32G) ha-server-2: sda=DRBD data (32G), sdb=OS (50G) The DRBD resource config was using a shared disk=/dev/sdb which targeted the OS disk on ha-server-2, causing drbdmeta and drbdadm up to operate on the mounted root filesystem (hence "Device or resource busy"). Changes: - variables.nix: add haServer1DrbdDisk/haServer2DrbdDisk - cluster-config.nix: move volume block inside per-host on{} sections so each node uses the correct backing disk - cluster-init.sh: use NODE1_DRBD_DISK/NODE2_DRBD_DISK variables; add runtime check that patches /etc/drbd.d/*.res on the running nodes if the deployed config points to the wrong disk (workaround for VMs built before this fix; redeploy with --force-rebuild to make permanent) Co-Authored-By: Claude Sonnet 4.6 --- modules/ha/cluster-config.nix | 16 ++++++++++------ scripts/ha/cluster-init.sh | 36 ++++++++++++++++++++++++++++------- variables.nix | 5 +++++ 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix index 37b6eb9..7e6055f 100644 --- a/modules/ha/cluster-config.nix +++ b/modules/ha/cluster-config.nix @@ -68,18 +68,22 @@ } resource ha-data { - volume 0 { - device /dev/drbd0; - disk /dev/sdb; - meta-disk internal; - } - on ${vars.haServer1Host} { address ${vars.haServer1StorageIp}:${toString vars.ports.haServerDrbd}; + volume 0 { + device /dev/drbd0; + disk ${vars.haServer1DrbdDisk}; + meta-disk internal; + } } on ${vars.haServer2Host} { address ${vars.haServer2StorageIp}:${toString vars.ports.haServerDrbd}; + volume 0 { + device /dev/drbd0; + disk ${vars.haServer2DrbdDisk}; + meta-disk internal; + } } } ''; diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh index 3528dcb..f5cc008 100755 --- a/scripts/ha/cluster-init.sh +++ b/scripts/ha/cluster-init.sh @@ -32,6 +32,11 @@ ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}" # vars.haIscsiIqn ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img" ISCSI_LUN_SIZE="10G" DRBD_DEVICE="/dev/drbd0" +# DRBD backing disk per node — disk ordering can differ between Proxmox VMs. +# Defaults match haServer{1,2}DrbdDisk in variables.nix. Override if your +# VM was created with disks in a different order. +NODE1_DRBD_DISK="${NODE1_DRBD_DISK:-/dev/sdb}" +NODE2_DRBD_DISK="${NODE2_DRBD_DISK:-/dev/sda}" VMID_NODE1="${VMID_NODE1:-}" # set by deploy.sh; needed for STONITH VMID_NODE2="${VMID_NODE2:-}" PVE_HOST="${PVE_HOST:-pve1.sweet.home}" @@ -149,7 +154,7 @@ done # ── 2. DRBD initialisation ──────────────────────────────────────────────── # Put both nodes in Pacemaker standby before touching DRBD metadata. # Without this, the OCF DRBD agent races: it sees drbdadm-down as a failure -# and immediately calls drbdadm-up again, leaving /dev/sdb busy when +# and immediately calls drbdadm-up again, leaving the backing disk busy when # create-md / write-dev-uuid runs. On a fresh cluster with no resources # configured this is a no-op; on a re-run it stops the race. log "Setting both nodes to Pacemaker standby for DRBD metadata init..." @@ -175,25 +180,42 @@ log "Detaching DRBD on $NODE2..." n2_ssh "drbdadm down ha-data 2>/dev/null || true" sleep 2 +# Verify the DRBD config on each node targets the correct backing disk. +# Proxmox VM disk ordering is not guaranteed — the data disk may appear as +# /dev/sda on one node and /dev/sdb on the other. If the deployed NixOS +# config was generated before variables.nix had per-node disk assignments, the +# config may point to the wrong device. Patch it in-place so drbdadm up +# attaches to the right disk; rebuild+redeploy to make this permanent. +n1_conf_disk=$(drbdadm sh-ll-dev ha-data 2>/dev/null || true) +if [[ -n "$n1_conf_disk" && "$n1_conf_disk" != "$NODE1_DRBD_DISK" ]]; then + warn "$NODE1 DRBD config says disk=$n1_conf_disk; patching to $NODE1_DRBD_DISK (redeploy to make permanent)" + sed -i "s|${n1_conf_disk}|${NODE1_DRBD_DISK}|g" /etc/drbd.d/*.res 2>/dev/null || true +fi +n2_conf_disk=$(n2_ssh "drbdadm sh-ll-dev ha-data 2>/dev/null" 2>/dev/null || true) +if [[ -n "$n2_conf_disk" && "$n2_conf_disk" != "$NODE2_DRBD_DISK" ]]; then + warn "$NODE2 DRBD config says disk=$n2_conf_disk; patching to $NODE2_DRBD_DISK (redeploy to make permanent)" + n2_ssh "sed -i 's|${n2_conf_disk}|${NODE2_DRBD_DISK}|g' /etc/drbd.d/*.res" 2>/dev/null || true +fi + log "Initialising DRBD metadata on $NODE1..." # Use drbdmeta --force directly for BOTH create-md and write-dev-uuid. # drbdadm create-md --force passes --force to drbdmeta create-md but NOT to # the write-dev-uuid sub-call it makes internally, so write-dev-uuid fails when -# /dev/sdb is still busy (udev auto-attach, stale DRBD state, etc.) and stdin -# is not a TTY: "stdin not a TTY, not waiting for confirmation" → exit 20. +# the backing disk is still busy and stdin is not a TTY: +# "stdin not a TTY, not waiting for confirmation" → exit 20. # Calling drbdmeta --force directly bypasses the exclusive-open confirmation on # both steps without needing a TTY, regardless of whether the device is busy. if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate"; then UUID1=$(_rand_uuid) - drbdmeta --force 0 v08 /dev/sdb internal create-md - drbdmeta --force 0 v08 /dev/sdb internal write-dev-uuid "$UUID1" + drbdmeta --force 0 v08 "${NODE1_DRBD_DISK}" internal create-md + drbdmeta --force 0 v08 "${NODE1_DRBD_DISK}" internal write-dev-uuid "$UUID1" fi log "Initialising DRBD metadata on $NODE2..." if ! n2_ssh "drbdadm dstate ha-data 2>/dev/null | grep -q UpToDate" 2>/dev/null; then UUID2=$(n2_ssh "cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' | cut -c1-16 | tr '[:lower:]' '[:upper:]'") - n2_ssh "drbdmeta --force 0 v08 /dev/sdb internal create-md" - n2_ssh "drbdmeta --force 0 v08 /dev/sdb internal write-dev-uuid ${UUID2}" + n2_ssh "drbdmeta --force 0 v08 ${NODE2_DRBD_DISK} internal create-md" + n2_ssh "drbdmeta --force 0 v08 ${NODE2_DRBD_DISK} internal write-dev-uuid ${UUID2}" fi log "Bringing up DRBD on both nodes..." diff --git a/variables.nix b/variables.nix index 195b4df..6522349 100644 --- a/variables.nix +++ b/variables.nix @@ -108,6 +108,11 @@ haStoragePrefixLength = 29; # storage subnet prefix length (/29) haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage"; + # DRBD backing device on each node. Disk ordering can differ between Proxmox + # VMs depending on the order disks were added; these must match the actual + # block device that is NOT the OS disk on each node (verify with lsblk). + haServer1DrbdDisk = "/dev/sdb"; # data disk on ha-server-1 (OS disk is /dev/sda) + haServer2DrbdDisk = "/dev/sda"; # data disk on ha-server-2 (OS disk is /dev/sdb) # Storage storageRoot = "/tank"; # ZFS pool root on `server`