From ead4f558052e0026448c8362519f77230093a952 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Wed, 29 Jul 2026 00:11:13 +1000 Subject: [PATCH] fix(ha): use stable by-id disk path for DRBD instead of /dev/sd* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /dev/sda and /dev/sdb are assigned by the OS based on Proxmox disk-add order, which is not consistent across VMs. Use the SCSI controller path instead — drive-scsi1 is always the dedicated data disk on all HA nodes regardless of which sda/sdb it gets assigned to. variables.nix: replace per-node haServer{1,2}DrbdDisk with a single haServerDrbdDisk using /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1. cluster-config.nix: revert to shared volume block (per-node block was needed for /dev/sd* but the by-id path is identical on both nodes). cluster-init.sh: - single DRBD_DISK variable (matching haServerDrbdDisk) - robust /etc/drbd.conf patch: NixOS manages this file as a symlink to a read-only Nix store path; cp --remove-destination breaks the symlink before sed -i so the edit actually takes effect - scp helper script to NODE2 rather than bash -c over SSH to avoid quoting complexity Co-Authored-By: Claude Sonnet 4.6 --- modules/ha/cluster-config.nix | 16 ++++------ scripts/ha/cluster-init.sh | 56 +++++++++++++++++++++-------------- variables.nix | 10 +++---- 3 files changed, 44 insertions(+), 38 deletions(-) diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix index 7e6055f..76ad56b 100644 --- a/modules/ha/cluster-config.nix +++ b/modules/ha/cluster-config.nix @@ -68,22 +68,18 @@ } resource ha-data { + volume 0 { + device /dev/drbd0; + disk ${vars.haServerDrbdDisk}; + meta-disk internal; + } + on ${vars.haServer1Host} { address ${vars.haServer1StorageIp}:${toString vars.ports.haServerDrbd}; - volume 0 { - device /dev/drbd0; - disk ${vars.haServer1DrbdDisk}; - meta-disk internal; - } } on ${vars.haServer2Host} { address ${vars.haServer2StorageIp}:${toString vars.ports.haServerDrbd}; - volume 0 { - device /dev/drbd0; - disk ${vars.haServer2DrbdDisk}; - meta-disk internal; - } } } ''; diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh index f5cc008..9495052 100755 --- a/scripts/ha/cluster-init.sh +++ b/scripts/ha/cluster-init.sh @@ -32,11 +32,11 @@ ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}" # vars.haIscsiIqn ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img" ISCSI_LUN_SIZE="10G" DRBD_DEVICE="/dev/drbd0" -# DRBD backing disk per node — disk ordering can differ between Proxmox VMs. -# Defaults match haServer{1,2}DrbdDisk in variables.nix. Override if your -# VM was created with disks in a different order. -NODE1_DRBD_DISK="${NODE1_DRBD_DISK:-/dev/sdb}" -NODE2_DRBD_DISK="${NODE2_DRBD_DISK:-/dev/sda}" +# DRBD backing disk — by-id path that resolves correctly on both nodes +# regardless of whether the OS-level name is sda or sdb (Proxmox VM disk +# ordering is not guaranteed). Matches haServerDrbdDisk in variables.nix. +# Override DRBD_DISK if your hardware uses a different controller/slot path. +DRBD_DISK="${DRBD_DISK:-/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1}" VMID_NODE1="${VMID_NODE1:-}" # set by deploy.sh; needed for STONITH VMID_NODE2="${VMID_NODE2:-}" PVE_HOST="${PVE_HOST:-pve1.sweet.home}" @@ -180,22 +180,32 @@ log "Detaching DRBD on $NODE2..." n2_ssh "drbdadm down ha-data 2>/dev/null || true" sleep 2 -# Verify the DRBD config on each node targets the correct backing disk. -# Proxmox VM disk ordering is not guaranteed — the data disk may appear as -# /dev/sda on one node and /dev/sdb on the other. If the deployed NixOS -# config was generated before variables.nix had per-node disk assignments, the -# config may point to the wrong device. Patch it in-place so drbdadm up -# attaches to the right disk; rebuild+redeploy to make this permanent. -n1_conf_disk=$(drbdadm sh-ll-dev ha-data 2>/dev/null || true) -if [[ -n "$n1_conf_disk" && "$n1_conf_disk" != "$NODE1_DRBD_DISK" ]]; then - warn "$NODE1 DRBD config says disk=$n1_conf_disk; patching to $NODE1_DRBD_DISK (redeploy to make permanent)" - sed -i "s|${n1_conf_disk}|${NODE1_DRBD_DISK}|g" /etc/drbd.d/*.res 2>/dev/null || true +# Ensure /etc/drbd.conf on both nodes points to DRBD_DISK (the stable by-id +# path). VMs built before this fix may have /dev/sda or /dev/sdb hardcoded. +# NixOS makes /etc/drbd.conf a symlink into the read-only Nix store, so +# sed -i on the symlink target would fail — we break the symlink first with +# cp --remove-destination, creating a regular writable copy. +# Rebuild+redeploy (--force-rebuild) to make this permanent. +_PATCH_DRBD=$(mktemp) +cat > "$_PATCH_DRBD" << 'PATCHEOF' +#!/bin/bash +WANT="$1" +conf=/etc/drbd.conf +if [[ -L "$conf" ]]; then + cp --remove-destination "$(readlink -f "$conf")" "$conf" fi -n2_conf_disk=$(n2_ssh "drbdadm sh-ll-dev ha-data 2>/dev/null" 2>/dev/null || true) -if [[ -n "$n2_conf_disk" && "$n2_conf_disk" != "$NODE2_DRBD_DISK" ]]; then - warn "$NODE2 DRBD config says disk=$n2_conf_disk; patching to $NODE2_DRBD_DISK (redeploy to make permanent)" - n2_ssh "sed -i 's|${n2_conf_disk}|${NODE2_DRBD_DISK}|g' /etc/drbd.d/*.res" 2>/dev/null || true +cur=$(drbdadm sh-ll-dev ha-data 2>/dev/null | head -1 || true) +if [[ -n "$cur" && "$cur" != "$WANT" ]]; then + echo "[cluster-init] WARNING: patching $conf: $cur → $WANT (rebuild to make permanent)" + sed -i "s,${cur},${WANT},g" "$conf" fi +PATCHEOF +chmod +x "$_PATCH_DRBD" +bash "$_PATCH_DRBD" "$DRBD_DISK" +n2_scp "$_PATCH_DRBD" "/tmp/patch-drbd-disk.sh" +n2_ssh "bash /tmp/patch-drbd-disk.sh ${DRBD_DISK}" +n2_ssh "rm -f /tmp/patch-drbd-disk.sh" +rm -f "$_PATCH_DRBD" log "Initialising DRBD metadata on $NODE1..." # Use drbdmeta --force directly for BOTH create-md and write-dev-uuid. @@ -207,15 +217,15 @@ log "Initialising DRBD metadata on $NODE1..." # both steps without needing a TTY, regardless of whether the device is busy. if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate"; then UUID1=$(_rand_uuid) - drbdmeta --force 0 v08 "${NODE1_DRBD_DISK}" internal create-md - drbdmeta --force 0 v08 "${NODE1_DRBD_DISK}" internal write-dev-uuid "$UUID1" + drbdmeta --force 0 v08 "${DRBD_DISK}" internal create-md + drbdmeta --force 0 v08 "${DRBD_DISK}" internal write-dev-uuid "$UUID1" fi log "Initialising DRBD metadata on $NODE2..." if ! n2_ssh "drbdadm dstate ha-data 2>/dev/null | grep -q UpToDate" 2>/dev/null; then UUID2=$(n2_ssh "cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' | cut -c1-16 | tr '[:lower:]' '[:upper:]'") - n2_ssh "drbdmeta --force 0 v08 ${NODE2_DRBD_DISK} internal create-md" - n2_ssh "drbdmeta --force 0 v08 ${NODE2_DRBD_DISK} internal write-dev-uuid ${UUID2}" + n2_ssh "drbdmeta --force 0 v08 ${DRBD_DISK} internal create-md" + n2_ssh "drbdmeta --force 0 v08 ${DRBD_DISK} internal write-dev-uuid ${UUID2}" fi log "Bringing up DRBD on both nodes..." diff --git a/variables.nix b/variables.nix index 6522349..4d0595e 100644 --- a/variables.nix +++ b/variables.nix @@ -108,11 +108,11 @@ haStoragePrefixLength = 29; # storage subnet prefix length (/29) haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage"; - # DRBD backing device on each node. Disk ordering can differ between Proxmox - # VMs depending on the order disks were added; these must match the actual - # block device that is NOT the OS disk on each node (verify with lsblk). - haServer1DrbdDisk = "/dev/sdb"; # data disk on ha-server-1 (OS disk is /dev/sda) - haServer2DrbdDisk = "/dev/sda"; # data disk on ha-server-2 (OS disk is /dev/sdb) + # DRBD backing disk — identified by SCSI controller path so it resolves to the + # correct block device regardless of OS-level naming (sda vs sdb can differ + # between Proxmox VMs depending on disk-add order). drive-scsi1 is always the + # dedicated data disk on all HA nodes; drive-scsi0 is the OS disk. + haServerDrbdDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1"; # Storage storageRoot = "/tank"; # ZFS pool root on `server`