fix(ha): use stable by-id disk path for DRBD instead of /dev/sd*

/dev/sda and /dev/sdb are assigned by the OS based on Proxmox disk-add
order, which is not consistent across VMs. Use the SCSI controller path
instead — drive-scsi1 is always the dedicated data disk on all HA nodes
regardless of which sda/sdb it gets assigned to.

variables.nix: replace per-node haServer{1,2}DrbdDisk with a single
haServerDrbdDisk using /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1.

cluster-config.nix: revert to shared volume block (per-node block was
needed for /dev/sd* but the by-id path is identical on both nodes).

cluster-init.sh:
- single DRBD_DISK variable (matching haServerDrbdDisk)
- robust /etc/drbd.conf patch: NixOS manages this file as a symlink to
  a read-only Nix store path; cp --remove-destination breaks the symlink
  before sed -i so the edit actually takes effect
- scp helper script to NODE2 rather than bash -c over SSH to avoid
  quoting complexity

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

modules/ha/cluster-config.nix

@@ -68,22 +68,18 @@
       }
 
       resource ha-data {
+        volume 0 {
+          device    /dev/drbd0;
+          disk      ${vars.haServerDrbdDisk};
+          meta-disk internal;
+        }
+
         on ${vars.haServer1Host} {
           address ${vars.haServer1StorageIp}:${toString vars.ports.haServerDrbd};
-          volume 0 {
-            device    /dev/drbd0;
-            disk      ${vars.haServer1DrbdDisk};
-            meta-disk internal;
-          }
         }
 
         on ${vars.haServer2Host} {
           address ${vars.haServer2StorageIp}:${toString vars.ports.haServerDrbd};
-          volume 0 {
-            device    /dev/drbd0;
-            disk      ${vars.haServer2DrbdDisk};
-            meta-disk internal;
-          }
         }
       }
     '';

scripts/ha/cluster-init.sh

@@ -32,11 +32,11 @@ ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}"  # vars.haIscsiIqn
 ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
 ISCSI_LUN_SIZE="10G"
 DRBD_DEVICE="/dev/drbd0"
-# DRBD backing disk per node — disk ordering can differ between Proxmox VMs.
-# Defaults match haServer{1,2}DrbdDisk in variables.nix.  Override if your
-# VM was created with disks in a different order.
-NODE1_DRBD_DISK="${NODE1_DRBD_DISK:-/dev/sdb}"
-NODE2_DRBD_DISK="${NODE2_DRBD_DISK:-/dev/sda}"
+# DRBD backing disk — by-id path that resolves correctly on both nodes
+# regardless of whether the OS-level name is sda or sdb (Proxmox VM disk
+# ordering is not guaranteed). Matches haServerDrbdDisk in variables.nix.
+# Override DRBD_DISK if your hardware uses a different controller/slot path.
+DRBD_DISK="${DRBD_DISK:-/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1}"
 VMID_NODE1="${VMID_NODE1:-}"            # set by deploy.sh; needed for STONITH
 VMID_NODE2="${VMID_NODE2:-}"
 PVE_HOST="${PVE_HOST:-pve1.sweet.home}"
@@ -180,22 +180,32 @@ log "Detaching DRBD on $NODE2..."
 n2_ssh "drbdadm down ha-data 2>/dev/null || true"
 sleep 2
 
-# Verify the DRBD config on each node targets the correct backing disk.
-# Proxmox VM disk ordering is not guaranteed — the data disk may appear as
-# /dev/sda on one node and /dev/sdb on the other.  If the deployed NixOS
-# config was generated before variables.nix had per-node disk assignments, the
-# config may point to the wrong device.  Patch it in-place so drbdadm up
-# attaches to the right disk; rebuild+redeploy to make this permanent.
-n1_conf_disk=$(drbdadm sh-ll-dev ha-data 2>/dev/null || true)
-if [[ -n "$n1_conf_disk" && "$n1_conf_disk" != "$NODE1_DRBD_DISK" ]]; then
-  warn "$NODE1 DRBD config says disk=$n1_conf_disk; patching to $NODE1_DRBD_DISK (redeploy to make permanent)"
-  sed -i "s|${n1_conf_disk}|${NODE1_DRBD_DISK}|g" /etc/drbd.d/*.res 2>/dev/null || true
+# Ensure /etc/drbd.conf on both nodes points to DRBD_DISK (the stable by-id
+# path).  VMs built before this fix may have /dev/sda or /dev/sdb hardcoded.
+# NixOS makes /etc/drbd.conf a symlink into the read-only Nix store, so
+# sed -i on the symlink target would fail — we break the symlink first with
+# cp --remove-destination, creating a regular writable copy.
+# Rebuild+redeploy (--force-rebuild) to make this permanent.
+_PATCH_DRBD=$(mktemp)
+cat > "$_PATCH_DRBD" << 'PATCHEOF'
+#!/bin/bash
+WANT="$1"
+conf=/etc/drbd.conf
+if [[ -L "$conf" ]]; then
+  cp --remove-destination "$(readlink -f "$conf")" "$conf"
 fi
-n2_conf_disk=$(n2_ssh "drbdadm sh-ll-dev ha-data 2>/dev/null" 2>/dev/null || true)
-if [[ -n "$n2_conf_disk" && "$n2_conf_disk" != "$NODE2_DRBD_DISK" ]]; then
-  warn "$NODE2 DRBD config says disk=$n2_conf_disk; patching to $NODE2_DRBD_DISK (redeploy to make permanent)"
-  n2_ssh "sed -i 's|${n2_conf_disk}|${NODE2_DRBD_DISK}|g' /etc/drbd.d/*.res" 2>/dev/null || true
+cur=$(drbdadm sh-ll-dev ha-data 2>/dev/null | head -1 || true)
+if [[ -n "$cur" && "$cur" != "$WANT" ]]; then
+  echo "[cluster-init] WARNING: patching $conf: $cur → $WANT (rebuild to make permanent)"
+  sed -i "s,${cur},${WANT},g" "$conf"
 fi
+PATCHEOF
+chmod +x "$_PATCH_DRBD"
+bash "$_PATCH_DRBD" "$DRBD_DISK"
+n2_scp "$_PATCH_DRBD" "/tmp/patch-drbd-disk.sh"
+n2_ssh "bash /tmp/patch-drbd-disk.sh ${DRBD_DISK}"
+n2_ssh "rm -f /tmp/patch-drbd-disk.sh"
+rm -f "$_PATCH_DRBD"
 
 log "Initialising DRBD metadata on $NODE1..."
 # Use drbdmeta --force directly for BOTH create-md and write-dev-uuid.
@@ -207,15 +217,15 @@ log "Initialising DRBD metadata on $NODE1..."
 # both steps without needing a TTY, regardless of whether the device is busy.
 if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate"; then
   UUID1=$(_rand_uuid)
-  drbdmeta --force 0 v08 "${NODE1_DRBD_DISK}" internal create-md
-  drbdmeta --force 0 v08 "${NODE1_DRBD_DISK}" internal write-dev-uuid "$UUID1"
+  drbdmeta --force 0 v08 "${DRBD_DISK}" internal create-md
+  drbdmeta --force 0 v08 "${DRBD_DISK}" internal write-dev-uuid "$UUID1"
 fi
 
 log "Initialising DRBD metadata on $NODE2..."
 if ! n2_ssh "drbdadm dstate ha-data 2>/dev/null | grep -q UpToDate" 2>/dev/null; then
   UUID2=$(n2_ssh "cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' | cut -c1-16 | tr '[:lower:]' '[:upper:]'")
-  n2_ssh "drbdmeta --force 0 v08 ${NODE2_DRBD_DISK} internal create-md"
-  n2_ssh "drbdmeta --force 0 v08 ${NODE2_DRBD_DISK} internal write-dev-uuid ${UUID2}"
+  n2_ssh "drbdmeta --force 0 v08 ${DRBD_DISK} internal create-md"
+  n2_ssh "drbdmeta --force 0 v08 ${DRBD_DISK} internal write-dev-uuid ${UUID2}"
 fi
 
 log "Bringing up DRBD on both nodes..."

secrets/ha-corosync-authkey

@@ -1,22 +1,22 @@
 {
-	"data": "ENC[AES256_GCM,data:Dqb0FfDXcSofPjHC6wuSjjHJ1VLtZ4RMqU71xBgxOuR+mV6fFP0LoNZcZp0an28XHyf3qLDwnGWvPMf8d9kYNpP0WC+nEqCDvtQjkraJ77j5y57+UNOqdM3tkJ85z5Cm0cQd2DoV4es/mtX6jzDotWW06ESCHtpcefz5SZ4PsYREIT09LmnHdJQ1sFYS0SQxtGa8oLSu7fcobgvV043GS35pq9BICHpCcUzr2LgpDFDpNGU1vSWN9QmwnmKpuf3kQKOYmCJLFsOn96cGnMbgsWc6U3zMOTwOHxgsracG169hlSS8fH7xLWWNgDDq+ILmREsWac5K680j98UXrSbn9g==,iv:iPlhl0GYlLFZOxdU+cHzCoVEkKYbg/X1XTP0lf0Mo0M=,tag:64U+dJ/vO3Q2BMkrtNI6Gw==,type:str]",
+	"data": "ENC[AES256_GCM,data:OhP8SVhryu47dc6zPBaYPrdoMzzRcyIyh82iirbw7LnJuyRfwYzb6VrLkHLXsqZRxb0UzQ7HMNt/o42vvbXIG0nFgfGGfVaaRgMo9ib0ThzldvLGgC23QcgH5oJvOYjzWDQ2FXMVtYzW2ibY80ZsqJUJUj/PyzHDrEi10vTDwPR1VAuEfc2rl05aJGm54dun1fUFO1EqucvjC6T5qbikcQ8duIz5M/1YXhWwWU+8FcX4ebVdbIGM6AqIy4pe8fBgQu5cTJmDUPt/OGAkaftvMV/MYobA9pJ1zhKUPx9yXLHDrQqUHKMCVv4qkpiM1Tlm5g+klqerKM7lm21VG0pn8g==,iv:u9XfPNM/g1DUoEB1MzIbZ01IUj49D1NMXRNGgPOdhaM=,tag:S4jCBUSLOEAVT+ZXetMoVg==,type:str]",
 	"sops": {
 		"age": [
 			{
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpcjBTMVBHZE1MSWhuakVo\nLzJiYm0xYzdzL2RETllTSDBqSU5qZ05rR3g4CjJGV21aU0xSbzlJekg5RDFtUVdX\nSjRWRUdLRnZTVHk5U2pqVytQbXQySUUKLS0tIFVJZXp5Vll0aUczR1pRT3RSeitX\nckYvSFZoTmNrLzgrNTUyYk9vTkJ5UzQK3f5ShqRPTfdonFkHfXc7RWDgBZlhX1+u\npyWitr+t8O26hTPNBDy3jA1mascupr7r8VCVQkgU6rA8ujitAK7x9g==\n-----END AGE ENCRYPTED FILE-----\n",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxQXYvbk5IbTdDZnMwOENs\nU1RMR0J5Y3Mza3pwREc4bDMyblRJUWFaTTJnCkZ1czZKd1dIYUlsRStNanlmYm9i\nd1QrZ0xxWDJad2tsVmF6MjFZa2N4NWcKLS0tIFlxdW15QVljZFJFWWhEaXl2aG4w\ndlIzUGpvVlRDaVJ2QlRHbDhaN1MyYlkKe30ld5MN7DFND+FJLGxghOblkxbq7OPG\nxT5cNbMaewu4JRly3A0EGtqWTjum7K0KRA5CDomEmV+DAg5Vk3Y5tA==\n-----END AGE ENCRYPTED FILE-----\n",
 				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
 			},
 			{
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxZUZXZkx1bVNvUnZMRFRQ\nd2N4QlBRTnoyaHpYWjZxN3JvUUJybW81Wm1BCm9oanJSM1RJemhtZUpIczIrYzFq\neVVjTnFkeUgvaVhDRGtzczJqdkdzL3MKLS0tIGVBRTZHWmw5aE9kcmpTMkkzbFg0\nT0dRVGRoYk1XWnBOWk84QTZzOFZ6UlUKpJieegDxYg5HmhH7HUptoff0UqYqUsIc\nBq3b5ChaZwSScbQzYz+18kBvdGgBEx4fB1q8vY+8SC+wJ5MVMh8lSQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKRm1TRzd0WHNSNUlrMVk3\nV0d0cXFVUWFkQ0FkR2gyWmhJR2dneVRQU25NCmt3UGU2Qy9zTVpNbDhvSTBncG83\nVk40T2JRNVZnODRQZmQ1WlN2Zm9YOGcKLS0tIGRtVzUyem42K0Y5a3BZeGlzWEd3\nN0lkMmo4Yjh0eWxTS2lWUEN6dUFJejgKukeA9kNDKkMIGsiuz9I+Ym9+Z/uZNPsT\npJ+ETkPSKW0gi6qZpr7BqN1KottbhMgFv8WXrtv+tOSIQUZl8eCVNw==\n-----END AGE ENCRYPTED FILE-----\n",
 				"recipient": "age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6"
 			},
 			{
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNWUxqT3BxSXF5bkxsUTRy\nTWZVei9JTHFhcmNNUkRjcklzV0Y1VllXQ0hRCjVTRmQ4WHFmUnM0RzNKYnF1dE4y\nOU1mQ3ZDcjZoVDd4VnlodHR5c1Q2UTgKLS0tIGhhbU1kWmlrWHZPVVZMOVU3RWFV\nM0txajIwK01NNElITXZDWU9KMGJDOVkK8k5hLKWoFEvfNT6TctSHPuIAZu9xj4t8\n5unQOFkAShHx6VcTz0vM8xMGa7bvTfoDHce/HXN42vbuVhvYJUzJRg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNNS9ES0l2VDhjN2c1RWhl\nNFN5ZmhkUlBiNE9EbVlNb2RzaXRicDRpSG5vClBCRmJhTklCY2haV3VYNENPeHZH\nWGZqOW1kWFhwQkw0WVdBNjJUalEzMmsKLS0tIDdiVmdDZDY2YzJuT2tnZHpKRCtK\nQzIwYTBnZUhYc081S3czWjNScjBHeDQKzzihvBhwwpONkg9p66xpYNn4NAKQxLri\n+KhqKFmHw5vPA9aG778adduNzckPs7JquSHCug74OTweoxRVw0gVgA==\n-----END AGE ENCRYPTED FILE-----\n",
 				"recipient": "age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r"
 			}
 		],
-		"lastmodified": "2026-07-28T13:48:23Z",
-		"mac": "ENC[AES256_GCM,data:+2LDHPXVbrjBfIFHjo5Oh+enWy3j0zt85+MMxbMIKOtHOMadXuodlfJN7Igr6GKlMxsP8KvCemfTTL07xuS6T69mrWt8Lpsc/SmA8Xabl99BXtT9oAjD8PlZ85DjBelhuILZ2j1dMW9b1P3b5qfiLvRKnGXw5gqZvuyKmLI9q/k=,iv:FTCyYwi813j3UVMnHPCmMcJwqSkHO4NELsQ7oZlBZk0=,tag:C0luCrScx+23u8RJUeHZYg==,type:str]",
+		"lastmodified": "2026-07-28T14:02:48Z",
+		"mac": "ENC[AES256_GCM,data:rqcQW9NP/5PvoatpgjHBx4A6pCw4HKWUSfewP1Mk7+ywBUz869uhIMWW1PYmdjGiX6pre14ba3u7mQ1eZA5FcxoxgCgfjVDH3Emn98d81zMeZRBdCs/D1Fopq5o+0zml/BVXoITrZiEF+AeVkeFKOGndiGgYbrot8PZEp8z27Ms=,iv:9+0MR52NdL7ockzp8QCfM2kBu9pclYZK0nHYbgNsEq4=,tag:AIQo0imL3Kl5xymcVbnTug==,type:str]",
 		"version": "3.13.3"
 	}
 }

variables.nix

@@ -108,11 +108,11 @@
   haStoragePrefixLength = 29; # storage subnet prefix length (/29)
   haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node
   haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage";
-  # DRBD backing device on each node. Disk ordering can differ between Proxmox
-  # VMs depending on the order disks were added; these must match the actual
-  # block device that is NOT the OS disk on each node (verify with lsblk).
-  haServer1DrbdDisk = "/dev/sdb"; # data disk on ha-server-1 (OS disk is /dev/sda)
-  haServer2DrbdDisk = "/dev/sda"; # data disk on ha-server-2 (OS disk is /dev/sdb)
+  # DRBD backing disk — identified by SCSI controller path so it resolves to the
+  # correct block device regardless of OS-level naming (sda vs sdb can differ
+  # between Proxmox VMs depending on disk-add order). drive-scsi1 is always the
+  # dedicated data disk on all HA nodes; drive-scsi0 is the OS disk.
+  haServerDrbdDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1";
 
   # Storage
   storageRoot = "/tank"; # ZFS pool root on `server`