test-lab: add two-node HA file-server test cluster config

Disposable test VMs (ha-test-node1 / ha-test-node2, VMIDs 200/201 on pve1)
to evaluate whether the DRBD + XFS + LIO + Corosync + Pacemaker stack
runs correctly on NixOS before deciding NixOS vs Debian for production.

Includes:
- test-lab/ha/disko.nix: 20G boot disk layout (smaller than production)
- test-lab/ha/common.nix: shared HA stack (drbd, corosync, pacemaker,
  targetcli-fb, xfsprogs), OCF PATH workaround for nixpkgs#207891
- test-lab/ha/node1.nix / node2.nix: per-node hostname + static IP
- test-lab/ha/fence-pve-ssh.py: Proxmox SSH fence agent for STONITH
- test-lab/ha/cluster-init.sh: one-shot cluster bootstrap script
- test-lab/ha/cluster-enable-stonith.sh: enables STONITH post-key-deploy
- flake.nix: adds ha-test-node1 / ha-test-node2 nixosConfigurations
  (bypasses mkTarget / clan-core / sops-nix — test-only)

These VMs must be destroyed once acceptance testing is complete.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

flake.nix

@@ -132,6 +132,34 @@
         lxc-tor-relay = mkTarget { platform = "lxc"; buildType = "tor-relay"; hostPath = ./hosts/tor-relay/host.nix; };
       };
 
+      # ── HA test lab VMs ─────────────────────────────────────────────────
+      # Two throwaway NixOS VMs to test the DRBD + XFS + LIO + Pacemaker
+      # stack.  Deliberately bypass mkTarget (no clan-core, no sops-nix,
+      # no home-manager) — these are disposable and must be destroyed once
+      # the acceptance tests are done.
+      # Build disk images with:
+      #   nix build .#nixosConfigurations.ha-test-node1.config.system.build.diskoImagesScript
+      # then run the script to produce ha-test-node1.raw (import with qm importdisk).
+      haTestTargets =
+        let
+          haNode = { nodeModule }: nixpkgs.lib.nixosSystem {
+            inherit system;
+            modules = [
+              inputs.disko.nixosModules.disko
+              ./modules/hardware-configuration/vm/proxmox.nix
+              ./modules/boot/efi.nix
+              ./test-lab/ha/disko.nix
+              ./test-lab/ha/common.nix
+              nodeModule
+            ];
+            specialArgs = { inherit vars; };
+          };
+        in
+        {
+          ha-test-node1 = haNode { nodeModule = ./test-lab/ha/node1.nix; };
+          ha-test-node2 = haNode { nodeModule = ./test-lab/ha/node2.nix; };
+        };
+
       # Auto-install environments (migrated from the former nix-auto-installer
       # flake): a self-contained NixOS installer that boots, discovers this
       # flake's own nixosConfigurations over the network, and runs
@@ -207,7 +235,7 @@
     in
     {
 
-      nixosConfigurations = generatedTargets // installerTargets;
+      nixosConfigurations = generatedTargets // haTestTargets // installerTargets;
 
       # Buildable auto-installer artifacts (`nix build .#<name>`). No `lxc`
       # variant (installer-boots-as-an-LXC-container) or `all` bundle

test-lab/ha/cluster-enable-stonith.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# cluster-enable-stonith.sh — enable STONITH fence agent after fence key is deployed
+# Run from ha-test-node1 as root, AFTER:
+#   - /etc/fence-pve-ssh-key exists on both nodes
+#   - The fence public key is in authorized_keys on pve1.sweet.home
+set -euo pipefail
+
+VMID_NODE1="200"
+VMID_NODE2="201"
+PVE_HOST="pve1.sweet.home"
+PVE_USER="wayne"
+FENCE_KEY="/etc/fence-pve-ssh-key"
+FENCE_SCRIPT="/usr/lib/ocf/resource.d/heartbeat/fence_pve_ssh"
+
+log() { echo "[stonith-setup] $*"; }
+die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; }
+
+[[ $(id -u) -eq 0 ]] || die "must run as root"
+
+[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY"
+[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT"
+
+log "Verifying fence agent can reach ${PVE_HOST}..."
+if ! ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \
+     -o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" "sudo /usr/sbin/qm list" &>/dev/null; then
+  die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo"
+fi
+log "Fence agent SSH connectivity confirmed"
+
+log "Creating Pacemaker STONITH resource..."
+cibadmin --create --scope resources --xml-text "
+<primitive id=\"stonith-pve-node1\" class=\"stonith\" type=\"external/fence_pve_ssh\">
+  <instance_attributes id=\"stonith-pve-node1-attrs\">
+    <nvpair id=\"stonith-node1-plug\" name=\"plug\" value=\"ha-test-node1\"/>
+    <nvpair id=\"stonith-node1-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
+    <nvpair id=\"stonith-node1-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
+    <nvpair id=\"stonith-node1-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
+    <nvpair id=\"stonith-node1-vmid-node1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
+    <nvpair id=\"stonith-node1-vmid-node2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
+    <nvpair id=\"stonith-node1-pcmk_host_list\" name=\"pcmk_host_list\" value=\"ha-test-node1\"/>
+  </instance_attributes>
+  <operations>
+    <op id=\"stonith-node1-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
+  </operations>
+</primitive>
+" 2>/dev/null || true
+
+cibadmin --create --scope resources --xml-text "
+<primitive id=\"stonith-pve-node2\" class=\"stonith\" type=\"external/fence_pve_ssh\">
+  <instance_attributes id=\"stonith-pve-node2-attrs\">
+    <nvpair id=\"stonith-node2-plug\" name=\"plug\" value=\"ha-test-node2\"/>
+    <nvpair id=\"stonith-node2-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
+    <nvpair id=\"stonith-node2-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
+    <nvpair id=\"stonith-node2-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
+    <nvpair id=\"stonith-node2-vmid-node1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
+    <nvpair id=\"stonith-node2-vmid-node2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
+    <nvpair id=\"stonith-node2-pcmk_host_list\" name=\"pcmk_host_list\" value=\"ha-test-node2\"/>
+  </instance_attributes>
+  <operations>
+    <op id=\"stonith-node2-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
+  </operations>
+</primitive>
+" 2>/dev/null || true
+
+log "Enabling STONITH..."
+crm_attribute -t crm_config -n stonith-enabled -v true
+
+# Restore quorum policy to stop (needed with STONITH)
+crm_attribute -t crm_config -n no-quorum-policy -v stop
+
+log "STONITH enabled. Testing fence agent..."
+if stonith_admin --list-devices; then
+  log "Fence devices listed successfully"
+else
+  log "WARNING: fence device list failed — check stonith config"
+fi
+
+log "STONITH setup complete. Cluster is now fully HA."

test-lab/ha/cluster-init.sh

@@ -0,0 +1,239 @@
+#!/usr/bin/env bash
+# cluster-init.sh — one-time HA cluster initialisation script
+#
+# Run this ONCE from node1 AFTER both VMs are booted and have SSH access.
+# It:
+#   1. Waits for corosync quorum on both nodes
+#   2. Initialises DRBD metadata and promotes node1 to primary
+#   3. Creates XFS filesystem on /dev/drbd0
+#   4. Configures targetcli / LIO iSCSI target (with a file-backed LUN)
+#   5. Configures the Pacemaker resource group
+#   6. Optionally enables the STONITH fence agent (requires fence SSH key)
+#
+# Prerequisites:
+#   - Both VMs booted with the ha-test config
+#   - fence-pve-ssh-key distributed to /etc/fence-pve-ssh-key on both nodes
+#   - Run as root on ha-test-node1
+set -euo pipefail
+
+NODE1_IP="192.168.2.200"
+NODE2_IP="192.168.2.201"
+VIP="192.168.2.202"
+DRBD_DEVICE="/dev/drbd0"
+XFS_MOUNT="/mnt/ha-data"
+ISCSI_IQN="iqn.2026-01.local.ha-test:storage"
+ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
+ISCSI_LUN_SIZE="1G"   # small test LUN
+VMID_NODE1="200"
+VMID_NODE2="201"
+PVE_HOST="pve1.sweet.home"
+PVE_USER="wayne"
+FENCE_KEY="/etc/fence-pve-ssh-key"
+
+log() { echo "[cluster-init] $*"; }
+die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; }
+
+[[ $(id -u) -eq 0 ]] || die "must run as root"
+[[ "$(hostname)" == "ha-test-node1" ]] || die "must run on ha-test-node1"
+
+# ── 1. Wait for corosync quorum ──────────────────────────────────────────
+log "Waiting for corosync quorum..."
+for i in $(seq 1 30); do
+  if corosync-quorumtool -q &>/dev/null; then
+    log "Quorum established"
+    break
+  fi
+  [[ $i -eq 30 ]] && die "corosync quorum not established after 30s"
+  sleep 2
+done
+
+log "Waiting for pacemaker to start..."
+for i in $(seq 1 30); do
+  if crm_mon -1 &>/dev/null; then
+    log "Pacemaker running"
+    break
+  fi
+  [[ $i -eq 30 ]] && die "pacemaker not running after 60s"
+  sleep 2
+done
+
+# ── 2. Initialise DRBD ───────────────────────────────────────────────────
+log "Initialising DRBD metadata on node1..."
+if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent"; then
+  drbdadm create-md ha-data --force
+fi
+
+log "Initialising DRBD metadata on node2..."
+if ! ssh "root@${NODE2_IP}" "drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent'"; then
+  ssh "root@${NODE2_IP}" "drbdadm create-md ha-data --force"
+fi
+
+log "Bringing up DRBD on both nodes..."
+drbdadm up ha-data || true
+ssh "root@${NODE2_IP}" "drbdadm up ha-data" || true
+
+log "Forcing node1 to DRBD primary (initial sync)..."
+drbdadm primary ha-data --force
+
+log "Waiting for DRBD to finish initial sync..."
+for i in $(seq 1 120); do
+  state=$(drbdadm dstate ha-data)
+  if echo "$state" | grep -q "UpToDate"; then
+    log "DRBD sync complete: $state"
+    break
+  fi
+  log "  DRBD state: $state (${i}/120s)"
+  [[ $i -eq 120 ]] && die "DRBD did not sync within 120s"
+  sleep 1
+done
+
+# ── 3. XFS filesystem ────────────────────────────────────────────────────
+log "Creating XFS on ${DRBD_DEVICE}..."
+if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
+  mkfs.xfs "${DRBD_DEVICE}"
+fi
+
+log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
+mkdir -p "${XFS_MOUNT}"
+mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
+
+# ── 4. iSCSI LUN (file-backed) ───────────────────────────────────────────
+log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..."
+if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then
+  fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}"
+fi
+
+log "Configuring LIO iSCSI target via targetcli..."
+# This produces a /etc/target/saveconfig.json that the targetctl service loads.
+# The commands create an iSCSI target backed by the file we just created.
+targetcli <<'EOF'
+/backstores/fileio create name=ha-lun0 file_or_dev=/mnt/ha-data/iscsi-lun.img size=0 write_back=false
+/iscsi create iqn.2026-01.local.ha-test:storage
+/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/luns create /backstores/fileio/ha-lun0
+/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/portals create 192.168.2.202
+/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute authentication=0
+/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute demo_mode_write_protect=0
+saveconfig /etc/target/saveconfig.json
+EOF
+
+log "Unmounting ${XFS_MOUNT} (Pacemaker will manage it)..."
+umount "${XFS_MOUNT}"
+
+log "Promoting DRBD back to secondary (Pacemaker manages primary role)..."
+drbdadm secondary ha-data
+
+# ── 5. Pacemaker resources ───────────────────────────────────────────────
+log "Configuring Pacemaker..."
+
+# Disable STONITH initially — enable once fence key is deployed
+crm_attribute -t crm_config -n stonith-enabled -v false
+
+# Disable quorum policy for two-node cluster (no-quorum-policy=ignore so
+# the surviving node can promote without a quorum device)
+crm_attribute -t crm_config -n no-quorum-policy -v ignore
+
+# Cluster resources:
+#   1. drbd-ha      — manages DRBD primary/secondary role
+#   2. xfs-mount    — XFS mount on /mnt/ha-data
+#   3. iscsi-target — LIO target service (systemd class)
+#   4. vip          — floating VIP 192.168.2.202
+
+log "Creating DRBD master/slave resource..."
+cibadmin --replace --scope resources --xml-text "
+<resources>
+  <master id=\"drbd-ha-ms\" globally-unique=\"false\">
+    <primitive id=\"drbd-ha\" class=\"ocf\" type=\"drbd\" provider=\"heartbeat\">
+      <instance_attributes id=\"drbd-ha-attrs\">
+        <nvpair id=\"drbd-ha-drbd_resource\" name=\"drbd_resource\" value=\"ha-data\"/>
+      </instance_attributes>
+      <operations>
+        <op id=\"drbd-ha-start\" name=\"start\" interval=\"0\" timeout=\"240s\"/>
+        <op id=\"drbd-ha-stop\" name=\"stop\" interval=\"0\" timeout=\"120s\"/>
+        <op id=\"drbd-ha-promote\" name=\"promote\" interval=\"0\" timeout=\"90s\"/>
+        <op id=\"drbd-ha-demote\" name=\"demote\" interval=\"0\" timeout=\"90s\"/>
+        <op id=\"drbd-ha-monitor-master\" name=\"monitor\" interval=\"20s\" timeout=\"20s\" role=\"Master\"/>
+        <op id=\"drbd-ha-monitor-slave\" name=\"monitor\" interval=\"30s\" timeout=\"20s\" role=\"Slave\"/>
+      </operations>
+    </primitive>
+    <meta_attributes id=\"drbd-ha-ms-meta\">
+      <nvpair id=\"drbd-ha-ms-master-max\" name=\"master-max\" value=\"1\"/>
+      <nvpair id=\"drbd-ha-ms-master-node-max\" name=\"master-node-max\" value=\"1\"/>
+      <nvpair id=\"drbd-ha-ms-clone-max\" name=\"clone-max\" value=\"2\"/>
+      <nvpair id=\"drbd-ha-ms-clone-node-max\" name=\"clone-node-max\" value=\"1\"/>
+      <nvpair id=\"drbd-ha-ms-notify\" name=\"notify\" value=\"true\"/>
+      <nvpair id=\"drbd-ha-ms-interleave\" name=\"interleave\" value=\"true\"/>
+    </meta_attributes>
+  </master>
+
+  <primitive id=\"xfs-mount\" class=\"ocf\" type=\"Filesystem\" provider=\"heartbeat\">
+    <instance_attributes id=\"xfs-mount-attrs\">
+      <nvpair id=\"xfs-mount-device\" name=\"device\" value=\"/dev/drbd0\"/>
+      <nvpair id=\"xfs-mount-directory\" name=\"directory\" value=\"${XFS_MOUNT}\"/>
+      <nvpair id=\"xfs-mount-fstype\" name=\"fstype\" value=\"xfs\"/>
+      <nvpair id=\"xfs-mount-options\" name=\"options\" value=\"defaults\"/>
+    </instance_attributes>
+    <operations>
+      <op id=\"xfs-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
+      <op id=\"xfs-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
+      <op id=\"xfs-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
+    </operations>
+  </primitive>
+
+  <primitive id=\"iscsi-target\" class=\"systemd\" type=\"targetctl\">
+    <operations>
+      <op id=\"iscsi-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
+      <op id=\"iscsi-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
+      <op id=\"iscsi-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
+    </operations>
+  </primitive>
+
+  <primitive id=\"vip\" class=\"ocf\" type=\"IPaddr2\" provider=\"heartbeat\">
+    <instance_attributes id=\"vip-attrs\">
+      <nvpair id=\"vip-ip\" name=\"ip\" value=\"${VIP}\"/>
+      <nvpair id=\"vip-cidr\" name=\"cidr_netmask\" value=\"24\"/>
+    </instance_attributes>
+    <operations>
+      <op id=\"vip-start\" name=\"start\" interval=\"0\" timeout=\"20s\"/>
+      <op id=\"vip-stop\" name=\"stop\" interval=\"0\" timeout=\"20s\"/>
+      <op id=\"vip-monitor\" name=\"monitor\" interval=\"10s\" timeout=\"20s\"/>
+    </operations>
+  </primitive>
+</resources>
+"
+
+log "Adding ordering and colocation constraints..."
+# All resources on the same node as DRBD master
+cibadmin --create --scope constraints --xml-text "
+<constraints>
+  <rsc_order id=\"order-drbd-xfs\" first=\"drbd-ha-ms\" first-action=\"promote\" then=\"xfs-mount\" then-action=\"start\"/>
+  <rsc_order id=\"order-xfs-iscsi\" first=\"xfs-mount\" then=\"iscsi-target\"/>
+  <rsc_order id=\"order-iscsi-vip\" first=\"iscsi-target\" then=\"vip\"/>
+  <rsc_colocation id=\"coloc-all-with-drbd\" rsc=\"xfs-mount\" with-rsc=\"drbd-ha-ms\" with-rsc-role=\"Master\" score=\"INFINITY\"/>
+  <rsc_colocation id=\"coloc-iscsi-with-xfs\" rsc=\"iscsi-target\" with-rsc=\"xfs-mount\" score=\"INFINITY\"/>
+  <rsc_colocation id=\"coloc-vip-with-iscsi\" rsc=\"vip\" with-rsc=\"iscsi-target\" score=\"INFINITY\"/>
+</constraints>
+"
+
+log "Resource group configured. Waiting for resources to start..."
+for i in $(seq 1 60); do
+  if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then
+    log "VIP is up: $(crm_resource -r vip --locate)"
+    break
+  fi
+  [[ $i -eq 60 ]] && { log "WARNING: VIP not up after 60s — check crm_mon"; break; }
+  sleep 2
+done
+
+log ""
+log "═══════════════════════════════════════════════════════"
+log " HA cluster initialised. Next steps:"
+log ""
+log "  - Verify: crm_mon -1"
+log "  - Test iSCSI: iscsiadm -m discovery -t sendtargets -p ${VIP}"
+log ""
+log "  To enable STONITH (after deploying fence key):"
+log "    1. Copy fence-pve-ssh.py to /usr/lib/ocf/resource.d/heartbeat/ on both nodes"
+log "    2. Distribute /etc/fence-pve-ssh-key to both nodes"
+log "    3. Add public key to authorized_keys on ${PVE_HOST}"
+log "    4. Run: ./cluster-enable-stonith.sh"
+log "═══════════════════════════════════════════════════════"

test-lab/ha/common.nix

@@ -0,0 +1,256 @@
+# Shared HA stack config for both test nodes.
+# These are throwaway test VMs — not production hosts.
+# No sops-nix, no clan, no home-manager.
+{ lib, pkgs, vars, ... }:
+
+let
+  node1Ip = "192.168.2.200";
+  node2Ip = "192.168.2.201";
+  drbdPort = 7789;
+
+  # Test-only corosync authkey (128 bytes minimum).
+  # Not secret — this is a disposable test cluster, not production.
+  testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+
+  # OCF agent path (from pacemaker's --with-ocfdir).
+  # We expose this on PATH so custom scripts can find each other.
+  ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d";
+
+  # Concatenated PATH that includes all binaries OCF agents and pacemaker
+  # lrmd children need.  This is the workaround for nixpkgs#207891 (PATH
+  # not set correctly for OCF agent child processes).
+  ocfBinPath = lib.concatStringsSep ":" [
+    "${pkgs.iproute2}/bin"
+    "${pkgs.iproute2}/sbin"
+    "${pkgs.iputils}/bin"
+    "${pkgs.util-linux}/bin"
+    "${pkgs.util-linux}/sbin"
+    "${pkgs.gawk}/bin"
+    "${pkgs.gnugrep}/bin"
+    "${pkgs.gnused}/bin"
+    "${pkgs.coreutils}/bin"
+    "${pkgs.bash}/bin"
+    "${pkgs.procps}/bin"
+    "${pkgs.xfsprogs}/bin"
+    "${pkgs.drbd}/bin"
+    "${pkgs.targetcli-fb}/bin"
+    "${pkgs.python3}/bin"
+    "/run/current-system/sw/bin"
+    "/run/current-system/sw/sbin"
+    "/usr/local/sbin"
+    "/usr/local/bin"
+    "/usr/sbin"
+    "/usr/bin"
+    "/sbin"
+    "/bin"
+  ];
+in
+{
+  system.stateVersion = "26.05";
+
+  # ── Hardware (Proxmox VM) ──────────────────────────────────────────────
+  imports = [
+    ../../modules/hardware-configuration/vm/proxmox.nix
+    ../../modules/boot/efi.nix
+  ];
+
+  # ── Nix settings ──────────────────────────────────────────────────────
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+
+  # ── SSH ───────────────────────────────────────────────────────────────
+  services.openssh = {
+    enable = true;
+    settings.PermitRootLogin = "yes";
+  };
+  users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey ];
+
+  # ── Networking ────────────────────────────────────────────────────────
+  networking.useDHCP = false;
+  networking.defaultGateway = "192.168.2.1";
+  networking.nameservers = [ "192.168.2.1" "8.8.8.8" ];
+
+  # ── DRBD ──────────────────────────────────────────────────────────────
+  services.drbd.enable = true;
+  services.drbd.config = ''
+    global {
+      usage-count yes;
+    }
+
+    common {
+      net {
+        protocol C;
+        ping-int 1;
+        verify-alg sha256;
+        after-sb-0pri discard-zero-changes;
+        after-sb-1pri discard-secondary;
+      }
+      disk {
+        # resource-only: DRBD itself won't fence (Pacemaker handles STONITH);
+        # the DRBD resource agent uses fencing to guard primary promotion.
+        fencing resource-only;
+      }
+      handlers {
+        # Called by DRBD when a split-brain is detected and this node is
+        # in the secondary role — notifies pacemaker to fence the split node.
+        split-brain "/usr/lib/drbd/notify-split-brain.sh root";
+        before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k";
+        after-resync-target "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh";
+      }
+    }
+
+    resource ha-data {
+      volume 0 {
+        device    /dev/drbd0;
+        disk      /dev/sdb;   # scsi1 in Proxmox VM → sdb
+        meta-disk internal;
+      }
+
+      on ha-test-node1 {
+        address ${node1Ip}:${toString drbdPort};
+      }
+
+      on ha-test-node2 {
+        address ${node2Ip}:${toString drbdPort};
+      }
+    }
+  '';
+
+  # ── Corosync ──────────────────────────────────────────────────────────
+  services.corosync = {
+    enable = true;
+    clusterName = "ha-test";
+    nodelist = [
+      { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; }
+      { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; }
+    ];
+  };
+
+  # Corosync authkey (test-only, not secret — generated with
+  # `corosync-keygen` for production).
+  environment.etc."corosync/authkey" = {
+    source = builtins.toFile "authkey" testAuthKey;
+    mode = "0400";
+  };
+
+  # ── Pacemaker ─────────────────────────────────────────────────────────
+  services.pacemaker.enable = true;
+
+  # Fix for nixpkgs#207891:
+  # 1. Ensure CIB directories are owned by hacluster before starting.
+  #    The stock module sets StateDirectory=pacemaker (owned by root);
+  #    pacemaker internally drops to hacluster but needs to write there.
+  # 2. Set PATH so OCF agent child processes can find all required binaries.
+  systemd.services.pacemaker.serviceConfig = {
+    ExecStartPre = [
+      "${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'"
+    ];
+  };
+  systemd.services.pacemaker.environment = {
+    PATH = lib.mkForce ocfBinPath;
+    # Expose OCF root so pacemaker and lrmd agree on where agents live.
+    OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
+  };
+
+  # pacemaker-execd is the local resource executor that calls OCF agents.
+  # Give it the same PATH so OCF scripts can find all required binaries.
+  systemd.services.pacemaker-execd.environment = {
+    PATH = lib.mkForce ocfBinPath;
+    OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
+  };
+
+  # ── LIO / iSCSI target ────────────────────────────────────────────────
+  # targetcli-fb is the management tool; actual kernel support is via
+  # the LIO modules.  We add a systemd service that saves/restores the
+  # target configuration so Pacemaker can trigger it via a systemd-class
+  # resource.
+  boot.kernelModules = [
+    "target_core_mod"
+    "iscsi_target_mod"
+    "target_core_file"
+    "target_core_pscsi"
+    "target_core_user"
+    "configfs"
+  ];
+
+  # configfs must be mounted for rtslib/targetcli to work
+  systemd.mounts = [{
+    where = "/sys/kernel/config";
+    what = "configfs";
+    type = "configfs";
+    wantedBy = [ "multi-user.target" ];
+    before = [ "targetctl.service" ];
+  }];
+
+  # targetctl: save/restore LIO configuration (mirrors Debian's package)
+  systemd.services.targetctl = {
+    description = "LIO iSCSI target config save/restore";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "sys-kernel-config.mount" "network.target" ];
+    requires = [ "sys-kernel-config.mount" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json";
+      ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json";
+    };
+    unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
+  };
+
+  # ── Packages ──────────────────────────────────────────────────────────
+  environment.systemPackages = with pkgs; [
+    # HA stack
+    corosync        # corosync-cfgtool, corosync-quorumtool
+    pacemaker       # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI
+    drbd            # drbdadm, drbdsetup, drbdmon
+    ocf-resource-agents  # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …)
+
+    # Storage
+    xfsprogs        # mkfs.xfs, xfs_admin, xfs_info
+    targetcli-fb    # targetcli shell + targetctl
+
+    # Networking / debug
+    iproute2        # ip, ss
+    iputils         # ping
+    tcpdump
+    lsof
+
+    # Scripting / config
+    python3
+    curl
+    jq
+    vim
+    htop
+  ];
+
+  # ── Firewall ──────────────────────────────────────────────────────────
+  networking.firewall = {
+    enable = true;
+    allowedTCPPorts = [
+      22      # SSH
+      3260    # iSCSI
+      3121    # pacemaker-remoted
+      2224    # pcsd
+      drbdPort # DRBD replication
+    ];
+    allowedUDPPorts = [
+      5404    # corosync cluster
+      5405    # corosync cluster
+      5407    # corosync crypto
+    ];
+    # Corosync uses ports 5404-5407 UDP; allow them on the cluster net
+    extraCommands = ''
+      iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT
+      iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT
+    '';
+  };
+
+  # ── tmpfiles: target config dir ────────────────────────────────────────
+  systemd.tmpfiles.rules = [
+    "d /etc/target 0750 root root -"
+    "f /etc/target/saveconfig.json 0640 root root -"
+  ];
+
+  # ── Locale / time ─────────────────────────────────────────────────────
+  time.timeZone = vars.timeZone;
+  i18n.defaultLocale = "en_AU.UTF-8";
+}

test-lab/ha/disko.nix

@@ -0,0 +1,49 @@
+{ config, ... }:
+
+# Smaller disk layout for throwaway test VMs (20G vs production 50G).
+# Same partition scheme as modules/disko/proxmox.nix: GPT, ESP + swap + ext4 root.
+# Only covers the boot disk (scsi0 → /dev/sda).  The DRBD data disk
+# (scsi1 → /dev/sdb) is left raw — drbdadm create-md initialises it.
+{
+  disko.devices.disk.main = {
+    type = "disk";
+    device = "/dev/sda";
+    imageSize = "20G";
+    imageName = config.networking.hostName;
+
+    content = {
+      type = "gpt";
+      partitions = {
+        esp = {
+          priority = 1;
+          name = "ESP";
+          size = "512M";
+          type = "EF00";
+          content = {
+            type = "filesystem";
+            format = "vfat";
+            mountpoint = "/boot";
+            mountOptions = [ "umask=0077" ];
+            extraArgs = [ "-F" "32" "-n" "boot" ];
+          };
+        };
+        swap = {
+          size = "2G";
+          content = {
+            type = "swap";
+            randomEncryption = false;
+          };
+        };
+        root = {
+          size = "100%";
+          content = {
+            type = "filesystem";
+            format = "ext4";
+            mountpoint = "/";
+            extraArgs = [ "-L" "nixos" ];
+          };
+        };
+      };
+    };
+  };
+}

test-lab/ha/fence-pve-ssh.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""
+fence_pve_ssh - Proxmox VE SSH fence agent for Pacemaker.
+
+Uses SSH to reach pve1.sweet.home and run 'qm stop/start <vmid>'.
+Designed for test-lab HA cluster only — not for production.
+
+Configuration (as pacemaker stonith resource attributes):
+  pve_host     Proxmox host to SSH to (default: pve1.sweet.home)
+  pve_user     SSH user (default: wayne)
+  key_file     SSH private key path (default: /etc/fence-pve-ssh-key)
+  vmid_node1   VMID for ha-test-node1 (e.g. 200)
+  vmid_node2   VMID for ha-test-node2 (e.g. 201)
+  plug         Node name to act on (set by pacemaker: ha-test-node1 or ha-test-node2)
+  action       Action: off|on|reboot|status|list|metadata
+"""
+
+import argparse
+import subprocess
+import sys
+import os
+
+
+METADATA = """<?xml version="1.0" ?>
+<resource-agent name="fence_pve_ssh" shortdesc="Proxmox VE SSH fence agent (test lab)">
+  <longdesc>Fences a VM on a Proxmox VE host by SSHing to the PVE host and
+  running qm stop/start.  For test use only.</longdesc>
+  <vendor-url>https://proxmox.com</vendor-url>
+  <parameters>
+    <parameter name="action" required="1" unique="0">
+      <getopt mixed="-a, --action=[action]"/>
+      <content type="string" default="reboot"/>
+      <shortdesc lang="en">Fencing action: off|on|reboot|status|list</shortdesc>
+    </parameter>
+    <parameter name="plug" required="0" unique="0">
+      <getopt mixed="-n, --plug=[nodename]"/>
+      <content type="string"/>
+      <shortdesc lang="en">Cluster node name to fence</shortdesc>
+    </parameter>
+    <parameter name="pve_host" required="0" unique="0">
+      <getopt mixed="--pve-host=[host]"/>
+      <content type="string" default="pve1.sweet.home"/>
+      <shortdesc lang="en">Proxmox VE host to SSH to</shortdesc>
+    </parameter>
+    <parameter name="pve_user" required="0" unique="0">
+      <getopt mixed="--pve-user=[user]"/>
+      <content type="string" default="wayne"/>
+      <shortdesc lang="en">SSH user on the Proxmox host</shortdesc>
+    </parameter>
+    <parameter name="key_file" required="0" unique="0">
+      <getopt mixed="--key-file=[path]"/>
+      <content type="string" default="/etc/fence-pve-ssh-key"/>
+      <shortdesc lang="en">SSH private key file path</shortdesc>
+    </parameter>
+    <parameter name="vmid_node1" required="1" unique="0">
+      <getopt mixed="--vmid-node1=[vmid]"/>
+      <content type="string"/>
+      <shortdesc lang="en">VMID for ha-test-node1</shortdesc>
+    </parameter>
+    <parameter name="vmid_node2" required="1" unique="0">
+      <getopt mixed="--vmid-node2=[vmid]"/>
+      <content type="string"/>
+      <shortdesc lang="en">VMID for ha-test-node2</shortdesc>
+    </parameter>
+  </parameters>
+  <actions>
+    <action name="off" timeout="60s"/>
+    <action name="on" timeout="60s"/>
+    <action name="reboot" timeout="60s"/>
+    <action name="status" timeout="30s"/>
+    <action name="list" timeout="10s"/>
+    <action name="metadata" timeout="5s"/>
+  </actions>
+</resource-agent>
+"""
+
+
+def parse_args():
+    p = argparse.ArgumentParser(add_help=False)
+    p.add_argument("-a", "--action", default="reboot")
+    p.add_argument("-n", "--plug")
+    p.add_argument("--pve-host", default="pve1.sweet.home")
+    p.add_argument("--pve-user", default="wayne")
+    p.add_argument("--key-file", default="/etc/fence-pve-ssh-key")
+    p.add_argument("--vmid-node1")
+    p.add_argument("--vmid-node2")
+    # Allow remaining unknown args (pacemaker may pass extra ones)
+    return p.parse_known_args()[0]
+
+
+def ssh(pve_host, pve_user, key_file, cmd):
+    result = subprocess.run(
+        [
+            "ssh",
+            "-i", key_file,
+            "-o", "StrictHostKeyChecking=no",
+            "-o", "BatchMode=yes",
+            "-o", "ConnectTimeout=10",
+            f"{pve_user}@{pve_host}",
+            cmd,
+        ],
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    return result
+
+
+def get_vmid(args):
+    node = args.plug
+    if not node:
+        print("ERROR: --plug not specified", file=sys.stderr)
+        sys.exit(1)
+    mapping = {
+        "ha-test-node1": args.vmid_node1,
+        "ha-test-node2": args.vmid_node2,
+    }
+    vmid = mapping.get(node)
+    if not vmid:
+        print(f"ERROR: unknown node '{node}'", file=sys.stderr)
+        sys.exit(1)
+    return vmid
+
+
+def main():
+    args = parse_args()
+    action = args.action.lower()
+
+    if action == "metadata":
+        print(METADATA)
+        sys.exit(0)
+
+    if action == "list":
+        if args.vmid_node1:
+            print("ha-test-node1")
+        if args.vmid_node2:
+            print("ha-test-node2")
+        sys.exit(0)
+
+    vmid = get_vmid(args)
+
+    if not os.path.exists(args.key_file):
+        print(f"ERROR: SSH key not found at {args.key_file}", file=sys.stderr)
+        sys.exit(1)
+
+    if action in ("off", "reboot"):
+        print(f"Stopping VM {vmid} ({args.plug}) on {args.pve_host}...")
+        r = ssh(args.pve_host, args.pve_user, args.key_file,
+                f"sudo /usr/sbin/qm stop {vmid}")
+        if r.returncode != 0:
+            print(f"ERROR stopping VM: {r.stderr}", file=sys.stderr)
+            sys.exit(1)
+        print(f"VM {vmid} stopped")
+
+    if action in ("on", "reboot"):
+        print(f"Starting VM {vmid} ({args.plug}) on {args.pve_host}...")
+        r = ssh(args.pve_host, args.pve_user, args.key_file,
+                f"sudo /usr/sbin/qm start {vmid}")
+        if r.returncode != 0:
+            print(f"ERROR starting VM: {r.stderr}", file=sys.stderr)
+            sys.exit(1)
+        print(f"VM {vmid} started")
+
+    if action == "status":
+        r = ssh(args.pve_host, args.pve_user, args.key_file,
+                f"sudo /usr/sbin/qm status {vmid}")
+        if r.returncode != 0:
+            print(f"ERROR querying VM status: {r.stderr}", file=sys.stderr)
+            sys.exit(1)
+        # qm status returns "status: running" or "status: stopped"
+        status_line = r.stdout.strip()
+        print(status_line)
+        if "stopped" in status_line:
+            sys.exit(2)   # pacemaker interprets exit 2 as "off"
+        sys.exit(0)       # running = exit 0
+
+
+if __name__ == "__main__":
+    main()

test-lab/ha/node1.nix

@@ -0,0 +1,12 @@
+{ ... }:
+
+# ha-test-node1: VMID 200, 192.168.2.200/24
+{
+  networking.hostName = "ha-test-node1";
+  networking.hostId = "a1b2c3d4";   # random, required by ZFS (not used here) but harmless
+
+  networking.interfaces.ens18.ipv4.addresses = [{
+    address = "192.168.2.200";
+    prefixLength = 24;
+  }];
+}

test-lab/ha/node2.nix

@@ -0,0 +1,12 @@
+{ ... }:
+
+# ha-test-node2: VMID 201, 192.168.2.201/24
+{
+  networking.hostName = "ha-test-node2";
+  networking.hostId = "e5f6a7b8";   # random, required by ZFS (not used here) but harmless
+
+  networking.interfaces.ens18.ipv4.addresses = [{
+    address = "192.168.2.201";
+    prefixLength = 24;
+  }];
+}