feat(ha): promote HA file server to production flake targets

Adds proxmox-ha-server-1 and proxmox-ha-server-2 as real mkTarget entries
alongside the existing proxmox-server, backed by a new ha-server build type.

New modules
  modules/ha/cluster-config.nix — DRBD resource + corosync nodelist sourced
    from vars (haServer1Host/Ip, haServer2Host/Ip); resource-only fencing for
    production STONITH; HA port firewall rules for DRBD, iSCSI, Corosync, pcsd
  modules/build-types/ha-server.nix — imports pacemaker-stack + iscsi-target
    + cluster-config + beszel; NFS exports from vars.haStorageRoot (XFS-over-DRBD
    mount); nfs-server.service.wantedBy force-cleared so Pacemaker controls
    start/stop on the Active node only

New hosts
  hosts/ha-server-{1,2}/host.nix — static IP from vars, unique hostId; sops
    secrets (beszel, corosync authkey) are TODOs pending sync-host-keys.sh

variables.nix
  haServer1/2Host, haServer1/2Ip, haServerVip, haStorageRoot, haIscsiIqn
  ports.haServerDrbd/Iscsi/Corosync{1,2,Crypto}/PacemakerRemoted/Pcsd

scripts/ha/ (migrated + updated from test-lab/ha/)
  cluster-init.sh — generates corosync authkey, initialises DRBD/XFS/iSCSI,
    creates NFS dataset dirs, configures Pacemaker with DRBD + XFS + iSCSI
    + nfs-server + VIP; STONITH disabled initially (enable separately)
  cluster-enable-stonith.sh — enables fence_pve_ssh STONITH after key deploy
  fence-pve-ssh.py — Proxmox SSH fence agent (node names updated to ha-server-1/2)
  acceptance-tests.sh — T1–T7 production acceptance tests

test-lab/ha/ removed — all Nix config moved to modules/ha/ and
  modules/build-types/; scripts moved to scripts/ha/

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8

scripts/ha/acceptance-tests.sh

@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+# acceptance-tests.sh — HA cluster acceptance tests (T1–T7)
+#
+# Run from a host with SSH access to both HA nodes (or from node1 itself).
+# All 7 tests must pass before considering the cluster production-ready.
+# Test values below must match variables.nix haServer* values.
+set -euo pipefail
+
+# ── Configuration ─────────────────────────────────────────────────────────
+NODE1="ha-server-1"
+NODE2="ha-server-2"
+NODE1_IP="192.168.2.200"   # vars.haServer1Ip
+NODE2_IP="192.168.2.201"   # vars.haServer2Ip
+VIP="192.168.2.202"        # vars.haServerVip
+XFS_MOUNT="/srv/ha-data"   # vars.haStorageRoot
+ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage"  # vars.haIscsiIqn
+# ──────────────────────────────────────────────────────────────────────────
+
+PASS=0
+FAIL=0
+RESULTS=()
+
+pass() { echo "  PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); }
+fail() { echo "  FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); }
+
+n1() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; }
+n2() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; }
+
+echo "════════════════════════════════════════════════════"
+echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
+echo "════════════════════════════════════════════════════"
+
+# ── T1: Corosync quorum established ──────────────────────────────────────
+echo ""
+echo "[T1] Corosync quorum"
+if n1 "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
+  pass "cluster has quorum"
+else
+  fail "cluster does not have quorum — check corosync on both nodes"
+fi
+
+# ── T2: DRBD Primary on node1, Secondary on node2 ────────────────────────
+echo ""
+echo "[T2] DRBD roles"
+DRBD_ROLE=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
+if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then
+  pass "DRBD Primary on $NODE1 ($DRBD_ROLE)"
+else
+  fail "unexpected DRBD role on $NODE1: $DRBD_ROLE (expected Primary/Secondary)"
+fi
+
+DRBD_DSTATE=$(n1 "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
+if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then
+  pass "DRBD disk state UpToDate ($DRBD_DSTATE)"
+else
+  fail "DRBD disk not UpToDate: $DRBD_DSTATE"
+fi
+
+# ── T3: XFS mounted at haStorageRoot on the Active node ──────────────────
+echo ""
+echo "[T3] XFS mount"
+if n1 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+  pass "XFS mounted at ${XFS_MOUNT} on $NODE1"
+else
+  fail "XFS not mounted at ${XFS_MOUNT} on $NODE1"
+fi
+
+if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+  fail "XFS unexpectedly mounted on $NODE2 (should only be on Active node)"
+else
+  pass "XFS not mounted on $NODE2 (correct — Secondary)"
+fi
+
+# ── T4: iSCSI target visible on both nodes ────────────────────────────────
+echo ""
+echo "[T4] iSCSI target"
+IQN_COUNT=$(n1 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
+if [[ "$IQN_COUNT" -ge 1 ]]; then
+  pass "iSCSI IQN active on $NODE1 ($IQN_COUNT target(s))"
+else
+  fail "no iSCSI IQN active on $NODE1"
+fi
+
+# iSCSI discovery from node2 via VIP
+if n2 "iscsiadm -m discovery -t sendtargets -p '${VIP}' 2>/dev/null | grep -q '${ISCSI_IQN}'"; then
+  pass "iSCSI target discoverable from $NODE2 via VIP ${VIP}"
+else
+  fail "iSCSI target not discoverable from $NODE2 via ${VIP}"
+fi
+
+# ── T5: Failover — standby node1, verify resources move to node2 ──────────
+echo ""
+echo "[T5] Failover (standby $NODE1)"
+MYNODE=$(n1 "crm_node -n" 2>/dev/null || echo "")
+n1 "crm_standby -N '${MYNODE}' -v on" 2>/dev/null || true
+echo "  Waiting up to 30 s for resources to move to $NODE2..."
+MOVED=false
+for i in $(seq 1 30); do
+  if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+    MOVED=true
+    echo "  Resources moved in ${i}s"
+    break
+  fi
+  sleep 1
+done
+
+if $MOVED; then
+  pass "XFS mounted on $NODE2 after failover"
+  IQN_ON_N2=$(n2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
+  [[ "$IQN_ON_N2" -ge 1 ]] \
+    && pass "iSCSI target active on $NODE2 after failover" \
+    || fail "iSCSI target NOT active on $NODE2 after failover"
+else
+  fail "XFS did not mount on $NODE2 within 30 s — failover incomplete"
+fi
+
+# ── T6: Data integrity — file written pre-failover readable post-failover ─
+echo ""
+echo "[T6] Data integrity"
+# Write a test file on node2 (now Active) and verify its content
+TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$"
+TEST_CONTENT="ha-acceptance-test-$(date +%s)"
+n2 "echo '${TEST_CONTENT}' > '${TEST_FILE}'" 2>/dev/null || true
+READBACK=$(n2 "cat '${TEST_FILE}' 2>/dev/null" || echo "")
+if [[ "$READBACK" == "$TEST_CONTENT" ]]; then
+  pass "test file written and read back correctly on $NODE2"
+else
+  fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')"
+fi
+n2 "rm -f '${TEST_FILE}'" 2>/dev/null || true
+
+# ── T7: Node rejoin — un-standby node1, verify cluster is healthy ─────────
+echo ""
+echo "[T7] Node rejoin"
+n1 "crm_standby -N '${MYNODE}' -v off" 2>/dev/null || true
+n1 "crm_resource --cleanup" 2>/dev/null || true
+sleep 5
+
+ONLINE_NODES=$(n2 "crm_mon -1 2>/dev/null | grep -c 'Online:'" || echo "0")
+if n1 "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
+  pass "$NODE1 rejoined — cluster has quorum"
+else
+  fail "$NODE1 did not rejoin with quorum"
+fi
+
+DRBD_ROLE_AFTER=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
+if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then
+  pass "$NODE1 is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
+else
+  fail "unexpected DRBD role on $NODE1 after rejoin: $DRBD_ROLE_AFTER"
+fi
+
+# ── Summary ───────────────────────────────────────────────────────────────
+echo ""
+echo "════════════════════════════════════════════════════"
+echo " Results: ${PASS} PASS, ${FAIL} FAIL"
+echo "════════════════════════════════════════════════════"
+for r in "${RESULTS[@]}"; do echo "  $r"; done
+echo ""
+
+if [[ "$FAIL" -eq 0 ]]; then
+  echo "ALL PASS — cluster is production-ready."
+  exit 0
+else
+  echo "SOME TESTS FAILED — investigate before deploying."
+  exit 1
+fi

scripts/ha/cluster-enable-stonith.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# cluster-enable-stonith.sh — enable STONITH fence agent after the fence SSH
+# key is deployed to both nodes and authorised on the Proxmox host.
+#
+# Run from ha-server-1 as root AFTER:
+#   - /etc/pacemaker/fence_pve_ssh exists on both nodes (chmod +x)
+#     (copy from scripts/ha/fence-pve-ssh.py)
+#   - /etc/fence-pve-ssh-key (SSH private key) exists on both nodes
+#   - The corresponding public key is in authorized_keys on PVE_HOST
+#   - VMID_NODE1 / VMID_NODE2 filled in below
+set -euo pipefail
+
+# ── Configuration ─────────────────────────────────────────────────────────
+NODE1="ha-server-1"
+NODE2="ha-server-2"
+VMID_NODE1=""          # FILL IN: Proxmox VMID for ha-server-1
+VMID_NODE2=""          # FILL IN: Proxmox VMID for ha-server-2
+PVE_HOST="pve1.sweet.home"
+PVE_USER="wayne"
+FENCE_KEY="/etc/fence-pve-ssh-key"
+FENCE_SCRIPT="/etc/pacemaker/fence_pve_ssh"
+# ──────────────────────────────────────────────────────────────────────────
+
+log() { echo "[stonith-setup] $*"; }
+die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; }
+
+[[ $(id -u) -eq 0 ]] || die "must run as root"
+[[ -n "$VMID_NODE1" ]] || die "VMID_NODE1 not set — edit this script"
+[[ -n "$VMID_NODE2" ]] || die "VMID_NODE2 not set — edit this script"
+[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY"
+[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT"
+
+log "Verifying fence agent can reach ${PVE_HOST}..."
+ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \
+    -o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" \
+    "sudo /usr/sbin/qm list" &>/dev/null \
+  || die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo"
+log "Fence agent SSH connectivity confirmed"
+
+log "Creating Pacemaker STONITH resources..."
+cibadmin --create --scope resources --xml-text "
+<primitive id=\"stonith-${NODE1}\" class=\"stonith\" type=\"external/fence_pve_ssh\">
+  <instance_attributes id=\"stonith-${NODE1}-attrs\">
+    <nvpair id=\"stonith-${NODE1}-plug\" name=\"plug\" value=\"${NODE1}\"/>
+    <nvpair id=\"stonith-${NODE1}-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
+    <nvpair id=\"stonith-${NODE1}-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
+    <nvpair id=\"stonith-${NODE1}-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
+    <nvpair id=\"stonith-${NODE1}-vmid1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
+    <nvpair id=\"stonith-${NODE1}-vmid2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
+    <nvpair id=\"stonith-${NODE1}-host-list\" name=\"pcmk_host_list\" value=\"${NODE1}\"/>
+  </instance_attributes>
+  <operations>
+    <op id=\"stonith-${NODE1}-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
+  </operations>
+</primitive>
+" 2>/dev/null || true
+
+cibadmin --create --scope resources --xml-text "
+<primitive id=\"stonith-${NODE2}\" class=\"stonith\" type=\"external/fence_pve_ssh\">
+  <instance_attributes id=\"stonith-${NODE2}-attrs\">
+    <nvpair id=\"stonith-${NODE2}-plug\" name=\"plug\" value=\"${NODE2}\"/>
+    <nvpair id=\"stonith-${NODE2}-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
+    <nvpair id=\"stonith-${NODE2}-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
+    <nvpair id=\"stonith-${NODE2}-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
+    <nvpair id=\"stonith-${NODE2}-vmid1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
+    <nvpair id=\"stonith-${NODE2}-vmid2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
+    <nvpair id=\"stonith-${NODE2}-host-list\" name=\"pcmk_host_list\" value=\"${NODE2}\"/>
+  </instance_attributes>
+  <operations>
+    <op id=\"stonith-${NODE2}-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
+  </operations>
+</primitive>
+" 2>/dev/null || true
+
+log "Enabling STONITH and restoring quorum policy..."
+crm_attribute -t crm_config -n stonith-enabled -v true
+crm_attribute -t crm_config -n no-quorum-policy -v stop
+
+log "DRBD fencing mode must also be updated to resource-only (already the"
+log "default in cluster-config.nix; confirm with: cat /etc/drbd.d/ha-data.conf)"
+
+log "Testing fence agent..."
+stonith_admin --list-devices && log "Fence devices listed successfully." \
+  || warn "stonith_admin --list-devices failed — check config"
+
+log "STONITH enabled. Cluster is now fully HA."

scripts/ha/cluster-init.sh

@@ -0,0 +1,284 @@
+#!/usr/bin/env bash
+# cluster-init.sh — one-time HA cluster initialisation script
+#
+# Run ONCE from ha-server-1 as root AFTER both VMs are booted and have SSH
+# access.  It:
+#   1. Generates and distributes the corosync authkey
+#   2. Waits for corosync quorum and pacemaker
+#   3. Initialises DRBD metadata, promotes node1 to primary
+#   4. Creates XFS on /dev/drbd0 and mounts it
+#   5. Creates the directory tree and iSCSI LUN backing file
+#   6. Configures LIO iSCSI target (file-backed LUN)
+#   7. Configures Pacemaker resources: DRBD → XFS → iSCSI → NFS → VIP
+#
+# Prerequisites:
+#   - Both VMs booted with the ha-server config (nixos-rebuild done)
+#   - SSH key access from node1 to root@NODE2_IP
+#   - VMID_NODE1 / VMID_NODE2 filled in below (needed for STONITH setup;
+#     cluster starts without STONITH, which you enable separately via
+#     scripts/ha/cluster-enable-stonith.sh)
+#   - Run as root on ha-server-1
+set -euo pipefail
+
+# ── Configuration ─────────────────────────────────────────────────────────
+# These must match variables.nix haServer* values and the Proxmox VMID
+# assignments.  Update before running.
+NODE1="ha-server-1"
+NODE2="ha-server-2"
+NODE1_IP="192.168.2.200"   # vars.haServer1Ip
+NODE2_IP="192.168.2.201"   # vars.haServer2Ip
+VIP="192.168.2.202"        # vars.haServerVip
+XFS_MOUNT="/srv/ha-data"   # vars.haStorageRoot
+ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage"  # vars.haIscsiIqn
+ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
+ISCSI_LUN_SIZE="10G"
+DRBD_DEVICE="/dev/drbd0"
+VMID_NODE1=""              # FILL IN: Proxmox VMID for ha-server-1
+VMID_NODE2=""              # FILL IN: Proxmox VMID for ha-server-2
+PVE_HOST="pve1.sweet.home"
+PVE_USER="wayne"
+
+# NFS dataset subdirectories to create under XFS_MOUNT.
+# Must mirror vars.nfsShares subpath values in variables.nix.
+NFS_SUBDIRS=(
+  "docker/config"
+  "docker/volumes"
+  "docker/databases"
+  "docker/nextcloud-data"
+  "raspi/volumes"
+  "proxmox/iso"
+  "proxmox/lxc"
+  "pxe-boot/images"
+)
+# ──────────────────────────────────────────────────────────────────────────
+
+log()  { echo "[cluster-init] $*"; }
+die()  { echo "[cluster-init] ERROR: $*" >&2; exit 1; }
+warn() { echo "[cluster-init] WARNING: $*" >&2; }
+
+[[ $(id -u) -eq 0 ]] || die "must run as root"
+[[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1"
+
+# ── 0. Corosync authkey ───────────────────────────────────────────────────
+AUTHKEY="/etc/corosync/authkey"
+mkdir -p /etc/corosync
+if [[ ! -f "$AUTHKEY" ]]; then
+  log "Generating corosync authkey..."
+  corosync-keygen -k "$AUTHKEY"
+  chmod 0400 "$AUTHKEY"
+fi
+log "Distributing authkey to $NODE2..."
+ssh "root@${NODE2_IP}" "mkdir -p /etc/corosync"
+scp -q "$AUTHKEY" "root@${NODE2_IP}:${AUTHKEY}"
+ssh "root@${NODE2_IP}" "chmod 0400 '${AUTHKEY}'"
+
+log "Restarting corosync on both nodes..."
+systemctl restart corosync
+ssh "root@${NODE2_IP}" "systemctl restart corosync"
+sleep 3
+
+# ── 1. Corosync quorum ────────────────────────────────────────────────────
+log "Waiting for corosync quorum..."
+for i in $(seq 1 30); do
+  if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then
+    log "Quorum established"
+    break
+  fi
+  [[ $i -eq 30 ]] && die "corosync quorum not established after 60 s"
+  sleep 2
+done
+
+log "Waiting for pacemaker..."
+for i in $(seq 1 30); do
+  if crm_mon -1 &>/dev/null; then
+    log "Pacemaker running"
+    break
+  fi
+  [[ $i -eq 30 ]] && die "pacemaker not running after 60 s"
+  sleep 2
+done
+
+# ── 2. DRBD initialisation ────────────────────────────────────────────────
+log "Initialising DRBD metadata on $NODE1..."
+if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then
+  drbdadm create-md ha-data --force
+fi
+
+log "Initialising DRBD metadata on $NODE2..."
+ssh "root@${NODE2_IP}" "
+  if ! drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent\|Diskless'; then
+    drbdadm create-md ha-data --force
+  fi
+"
+
+log "Bringing up DRBD on both nodes..."
+drbdadm up ha-data 2>/dev/null || true
+ssh "root@${NODE2_IP}" "drbdadm up ha-data 2>/dev/null" || true
+
+log "Forcing $NODE1 to DRBD Primary for initial sync..."
+drbdadm primary ha-data --force
+
+log "Waiting for DRBD to finish initial sync (this may take several minutes)..."
+for i in $(seq 1 300); do
+  state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
+  if echo "$state" | grep -q "UpToDate/UpToDate"; then
+    log "DRBD sync complete: $state"
+    break
+  fi
+  [[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)"
+  sleep 1
+done
+
+# ── 3. XFS filesystem ─────────────────────────────────────────────────────
+log "Creating XFS on ${DRBD_DEVICE}..."
+if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
+  mkfs.xfs -f "${DRBD_DEVICE}"
+fi
+
+log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
+mkdir -p "${XFS_MOUNT}"
+mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
+
+# ── 4. NFS dataset directories ────────────────────────────────────────────
+log "Creating NFS dataset directories..."
+for subdir in "${NFS_SUBDIRS[@]}"; do
+  mkdir -p "${XFS_MOUNT}/${subdir}"
+done
+
+# ── 5. iSCSI LUN backing file ─────────────────────────────────────────────
+log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..."
+if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then
+  fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}"
+fi
+
+# ── 6. LIO iSCSI target ───────────────────────────────────────────────────
+log "Configuring LIO iSCSI target via targetcli..."
+targetcli <<EOF
+/backstores/fileio create name=ha-lun0 file_or_dev=${ISCSI_LUN_FILE} size=0 write_back=false
+/iscsi create ${ISCSI_IQN}
+/iscsi/${ISCSI_IQN}/tpg1/luns create /backstores/fileio/ha-lun0
+/iscsi/${ISCSI_IQN}/tpg1/portals create ${VIP}
+/iscsi/${ISCSI_IQN}/tpg1 set attribute authentication=0
+/iscsi/${ISCSI_IQN}/tpg1 set attribute demo_mode_write_protect=0
+saveconfig /etc/target/saveconfig.json
+EOF
+
+log "Distributing iSCSI saveconfig to $NODE2..."
+scp -q /etc/target/saveconfig.json "root@${NODE2_IP}:/etc/target/saveconfig.json"
+
+log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..."
+umount "${XFS_MOUNT}"
+
+log "Demoting DRBD to Secondary — Pacemaker manages primary role..."
+drbdadm secondary ha-data
+
+# ── 7. Pacemaker resources ────────────────────────────────────────────────
+log "Configuring Pacemaker cluster properties..."
+crm_attribute -t crm_config -n stonith-enabled -v false
+crm_attribute -t crm_config -n no-quorum-policy -v ignore
+
+log "Creating DRBD promotable clone resource..."
+cibadmin --replace --scope resources --xml-text "
+<resources>
+  <clone id=\"ms-drbd0\" globally-unique=\"false\">
+    <meta_attributes id=\"ms-drbd0-meta\">
+      <nvpair id=\"ms-drbd0-promotable\" name=\"promotable\" value=\"true\"/>
+      <nvpair id=\"ms-drbd0-master-max\" name=\"master-max\" value=\"1\"/>
+      <nvpair id=\"ms-drbd0-master-node-max\" name=\"master-node-max\" value=\"1\"/>
+      <nvpair id=\"ms-drbd0-clone-max\" name=\"clone-max\" value=\"2\"/>
+      <nvpair id=\"ms-drbd0-clone-node-max\" name=\"clone-node-max\" value=\"1\"/>
+      <nvpair id=\"ms-drbd0-notify\" name=\"notify\" value=\"true\"/>
+      <nvpair id=\"ms-drbd0-interleave\" name=\"interleave\" value=\"true\"/>
+    </meta_attributes>
+    <primitive id=\"drbd0\" class=\"ocf\" type=\"drbd\" provider=\"linbit\">
+      <instance_attributes id=\"drbd0-attrs\">
+        <nvpair id=\"drbd0-resource\" name=\"drbd_resource\" value=\"ha-data\"/>
+      </instance_attributes>
+      <operations>
+        <op id=\"drbd0-start\" name=\"start\" interval=\"0\" timeout=\"240s\"/>
+        <op id=\"drbd0-stop\" name=\"stop\" interval=\"0\" timeout=\"120s\"/>
+        <op id=\"drbd0-promote\" name=\"promote\" interval=\"0\" timeout=\"90s\"/>
+        <op id=\"drbd0-demote\" name=\"demote\" interval=\"0\" timeout=\"90s\"/>
+        <op id=\"drbd0-monitor-master\" name=\"monitor\" interval=\"20s\" timeout=\"20s\" role=\"Promoted\"/>
+        <op id=\"drbd0-monitor-slave\" name=\"monitor\" interval=\"30s\" timeout=\"20s\" role=\"Unpromoted\"/>
+      </operations>
+    </primitive>
+  </clone>
+  <group id=\"ha-group\">
+    <primitive id=\"xfs-data\" class=\"ocf\" type=\"Filesystem\" provider=\"heartbeat\">
+      <instance_attributes id=\"xfs-data-attrs\">
+        <nvpair id=\"xfs-data-device\" name=\"device\" value=\"${DRBD_DEVICE}\"/>
+        <nvpair id=\"xfs-data-directory\" name=\"directory\" value=\"${XFS_MOUNT}\"/>
+        <nvpair id=\"xfs-data-fstype\" name=\"fstype\" value=\"xfs\"/>
+        <nvpair id=\"xfs-data-options\" name=\"options\" value=\"defaults\"/>
+        <nvpair id=\"xfs-data-force_unmount\" name=\"force_unmount\" value=\"false\"/>
+      </instance_attributes>
+      <operations>
+        <op id=\"xfs-data-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
+        <op id=\"xfs-data-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
+        <op id=\"xfs-data-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
+      </operations>
+    </primitive>
+    <primitive id=\"iscsi-target\" class=\"systemd\" type=\"targetctl\">
+      <operations>
+        <op id=\"iscsi-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
+        <op id=\"iscsi-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
+        <op id=\"iscsi-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
+      </operations>
+    </primitive>
+    <primitive id=\"nfs-server\" class=\"systemd\" type=\"nfs-server\">
+      <operations>
+        <op id=\"nfs-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
+        <op id=\"nfs-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
+        <op id=\"nfs-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"40s\"/>
+      </operations>
+    </primitive>
+    <primitive id=\"vip\" class=\"ocf\" type=\"IPaddr2\" provider=\"heartbeat\">
+      <instance_attributes id=\"vip-attrs\">
+        <nvpair id=\"vip-ip\" name=\"ip\" value=\"${VIP}\"/>
+        <nvpair id=\"vip-cidr\" name=\"cidr_netmask\" value=\"24\"/>
+      </instance_attributes>
+      <operations>
+        <op id=\"vip-start\" name=\"start\" interval=\"0\" timeout=\"20s\"/>
+        <op id=\"vip-stop\" name=\"stop\" interval=\"0\" timeout=\"20s\"/>
+        <op id=\"vip-monitor\" name=\"monitor\" interval=\"10s\" timeout=\"20s\"/>
+      </operations>
+    </primitive>
+  </group>
+</resources>
+"
+
+log "Adding ordering and colocation constraints..."
+cibadmin --create --scope constraints --xml-text "
+<constraints>
+  <rsc_order id=\"order-drbd-group\" first=\"ms-drbd0\" first-action=\"promote\" then=\"ha-group\" then-action=\"start\"/>
+  <rsc_colocation id=\"coloc-group-with-drbd\" rsc=\"ha-group\" with-rsc=\"ms-drbd0\" with-rsc-role=\"Master\" score=\"INFINITY\"/>
+</constraints>
+"
+
+log "Waiting for resources to start..."
+for i in $(seq 1 60); do
+  if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then
+    log "VIP is up: $(crm_resource -r vip --locate)"
+    break
+  fi
+  [[ $i -eq 60 ]] && { warn "VIP not up after 120 s — check: crm_mon -1"; break; }
+  sleep 2
+done
+
+log ""
+log "═══════════════════════════════════════════════════════════════"
+log " HA cluster initialised."
+log ""
+log "  crm_mon -1                         — cluster status"
+log "  iscsiadm -m discovery -t st -p ${VIP}  — verify iSCSI target"
+log "  showmount -e ${VIP}                    — verify NFS exports"
+log ""
+log "  To enable STONITH (after deploying fence SSH key):"
+log "    1. Fill in VMID_NODE1 / VMID_NODE2 in cluster-enable-stonith.sh"
+log "    2. Copy scripts/ha/fence-pve-ssh.py to /etc/pacemaker/fence_pve_ssh"
+log "       on both nodes (chmod +x)"
+log "    3. Generate and distribute the fence SSH key"
+log "       (see docs or cluster-enable-stonith.sh header)"
+log "    4. bash scripts/ha/cluster-enable-stonith.sh"
+log "═══════════════════════════════════════════════════════════════"

scripts/ha/fence-pve-ssh.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""
+fence_pve_ssh - Proxmox VE SSH fence agent for Pacemaker.
+
+Uses SSH to reach the Proxmox host and run 'qm stop/start <vmid>'.
+Deploy to /etc/pacemaker/fence_pve_ssh on both HA nodes (chmod +x).
+
+Configuration (as pacemaker stonith resource attributes):
+  pve_host     Proxmox host to SSH to (default: pve1.sweet.home)
+  pve_user     SSH user (default: wayne)
+  key_file     SSH private key path (default: /etc/fence-pve-ssh-key)
+  vmid_node1   VMID for ha-server-1
+  vmid_node2   VMID for ha-server-2
+  plug         Node name to act on (set by pacemaker: ha-server-1 or ha-server-2)
+  action       Action: off|on|reboot|status|list|metadata
+"""
+
+import argparse
+import subprocess
+import sys
+import os
+
+
+METADATA = """<?xml version="1.0" ?>
+<resource-agent name="fence_pve_ssh" shortdesc="Proxmox VE SSH fence agent (test lab)">
+  <longdesc>Fences a VM on a Proxmox VE host by SSHing to the PVE host and
+  running qm stop/start.  For test use only.</longdesc>
+  <vendor-url>https://proxmox.com</vendor-url>
+  <parameters>
+    <parameter name="action" required="1" unique="0">
+      <getopt mixed="-a, --action=[action]"/>
+      <content type="string" default="reboot"/>
+      <shortdesc lang="en">Fencing action: off|on|reboot|status|list</shortdesc>
+    </parameter>
+    <parameter name="plug" required="0" unique="0">
+      <getopt mixed="-n, --plug=[nodename]"/>
+      <content type="string"/>
+      <shortdesc lang="en">Cluster node name to fence</shortdesc>
+    </parameter>
+    <parameter name="pve_host" required="0" unique="0">
+      <getopt mixed="--pve-host=[host]"/>
+      <content type="string" default="pve1.sweet.home"/>
+      <shortdesc lang="en">Proxmox VE host to SSH to</shortdesc>
+    </parameter>
+    <parameter name="pve_user" required="0" unique="0">
+      <getopt mixed="--pve-user=[user]"/>
+      <content type="string" default="wayne"/>
+      <shortdesc lang="en">SSH user on the Proxmox host</shortdesc>
+    </parameter>
+    <parameter name="key_file" required="0" unique="0">
+      <getopt mixed="--key-file=[path]"/>
+      <content type="string" default="/etc/fence-pve-ssh-key"/>
+      <shortdesc lang="en">SSH private key file path</shortdesc>
+    </parameter>
+    <parameter name="vmid_node1" required="1" unique="0">
+      <getopt mixed="--vmid-node1=[vmid]"/>
+      <content type="string"/>
+      <shortdesc lang="en">VMID for ha-test-node1</shortdesc>
+    </parameter>
+    <parameter name="vmid_node2" required="1" unique="0">
+      <getopt mixed="--vmid-node2=[vmid]"/>
+      <content type="string"/>
+      <shortdesc lang="en">VMID for ha-test-node2</shortdesc>
+    </parameter>
+  </parameters>
+  <actions>
+    <action name="off" timeout="60s"/>
+    <action name="on" timeout="60s"/>
+    <action name="reboot" timeout="60s"/>
+    <action name="status" timeout="30s"/>
+    <action name="list" timeout="10s"/>
+    <action name="metadata" timeout="5s"/>
+  </actions>
+</resource-agent>
+"""
+
+
+def parse_args():
+    p = argparse.ArgumentParser(add_help=False)
+    p.add_argument("-a", "--action", default="reboot")
+    p.add_argument("-n", "--plug")
+    p.add_argument("--pve-host", default="pve1.sweet.home")
+    p.add_argument("--pve-user", default="wayne")
+    p.add_argument("--key-file", default="/etc/fence-pve-ssh-key")
+    p.add_argument("--vmid-node1")
+    p.add_argument("--vmid-node2")
+    # Allow remaining unknown args (pacemaker may pass extra ones)
+    return p.parse_known_args()[0]
+
+
+def ssh(pve_host, pve_user, key_file, cmd):
+    result = subprocess.run(
+        [
+            "ssh",
+            "-i", key_file,
+            "-o", "StrictHostKeyChecking=no",
+            "-o", "BatchMode=yes",
+            "-o", "ConnectTimeout=10",
+            f"{pve_user}@{pve_host}",
+            cmd,
+        ],
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    return result
+
+
+def get_vmid(args):
+    node = args.plug
+    if not node:
+        print("ERROR: --plug not specified", file=sys.stderr)
+        sys.exit(1)
+    mapping = {
+        "ha-server-1": args.vmid_node1,
+        "ha-server-2": args.vmid_node2,
+    }
+    vmid = mapping.get(node)
+    if not vmid:
+        print(f"ERROR: unknown node '{node}'", file=sys.stderr)
+        sys.exit(1)
+    return vmid
+
+
+def main():
+    args = parse_args()
+    action = args.action.lower()
+
+    if action == "metadata":
+        print(METADATA)
+        sys.exit(0)
+
+    if action == "list":
+        if args.vmid_node1:
+            print("ha-server-1")
+        if args.vmid_node2:
+            print("ha-server-2")
+        sys.exit(0)
+
+    vmid = get_vmid(args)
+
+    if not os.path.exists(args.key_file):
+        print(f"ERROR: SSH key not found at {args.key_file}", file=sys.stderr)
+        sys.exit(1)
+
+    if action in ("off", "reboot"):
+        print(f"Stopping VM {vmid} ({args.plug}) on {args.pve_host}...")
+        r = ssh(args.pve_host, args.pve_user, args.key_file,
+                f"sudo /usr/sbin/qm stop {vmid}")
+        if r.returncode != 0:
+            print(f"ERROR stopping VM: {r.stderr}", file=sys.stderr)
+            sys.exit(1)
+        print(f"VM {vmid} stopped")
+
+    if action in ("on", "reboot"):
+        print(f"Starting VM {vmid} ({args.plug}) on {args.pve_host}...")
+        r = ssh(args.pve_host, args.pve_user, args.key_file,
+                f"sudo /usr/sbin/qm start {vmid}")
+        if r.returncode != 0:
+            print(f"ERROR starting VM: {r.stderr}", file=sys.stderr)
+            sys.exit(1)
+        print(f"VM {vmid} started")
+
+    if action == "status":
+        r = ssh(args.pve_host, args.pve_user, args.key_file,
+                f"sudo /usr/sbin/qm status {vmid}")
+        if r.returncode != 0:
+            print(f"ERROR querying VM status: {r.stderr}", file=sys.stderr)
+            sys.exit(1)
+        # qm status returns "status: running" or "status: stopped"
+        status_line = r.stdout.strip()
+        print(status_line)
+        if "stopped" in status_line:
+            sys.exit(2)   # pacemaker interprets exit 2 as "off"
+        sys.exit(0)       # running = exit 0
+
+
+if __name__ == "__main__":
+    main()