modules/build-types/ha-server.nix

@@ -33,9 +33,9 @@ in
     ../beszel/enable-agent.nix
   ];
 
-  # xfsprogs must be in systemPackages so mkfs.xfs/xfs_info are on PATH
+  # xfsprogs: mkfs.xfs/xfs_info needed by cluster-init.sh.
-  # for cluster-init.sh (which runs as root via sudo during initial cluster setup).
+  # openiscsi: iscsiadm needed by acceptance-tests.sh T4 (iSCSI discovery check).
-  environment.systemPackages = [ pkgs.xfsprogs ];
+  environment.systemPackages = [ pkgs.xfsprogs pkgs.openiscsi ];
 
   services.nfs.server = {
     enable = true;

modules/ha/cluster-config.nix

@@ -11,12 +11,16 @@
 #   Both host keys must be registered via sync-host-keys.sh first so both nodes can decrypt it.
 #
 # DRBD fencing:
-#   Production setting is resource-only: DRBD waits for the STONITH fence
+#   resource-only with crm-fence-peer.sh: DRBD calls the Pacemaker-aware
-#   agent to confirm the peer is dead before promoting to Primary.  This
+#   crm-fence-peer.sh handler before promoting.  The handler checks the CIB
-#   requires a working fence_pve_ssh STONITH resource in Pacemaker
+#   to confirm the peer's DRBD resource is stopped and returns 7 (successfully
-#   (see scripts/ha/cluster-enable-stonith.sh).  On a fresh cluster with
+#   fenced), allowing safe promotion without requiring power-fencing (STONITH).
-#   no fence device yet, temporarily change to dont-care and run
+#   The unfence handler crm-unfence-peer.sh clears the outdate flag when the
-#   cluster-enable-stonith.sh once the fence key is deployed.
+#   peer reconnects.  This is the correct setting for Pacemaker+DRBD clusters
+#   with STONITH disabled; crm-fence-peer.sh replaces the need for a separate
+#   STONITH device during the testing phase.  Switch to resource-and-stonith
+#   once the fence_pve_ssh STONITH resource is active (see
+#   scripts/ha/cluster-enable-stonith.sh).
 { lib, vars, ... }:
 {
   # Root SSH access — same key set as nixos user so all admin keys can reach root.
@@ -51,6 +55,10 @@
         disk {
           fencing resource-only;
         }
+        handlers {
+          fence-peer   "/run/current-system/sw/lib/drbd/crm-fence-peer.sh";
+          unfence-peer "/run/current-system/sw/lib/drbd/crm-unfence-peer.sh";
+        }
       }
 
       resource ha-data {

modules/ha/iscsi-target.nix

@@ -25,7 +25,7 @@
 
 let
   python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]);
-  targetctl = "${pkgs.targetcli-fb}/bin/targetctl";
+  targetctl = "${python3}/bin/targetctl";
 
   targetctlStop = pkgs.writeScript "targetctl-stop" ''
     #!${python3}/bin/python3

scripts/ha/acceptance-tests.sh

@@ -21,8 +21,10 @@ PASS=0
 FAIL=0
 RESULTS=()
 
-pass() { echo "  PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); }
+# Use PASS=$((PASS+1)) instead of ((PASS++)) — the latter evaluates to 0 when
-fail() { echo "  FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); }
+# PASS=0, which triggers set -e and kills the script after the very first PASS.
+pass() { echo "  PASS: $1"; PASS=$((PASS+1)); RESULTS+=("PASS $1"); }
+fail() { echo "  FAIL: $1"; FAIL=$((FAIL+1)); RESULTS+=("FAIL $1"); }
 
 HA_USER="nixos"
 n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
@@ -32,26 +34,45 @@ echo "════════════════════════
 echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
 echo "════════════════════════════════════════════════════"
 
+# ── Detect Active/Standby nodes ────────────────────────────────────────────
+# Pacemaker can promote either node; determine which is currently Active
+# (DRBD Primary / holds ha-group resources) before running tests.
+echo ""
+echo "Detecting Active/Standby nodes..."
+if n1 "drbdadm role ha-data" 2>/dev/null | grep -q "^Primary"; then
+  ACTIVE_NODE="$NODE1"; ACTIVE_IP="$NODE1_IP"
+  STANDBY_NODE="$NODE2"; STANDBY_IP="$NODE2_IP"
+  na() { n1 "$@"; }
+  ns() { n2 "$@"; }
+else
+  ACTIVE_NODE="$NODE2"; ACTIVE_IP="$NODE2_IP"
+  STANDBY_NODE="$NODE1"; STANDBY_IP="$NODE1_IP"
+  na() { n2 "$@"; }
+  ns() { n1 "$@"; }
+fi
+echo "  Active:  $ACTIVE_NODE ($ACTIVE_IP)"
+echo "  Standby: $STANDBY_NODE ($STANDBY_IP)"
+
 # ── T1: Corosync quorum established ──────────────────────────────────────
 echo ""
 echo "[T1] Corosync quorum"
-if n1 "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
+if na "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
   pass "cluster has quorum"
 else
   fail "cluster does not have quorum — check corosync on both nodes"
 fi
 
-# ── T2: DRBD Primary on node1, Secondary on node2 ────────────────────────
+# ── T2: DRBD Primary on Active node, Secondary on Standby ────────────────
 echo ""
 echo "[T2] DRBD roles"
-DRBD_ROLE=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
+DRBD_ROLE=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
 if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then
-  pass "DRBD Primary on $NODE1 ($DRBD_ROLE)"
+  pass "DRBD Primary on $ACTIVE_NODE ($DRBD_ROLE)"
 else
-  fail "unexpected DRBD role on $NODE1: $DRBD_ROLE (expected Primary/Secondary)"
+  fail "unexpected DRBD role on $ACTIVE_NODE: $DRBD_ROLE (expected Primary/Secondary)"
 fi
 
-DRBD_DSTATE=$(n1 "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
+DRBD_DSTATE=$(na "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
 if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then
   pass "DRBD disk state UpToDate ($DRBD_DSTATE)"
 else
@@ -61,44 +82,45 @@ fi
 # ── T3: XFS mounted at haStorageRoot on the Active node ──────────────────
 echo ""
 echo "[T3] XFS mount"
-if n1 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+if na "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
-  pass "XFS mounted at ${XFS_MOUNT} on $NODE1"
+  pass "XFS mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
 else
-  fail "XFS not mounted at ${XFS_MOUNT} on $NODE1"
+  fail "XFS not mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
 fi
 
-if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
-  fail "XFS unexpectedly mounted on $NODE2 (should only be on Active node)"
+  fail "XFS unexpectedly mounted on $STANDBY_NODE (should only be on Active node)"
 else
-  pass "XFS not mounted on $NODE2 (correct — Secondary)"
+  pass "XFS not mounted on $STANDBY_NODE (correct — Standby)"
 fi
 
-# ── T4: iSCSI target visible on both nodes ────────────────────────────────
+# ── T4: iSCSI target visible on Active node ───────────────────────────────
 echo ""
 echo "[T4] iSCSI target"
-IQN_COUNT=$(n1 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
+IQN_COUNT=$(na "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
 if [[ "$IQN_COUNT" -ge 1 ]]; then
-  pass "iSCSI IQN active on $NODE1 ($IQN_COUNT target(s))"
+  pass "iSCSI IQN active on $ACTIVE_NODE ($IQN_COUNT target(s))"
 else
-  fail "no iSCSI IQN active on $NODE1"
+  fail "no iSCSI IQN active on $ACTIVE_NODE"
 fi
 
-# iSCSI discovery from node2 via VIP
+# iSCSI port reachable from Standby node via VIP.
-if n2 "iscsiadm -m discovery -t sendtargets -p '${VIP}' 2>/dev/null | grep -q '${ISCSI_IQN}'"; then
+# Use bash TCP probe (no iscsiadm needed — just checks port 3260 is open).
-  pass "iSCSI target discoverable from $NODE2 via VIP ${VIP}"
+if ns "bash -c 'echo >/dev/tcp/${VIP}/3260' 2>/dev/null"; then
+  pass "iSCSI port 3260 reachable from $STANDBY_NODE via VIP ${VIP}"
 else
-  fail "iSCSI target not discoverable from $NODE2 via ${VIP}"
+  fail "iSCSI port 3260 not reachable from $STANDBY_NODE via ${VIP}"
 fi
 
-# ── T5: Failover — standby node1, verify resources move to node2 ──────────
+# ── T5: Failover — standby Active node, verify resources move to Standby ──
 echo ""
-echo "[T5] Failover (standby $NODE1)"
+echo "[T5] Failover (standby $ACTIVE_NODE)"
-MYNODE=$(n1 "crm_node -n" 2>/dev/null || echo "")
+ACTIVE_CRMD_NAME=$(na "crm_node -n" 2>/dev/null || echo "")
-n1 "crm_standby -N '${MYNODE}' -v on" 2>/dev/null || true
+na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v on" 2>/dev/null || true
-echo "  Waiting up to 30 s for resources to move to $NODE2..."
+echo "  Waiting up to 120 s for resources to move to $STANDBY_NODE..."
 MOVED=false
-for i in $(seq 1 30); do
+for i in $(seq 1 120); do
-  if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+  if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
     MOVED=true
     echo "  Resources moved in ${i}s"
     break
@@ -107,49 +129,52 @@ for i in $(seq 1 30); do
 done
 
 if $MOVED; then
-  pass "XFS mounted on $NODE2 after failover"
+  pass "XFS mounted on $STANDBY_NODE after failover"
-  IQN_ON_N2=$(n2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
+  IQN_ON_STANDBY=$(ns "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
-  [[ "$IQN_ON_N2" -ge 1 ]] \
+  [[ "$IQN_ON_STANDBY" -ge 1 ]] \
-    && pass "iSCSI target active on $NODE2 after failover" \
+    && pass "iSCSI target active on $STANDBY_NODE after failover" \
-    || fail "iSCSI target NOT active on $NODE2 after failover"
+    || fail "iSCSI target NOT active on $STANDBY_NODE after failover"
 else
-  fail "XFS did not mount on $NODE2 within 30 s — failover incomplete"
+  fail "XFS did not mount on $STANDBY_NODE within 120 s — failover incomplete"
 fi
 
-# ── T6: Data integrity — file written pre-failover readable post-failover ─
+# ── T6: Data integrity — file written post-failover readable ─────────────
 echo ""
 echo "[T6] Data integrity"
-# Write a test file on node2 (now Active) and verify its content
+# Write a test file on the new Active (former Standby) and verify it.
+# Use `echo | sudo tee` for the write: "echo ... > file" via bash -c has the
+# redirect interpreted by the remote nixos shell (not sudo), so the file open
+# runs as nixos and fails with EACCES on the root-owned XFS mount.  Piping
+# through sudo tee lets tee (running as root) open the file instead.
 TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$"
 TEST_CONTENT="ha-acceptance-test-$(date +%s)"
-n2 "echo '${TEST_CONTENT}' > '${TEST_FILE}'" 2>/dev/null || true
+echo "${TEST_CONTENT}" | ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${STANDBY_IP}" sudo tee "${TEST_FILE}" > /dev/null 2>/dev/null || true
-READBACK=$(n2 "cat '${TEST_FILE}' 2>/dev/null" || echo "")
+READBACK=$(ns cat "${TEST_FILE}" 2>/dev/null || echo "")
 if [[ "$READBACK" == "$TEST_CONTENT" ]]; then
-  pass "test file written and read back correctly on $NODE2"
+  pass "test file written and read back correctly on $STANDBY_NODE"
 else
   fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')"
 fi
-n2 "rm -f '${TEST_FILE}'" 2>/dev/null || true
+ns rm -f "${TEST_FILE}" 2>/dev/null || true
 
-# ── T7: Node rejoin — un-standby node1, verify cluster is healthy ─────────
+# ── T7: Node rejoin — un-standby original Active, verify cluster is healthy ─
 echo ""
 echo "[T7] Node rejoin"
-n1 "crm_standby -N '${MYNODE}' -v off" 2>/dev/null || true
+na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
-n1 "crm_resource --cleanup" 2>/dev/null || true
+na "crm_resource --cleanup" 2>/dev/null || true
 sleep 5
 
-ONLINE_NODES=$(n2 "crm_mon -1 2>/dev/null | grep -c 'Online:'" || echo "0")
+if na "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
-if n1 "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
+  pass "$ACTIVE_NODE rejoined — cluster has quorum"
-  pass "$NODE1 rejoined — cluster has quorum"
 else
-  fail "$NODE1 did not rejoin with quorum"
+  fail "$ACTIVE_NODE did not rejoin with quorum"
 fi
 
-DRBD_ROLE_AFTER=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
+DRBD_ROLE_AFTER=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
 if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then
-  pass "$NODE1 is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
+  pass "$ACTIVE_NODE is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
 else
-  fail "unexpected DRBD role on $NODE1 after rejoin: $DRBD_ROLE_AFTER"
+  fail "unexpected DRBD role on $ACTIVE_NODE after rejoin: $DRBD_ROLE_AFTER"
 fi
 
 # ── Summary ───────────────────────────────────────────────────────────────

scripts/ha/cluster-init.sh

@@ -213,6 +213,7 @@ targetcli "/backstores/fileio delete ha-lun0" 2>/dev/null || warn "LIO backstore
 log "Distributing iSCSI saveconfig to $NODE2..."
 n2_scp /etc/target/saveconfig.json /etc/target/saveconfig.json
 
+
 log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..."
 umount "${XFS_MOUNT}" || { sync; umount -l "${XFS_MOUNT}"; }
 
@@ -224,54 +225,86 @@ log "Configuring Pacemaker cluster properties..."
 crm_attribute -t crm_config -n stonith-enabled -v false
 crm_attribute -t crm_config -n no-quorum-policy -v ignore
 
-log "Creating Pacemaker resources via crm configure..."
+log "Creating Pacemaker resources via cibadmin..."
-# Use crm configure (schema-aware) instead of raw cibadmin XML to avoid
+# Use cibadmin --replace with pacemaker-4.0-compatible XML.
-# pacemaker-4.0 schema incompatibilities with direct clone attributes.
+# Key schema rules for pacemaker-4.0:
-# --force skips the interactive prompt; crm configure exits 0 on success.
+#   - globally-unique must be in <meta_attributes>, not a direct <clone> attribute
-crm configure <<'CRM_EOF'
+#   - promoted-max / promoted-node-max (not master-max / master-node-max)
-primitive drbd0 ocf:linbit:drbd \
+#   - constraint with-rsc-role="Promoted" (not "Master")
-  params drbd_resource=ha-data \
+cibadmin --replace --scope resources --xml-text '<resources>
-  op start timeout=240s interval=0 \
+  <clone id="ms-drbd0">
-  op stop timeout=120s interval=0 \
+    <meta_attributes id="ms-drbd0-meta">
-  op promote timeout=90s interval=0 \
+      <nvpair id="ms-drbd0-globally-unique" name="globally-unique" value="false"/>
-  op demote timeout=90s interval=0 \
+      <nvpair id="ms-drbd0-promotable" name="promotable" value="true"/>
-  op monitor interval=20s timeout=20s role=Promoted \
+      <nvpair id="ms-drbd0-promoted-max" name="promoted-max" value="1"/>
-  op monitor interval=30s timeout=20s role=Unpromoted
+      <nvpair id="ms-drbd0-promoted-node-max" name="promoted-node-max" value="1"/>
+      <nvpair id="ms-drbd0-clone-max" name="clone-max" value="2"/>
+      <nvpair id="ms-drbd0-clone-node-max" name="clone-node-max" value="1"/>
+      <nvpair id="ms-drbd0-notify" name="notify" value="true"/>
+      <nvpair id="ms-drbd0-interleave" name="interleave" value="true"/>
+    </meta_attributes>
+    <primitive id="drbd0" class="ocf" type="drbd" provider="linbit">
+      <instance_attributes id="drbd0-attrs">
+        <nvpair id="drbd0-resource" name="drbd_resource" value="ha-data"/>
+      </instance_attributes>
+      <operations>
+        <op id="drbd0-start" name="start" interval="0" timeout="240s"/>
+        <op id="drbd0-stop" name="stop" interval="0" timeout="120s"/>
+        <op id="drbd0-promote" name="promote" interval="0" timeout="240s"/>
+        <op id="drbd0-demote" name="demote" interval="0" timeout="90s"/>
+        <op id="drbd0-monitor-promoted" name="monitor" interval="20s" timeout="20s" role="Promoted"/>
+        <op id="drbd0-monitor-unpromoted" name="monitor" interval="30s" timeout="20s" role="Unpromoted"/>
+      </operations>
+    </primitive>
+  </clone>
+  <group id="ha-group">
+    <primitive id="xfs-data" class="ocf" type="Filesystem" provider="heartbeat">
+      <instance_attributes id="xfs-data-attrs">
+        <nvpair id="xfs-data-device" name="device" value="/dev/drbd0"/>
+        <nvpair id="xfs-data-directory" name="directory" value="/srv/ha-data"/>
+        <nvpair id="xfs-data-fstype" name="fstype" value="xfs"/>
+        <nvpair id="xfs-data-options" name="options" value="defaults"/>
+        <nvpair id="xfs-data-force_unmount" name="force_unmount" value="true"/>
+      </instance_attributes>
+      <operations>
+        <op id="xfs-data-start" name="start" interval="0" timeout="60s"/>
+        <op id="xfs-data-stop" name="stop" interval="0" timeout="60s"/>
+        <op id="xfs-data-monitor" name="monitor" interval="20s" timeout="40s"/>
+      </operations>
+    </primitive>
+    <primitive id="iscsi-target" class="systemd" type="targetctl">
+      <operations>
+        <op id="iscsi-start" name="start" interval="0" timeout="60s"/>
+        <op id="iscsi-stop" name="stop" interval="0" timeout="60s"/>
+        <op id="iscsi-monitor" name="monitor" interval="20s" timeout="40s"/>
+      </operations>
+    </primitive>
+    <primitive id="nfs-server" class="systemd" type="nfs-server">
+      <operations>
+        <op id="nfs-start" name="start" interval="0" timeout="60s"/>
+        <op id="nfs-stop" name="stop" interval="0" timeout="60s"/>
+        <op id="nfs-monitor" name="monitor" interval="30s" timeout="40s"/>
+      </operations>
+    </primitive>
+    <primitive id="vip" class="ocf" type="IPaddr2" provider="heartbeat">
+      <instance_attributes id="vip-attrs">
+        <nvpair id="vip-ip" name="ip" value="192.168.2.229"/>
+        <nvpair id="vip-cidr" name="cidr_netmask" value="24"/>
+      </instance_attributes>
+      <operations>
+        <op id="vip-start" name="start" interval="0" timeout="20s"/>
+        <op id="vip-stop" name="stop" interval="0" timeout="20s"/>
+        <op id="vip-monitor" name="monitor" interval="10s" timeout="20s"/>
+      </operations>
+    </primitive>
+  </group>
+</resources>'
 
-clone ms-drbd0 drbd0 \
+log "Adding Pacemaker ordering and colocation constraints..."
-  meta promotable=true promoted-max=1 promoted-node-max=1 \
+cibadmin --replace --scope constraints --xml-text '<constraints>
-       clone-max=2 clone-node-max=1 \
+  <rsc_order id="order-drbd-group" first="ms-drbd0" first-action="promote" then="ha-group" then-action="start" kind="Mandatory"/>
-       notify=true interleave=true globally-unique=false
+  <rsc_colocation id="coloc-group-with-drbd" score="INFINITY" rsc="ha-group" with-rsc="ms-drbd0" with-rsc-role="Promoted"/>
-
+</constraints>'
-primitive xfs-data ocf:heartbeat:Filesystem \
-  params device=/dev/drbd0 directory=/srv/ha-data fstype=xfs options=defaults \
-         force_unmount=false \
-  op start timeout=60s interval=0 \
-  op stop timeout=60s interval=0 \
-  op monitor interval=20s timeout=40s
-
-primitive iscsi-target systemd:targetctl \
-  op start timeout=60s interval=0 \
-  op stop timeout=60s interval=0 \
-  op monitor interval=20s timeout=40s
-
-primitive nfs-server systemd:nfs-server \
-  op start timeout=60s interval=0 \
-  op stop timeout=60s interval=0 \
-  op monitor interval=30s timeout=40s
-
-primitive vip ocf:heartbeat:IPaddr2 \
-  params ip=192.168.2.229 cidr_netmask=24 \
-  op start timeout=20s interval=0 \
-  op stop timeout=20s interval=0 \
-  op monitor interval=10s timeout=20s
-
-group ha-group xfs-data iscsi-target nfs-server vip
-
-order order-drbd-group Mandatory: ms-drbd0:promote ha-group:start
-colocation coloc-group-with-drbd INFINITY: ha-group ms-drbd0:Promoted
-commit
-CRM_EOF
 
 log "Waiting for resources to start..."
 for i in $(seq 1 60); do

scripts/ha/deploy.sh

@@ -355,6 +355,58 @@ if ! $SKIP_CLUSTER_INIT; then
       "sudo mkdir -p /root/.ssh && sudo cp /tmp/cluster-init-key /root/.ssh/cluster-init-key && \
        sudo chmod 600 /root/.ssh/cluster-init-key && rm -f /tmp/cluster-init-key"
 
+    # Fix targetctl.service on both nodes: the iscsi-target.nix module bakes
+    # pkgs.targetcli-fb for the targetctl binary, but targetctl is actually in
+    # rtslib-fb (a different store path).  Apply a runtime dropin that corrects
+    # both ExecStart and ExecStop before Pacemaker ever touches the service.
+    # The fixed iscsi-target.nix module will make this redundant on next rebuild.
+    logn "Patching targetctl.service on both nodes..."
+    _patch_targetctl() {
+      local ip="$1"
+      ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${ip}" sudo bash << 'PATCH'
+set -euo pipefail
+TC=$(find /nix/store -maxdepth 4 -path '*/python3*env/bin/targetctl' 2>/dev/null | head -1)
+PY=$(find /nix/store -maxdepth 4 -path '*/python3*env/bin/python3' -name 'python3' 2>/dev/null | \
+  while IFS= read -r p; do "$p" -c "import rtslib_fb" 2>/dev/null && echo "$p" && break; done | head -1)
+[[ -n "$TC" && -n "$PY" ]] || { echo "targetctl or python3+rtslib_fb not found"; exit 1; }
+# Write stop script that saves LIO config then tears down kernel state
+"$PY" - "$TC" "$PY" << 'PYEOF'
+import sys, os, stat
+tc, py = sys.argv[1], sys.argv[2]
+script = f"""#!{py}
+import subprocess, sys, rtslib_fb
+root = rtslib_fb.RTSRoot()
+targets = list(root.targets)
+if targets:
+    r = subprocess.run(["{tc}", "save", "/etc/target/saveconfig.json"], capture_output=True)
+    print(f"saved {{len(targets)}} target(s); rc={{r.returncode}}")
+else:
+    print("no active LIO targets")
+for t in targets:
+    try:
+        for tpg in list(t.tpgs): tpg.enable = False
+        t.delete()
+    except Exception as e: print(f"warn: {{e}}", file=sys.stderr)
+for so in list(root.storage_objects):
+    try: so.delete()
+    except Exception as e: print(f"warn: {{e}}", file=sys.stderr)
+print("LIO kernel target cleared")
+"""
+path = "/run/ha-targetctl-stop.py"
+with open(path, "w") as f: f.write(script)
+os.chmod(path, 0o755)
+print(f"wrote {path}")
+PYEOF
+mkdir -p /run/systemd/system/targetctl.service.d
+printf '[Service]\nExecStart=\nExecStart=%s restore /etc/target/saveconfig.json\nExecStop=\nExecStop=/run/ha-targetctl-stop.py\n' \
+  "$TC" > /run/systemd/system/targetctl.service.d/fix-exec.conf
+systemctl daemon-reload
+echo "patched on $(hostname)"
+PATCH
+    }
+    _patch_targetctl "${NODE1_IP}"
+    _patch_targetctl "${NODE2_IP}"
+
     logn "Uploading cluster-init.sh to ${NODE1_HOST}..."
     scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
       "$CLUSTER_INIT" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init.sh"