feat(ha): add NixOS modules for DRBD+XFS+LIO+Corosync+Pacemaker HA stack

All 7 acceptance tests pass on live NixOS 25.11 VMs (VMIDs 200/201 on
pve1).  Failover completes in ~5 s with data integrity verified.

modules/ha/pacemaker-stack.nix — fixes four NixOS-specific breakages:
  - systemd StateDirectory resets /var/lib/pacemaker to root:root; removed
    and replaced with ExecStartPre to create/chown dirs as hacluster
  - HA_SBIN_DIR points to a non-existent Nix store path; overridden to
    /run/current-system/sw/bin so crm_master resolves correctly
  - OCF agents need an explicit broad PATH (iproute2, util-linux, xfsprogs,
    drbd, bash, etc.) — NixOS services have no implicit PATH
  - FUSER=true bypasses the psmisc fuser check_binary call in the
    Filesystem OCF agent (psmisc not installed on minimal hosts)

modules/ha/iscsi-target.nix — LIO iSCSI target via targetctl with a
Python/rtslib_fb ExecStop that explicitly clears the kernel LIO state
(not just saves JSON), so the XFS backing store's file descriptor is
released before umount — preventing EBUSY stop timeouts on failover.
Includes an empty-config guard so the secondary node never overwrites
the primary's saveconfig.json with an empty one.

test-lab/ha/common.nix — updated to import both modules, use fencing
dont-care (no STONITH in test lab), omit LVM handlers (non-existent on
NixOS paths), and merge repeated services/networking attr sets to satisfy
statix W20.  test-lab/ha/acceptance-tests.sh — final v4 with crm_standby
fix (pacemaker 3.x API).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8

modules/ha/iscsi-target.nix

@@ -0,0 +1,99 @@
+# LIO iSCSI target service (targetctl) for NixOS HA clusters.
+#
+# Provides the targetctl.service that saves/restores LIO configuration from
+# /etc/target/saveconfig.json.  Pacemaker manages this service via its
+# systemd resource agent (class="systemd" type="targetctl").
+#
+# Why ExecStop is not simply "targetctl save":
+#   targetctl save writes the LIO config to JSON but does NOT remove the LIO
+#   target from the kernel's configfs.  As a result, any fileio backing store
+#   that LIO has open (e.g. iscsi-lun.img on an XFS-over-DRBD filesystem)
+#   stays referenced in the kernel.  The subsequent XFS umount from the
+#   Filesystem OCF resource then returns EBUSY and either hangs for the full
+#   op-stop timeout or fails outright, blocking the entire failover.
+#
+#   The ExecStop script here additionally tears down the kernel LIO state
+#   via rtslib_fb after saving, so the backing-store file descriptor is
+#   released and umount succeeds immediately.
+#
+# Empty-config guard:
+#   The save step is skipped when no iSCSI targets are currently active.
+#   This prevents the secondary node (where LIO was never started) from
+#   overwriting a valid saveconfig.json with an empty one when Pacemaker
+#   stops the iscsi-target resource as part of a failover or cleanup.
+{ pkgs, ... }:
+
+let
+  python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]);
+  targetctl = "${pkgs.targetcli-fb}/bin/targetctl";
+
+  targetctlStop = pkgs.writeScript "targetctl-stop" ''
+    #!${python3}/bin/python3
+    import subprocess, sys
+    import rtslib_fb
+
+    root = rtslib_fb.RTSRoot()
+    targets = list(root.targets)
+    if targets:
+        subprocess.run(
+            ["${targetctl}", "save", "/etc/target/saveconfig.json"],
+            capture_output=True,
+        )
+        print(f"saved {len(targets)} iSCSI target(s)")
+    else:
+        print("no active LIO targets — saveconfig.json unchanged")
+
+    for target in targets:
+        try:
+            for tpg in list(target.tpgs):
+                tpg.enable = False
+            target.delete()
+        except Exception as e:
+            print(f"warn (target): {e}", file=sys.stderr)
+    for so in list(root.storage_objects):
+        try:
+            so.delete()
+        except Exception as e:
+            print(f"warn (backstore): {e}", file=sys.stderr)
+    print("LIO kernel target cleared")
+  '';
+in
+{
+  boot.kernelModules = [
+    "target_core_mod"
+    "iscsi_target_mod"
+    "target_core_file"
+    "target_core_pscsi"
+    "target_core_user"
+    "configfs"
+  ];
+
+  systemd = {
+    mounts = [{
+      where = "/sys/kernel/config";
+      what = "configfs";
+      type = "configfs";
+      wantedBy = [ "multi-user.target" ];
+      before = [ "targetctl.service" ];
+    }];
+    services.targetctl = {
+      description = "LIO iSCSI target config save/restore";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "sys-kernel-config.mount" "network.target" ];
+      requires = [ "sys-kernel-config.mount" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = "${targetctl} restore /etc/target/saveconfig.json";
+        ExecStop = "${targetctlStop}";
+      };
+      unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
+    };
+    tmpfiles.rules = [
+      "d /etc/target 0750 root root -"
+      "f /etc/target/saveconfig.json 0640 root root -"
+    ];
+  };
+
+  environment.systemPackages = [ pkgs.targetcli-fb ];
+}

modules/ha/pacemaker-stack.nix

@@ -0,0 +1,94 @@
+# Pacemaker + Corosync HA stack for NixOS with known-good workarounds.
+#
+# Issues fixed here (confirmed through live testing on NixOS 25.11):
+#
+# 1. StateDirectory ownership reset: systemd's StateDirectory=pacemaker
+#    creates /var/lib/pacemaker owned root:root.  pacemaker-based (the CIB
+#    daemon) runs as the hacluster user and calls pcmk__daemon_can_write,
+#    which requires the CIB directory to be owned by hacluster or be
+#    group-writable by haclient.  Workaround: remove StateDirectory and let
+#    ExecStartPre create every required subdirectory with correct ownership.
+#
+# 2. HA_SBIN_DIR wrong path: ocf-shellfuncs sets HA_SBIN_DIR to the Nix
+#    store path of the resource-agents derivation's /sbin, which doesn't
+#    exist.  The DRBD OCF agent uses ${HA_SBIN_DIR}/crm_master, so it exits
+#    127 without this override.  Fix: export HA_SBIN_DIR=/run/current-system/sw/bin.
+#
+# 3. Broad PATH for OCF agents: the resource executor (pacemaker-execd) runs
+#    OCF agent scripts as children.  NixOS provides no implicit PATH for
+#    system services; without an explicit PATH the agents can't find ip, ss,
+#    mount, umount, drbdadm, etc.
+#
+# 4. FUSER=true: the Filesystem OCF agent calls check_binary $FUSER (default:
+#    fuser from psmisc), which is not installed.  Setting FUSER=true makes
+#    check_binary succeed (true is always in PATH) and the subsequent
+#    "$FUSER -km $mountpoint" becomes a no-op.  Pair with force_unmount=false
+#    on each Filesystem resource unless you want lazy unmount behaviour.
+{ lib, pkgs, ... }:
+
+let
+  ocfBinPath = lib.concatStringsSep ":" [
+    "${pkgs.iproute2}/bin"
+    "${pkgs.iproute2}/sbin"
+    "${pkgs.iputils}/bin"
+    "${pkgs.util-linux}/bin"
+    "${pkgs.util-linux}/sbin"
+    "${pkgs.gawk}/bin"
+    "${pkgs.gnugrep}/bin"
+    "${pkgs.gnused}/bin"
+    "${pkgs.coreutils}/bin"
+    "${pkgs.bash}/bin"
+    "${pkgs.procps}/bin"
+    "${pkgs.xfsprogs}/bin"
+    "${pkgs.drbd}/bin"
+    "${pkgs.python3}/bin"
+    "/run/current-system/sw/bin"
+    "/run/current-system/sw/sbin"
+    "/usr/local/sbin"
+    "/usr/local/bin"
+    "/usr/sbin"
+    "/usr/bin"
+    "/sbin"
+    "/bin"
+  ];
+
+  # Single pre-start script: schemas symlink + directory ownership.
+  # Runs before pacemakerd so pacemaker-based finds hacluster-owned dirs.
+  preStartCmd = "${pkgs.bash}/bin/bash -c '"
+    + "ln -sfn ${pkgs.pacemaker}/share/pacemaker /var/lib/pacemaker/schemas; "
+    + "for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores "
+    + "/var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox "
+    + "/var/lib/pacemaker/hostcache; do "
+    + "mkdir -p \"\\$d\" && chown hacluster:pacemaker \"\\$d\" && chmod 2770 \"\\$d\"; "
+    + "done'";
+
+  ocfEnv = {
+    PATH = lib.mkForce ocfBinPath;
+    OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
+    HA_SBIN_DIR = "/run/current-system/sw/bin";
+    FUSER = "true";
+  };
+in
+{
+  users.groups.haclient = { };
+
+  services.corosync.enable = true;
+  services.pacemaker.enable = true;
+
+  systemd.services = {
+    pacemaker = {
+      serviceConfig = {
+        StateDirectory = lib.mkForce "";
+        ExecStartPre = lib.mkBefore [ preStartCmd ];
+      };
+      environment = ocfEnv;
+    };
+    pacemaker-execd.environment = ocfEnv;
+  };
+
+  environment.systemPackages = with pkgs; [
+    corosync
+    pacemaker
+    ocf-resource-agents
+  ];
+}

test-lab/ha/acceptance-tests.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+# HA File Server Acceptance Tests T1-T7
+# Failover (T5) uses pacemaker standby mode to gracefully move resources,
+# simulating what STONITH + node-restart does in production.
+# Note: A production cluster requires real STONITH (fence agent for the hypervisor).
+set -euo pipefail
+
+NODE2="root@192.168.2.201"
+VIP="192.168.2.202"
+IQN="iqn.2026-01.local.ha-test:storage"
+MOUNT="/srv/ha-data"
+PACEMAKERD="/nix/store/3v9sb74cg2qmpcyzb4h6fq0z8bvp5gw1-pacemaker-3.0.1/sbin/pacemakerd"
+MYNODE=$(hostname)
+
+SSH="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15"
+PASS=0; FAIL=0
+
+pass() { echo "[PASS] $1"; ((PASS++)) || true; }
+fail() { echo "[FAIL] $1"; ((FAIL++)) || true; }
+info() { echo "[INFO] $1"; }
+
+echo "=== HA File Server Acceptance Tests ==="
+echo "Node: $MYNODE  Date: $(date)"
+echo ""
+
+# T1: Corosync 2-node cluster with quorum
+echo "--- T1: Corosync cluster quorum ---"
+QUORATE=$(corosync-quorumtool -s 2>/dev/null | grep 'Quorate:' | awk '{print $2}')
+NODE_COUNT=$(corosync-quorumtool -s 2>/dev/null | grep '^Nodes:' | awk '{print $2}')
+if [ "$QUORATE" = "Yes" ] && [ "$NODE_COUNT" = "2" ]; then
+  pass "T1: Corosync quorate with 2 nodes"
+else
+  fail "T1: quorate=$QUORATE nodes=$NODE_COUNT"
+fi
+
+# T2: DRBD both UpToDate, replication Established
+echo "--- T2: DRBD replication healthy ---"
+DRBD_STATUS=$(drbdadm status ha-data 2>/dev/null)
+N1_ROLE=$(echo "$DRBD_STATUS" | grep '^ha-data role:' | awk -F: '{print $2}')
+N1_DISK=$(echo "$DRBD_STATUS" | grep -oP 'disk:\K\S+' | head -1)
+N2_DISK=$(echo "$DRBD_STATUS" | grep -oP 'peer-disk:\K\S+' | head -1)
+REPL=$(echo "$DRBD_STATUS" | grep -oP 'replication:\K\S+' | head -1)
+info "DRBD: role=$N1_ROLE local_disk=$N1_DISK peer_disk=$N2_DISK replication=$REPL"
+if [ "$N1_DISK" = "UpToDate" ] && [ "$N2_DISK" = "UpToDate" ] && [ "$REPL" = "Established" ]; then
+  pass "T2: DRBD both UpToDate, Established (role=$N1_ROLE)"
+else
+  fail "T2: DRBD issue: local=$N1_DISK peer=$N2_DISK replication=$REPL"
+fi
+
+# T3: XFS mounted on primary
+echo "--- T3: XFS mount on primary ---"
+if mountpoint -q $MOUNT && df -t xfs $MOUNT &>/dev/null; then
+  FSINFO=$(df -h $MOUNT | tail -1)
+  pass "T3: XFS mounted at $MOUNT: $FSINFO"
+else
+  fail "T3: XFS not mounted at $MOUNT"
+fi
+
+# T4: iSCSI active on primary, VIP responds on port 3260
+echo "--- T4: iSCSI target active ---"
+ACTIVE_IQN=$(ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep iqn | head -1)
+if [ "$ACTIVE_IQN" = "$IQN" ] && nc -w3 $VIP 3260 < /dev/null 2>/dev/null; then
+  pass "T4: iSCSI $IQN active, port 3260 open on VIP $VIP"
+elif [ "$ACTIVE_IQN" = "$IQN" ]; then
+  fail "T4: iSCSI IQN active but port 3260 not reachable on VIP"
+else
+  fail "T4: iSCSI not active (got '$ACTIVE_IQN')"
+fi
+
+# Pre-T5: write test file for data integrity check
+echo "--- Pre-T5: writing test data ---"
+TESTFILE="$MOUNT/failover-test.txt"
+TESTDATA="FAILOVER_INTEGRITY_$(date +%s)"
+echo "$TESTDATA" > "$TESTFILE"
+sync
+info "Wrote: $TESTFILE (data: $TESTDATA)"
+
+# T5: Failover — put this node into pacemaker standby, forcing resource migration
+# (In production this is triggered by real STONITH; standby simulates the result.)
+echo "--- T5: Failover (pacemaker standby + node isolation) ---"
+info "Putting $MYNODE into standby mode (triggers resource migration to node2)..."
+crm_standby -N "$MYNODE" -v on 2>&1 || true
+
+FAILOVER_OK=false
+info "Waiting up to 90s for node2 failover..."
+for i in $(seq 1 18); do
+  sleep 5
+  STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "UNREACHABLE")
+  if echo "$STATUS" | grep -q "Started ha-test-node2"; then
+    FAILOVER_OK=true
+    info "Failover complete at $((i*5))s"
+    echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted'
+    break
+  fi
+done
+
+if $FAILOVER_OK; then
+  N2_DRBD=$($SSH $NODE2 "drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print \$2}'" 2>/dev/null || echo "unknown")
+  N2_MOUNT=$($SSH $NODE2 "mountpoint -q $MOUNT && echo 'mounted' || echo 'not-mounted'" 2>/dev/null || echo "unknown")
+  N2_ISCSI=$($SSH $NODE2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" 2>/dev/null || echo "0")
+  N2_VIP=$($SSH $NODE2 "ip addr show | grep -c '$VIP'" 2>/dev/null || echo "0")
+  info "Node2: DRBD=$N2_DRBD mount=$N2_MOUNT iSCSI_IQNs=$N2_ISCSI VIP=$N2_VIP"
+  
+  FAILS=0
+  [ "$N2_DRBD" = "Primary" ]  || { info "FAIL: DRBD not Primary on node2"; ((FAILS++)) || true; }
+  [ "$N2_MOUNT" = "mounted" ] || { info "FAIL: XFS not mounted on node2"; ((FAILS++)) || true; }
+  [ "$N2_ISCSI" -ge "1" ] 2>/dev/null || { info "FAIL: iSCSI not active on node2"; ((FAILS++)) || true; }
+  [ "$N2_VIP" -ge "1" ] 2>/dev/null || { info "FAIL: VIP not on node2"; ((FAILS++)) || true; }
+  
+  if [ $FAILS -eq 0 ]; then
+    pass "T5: Failover complete — DRBD Primary, XFS, iSCSI, VIP all on node2"
+  else
+    fail "T5: Partial failover ($FAILS sub-checks failed)"
+  fi
+else
+  fail "T5: No failover detected within 90s"
+fi
+
+# T7: Data integrity — test file readable on node2 after failover
+echo "--- T7: Data integrity after failover ---"
+if $SSH $NODE2 "grep -q '$TESTDATA' $TESTFILE 2>/dev/null"; then
+  pass "T7: Test data intact on node2 after failover"
+else
+  ACTUAL=$($SSH $NODE2 "cat $TESTFILE 2>/dev/null || echo FILE_MISSING" 2>/dev/null || echo "SSH_FAIL")
+  fail "T7: Data integrity check failed (expected '$TESTDATA', got '$ACTUAL')"
+fi
+
+# T6: Node rejoin — take node out of standby
+echo "--- T6: Node rejoin ---"
+info "Taking $MYNODE out of standby..."
+crm_standby -N "$MYNODE" -v off 2>&1 || true
+
+REJOIN_OK=false
+for i in $(seq 1 12); do
+  sleep 5
+  # Check if this node is back online (no longer standby)
+  STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "")
+  if echo "$STATUS" | grep -q "Online:.*$MYNODE"; then
+    REJOIN_OK=true
+    info "Rejoined at $((i*5))s"
+    echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted'
+    break
+  fi
+done
+
+if $REJOIN_OK; then
+  sleep 5
+  N1_DRBD=$(drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print $2}')
+  N1_DISK=$(drbdadm status ha-data 2>/dev/null | grep -oP 'disk:\K\S+' | head -1)
+  info "Node1 DRBD after rejoin: role=$N1_DRBD disk=$N1_DISK"
+  pass "T6: Node rejoined cluster (DRBD role=$N1_DRBD, disk=$N1_DISK)"
+else
+  fail "T6: Node did not rejoin within 60s"
+fi
+
+echo ""
+echo "========================================"
+echo "RESULTS: $PASS passed, $FAIL failed"
+echo "========================================"
+echo ""
+echo "NOTES:"
+echo "  T5 uses pacemaker standby to simulate failover (production needs STONITH"
+echo "  fence agent, e.g. fence_pve_ssh, to crash the VM — same requirement on Debian)"
+echo ""
+if [ $FAIL -eq 0 ]; then
+  echo "VERDICT: ALL TESTS PASSED → Deliverable A (NixOS modules)"
+  exit 0
+else
+  echo "VERDICT: $FAIL TEST(S) FAILED → review above"
+  exit 1
+fi

test-lab/ha/common.nix

@@ -11,39 +11,6 @@ let
   # Test-only corosync authkey (128 bytes = 1024 bits minimum for corosync).
   # Not secret — this is a disposable test cluster, not production.
   testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
-
-  # OCF agent path (from pacemaker's --with-ocfdir).
-  # We expose this on PATH so custom scripts can find each other.
-  ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d";
-
-  # Concatenated PATH that includes all binaries OCF agents and pacemaker
-  # lrmd children need.  This is the workaround for nixpkgs#207891 (PATH
-  # not set correctly for OCF agent child processes).
-  ocfBinPath = lib.concatStringsSep ":" [
-    "${pkgs.iproute2}/bin"
-    "${pkgs.iproute2}/sbin"
-    "${pkgs.iputils}/bin"
-    "${pkgs.util-linux}/bin"
-    "${pkgs.util-linux}/sbin"
-    "${pkgs.gawk}/bin"
-    "${pkgs.gnugrep}/bin"
-    "${pkgs.gnused}/bin"
-    "${pkgs.coreutils}/bin"
-    "${pkgs.bash}/bin"
-    "${pkgs.procps}/bin"
-    "${pkgs.xfsprogs}/bin"
-    "${pkgs.drbd}/bin"
-    "${pkgs.targetcli-fb}/bin"
-    "${pkgs.python3}/bin"
-    "/run/current-system/sw/bin"
-    "/run/current-system/sw/sbin"
-    "/usr/local/sbin"
-    "/usr/local/bin"
-    "/usr/sbin"
-    "/usr/bin"
-    "/sbin"
-    "/bin"
-  ];
 in
 {
   system.stateVersion = "26.05";
@@ -52,33 +19,32 @@ in
   imports = [
     ../../modules/hardware-configuration/vm/proxmox.nix
     ../../modules/boot/efi.nix
+    ../../modules/ha/pacemaker-stack.nix
+    ../../modules/ha/iscsi-target.nix
   ];
 
   # ── Nix settings ──────────────────────────────────────────────────────
   nix.settings.experimental-features = [ "nix-command" "flakes" ];
 
-  # ── SSH ───────────────────────────────────────────────────────────────
-  services.openssh = {
-    enable = true;
-    settings.PermitRootLogin = "yes";
-  };
   users.users.root.openssh.authorizedKeys.keys = [
     vars.adminSshKey
     # Claude Code session key (this machine) — test-lab only
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
   ];
 
+  # ── Services ──────────────────────────────────────────────────────────
+  services = {
+    openssh = {
+      enable = true;
+      settings.PermitRootLogin = "yes";
+    };
+
     # Allow QEMU guest exec for key injection fallback
-  services.qemuGuest.enable = true;
+    qemuGuest.enable = true;
 
-  # ── Networking ────────────────────────────────────────────────────────
+    drbd = {
-  networking.useDHCP = false;
+      enable = true;
-  networking.defaultGateway = "192.168.2.1";
+      config = ''
-  networking.nameservers = [ "192.168.2.1" "8.8.8.8" ];
-
-  # ── DRBD ──────────────────────────────────────────────────────────────
-  services.drbd.enable = true;
-  services.drbd.config = ''
         global {
           usage-count yes;
         }
@@ -92,14 +58,18 @@ in
             after-sb-1pri discard-secondary;
           }
           disk {
-        # resource-only: DRBD itself won't fence (Pacemaker handles STONITH);
+            # dont-care: DRBD itself won't fence before promoting.  Production
-        # the DRBD resource agent uses fencing to guard primary promotion.
+            # clusters should use resource-only here and configure a STONITH
-        fencing resource-only;
+            # fence agent (e.g. fence_pve_ssh) in Pacemaker so DRBD can safely
-      }
+            # protect against split-brain without risking dual-Primary.
-      handlers {
+            # For this test cluster (no fence device) dont-care lets promotion
-        # NOTE: LVM-specific before/after-resync-target handlers omitted
+            # proceed; the DRBD kernel module still refuses dual-Primary without
-        # (raw /dev/sdb, no LVM). split-brain handler also omitted —
+            # allow-two-primaries in net {}.
-        # Pacemaker STONITH manages split-brain fencing for the test cluster.
+            fencing dont-care;
+            # LVM before/after-resync-target handlers omitted: the LVM snapshot
+            # scripts (/usr/lib/drbd/snapshot-resync-target-lvm.sh) don't exist
+            # on NixOS paths.  If present, DRBD calls them on resync and exits 127,
+            # dropping the peer connection and leaving the secondary Outdated.
           }
         }
 
@@ -119,16 +89,17 @@ in
           }
         }
       '';
+    };
 
-  # ── Corosync ──────────────────────────────────────────────────────────
+    # services.corosync.enable = true is set by modules/ha/pacemaker-stack.nix
-  services.corosync = {
+    corosync = {
-    enable = true;
       clusterName = "ha-test";
       nodelist = [
         { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; }
         { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; }
       ];
     };
+  };
 
   # Corosync authkey (test-only, not secret — generated with
   # `corosync-keygen` for production).
@@ -137,81 +108,13 @@ in
     mode = "0400";
   };
 
-  # ── Pacemaker ─────────────────────────────────────────────────────────
-  services.pacemaker.enable = true;
-
-  # Fix for nixpkgs#207891:
-  # 1. Ensure CIB directories are owned by hacluster before starting.
-  #    The stock module sets StateDirectory=pacemaker (owned by root);
-  #    pacemaker internally drops to hacluster but needs to write there.
-  # 2. Set PATH so OCF agent child processes can find all required binaries.
-  systemd.services.pacemaker.serviceConfig = {
-    ExecStartPre = [
-      "${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'"
-    ];
-  };
-  systemd.services.pacemaker.environment = {
-    PATH = lib.mkForce ocfBinPath;
-    # Expose OCF root so pacemaker and lrmd agree on where agents live.
-    OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
-  };
-
-  # pacemaker-execd is the local resource executor that calls OCF agents.
-  # Give it the same PATH so OCF scripts can find all required binaries.
-  systemd.services.pacemaker-execd.environment = {
-    PATH = lib.mkForce ocfBinPath;
-    OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
-  };
-
-  # ── LIO / iSCSI target ────────────────────────────────────────────────
-  # targetcli-fb is the management tool; actual kernel support is via
-  # the LIO modules.  We add a systemd service that saves/restores the
-  # target configuration so Pacemaker can trigger it via a systemd-class
-  # resource.
-  boot.kernelModules = [
-    "target_core_mod"
-    "iscsi_target_mod"
-    "target_core_file"
-    "target_core_pscsi"
-    "target_core_user"
-    "configfs"
-  ];
-
-  # configfs must be mounted for rtslib/targetcli to work
-  systemd.mounts = [{
-    where = "/sys/kernel/config";
-    what = "configfs";
-    type = "configfs";
-    wantedBy = [ "multi-user.target" ];
-    before = [ "targetctl.service" ];
-  }];
-
-  # targetctl: save/restore LIO configuration (mirrors Debian's package)
-  systemd.services.targetctl = {
-    description = "LIO iSCSI target config save/restore";
-    wantedBy = [ "multi-user.target" ];
-    after = [ "sys-kernel-config.mount" "network.target" ];
-    requires = [ "sys-kernel-config.mount" ];
-    serviceConfig = {
-      Type = "oneshot";
-      RemainAfterExit = true;
-      ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json";
-      ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json";
-    };
-    unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
-  };
-
   # ── Packages ──────────────────────────────────────────────────────────
+  # corosync, pacemaker, ocf-resource-agents, targetcli-fb already added
+  # by the ha/ modules; add the remaining stack-specific tools here.
   environment.systemPackages = with pkgs; [
-    # HA stack
-    corosync # corosync-cfgtool, corosync-quorumtool
-    pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI
-    drbd # drbdadm, drbdsetup, drbdmon
-    ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …)
-
     # Storage
+    drbd # drbdadm, drbdsetup, drbdmon
     xfsprogs # mkfs.xfs, xfs_admin, xfs_info
-    targetcli-fb # targetcli shell + targetctl
 
     # Networking / debug
     iproute2 # ip, ss
@@ -227,33 +130,32 @@ in
     htop
   ];
 
-  # ── Firewall ──────────────────────────────────────────────────────────
+  # ── Networking ────────────────────────────────────────────────────────
-  networking.firewall = {
+  networking = {
+    useDHCP = false;
+    defaultGateway = "192.168.2.1";
+    nameservers = [ "192.168.2.1" "8.8.8.8" ];
+
+    firewall = {
       enable = true;
       allowedTCPPorts = [
         22 # SSH
         3260 # iSCSI
         3121 # pacemaker-remoted
         2224 # pcsd
-      drbdPort # DRBD replication
+        drbdPort
       ];
       allowedUDPPorts = [
-      5404 # corosync cluster
+        5404 # corosync
-      5405 # corosync cluster
+        5405 # corosync
         5407 # corosync crypto
       ];
-    # Corosync uses ports 5404-5407 UDP; allow them on the cluster net
       extraCommands = ''
         iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT
         iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT
       '';
     };
-
+  };
-  # ── tmpfiles: target config dir ────────────────────────────────────────
-  systemd.tmpfiles.rules = [
-    "d /etc/target 0750 root root -"
-    "f /etc/target/saveconfig.json 0640 root root -"
-  ];
 
   # ── Locale / time ─────────────────────────────────────────────────────
   time.timeZone = vars.timeZone;