diff --git a/modules/ha/iscsi-target.nix b/modules/ha/iscsi-target.nix new file mode 100644 index 0000000..c39ec88 --- /dev/null +++ b/modules/ha/iscsi-target.nix @@ -0,0 +1,99 @@ +# LIO iSCSI target service (targetctl) for NixOS HA clusters. +# +# Provides the targetctl.service that saves/restores LIO configuration from +# /etc/target/saveconfig.json. Pacemaker manages this service via its +# systemd resource agent (class="systemd" type="targetctl"). +# +# Why ExecStop is not simply "targetctl save": +# targetctl save writes the LIO config to JSON but does NOT remove the LIO +# target from the kernel's configfs. As a result, any fileio backing store +# that LIO has open (e.g. iscsi-lun.img on an XFS-over-DRBD filesystem) +# stays referenced in the kernel. The subsequent XFS umount from the +# Filesystem OCF resource then returns EBUSY and either hangs for the full +# op-stop timeout or fails outright, blocking the entire failover. +# +# The ExecStop script here additionally tears down the kernel LIO state +# via rtslib_fb after saving, so the backing-store file descriptor is +# released and umount succeeds immediately. +# +# Empty-config guard: +# The save step is skipped when no iSCSI targets are currently active. +# This prevents the secondary node (where LIO was never started) from +# overwriting a valid saveconfig.json with an empty one when Pacemaker +# stops the iscsi-target resource as part of a failover or cleanup. +{ pkgs, ... }: + +let + python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]); + targetctl = "${pkgs.targetcli-fb}/bin/targetctl"; + + targetctlStop = pkgs.writeScript "targetctl-stop" '' + #!${python3}/bin/python3 + import subprocess, sys + import rtslib_fb + + root = rtslib_fb.RTSRoot() + targets = list(root.targets) + if targets: + subprocess.run( + ["${targetctl}", "save", "/etc/target/saveconfig.json"], + capture_output=True, + ) + print(f"saved {len(targets)} iSCSI target(s)") + else: + print("no active LIO targets — saveconfig.json unchanged") + + for target in targets: + try: + for tpg in list(target.tpgs): + tpg.enable = False + target.delete() + except Exception as e: + print(f"warn (target): {e}", file=sys.stderr) + for so in list(root.storage_objects): + try: + so.delete() + except Exception as e: + print(f"warn (backstore): {e}", file=sys.stderr) + print("LIO kernel target cleared") + ''; +in +{ + boot.kernelModules = [ + "target_core_mod" + "iscsi_target_mod" + "target_core_file" + "target_core_pscsi" + "target_core_user" + "configfs" + ]; + + systemd = { + mounts = [{ + where = "/sys/kernel/config"; + what = "configfs"; + type = "configfs"; + wantedBy = [ "multi-user.target" ]; + before = [ "targetctl.service" ]; + }]; + services.targetctl = { + description = "LIO iSCSI target config save/restore"; + wantedBy = [ "multi-user.target" ]; + after = [ "sys-kernel-config.mount" "network.target" ]; + requires = [ "sys-kernel-config.mount" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${targetctl} restore /etc/target/saveconfig.json"; + ExecStop = "${targetctlStop}"; + }; + unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json"; + }; + tmpfiles.rules = [ + "d /etc/target 0750 root root -" + "f /etc/target/saveconfig.json 0640 root root -" + ]; + }; + + environment.systemPackages = [ pkgs.targetcli-fb ]; +} diff --git a/modules/ha/pacemaker-stack.nix b/modules/ha/pacemaker-stack.nix new file mode 100644 index 0000000..8fcab2c --- /dev/null +++ b/modules/ha/pacemaker-stack.nix @@ -0,0 +1,94 @@ +# Pacemaker + Corosync HA stack for NixOS with known-good workarounds. +# +# Issues fixed here (confirmed through live testing on NixOS 25.11): +# +# 1. StateDirectory ownership reset: systemd's StateDirectory=pacemaker +# creates /var/lib/pacemaker owned root:root. pacemaker-based (the CIB +# daemon) runs as the hacluster user and calls pcmk__daemon_can_write, +# which requires the CIB directory to be owned by hacluster or be +# group-writable by haclient. Workaround: remove StateDirectory and let +# ExecStartPre create every required subdirectory with correct ownership. +# +# 2. HA_SBIN_DIR wrong path: ocf-shellfuncs sets HA_SBIN_DIR to the Nix +# store path of the resource-agents derivation's /sbin, which doesn't +# exist. The DRBD OCF agent uses ${HA_SBIN_DIR}/crm_master, so it exits +# 127 without this override. Fix: export HA_SBIN_DIR=/run/current-system/sw/bin. +# +# 3. Broad PATH for OCF agents: the resource executor (pacemaker-execd) runs +# OCF agent scripts as children. NixOS provides no implicit PATH for +# system services; without an explicit PATH the agents can't find ip, ss, +# mount, umount, drbdadm, etc. +# +# 4. FUSER=true: the Filesystem OCF agent calls check_binary $FUSER (default: +# fuser from psmisc), which is not installed. Setting FUSER=true makes +# check_binary succeed (true is always in PATH) and the subsequent +# "$FUSER -km $mountpoint" becomes a no-op. Pair with force_unmount=false +# on each Filesystem resource unless you want lazy unmount behaviour. +{ lib, pkgs, ... }: + +let + ocfBinPath = lib.concatStringsSep ":" [ + "${pkgs.iproute2}/bin" + "${pkgs.iproute2}/sbin" + "${pkgs.iputils}/bin" + "${pkgs.util-linux}/bin" + "${pkgs.util-linux}/sbin" + "${pkgs.gawk}/bin" + "${pkgs.gnugrep}/bin" + "${pkgs.gnused}/bin" + "${pkgs.coreutils}/bin" + "${pkgs.bash}/bin" + "${pkgs.procps}/bin" + "${pkgs.xfsprogs}/bin" + "${pkgs.drbd}/bin" + "${pkgs.python3}/bin" + "/run/current-system/sw/bin" + "/run/current-system/sw/sbin" + "/usr/local/sbin" + "/usr/local/bin" + "/usr/sbin" + "/usr/bin" + "/sbin" + "/bin" + ]; + + # Single pre-start script: schemas symlink + directory ownership. + # Runs before pacemakerd so pacemaker-based finds hacluster-owned dirs. + preStartCmd = "${pkgs.bash}/bin/bash -c '" + + "ln -sfn ${pkgs.pacemaker}/share/pacemaker /var/lib/pacemaker/schemas; " + + "for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores " + + "/var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox " + + "/var/lib/pacemaker/hostcache; do " + + "mkdir -p \"\\$d\" && chown hacluster:pacemaker \"\\$d\" && chmod 2770 \"\\$d\"; " + + "done'"; + + ocfEnv = { + PATH = lib.mkForce ocfBinPath; + OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; + HA_SBIN_DIR = "/run/current-system/sw/bin"; + FUSER = "true"; + }; +in +{ + users.groups.haclient = { }; + + services.corosync.enable = true; + services.pacemaker.enable = true; + + systemd.services = { + pacemaker = { + serviceConfig = { + StateDirectory = lib.mkForce ""; + ExecStartPre = lib.mkBefore [ preStartCmd ]; + }; + environment = ocfEnv; + }; + pacemaker-execd.environment = ocfEnv; + }; + + environment.systemPackages = with pkgs; [ + corosync + pacemaker + ocf-resource-agents + ]; +} diff --git a/test-lab/ha/acceptance-tests.sh b/test-lab/ha/acceptance-tests.sh new file mode 100755 index 0000000..651f1e9 --- /dev/null +++ b/test-lab/ha/acceptance-tests.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# HA File Server Acceptance Tests T1-T7 +# Failover (T5) uses pacemaker standby mode to gracefully move resources, +# simulating what STONITH + node-restart does in production. +# Note: A production cluster requires real STONITH (fence agent for the hypervisor). +set -euo pipefail + +NODE2="root@192.168.2.201" +VIP="192.168.2.202" +IQN="iqn.2026-01.local.ha-test:storage" +MOUNT="/srv/ha-data" +PACEMAKERD="/nix/store/3v9sb74cg2qmpcyzb4h6fq0z8bvp5gw1-pacemaker-3.0.1/sbin/pacemakerd" +MYNODE=$(hostname) + +SSH="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15" +PASS=0; FAIL=0 + +pass() { echo "[PASS] $1"; ((PASS++)) || true; } +fail() { echo "[FAIL] $1"; ((FAIL++)) || true; } +info() { echo "[INFO] $1"; } + +echo "=== HA File Server Acceptance Tests ===" +echo "Node: $MYNODE Date: $(date)" +echo "" + +# T1: Corosync 2-node cluster with quorum +echo "--- T1: Corosync cluster quorum ---" +QUORATE=$(corosync-quorumtool -s 2>/dev/null | grep 'Quorate:' | awk '{print $2}') +NODE_COUNT=$(corosync-quorumtool -s 2>/dev/null | grep '^Nodes:' | awk '{print $2}') +if [ "$QUORATE" = "Yes" ] && [ "$NODE_COUNT" = "2" ]; then + pass "T1: Corosync quorate with 2 nodes" +else + fail "T1: quorate=$QUORATE nodes=$NODE_COUNT" +fi + +# T2: DRBD both UpToDate, replication Established +echo "--- T2: DRBD replication healthy ---" +DRBD_STATUS=$(drbdadm status ha-data 2>/dev/null) +N1_ROLE=$(echo "$DRBD_STATUS" | grep '^ha-data role:' | awk -F: '{print $2}') +N1_DISK=$(echo "$DRBD_STATUS" | grep -oP 'disk:\K\S+' | head -1) +N2_DISK=$(echo "$DRBD_STATUS" | grep -oP 'peer-disk:\K\S+' | head -1) +REPL=$(echo "$DRBD_STATUS" | grep -oP 'replication:\K\S+' | head -1) +info "DRBD: role=$N1_ROLE local_disk=$N1_DISK peer_disk=$N2_DISK replication=$REPL" +if [ "$N1_DISK" = "UpToDate" ] && [ "$N2_DISK" = "UpToDate" ] && [ "$REPL" = "Established" ]; then + pass "T2: DRBD both UpToDate, Established (role=$N1_ROLE)" +else + fail "T2: DRBD issue: local=$N1_DISK peer=$N2_DISK replication=$REPL" +fi + +# T3: XFS mounted on primary +echo "--- T3: XFS mount on primary ---" +if mountpoint -q $MOUNT && df -t xfs $MOUNT &>/dev/null; then + FSINFO=$(df -h $MOUNT | tail -1) + pass "T3: XFS mounted at $MOUNT: $FSINFO" +else + fail "T3: XFS not mounted at $MOUNT" +fi + +# T4: iSCSI active on primary, VIP responds on port 3260 +echo "--- T4: iSCSI target active ---" +ACTIVE_IQN=$(ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep iqn | head -1) +if [ "$ACTIVE_IQN" = "$IQN" ] && nc -w3 $VIP 3260 < /dev/null 2>/dev/null; then + pass "T4: iSCSI $IQN active, port 3260 open on VIP $VIP" +elif [ "$ACTIVE_IQN" = "$IQN" ]; then + fail "T4: iSCSI IQN active but port 3260 not reachable on VIP" +else + fail "T4: iSCSI not active (got '$ACTIVE_IQN')" +fi + +# Pre-T5: write test file for data integrity check +echo "--- Pre-T5: writing test data ---" +TESTFILE="$MOUNT/failover-test.txt" +TESTDATA="FAILOVER_INTEGRITY_$(date +%s)" +echo "$TESTDATA" > "$TESTFILE" +sync +info "Wrote: $TESTFILE (data: $TESTDATA)" + +# T5: Failover — put this node into pacemaker standby, forcing resource migration +# (In production this is triggered by real STONITH; standby simulates the result.) +echo "--- T5: Failover (pacemaker standby + node isolation) ---" +info "Putting $MYNODE into standby mode (triggers resource migration to node2)..." +crm_standby -N "$MYNODE" -v on 2>&1 || true + +FAILOVER_OK=false +info "Waiting up to 90s for node2 failover..." +for i in $(seq 1 18); do + sleep 5 + STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "UNREACHABLE") + if echo "$STATUS" | grep -q "Started ha-test-node2"; then + FAILOVER_OK=true + info "Failover complete at $((i*5))s" + echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted' + break + fi +done + +if $FAILOVER_OK; then + N2_DRBD=$($SSH $NODE2 "drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print \$2}'" 2>/dev/null || echo "unknown") + N2_MOUNT=$($SSH $NODE2 "mountpoint -q $MOUNT && echo 'mounted' || echo 'not-mounted'" 2>/dev/null || echo "unknown") + N2_ISCSI=$($SSH $NODE2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" 2>/dev/null || echo "0") + N2_VIP=$($SSH $NODE2 "ip addr show | grep -c '$VIP'" 2>/dev/null || echo "0") + info "Node2: DRBD=$N2_DRBD mount=$N2_MOUNT iSCSI_IQNs=$N2_ISCSI VIP=$N2_VIP" + + FAILS=0 + [ "$N2_DRBD" = "Primary" ] || { info "FAIL: DRBD not Primary on node2"; ((FAILS++)) || true; } + [ "$N2_MOUNT" = "mounted" ] || { info "FAIL: XFS not mounted on node2"; ((FAILS++)) || true; } + [ "$N2_ISCSI" -ge "1" ] 2>/dev/null || { info "FAIL: iSCSI not active on node2"; ((FAILS++)) || true; } + [ "$N2_VIP" -ge "1" ] 2>/dev/null || { info "FAIL: VIP not on node2"; ((FAILS++)) || true; } + + if [ $FAILS -eq 0 ]; then + pass "T5: Failover complete — DRBD Primary, XFS, iSCSI, VIP all on node2" + else + fail "T5: Partial failover ($FAILS sub-checks failed)" + fi +else + fail "T5: No failover detected within 90s" +fi + +# T7: Data integrity — test file readable on node2 after failover +echo "--- T7: Data integrity after failover ---" +if $SSH $NODE2 "grep -q '$TESTDATA' $TESTFILE 2>/dev/null"; then + pass "T7: Test data intact on node2 after failover" +else + ACTUAL=$($SSH $NODE2 "cat $TESTFILE 2>/dev/null || echo FILE_MISSING" 2>/dev/null || echo "SSH_FAIL") + fail "T7: Data integrity check failed (expected '$TESTDATA', got '$ACTUAL')" +fi + +# T6: Node rejoin — take node out of standby +echo "--- T6: Node rejoin ---" +info "Taking $MYNODE out of standby..." +crm_standby -N "$MYNODE" -v off 2>&1 || true + +REJOIN_OK=false +for i in $(seq 1 12); do + sleep 5 + # Check if this node is back online (no longer standby) + STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "") + if echo "$STATUS" | grep -q "Online:.*$MYNODE"; then + REJOIN_OK=true + info "Rejoined at $((i*5))s" + echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted' + break + fi +done + +if $REJOIN_OK; then + sleep 5 + N1_DRBD=$(drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print $2}') + N1_DISK=$(drbdadm status ha-data 2>/dev/null | grep -oP 'disk:\K\S+' | head -1) + info "Node1 DRBD after rejoin: role=$N1_DRBD disk=$N1_DISK" + pass "T6: Node rejoined cluster (DRBD role=$N1_DRBD, disk=$N1_DISK)" +else + fail "T6: Node did not rejoin within 60s" +fi + +echo "" +echo "========================================" +echo "RESULTS: $PASS passed, $FAIL failed" +echo "========================================" +echo "" +echo "NOTES:" +echo " T5 uses pacemaker standby to simulate failover (production needs STONITH" +echo " fence agent, e.g. fence_pve_ssh, to crash the VM — same requirement on Debian)" +echo "" +if [ $FAIL -eq 0 ]; then + echo "VERDICT: ALL TESTS PASSED → Deliverable A (NixOS modules)" + exit 0 +else + echo "VERDICT: $FAIL TEST(S) FAILED → review above" + exit 1 +fi diff --git a/test-lab/ha/common.nix b/test-lab/ha/common.nix index 8b75d7b..c984e1c 100644 --- a/test-lab/ha/common.nix +++ b/test-lab/ha/common.nix @@ -11,39 +11,6 @@ let # Test-only corosync authkey (128 bytes = 1024 bits minimum for corosync). # Not secret — this is a disposable test cluster, not production. testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; - - # OCF agent path (from pacemaker's --with-ocfdir). - # We expose this on PATH so custom scripts can find each other. - ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d"; - - # Concatenated PATH that includes all binaries OCF agents and pacemaker - # lrmd children need. This is the workaround for nixpkgs#207891 (PATH - # not set correctly for OCF agent child processes). - ocfBinPath = lib.concatStringsSep ":" [ - "${pkgs.iproute2}/bin" - "${pkgs.iproute2}/sbin" - "${pkgs.iputils}/bin" - "${pkgs.util-linux}/bin" - "${pkgs.util-linux}/sbin" - "${pkgs.gawk}/bin" - "${pkgs.gnugrep}/bin" - "${pkgs.gnused}/bin" - "${pkgs.coreutils}/bin" - "${pkgs.bash}/bin" - "${pkgs.procps}/bin" - "${pkgs.xfsprogs}/bin" - "${pkgs.drbd}/bin" - "${pkgs.targetcli-fb}/bin" - "${pkgs.python3}/bin" - "/run/current-system/sw/bin" - "/run/current-system/sw/sbin" - "/usr/local/sbin" - "/usr/local/bin" - "/usr/sbin" - "/usr/bin" - "/sbin" - "/bin" - ]; in { system.stateVersion = "26.05"; @@ -52,82 +19,86 @@ in imports = [ ../../modules/hardware-configuration/vm/proxmox.nix ../../modules/boot/efi.nix + ../../modules/ha/pacemaker-stack.nix + ../../modules/ha/iscsi-target.nix ]; # ── Nix settings ────────────────────────────────────────────────────── nix.settings.experimental-features = [ "nix-command" "flakes" ]; - # ── SSH ─────────────────────────────────────────────────────────────── - services.openssh = { - enable = true; - settings.PermitRootLogin = "yes"; - }; users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey # Claude Code session key (this machine) — test-lab only "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos" ]; - # Allow QEMU guest exec for key injection fallback - services.qemuGuest.enable = true; + # ── Services ────────────────────────────────────────────────────────── + services = { + openssh = { + enable = true; + settings.PermitRootLogin = "yes"; + }; - # ── Networking ──────────────────────────────────────────────────────── - networking.useDHCP = false; - networking.defaultGateway = "192.168.2.1"; - networking.nameservers = [ "192.168.2.1" "8.8.8.8" ]; + # Allow QEMU guest exec for key injection fallback + qemuGuest.enable = true; - # ── DRBD ────────────────────────────────────────────────────────────── - services.drbd.enable = true; - services.drbd.config = '' - global { - usage-count yes; - } + drbd = { + enable = true; + config = '' + global { + usage-count yes; + } - common { - net { - protocol C; - ping-int 1; - verify-alg sha256; - after-sb-0pri discard-zero-changes; - after-sb-1pri discard-secondary; - } - disk { - # resource-only: DRBD itself won't fence (Pacemaker handles STONITH); - # the DRBD resource agent uses fencing to guard primary promotion. - fencing resource-only; - } - handlers { - # NOTE: LVM-specific before/after-resync-target handlers omitted - # (raw /dev/sdb, no LVM). split-brain handler also omitted — - # Pacemaker STONITH manages split-brain fencing for the test cluster. - } - } + common { + net { + protocol C; + ping-int 1; + verify-alg sha256; + after-sb-0pri discard-zero-changes; + after-sb-1pri discard-secondary; + } + disk { + # dont-care: DRBD itself won't fence before promoting. Production + # clusters should use resource-only here and configure a STONITH + # fence agent (e.g. fence_pve_ssh) in Pacemaker so DRBD can safely + # protect against split-brain without risking dual-Primary. + # For this test cluster (no fence device) dont-care lets promotion + # proceed; the DRBD kernel module still refuses dual-Primary without + # allow-two-primaries in net {}. + fencing dont-care; + # LVM before/after-resync-target handlers omitted: the LVM snapshot + # scripts (/usr/lib/drbd/snapshot-resync-target-lvm.sh) don't exist + # on NixOS paths. If present, DRBD calls them on resync and exits 127, + # dropping the peer connection and leaving the secondary Outdated. + } + } - resource ha-data { - volume 0 { - device /dev/drbd0; - disk /dev/sdb; # scsi1 in Proxmox VM → sdb - meta-disk internal; - } + resource ha-data { + volume 0 { + device /dev/drbd0; + disk /dev/sdb; # scsi1 in Proxmox VM → sdb + meta-disk internal; + } - on ha-test-node1 { - address ${node1Ip}:${toString drbdPort}; - } + on ha-test-node1 { + address ${node1Ip}:${toString drbdPort}; + } - on ha-test-node2 { - address ${node2Ip}:${toString drbdPort}; - } - } - ''; + on ha-test-node2 { + address ${node2Ip}:${toString drbdPort}; + } + } + ''; + }; - # ── Corosync ────────────────────────────────────────────────────────── - services.corosync = { - enable = true; - clusterName = "ha-test"; - nodelist = [ - { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; } - { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; } - ]; + # services.corosync.enable = true is set by modules/ha/pacemaker-stack.nix + corosync = { + clusterName = "ha-test"; + nodelist = [ + { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; } + { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; } + ]; + }; }; # Corosync authkey (test-only, not secret — generated with @@ -137,81 +108,13 @@ in mode = "0400"; }; - # ── Pacemaker ───────────────────────────────────────────────────────── - services.pacemaker.enable = true; - - # Fix for nixpkgs#207891: - # 1. Ensure CIB directories are owned by hacluster before starting. - # The stock module sets StateDirectory=pacemaker (owned by root); - # pacemaker internally drops to hacluster but needs to write there. - # 2. Set PATH so OCF agent child processes can find all required binaries. - systemd.services.pacemaker.serviceConfig = { - ExecStartPre = [ - "${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'" - ]; - }; - systemd.services.pacemaker.environment = { - PATH = lib.mkForce ocfBinPath; - # Expose OCF root so pacemaker and lrmd agree on where agents live. - OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; - }; - - # pacemaker-execd is the local resource executor that calls OCF agents. - # Give it the same PATH so OCF scripts can find all required binaries. - systemd.services.pacemaker-execd.environment = { - PATH = lib.mkForce ocfBinPath; - OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; - }; - - # ── LIO / iSCSI target ──────────────────────────────────────────────── - # targetcli-fb is the management tool; actual kernel support is via - # the LIO modules. We add a systemd service that saves/restores the - # target configuration so Pacemaker can trigger it via a systemd-class - # resource. - boot.kernelModules = [ - "target_core_mod" - "iscsi_target_mod" - "target_core_file" - "target_core_pscsi" - "target_core_user" - "configfs" - ]; - - # configfs must be mounted for rtslib/targetcli to work - systemd.mounts = [{ - where = "/sys/kernel/config"; - what = "configfs"; - type = "configfs"; - wantedBy = [ "multi-user.target" ]; - before = [ "targetctl.service" ]; - }]; - - # targetctl: save/restore LIO configuration (mirrors Debian's package) - systemd.services.targetctl = { - description = "LIO iSCSI target config save/restore"; - wantedBy = [ "multi-user.target" ]; - after = [ "sys-kernel-config.mount" "network.target" ]; - requires = [ "sys-kernel-config.mount" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json"; - ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json"; - }; - unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json"; - }; - # ── Packages ────────────────────────────────────────────────────────── + # corosync, pacemaker, ocf-resource-agents, targetcli-fb already added + # by the ha/ modules; add the remaining stack-specific tools here. environment.systemPackages = with pkgs; [ - # HA stack - corosync # corosync-cfgtool, corosync-quorumtool - pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI - drbd # drbdadm, drbdsetup, drbdmon - ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …) - # Storage + drbd # drbdadm, drbdsetup, drbdmon xfsprogs # mkfs.xfs, xfs_admin, xfs_info - targetcli-fb # targetcli shell + targetctl # Networking / debug iproute2 # ip, ss @@ -227,33 +130,32 @@ in htop ]; - # ── Firewall ────────────────────────────────────────────────────────── - networking.firewall = { - enable = true; - allowedTCPPorts = [ - 22 # SSH - 3260 # iSCSI - 3121 # pacemaker-remoted - 2224 # pcsd - drbdPort # DRBD replication - ]; - allowedUDPPorts = [ - 5404 # corosync cluster - 5405 # corosync cluster - 5407 # corosync crypto - ]; - # Corosync uses ports 5404-5407 UDP; allow them on the cluster net - extraCommands = '' - iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT - iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT - ''; - }; + # ── Networking ──────────────────────────────────────────────────────── + networking = { + useDHCP = false; + defaultGateway = "192.168.2.1"; + nameservers = [ "192.168.2.1" "8.8.8.8" ]; - # ── tmpfiles: target config dir ──────────────────────────────────────── - systemd.tmpfiles.rules = [ - "d /etc/target 0750 root root -" - "f /etc/target/saveconfig.json 0640 root root -" - ]; + firewall = { + enable = true; + allowedTCPPorts = [ + 22 # SSH + 3260 # iSCSI + 3121 # pacemaker-remoted + 2224 # pcsd + drbdPort + ]; + allowedUDPPorts = [ + 5404 # corosync + 5405 # corosync + 5407 # corosync crypto + ]; + extraCommands = '' + iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT + iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT + ''; + }; + }; # ── Locale / time ───────────────────────────────────────────────────── time.timeZone = vars.timeZone;