Archived
beatzaplentyandClaude Sonnet 4.6
724d9a45af
All 7 acceptance tests pass on live NixOS 25.11 VMs (VMIDs 200/201 on
pve1). Failover completes in ~5 s with data integrity verified.
modules/ha/pacemaker-stack.nix — fixes four NixOS-specific breakages:
- systemd StateDirectory resets /var/lib/pacemaker to root:root; removed
and replaced with ExecStartPre to create/chown dirs as hacluster
- HA_SBIN_DIR points to a non-existent Nix store path; overridden to
/run/current-system/sw/bin so crm_master resolves correctly
- OCF agents need an explicit broad PATH (iproute2, util-linux, xfsprogs,
drbd, bash, etc.) — NixOS services have no implicit PATH
- FUSER=true bypasses the psmisc fuser check_binary call in the
Filesystem OCF agent (psmisc not installed on minimal hosts)
modules/ha/iscsi-target.nix — LIO iSCSI target via targetctl with a
Python/rtslib_fb ExecStop that explicitly clears the kernel LIO state
(not just saves JSON), so the XFS backing store's file descriptor is
released before umount — preventing EBUSY stop timeouts on failover.
Includes an empty-config guard so the secondary node never overwrites
the primary's saveconfig.json with an empty one.
test-lab/ha/common.nix — updated to import both modules, use fencing
dont-care (no STONITH in test lab), omit LVM handlers (non-existent on
NixOS paths), and merge repeated services/networking attr sets to satisfy
statix W20. test-lab/ha/acceptance-tests.sh — final v4 with crm_standby
fix (pacemaker 3.x API).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
100 lines
3.3 KiB
Nix
100 lines
3.3 KiB
Nix
# LIO iSCSI target service (targetctl) for NixOS HA clusters.
|
|
#
|
|
# Provides the targetctl.service that saves/restores LIO configuration from
|
|
# /etc/target/saveconfig.json. Pacemaker manages this service via its
|
|
# systemd resource agent (class="systemd" type="targetctl").
|
|
#
|
|
# Why ExecStop is not simply "targetctl save":
|
|
# targetctl save writes the LIO config to JSON but does NOT remove the LIO
|
|
# target from the kernel's configfs. As a result, any fileio backing store
|
|
# that LIO has open (e.g. iscsi-lun.img on an XFS-over-DRBD filesystem)
|
|
# stays referenced in the kernel. The subsequent XFS umount from the
|
|
# Filesystem OCF resource then returns EBUSY and either hangs for the full
|
|
# op-stop timeout or fails outright, blocking the entire failover.
|
|
#
|
|
# The ExecStop script here additionally tears down the kernel LIO state
|
|
# via rtslib_fb after saving, so the backing-store file descriptor is
|
|
# released and umount succeeds immediately.
|
|
#
|
|
# Empty-config guard:
|
|
# The save step is skipped when no iSCSI targets are currently active.
|
|
# This prevents the secondary node (where LIO was never started) from
|
|
# overwriting a valid saveconfig.json with an empty one when Pacemaker
|
|
# stops the iscsi-target resource as part of a failover or cleanup.
|
|
{ pkgs, ... }:
|
|
|
|
let
|
|
python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]);
|
|
targetctl = "${pkgs.targetcli-fb}/bin/targetctl";
|
|
|
|
targetctlStop = pkgs.writeScript "targetctl-stop" ''
|
|
#!${python3}/bin/python3
|
|
import subprocess, sys
|
|
import rtslib_fb
|
|
|
|
root = rtslib_fb.RTSRoot()
|
|
targets = list(root.targets)
|
|
if targets:
|
|
subprocess.run(
|
|
["${targetctl}", "save", "/etc/target/saveconfig.json"],
|
|
capture_output=True,
|
|
)
|
|
print(f"saved {len(targets)} iSCSI target(s)")
|
|
else:
|
|
print("no active LIO targets — saveconfig.json unchanged")
|
|
|
|
for target in targets:
|
|
try:
|
|
for tpg in list(target.tpgs):
|
|
tpg.enable = False
|
|
target.delete()
|
|
except Exception as e:
|
|
print(f"warn (target): {e}", file=sys.stderr)
|
|
for so in list(root.storage_objects):
|
|
try:
|
|
so.delete()
|
|
except Exception as e:
|
|
print(f"warn (backstore): {e}", file=sys.stderr)
|
|
print("LIO kernel target cleared")
|
|
'';
|
|
in
|
|
{
|
|
boot.kernelModules = [
|
|
"target_core_mod"
|
|
"iscsi_target_mod"
|
|
"target_core_file"
|
|
"target_core_pscsi"
|
|
"target_core_user"
|
|
"configfs"
|
|
];
|
|
|
|
systemd = {
|
|
mounts = [{
|
|
where = "/sys/kernel/config";
|
|
what = "configfs";
|
|
type = "configfs";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "targetctl.service" ];
|
|
}];
|
|
services.targetctl = {
|
|
description = "LIO iSCSI target config save/restore";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "sys-kernel-config.mount" "network.target" ];
|
|
requires = [ "sys-kernel-config.mount" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = "${targetctl} restore /etc/target/saveconfig.json";
|
|
ExecStop = "${targetctlStop}";
|
|
};
|
|
unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
|
|
};
|
|
tmpfiles.rules = [
|
|
"d /etc/target 0750 root root -"
|
|
"f /etc/target/saveconfig.json 0640 root root -"
|
|
];
|
|
};
|
|
|
|
environment.systemPackages = [ pkgs.targetcli-fb ];
|
|
}
|