Archived
feat(ha): add NixOS modules for DRBD+XFS+LIO+Corosync+Pacemaker HA stack
All 7 acceptance tests pass on live NixOS 25.11 VMs (VMIDs 200/201 on
pve1). Failover completes in ~5 s with data integrity verified.
modules/ha/pacemaker-stack.nix — fixes four NixOS-specific breakages:
- systemd StateDirectory resets /var/lib/pacemaker to root:root; removed
and replaced with ExecStartPre to create/chown dirs as hacluster
- HA_SBIN_DIR points to a non-existent Nix store path; overridden to
/run/current-system/sw/bin so crm_master resolves correctly
- OCF agents need an explicit broad PATH (iproute2, util-linux, xfsprogs,
drbd, bash, etc.) — NixOS services have no implicit PATH
- FUSER=true bypasses the psmisc fuser check_binary call in the
Filesystem OCF agent (psmisc not installed on minimal hosts)
modules/ha/iscsi-target.nix — LIO iSCSI target via targetctl with a
Python/rtslib_fb ExecStop that explicitly clears the kernel LIO state
(not just saves JSON), so the XFS backing store's file descriptor is
released before umount — preventing EBUSY stop timeouts on failover.
Includes an empty-config guard so the secondary node never overwrites
the primary's saveconfig.json with an empty one.
test-lab/ha/common.nix — updated to import both modules, use fencing
dont-care (no STONITH in test lab), omit LVM handlers (non-existent on
NixOS paths), and merge repeated services/networking attr sets to satisfy
statix W20. test-lab/ha/acceptance-tests.sh — final v4 with crm_standby
fix (pacemaker 3.x API).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
This commit is contained in:
Claude Sonnet 4.6
@@ -0,0 +1,99 @@
|
||||
# LIO iSCSI target service (targetctl) for NixOS HA clusters.
|
||||
#
|
||||
# Provides the targetctl.service that saves/restores LIO configuration from
|
||||
# /etc/target/saveconfig.json. Pacemaker manages this service via its
|
||||
# systemd resource agent (class="systemd" type="targetctl").
|
||||
#
|
||||
# Why ExecStop is not simply "targetctl save":
|
||||
# targetctl save writes the LIO config to JSON but does NOT remove the LIO
|
||||
# target from the kernel's configfs. As a result, any fileio backing store
|
||||
# that LIO has open (e.g. iscsi-lun.img on an XFS-over-DRBD filesystem)
|
||||
# stays referenced in the kernel. The subsequent XFS umount from the
|
||||
# Filesystem OCF resource then returns EBUSY and either hangs for the full
|
||||
# op-stop timeout or fails outright, blocking the entire failover.
|
||||
#
|
||||
# The ExecStop script here additionally tears down the kernel LIO state
|
||||
# via rtslib_fb after saving, so the backing-store file descriptor is
|
||||
# released and umount succeeds immediately.
|
||||
#
|
||||
# Empty-config guard:
|
||||
# The save step is skipped when no iSCSI targets are currently active.
|
||||
# This prevents the secondary node (where LIO was never started) from
|
||||
# overwriting a valid saveconfig.json with an empty one when Pacemaker
|
||||
# stops the iscsi-target resource as part of a failover or cleanup.
|
||||
{ pkgs, ... }:
|
||||
|
||||
let
|
||||
python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]);
|
||||
targetctl = "${pkgs.targetcli-fb}/bin/targetctl";
|
||||
|
||||
targetctlStop = pkgs.writeScript "targetctl-stop" ''
|
||||
#!${python3}/bin/python3
|
||||
import subprocess, sys
|
||||
import rtslib_fb
|
||||
|
||||
root = rtslib_fb.RTSRoot()
|
||||
targets = list(root.targets)
|
||||
if targets:
|
||||
subprocess.run(
|
||||
["${targetctl}", "save", "/etc/target/saveconfig.json"],
|
||||
capture_output=True,
|
||||
)
|
||||
print(f"saved {len(targets)} iSCSI target(s)")
|
||||
else:
|
||||
print("no active LIO targets — saveconfig.json unchanged")
|
||||
|
||||
for target in targets:
|
||||
try:
|
||||
for tpg in list(target.tpgs):
|
||||
tpg.enable = False
|
||||
target.delete()
|
||||
except Exception as e:
|
||||
print(f"warn (target): {e}", file=sys.stderr)
|
||||
for so in list(root.storage_objects):
|
||||
try:
|
||||
so.delete()
|
||||
except Exception as e:
|
||||
print(f"warn (backstore): {e}", file=sys.stderr)
|
||||
print("LIO kernel target cleared")
|
||||
'';
|
||||
in
|
||||
{
|
||||
boot.kernelModules = [
|
||||
"target_core_mod"
|
||||
"iscsi_target_mod"
|
||||
"target_core_file"
|
||||
"target_core_pscsi"
|
||||
"target_core_user"
|
||||
"configfs"
|
||||
];
|
||||
|
||||
systemd = {
|
||||
mounts = [{
|
||||
where = "/sys/kernel/config";
|
||||
what = "configfs";
|
||||
type = "configfs";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [ "targetctl.service" ];
|
||||
}];
|
||||
services.targetctl = {
|
||||
description = "LIO iSCSI target config save/restore";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "sys-kernel-config.mount" "network.target" ];
|
||||
requires = [ "sys-kernel-config.mount" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = "${targetctl} restore /etc/target/saveconfig.json";
|
||||
ExecStop = "${targetctlStop}";
|
||||
};
|
||||
unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
|
||||
};
|
||||
tmpfiles.rules = [
|
||||
"d /etc/target 0750 root root -"
|
||||
"f /etc/target/saveconfig.json 0640 root root -"
|
||||
];
|
||||
};
|
||||
|
||||
environment.systemPackages = [ pkgs.targetcli-fb ];
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
# Pacemaker + Corosync HA stack for NixOS with known-good workarounds.
|
||||
#
|
||||
# Issues fixed here (confirmed through live testing on NixOS 25.11):
|
||||
#
|
||||
# 1. StateDirectory ownership reset: systemd's StateDirectory=pacemaker
|
||||
# creates /var/lib/pacemaker owned root:root. pacemaker-based (the CIB
|
||||
# daemon) runs as the hacluster user and calls pcmk__daemon_can_write,
|
||||
# which requires the CIB directory to be owned by hacluster or be
|
||||
# group-writable by haclient. Workaround: remove StateDirectory and let
|
||||
# ExecStartPre create every required subdirectory with correct ownership.
|
||||
#
|
||||
# 2. HA_SBIN_DIR wrong path: ocf-shellfuncs sets HA_SBIN_DIR to the Nix
|
||||
# store path of the resource-agents derivation's /sbin, which doesn't
|
||||
# exist. The DRBD OCF agent uses ${HA_SBIN_DIR}/crm_master, so it exits
|
||||
# 127 without this override. Fix: export HA_SBIN_DIR=/run/current-system/sw/bin.
|
||||
#
|
||||
# 3. Broad PATH for OCF agents: the resource executor (pacemaker-execd) runs
|
||||
# OCF agent scripts as children. NixOS provides no implicit PATH for
|
||||
# system services; without an explicit PATH the agents can't find ip, ss,
|
||||
# mount, umount, drbdadm, etc.
|
||||
#
|
||||
# 4. FUSER=true: the Filesystem OCF agent calls check_binary $FUSER (default:
|
||||
# fuser from psmisc), which is not installed. Setting FUSER=true makes
|
||||
# check_binary succeed (true is always in PATH) and the subsequent
|
||||
# "$FUSER -km $mountpoint" becomes a no-op. Pair with force_unmount=false
|
||||
# on each Filesystem resource unless you want lazy unmount behaviour.
|
||||
{ lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
ocfBinPath = lib.concatStringsSep ":" [
|
||||
"${pkgs.iproute2}/bin"
|
||||
"${pkgs.iproute2}/sbin"
|
||||
"${pkgs.iputils}/bin"
|
||||
"${pkgs.util-linux}/bin"
|
||||
"${pkgs.util-linux}/sbin"
|
||||
"${pkgs.gawk}/bin"
|
||||
"${pkgs.gnugrep}/bin"
|
||||
"${pkgs.gnused}/bin"
|
||||
"${pkgs.coreutils}/bin"
|
||||
"${pkgs.bash}/bin"
|
||||
"${pkgs.procps}/bin"
|
||||
"${pkgs.xfsprogs}/bin"
|
||||
"${pkgs.drbd}/bin"
|
||||
"${pkgs.python3}/bin"
|
||||
"/run/current-system/sw/bin"
|
||||
"/run/current-system/sw/sbin"
|
||||
"/usr/local/sbin"
|
||||
"/usr/local/bin"
|
||||
"/usr/sbin"
|
||||
"/usr/bin"
|
||||
"/sbin"
|
||||
"/bin"
|
||||
];
|
||||
|
||||
# Single pre-start script: schemas symlink + directory ownership.
|
||||
# Runs before pacemakerd so pacemaker-based finds hacluster-owned dirs.
|
||||
preStartCmd = "${pkgs.bash}/bin/bash -c '"
|
||||
+ "ln -sfn ${pkgs.pacemaker}/share/pacemaker /var/lib/pacemaker/schemas; "
|
||||
+ "for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores "
|
||||
+ "/var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox "
|
||||
+ "/var/lib/pacemaker/hostcache; do "
|
||||
+ "mkdir -p \"\\$d\" && chown hacluster:pacemaker \"\\$d\" && chmod 2770 \"\\$d\"; "
|
||||
+ "done'";
|
||||
|
||||
ocfEnv = {
|
||||
PATH = lib.mkForce ocfBinPath;
|
||||
OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
|
||||
HA_SBIN_DIR = "/run/current-system/sw/bin";
|
||||
FUSER = "true";
|
||||
};
|
||||
in
|
||||
{
|
||||
users.groups.haclient = { };
|
||||
|
||||
services.corosync.enable = true;
|
||||
services.pacemaker.enable = true;
|
||||
|
||||
systemd.services = {
|
||||
pacemaker = {
|
||||
serviceConfig = {
|
||||
StateDirectory = lib.mkForce "";
|
||||
ExecStartPre = lib.mkBefore [ preStartCmd ];
|
||||
};
|
||||
environment = ocfEnv;
|
||||
};
|
||||
pacemaker-execd.environment = ocfEnv;
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
corosync
|
||||
pacemaker
|
||||
ocf-resource-agents
|
||||
];
|
||||
}
|
||||
Reference in New Issue
Block a user