Archived
Merge pull request 'Worktree ha file server test' (#81) from worktree-ha-file-server-test into main
Reviewed-on: #81
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
# HA file server build type: DRBD + XFS + LIO iSCSI + NFS, managed by
|
||||
# Corosync + Pacemaker. Both ha-server-1 and ha-server-2 use this type.
|
||||
#
|
||||
# NFS start/stop:
|
||||
# services.nfs.server.enable = true configures /etc/exports, wires up
|
||||
# rpcbind, and loads kernel modules — but nfs-server.service.wantedBy is
|
||||
# force-cleared so systemd does NOT auto-start it at boot. Pacemaker's
|
||||
# ha-group resource group (configured by scripts/ha/cluster-init.sh)
|
||||
# starts and stops nfs-server as part of the failover sequence after the
|
||||
# XFS mount and iSCSI target are brought up on the new Active node.
|
||||
#
|
||||
# Beszel agent:
|
||||
# Enabled here via enable-agent.nix. The agent KEY (used to pair with
|
||||
# the Beszel hub) is not set yet — add it to hosts/ha-server-{1,2}/host.nix
|
||||
# under services.beszel.agent.environment.KEY once the hub accepts the
|
||||
# new agents, following the pattern in hosts/server/host.nix.
|
||||
{ lib, vars, ... }:
|
||||
{
|
||||
imports = [
|
||||
../ha/pacemaker-stack.nix
|
||||
../ha/iscsi-target.nix
|
||||
../ha/cluster-config.nix
|
||||
../beszel/enable-agent.nix
|
||||
];
|
||||
|
||||
services.nfs.server = {
|
||||
enable = true;
|
||||
exports = ''
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
'';
|
||||
};
|
||||
|
||||
# Pacemaker controls nfs-server — prevent systemd from starting it at boot
|
||||
# on both nodes (only the Active node should be serving NFS).
|
||||
systemd.services.nfs-server.wantedBy = lib.mkForce [ ];
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
# Cluster-wide HA config shared by both ha-server nodes.
|
||||
#
|
||||
# Covers everything that is identical on both nodes and references cluster
|
||||
# topology (node IPs, hostnames, DRBD resource). Per-node identity
|
||||
# (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix.
|
||||
#
|
||||
# Corosync authkey:
|
||||
# /etc/corosync/authkey (mode 0400) is managed by sops-nix below.
|
||||
# Bootstrap: run scripts/ha/cluster-init.sh on node1 to generate the key,
|
||||
# then encrypt it with: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey
|
||||
# Both host keys must be registered via sync-host-keys.sh first so both nodes can decrypt it.
|
||||
#
|
||||
# DRBD fencing:
|
||||
# Production setting is resource-only: DRBD waits for the STONITH fence
|
||||
# agent to confirm the peer is dead before promoting to Primary. This
|
||||
# requires a working fence_pve_ssh STONITH resource in Pacemaker
|
||||
# (see scripts/ha/cluster-enable-stonith.sh). On a fresh cluster with
|
||||
# no fence device yet, temporarily change to dont-care and run
|
||||
# cluster-enable-stonith.sh once the fence key is deployed.
|
||||
{ lib, vars, ... }:
|
||||
{
|
||||
services.drbd = {
|
||||
enable = true;
|
||||
config = ''
|
||||
global {
|
||||
usage-count yes;
|
||||
}
|
||||
|
||||
common {
|
||||
net {
|
||||
protocol C;
|
||||
ping-int 1;
|
||||
verify-alg sha256;
|
||||
after-sb-0pri discard-zero-changes;
|
||||
after-sb-1pri discard-secondary;
|
||||
}
|
||||
disk {
|
||||
fencing resource-only;
|
||||
}
|
||||
}
|
||||
|
||||
resource ha-data {
|
||||
volume 0 {
|
||||
device /dev/drbd0;
|
||||
disk /dev/sdb;
|
||||
meta-disk internal;
|
||||
}
|
||||
|
||||
on ${vars.haServer1Host} {
|
||||
address ${vars.haServer1Ip}:${toString vars.ports.haServerDrbd};
|
||||
}
|
||||
|
||||
on ${vars.haServer2Host} {
|
||||
address ${vars.haServer2Ip}:${toString vars.ports.haServerDrbd};
|
||||
}
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
# /etc/corosync/authkey — sops binary secret, identical on both nodes.
|
||||
# Decryptable by both ha-server host keys (added by sync-host-keys.sh).
|
||||
sops.secrets.corosync_authkey = {
|
||||
sopsFile = ../../secrets/ha-corosync-authkey;
|
||||
format = "binary";
|
||||
path = "/etc/corosync/authkey";
|
||||
mode = "0400";
|
||||
restartUnits = [ "corosync.service" ];
|
||||
};
|
||||
|
||||
# NixOS common config enables NetworkManager by default; HA cluster nodes
|
||||
# need stable static IPs with predictable interface names — NM is not suitable.
|
||||
networking.networkmanager.enable = lib.mkForce false;
|
||||
|
||||
# services.corosync.enable is set by modules/ha/pacemaker-stack.nix.
|
||||
services.corosync = {
|
||||
clusterName = "ha-cluster";
|
||||
nodelist = [
|
||||
{ nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1Ip ]; }
|
||||
{ nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2Ip ]; }
|
||||
];
|
||||
};
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [
|
||||
vars.ports.haServerIscsi
|
||||
vars.ports.haServerPacemakerRemoted
|
||||
vars.ports.haServerPcsd
|
||||
vars.ports.haServerDrbd
|
||||
vars.ports.nfsRpcbind
|
||||
vars.ports.nfsd
|
||||
vars.ports.nfsMountd
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
vars.ports.haServerCorosync1
|
||||
vars.ports.haServerCorosync2
|
||||
vars.ports.haServerCorosyncCrypto
|
||||
vars.ports.nfsRpcbind
|
||||
vars.ports.nfsd
|
||||
vars.ports.nfsMountd
|
||||
];
|
||||
extraCommands = ''
|
||||
iptables -A INPUT -s ${vars.haServer1Ip}/32 -j ACCEPT
|
||||
iptables -A INPUT -s ${vars.haServer2Ip}/32 -j ACCEPT
|
||||
'';
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
# LIO iSCSI target service (targetctl) for NixOS HA clusters.
|
||||
#
|
||||
# Provides the targetctl.service that saves/restores LIO configuration from
|
||||
# /etc/target/saveconfig.json. Pacemaker manages this service via its
|
||||
# systemd resource agent (class="systemd" type="targetctl").
|
||||
#
|
||||
# Why ExecStop is not simply "targetctl save":
|
||||
# targetctl save writes the LIO config to JSON but does NOT remove the LIO
|
||||
# target from the kernel's configfs. As a result, any fileio backing store
|
||||
# that LIO has open (e.g. iscsi-lun.img on an XFS-over-DRBD filesystem)
|
||||
# stays referenced in the kernel. The subsequent XFS umount from the
|
||||
# Filesystem OCF resource then returns EBUSY and either hangs for the full
|
||||
# op-stop timeout or fails outright, blocking the entire failover.
|
||||
#
|
||||
# The ExecStop script here additionally tears down the kernel LIO state
|
||||
# via rtslib_fb after saving, so the backing-store file descriptor is
|
||||
# released and umount succeeds immediately.
|
||||
#
|
||||
# Empty-config guard:
|
||||
# The save step is skipped when no iSCSI targets are currently active.
|
||||
# This prevents the secondary node (where LIO was never started) from
|
||||
# overwriting a valid saveconfig.json with an empty one when Pacemaker
|
||||
# stops the iscsi-target resource as part of a failover or cleanup.
|
||||
{ pkgs, ... }:
|
||||
|
||||
let
|
||||
python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]);
|
||||
targetctl = "${pkgs.targetcli-fb}/bin/targetctl";
|
||||
|
||||
targetctlStop = pkgs.writeScript "targetctl-stop" ''
|
||||
#!${python3}/bin/python3
|
||||
import subprocess, sys
|
||||
import rtslib_fb
|
||||
|
||||
root = rtslib_fb.RTSRoot()
|
||||
targets = list(root.targets)
|
||||
if targets:
|
||||
subprocess.run(
|
||||
["${targetctl}", "save", "/etc/target/saveconfig.json"],
|
||||
capture_output=True,
|
||||
)
|
||||
print(f"saved {len(targets)} iSCSI target(s)")
|
||||
else:
|
||||
print("no active LIO targets — saveconfig.json unchanged")
|
||||
|
||||
for target in targets:
|
||||
try:
|
||||
for tpg in list(target.tpgs):
|
||||
tpg.enable = False
|
||||
target.delete()
|
||||
except Exception as e:
|
||||
print(f"warn (target): {e}", file=sys.stderr)
|
||||
for so in list(root.storage_objects):
|
||||
try:
|
||||
so.delete()
|
||||
except Exception as e:
|
||||
print(f"warn (backstore): {e}", file=sys.stderr)
|
||||
print("LIO kernel target cleared")
|
||||
'';
|
||||
in
|
||||
{
|
||||
boot.kernelModules = [
|
||||
"target_core_mod"
|
||||
"iscsi_target_mod"
|
||||
"target_core_file"
|
||||
"target_core_pscsi"
|
||||
"target_core_user"
|
||||
"configfs"
|
||||
];
|
||||
|
||||
systemd = {
|
||||
mounts = [{
|
||||
where = "/sys/kernel/config";
|
||||
what = "configfs";
|
||||
type = "configfs";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [ "targetctl.service" ];
|
||||
}];
|
||||
services.targetctl = {
|
||||
description = "LIO iSCSI target config save/restore";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "sys-kernel-config.mount" "network.target" ];
|
||||
requires = [ "sys-kernel-config.mount" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = "${targetctl} restore /etc/target/saveconfig.json";
|
||||
ExecStop = "${targetctlStop}";
|
||||
};
|
||||
unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
|
||||
};
|
||||
tmpfiles.rules = [
|
||||
"d /etc/target 0750 root root -"
|
||||
"f /etc/target/saveconfig.json 0640 root root -"
|
||||
];
|
||||
};
|
||||
|
||||
environment.systemPackages = [ pkgs.targetcli-fb ];
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
# Pacemaker + Corosync HA stack for NixOS with known-good workarounds.
|
||||
#
|
||||
# Issues fixed here (confirmed through live testing on NixOS 25.11):
|
||||
#
|
||||
# 1. StateDirectory ownership reset: systemd's StateDirectory=pacemaker
|
||||
# creates /var/lib/pacemaker owned root:root. pacemaker-based (the CIB
|
||||
# daemon) runs as the hacluster user and calls pcmk__daemon_can_write,
|
||||
# which requires the CIB directory to be owned by hacluster or be
|
||||
# group-writable by haclient. Workaround: remove StateDirectory and let
|
||||
# ExecStartPre create every required subdirectory with correct ownership.
|
||||
#
|
||||
# 2. HA_SBIN_DIR wrong path: ocf-shellfuncs sets HA_SBIN_DIR to the Nix
|
||||
# store path of the resource-agents derivation's /sbin, which doesn't
|
||||
# exist. The DRBD OCF agent uses ${HA_SBIN_DIR}/crm_master, so it exits
|
||||
# 127 without this override. Fix: export HA_SBIN_DIR=/run/current-system/sw/bin.
|
||||
#
|
||||
# 3. Broad PATH for OCF agents: the resource executor (pacemaker-execd) runs
|
||||
# OCF agent scripts as children. NixOS provides no implicit PATH for
|
||||
# system services; without an explicit PATH the agents can't find ip, ss,
|
||||
# mount, umount, drbdadm, etc.
|
||||
#
|
||||
# 4. FUSER=true: the Filesystem OCF agent calls check_binary $FUSER (default:
|
||||
# fuser from psmisc), which is not installed. Setting FUSER=true makes
|
||||
# check_binary succeed (true is always in PATH) and the subsequent
|
||||
# "$FUSER -km $mountpoint" becomes a no-op. Pair with force_unmount=false
|
||||
# on each Filesystem resource unless you want lazy unmount behaviour.
|
||||
{ lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
ocfBinPath = lib.concatStringsSep ":" [
|
||||
"${pkgs.iproute2}/bin"
|
||||
"${pkgs.iproute2}/sbin"
|
||||
"${pkgs.iputils}/bin"
|
||||
"${pkgs.util-linux}/bin"
|
||||
"${pkgs.util-linux}/sbin"
|
||||
"${pkgs.gawk}/bin"
|
||||
"${pkgs.gnugrep}/bin"
|
||||
"${pkgs.gnused}/bin"
|
||||
"${pkgs.coreutils}/bin"
|
||||
"${pkgs.bash}/bin"
|
||||
"${pkgs.procps}/bin"
|
||||
"${pkgs.xfsprogs}/bin"
|
||||
"${pkgs.drbd}/bin"
|
||||
"${pkgs.python3}/bin"
|
||||
"/run/current-system/sw/bin"
|
||||
"/run/current-system/sw/sbin"
|
||||
"/usr/local/sbin"
|
||||
"/usr/local/bin"
|
||||
"/usr/sbin"
|
||||
"/usr/bin"
|
||||
"/sbin"
|
||||
"/bin"
|
||||
];
|
||||
|
||||
# Single pre-start script: schemas symlink + directory ownership.
|
||||
# Runs before pacemakerd so pacemaker-based finds hacluster-owned dirs.
|
||||
preStartCmd = "${pkgs.bash}/bin/bash -c '"
|
||||
+ "ln -sfn ${pkgs.pacemaker}/share/pacemaker /var/lib/pacemaker/schemas; "
|
||||
+ "for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores "
|
||||
+ "/var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox "
|
||||
+ "/var/lib/pacemaker/hostcache; do "
|
||||
+ "mkdir -p \"\\$d\" && chown hacluster:pacemaker \"\\$d\" && chmod 2770 \"\\$d\"; "
|
||||
+ "done'";
|
||||
|
||||
ocfEnv = {
|
||||
PATH = lib.mkForce ocfBinPath;
|
||||
OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
|
||||
HA_SBIN_DIR = "/run/current-system/sw/bin";
|
||||
FUSER = "true";
|
||||
};
|
||||
in
|
||||
{
|
||||
users.groups.haclient = { };
|
||||
|
||||
services.corosync.enable = true;
|
||||
services.pacemaker.enable = true;
|
||||
|
||||
systemd.services = {
|
||||
pacemaker = {
|
||||
serviceConfig = {
|
||||
StateDirectory = lib.mkForce "";
|
||||
ExecStartPre = lib.mkBefore [ preStartCmd ];
|
||||
};
|
||||
environment = ocfEnv;
|
||||
};
|
||||
pacemaker-execd.environment = ocfEnv;
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
corosync
|
||||
pacemaker
|
||||
ocf-resource-agents
|
||||
];
|
||||
}
|
||||
Reference in New Issue
Block a user