From 750121e9dd19477bc4180fcc44b174a4d4664eba Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 27 Jul 2026 05:46:04 +1000 Subject: [PATCH 1/7] test-lab: add two-node HA file-server test cluster config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disposable test VMs (ha-test-node1 / ha-test-node2, VMIDs 200/201 on pve1) to evaluate whether the DRBD + XFS + LIO + Corosync + Pacemaker stack runs correctly on NixOS before deciding NixOS vs Debian for production. Includes: - test-lab/ha/disko.nix: 20G boot disk layout (smaller than production) - test-lab/ha/common.nix: shared HA stack (drbd, corosync, pacemaker, targetcli-fb, xfsprogs), OCF PATH workaround for nixpkgs#207891 - test-lab/ha/node1.nix / node2.nix: per-node hostname + static IP - test-lab/ha/fence-pve-ssh.py: Proxmox SSH fence agent for STONITH - test-lab/ha/cluster-init.sh: one-shot cluster bootstrap script - test-lab/ha/cluster-enable-stonith.sh: enables STONITH post-key-deploy - flake.nix: adds ha-test-node1 / ha-test-node2 nixosConfigurations (bypasses mkTarget / clan-core / sops-nix — test-only) These VMs must be destroyed once acceptance testing is complete. Co-Authored-By: Claude Sonnet 4.6 --- flake.nix | 30 ++- test-lab/ha/cluster-enable-stonith.sh | 78 ++++++++ test-lab/ha/cluster-init.sh | 239 ++++++++++++++++++++++++ test-lab/ha/common.nix | 256 ++++++++++++++++++++++++++ test-lab/ha/disko.nix | 49 +++++ test-lab/ha/fence-pve-ssh.py | 179 ++++++++++++++++++ test-lab/ha/node1.nix | 12 ++ test-lab/ha/node2.nix | 12 ++ 8 files changed, 854 insertions(+), 1 deletion(-) create mode 100644 test-lab/ha/cluster-enable-stonith.sh create mode 100644 test-lab/ha/cluster-init.sh create mode 100644 test-lab/ha/common.nix create mode 100644 test-lab/ha/disko.nix create mode 100644 test-lab/ha/fence-pve-ssh.py create mode 100644 test-lab/ha/node1.nix create mode 100644 test-lab/ha/node2.nix diff --git a/flake.nix b/flake.nix index 47ac295..4788634 100644 --- a/flake.nix +++ b/flake.nix @@ -132,6 +132,34 @@ lxc-tor-relay = mkTarget { platform = "lxc"; buildType = "tor-relay"; hostPath = ./hosts/tor-relay/host.nix; }; }; + # ── HA test lab VMs ───────────────────────────────────────────────── + # Two throwaway NixOS VMs to test the DRBD + XFS + LIO + Pacemaker + # stack. Deliberately bypass mkTarget (no clan-core, no sops-nix, + # no home-manager) — these are disposable and must be destroyed once + # the acceptance tests are done. + # Build disk images with: + # nix build .#nixosConfigurations.ha-test-node1.config.system.build.diskoImagesScript + # then run the script to produce ha-test-node1.raw (import with qm importdisk). + haTestTargets = + let + haNode = { nodeModule }: nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + inputs.disko.nixosModules.disko + ./modules/hardware-configuration/vm/proxmox.nix + ./modules/boot/efi.nix + ./test-lab/ha/disko.nix + ./test-lab/ha/common.nix + nodeModule + ]; + specialArgs = { inherit vars; }; + }; + in + { + ha-test-node1 = haNode { nodeModule = ./test-lab/ha/node1.nix; }; + ha-test-node2 = haNode { nodeModule = ./test-lab/ha/node2.nix; }; + }; + # Auto-install environments (migrated from the former nix-auto-installer # flake): a self-contained NixOS installer that boots, discovers this # flake's own nixosConfigurations over the network, and runs @@ -207,7 +235,7 @@ in { - nixosConfigurations = generatedTargets // installerTargets; + nixosConfigurations = generatedTargets // haTestTargets // installerTargets; # Buildable auto-installer artifacts (`nix build .#`). No `lxc` # variant (installer-boots-as-an-LXC-container) or `all` bundle diff --git a/test-lab/ha/cluster-enable-stonith.sh b/test-lab/ha/cluster-enable-stonith.sh new file mode 100644 index 0000000..6c23081 --- /dev/null +++ b/test-lab/ha/cluster-enable-stonith.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# cluster-enable-stonith.sh — enable STONITH fence agent after fence key is deployed +# Run from ha-test-node1 as root, AFTER: +# - /etc/fence-pve-ssh-key exists on both nodes +# - The fence public key is in authorized_keys on pve1.sweet.home +set -euo pipefail + +VMID_NODE1="200" +VMID_NODE2="201" +PVE_HOST="pve1.sweet.home" +PVE_USER="wayne" +FENCE_KEY="/etc/fence-pve-ssh-key" +FENCE_SCRIPT="/usr/lib/ocf/resource.d/heartbeat/fence_pve_ssh" + +log() { echo "[stonith-setup] $*"; } +die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; } + +[[ $(id -u) -eq 0 ]] || die "must run as root" + +[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY" +[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT" + +log "Verifying fence agent can reach ${PVE_HOST}..." +if ! ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \ + -o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" "sudo /usr/sbin/qm list" &>/dev/null; then + die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo" +fi +log "Fence agent SSH connectivity confirmed" + +log "Creating Pacemaker STONITH resource..." +cibadmin --create --scope resources --xml-text " + + + + + + + + + + + + + + +" 2>/dev/null || true + +cibadmin --create --scope resources --xml-text " + + + + + + + + + + + + + + +" 2>/dev/null || true + +log "Enabling STONITH..." +crm_attribute -t crm_config -n stonith-enabled -v true + +# Restore quorum policy to stop (needed with STONITH) +crm_attribute -t crm_config -n no-quorum-policy -v stop + +log "STONITH enabled. Testing fence agent..." +if stonith_admin --list-devices; then + log "Fence devices listed successfully" +else + log "WARNING: fence device list failed — check stonith config" +fi + +log "STONITH setup complete. Cluster is now fully HA." diff --git a/test-lab/ha/cluster-init.sh b/test-lab/ha/cluster-init.sh new file mode 100644 index 0000000..4274740 --- /dev/null +++ b/test-lab/ha/cluster-init.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +# cluster-init.sh — one-time HA cluster initialisation script +# +# Run this ONCE from node1 AFTER both VMs are booted and have SSH access. +# It: +# 1. Waits for corosync quorum on both nodes +# 2. Initialises DRBD metadata and promotes node1 to primary +# 3. Creates XFS filesystem on /dev/drbd0 +# 4. Configures targetcli / LIO iSCSI target (with a file-backed LUN) +# 5. Configures the Pacemaker resource group +# 6. Optionally enables the STONITH fence agent (requires fence SSH key) +# +# Prerequisites: +# - Both VMs booted with the ha-test config +# - fence-pve-ssh-key distributed to /etc/fence-pve-ssh-key on both nodes +# - Run as root on ha-test-node1 +set -euo pipefail + +NODE1_IP="192.168.2.200" +NODE2_IP="192.168.2.201" +VIP="192.168.2.202" +DRBD_DEVICE="/dev/drbd0" +XFS_MOUNT="/mnt/ha-data" +ISCSI_IQN="iqn.2026-01.local.ha-test:storage" +ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img" +ISCSI_LUN_SIZE="1G" # small test LUN +VMID_NODE1="200" +VMID_NODE2="201" +PVE_HOST="pve1.sweet.home" +PVE_USER="wayne" +FENCE_KEY="/etc/fence-pve-ssh-key" + +log() { echo "[cluster-init] $*"; } +die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; } + +[[ $(id -u) -eq 0 ]] || die "must run as root" +[[ "$(hostname)" == "ha-test-node1" ]] || die "must run on ha-test-node1" + +# ── 1. Wait for corosync quorum ────────────────────────────────────────── +log "Waiting for corosync quorum..." +for i in $(seq 1 30); do + if corosync-quorumtool -q &>/dev/null; then + log "Quorum established" + break + fi + [[ $i -eq 30 ]] && die "corosync quorum not established after 30s" + sleep 2 +done + +log "Waiting for pacemaker to start..." +for i in $(seq 1 30); do + if crm_mon -1 &>/dev/null; then + log "Pacemaker running" + break + fi + [[ $i -eq 30 ]] && die "pacemaker not running after 60s" + sleep 2 +done + +# ── 2. Initialise DRBD ─────────────────────────────────────────────────── +log "Initialising DRBD metadata on node1..." +if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent"; then + drbdadm create-md ha-data --force +fi + +log "Initialising DRBD metadata on node2..." +if ! ssh "root@${NODE2_IP}" "drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent'"; then + ssh "root@${NODE2_IP}" "drbdadm create-md ha-data --force" +fi + +log "Bringing up DRBD on both nodes..." +drbdadm up ha-data || true +ssh "root@${NODE2_IP}" "drbdadm up ha-data" || true + +log "Forcing node1 to DRBD primary (initial sync)..." +drbdadm primary ha-data --force + +log "Waiting for DRBD to finish initial sync..." +for i in $(seq 1 120); do + state=$(drbdadm dstate ha-data) + if echo "$state" | grep -q "UpToDate"; then + log "DRBD sync complete: $state" + break + fi + log " DRBD state: $state (${i}/120s)" + [[ $i -eq 120 ]] && die "DRBD did not sync within 120s" + sleep 1 +done + +# ── 3. XFS filesystem ──────────────────────────────────────────────────── +log "Creating XFS on ${DRBD_DEVICE}..." +if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then + mkfs.xfs "${DRBD_DEVICE}" +fi + +log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..." +mkdir -p "${XFS_MOUNT}" +mount "${DRBD_DEVICE}" "${XFS_MOUNT}" + +# ── 4. iSCSI LUN (file-backed) ─────────────────────────────────────────── +log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..." +if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then + fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}" +fi + +log "Configuring LIO iSCSI target via targetcli..." +# This produces a /etc/target/saveconfig.json that the targetctl service loads. +# The commands create an iSCSI target backed by the file we just created. +targetcli <<'EOF' +/backstores/fileio create name=ha-lun0 file_or_dev=/mnt/ha-data/iscsi-lun.img size=0 write_back=false +/iscsi create iqn.2026-01.local.ha-test:storage +/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/luns create /backstores/fileio/ha-lun0 +/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/portals create 192.168.2.202 +/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute authentication=0 +/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute demo_mode_write_protect=0 +saveconfig /etc/target/saveconfig.json +EOF + +log "Unmounting ${XFS_MOUNT} (Pacemaker will manage it)..." +umount "${XFS_MOUNT}" + +log "Promoting DRBD back to secondary (Pacemaker manages primary role)..." +drbdadm secondary ha-data + +# ── 5. Pacemaker resources ─────────────────────────────────────────────── +log "Configuring Pacemaker..." + +# Disable STONITH initially — enable once fence key is deployed +crm_attribute -t crm_config -n stonith-enabled -v false + +# Disable quorum policy for two-node cluster (no-quorum-policy=ignore so +# the surviving node can promote without a quorum device) +crm_attribute -t crm_config -n no-quorum-policy -v ignore + +# Cluster resources: +# 1. drbd-ha — manages DRBD primary/secondary role +# 2. xfs-mount — XFS mount on /mnt/ha-data +# 3. iscsi-target — LIO target service (systemd class) +# 4. vip — floating VIP 192.168.2.202 + +log "Creating DRBD master/slave resource..." +cibadmin --replace --scope resources --xml-text " + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +" + +log "Adding ordering and colocation constraints..." +# All resources on the same node as DRBD master +cibadmin --create --scope constraints --xml-text " + + + + + + + + +" + +log "Resource group configured. Waiting for resources to start..." +for i in $(seq 1 60); do + if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then + log "VIP is up: $(crm_resource -r vip --locate)" + break + fi + [[ $i -eq 60 ]] && { log "WARNING: VIP not up after 60s — check crm_mon"; break; } + sleep 2 +done + +log "" +log "═══════════════════════════════════════════════════════" +log " HA cluster initialised. Next steps:" +log "" +log " - Verify: crm_mon -1" +log " - Test iSCSI: iscsiadm -m discovery -t sendtargets -p ${VIP}" +log "" +log " To enable STONITH (after deploying fence key):" +log " 1. Copy fence-pve-ssh.py to /usr/lib/ocf/resource.d/heartbeat/ on both nodes" +log " 2. Distribute /etc/fence-pve-ssh-key to both nodes" +log " 3. Add public key to authorized_keys on ${PVE_HOST}" +log " 4. Run: ./cluster-enable-stonith.sh" +log "═══════════════════════════════════════════════════════" diff --git a/test-lab/ha/common.nix b/test-lab/ha/common.nix new file mode 100644 index 0000000..01659e0 --- /dev/null +++ b/test-lab/ha/common.nix @@ -0,0 +1,256 @@ +# Shared HA stack config for both test nodes. +# These are throwaway test VMs — not production hosts. +# No sops-nix, no clan, no home-manager. +{ lib, pkgs, vars, ... }: + +let + node1Ip = "192.168.2.200"; + node2Ip = "192.168.2.201"; + drbdPort = 7789; + + # Test-only corosync authkey (128 bytes minimum). + # Not secret — this is a disposable test cluster, not production. + testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; + + # OCF agent path (from pacemaker's --with-ocfdir). + # We expose this on PATH so custom scripts can find each other. + ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d"; + + # Concatenated PATH that includes all binaries OCF agents and pacemaker + # lrmd children need. This is the workaround for nixpkgs#207891 (PATH + # not set correctly for OCF agent child processes). + ocfBinPath = lib.concatStringsSep ":" [ + "${pkgs.iproute2}/bin" + "${pkgs.iproute2}/sbin" + "${pkgs.iputils}/bin" + "${pkgs.util-linux}/bin" + "${pkgs.util-linux}/sbin" + "${pkgs.gawk}/bin" + "${pkgs.gnugrep}/bin" + "${pkgs.gnused}/bin" + "${pkgs.coreutils}/bin" + "${pkgs.bash}/bin" + "${pkgs.procps}/bin" + "${pkgs.xfsprogs}/bin" + "${pkgs.drbd}/bin" + "${pkgs.targetcli-fb}/bin" + "${pkgs.python3}/bin" + "/run/current-system/sw/bin" + "/run/current-system/sw/sbin" + "/usr/local/sbin" + "/usr/local/bin" + "/usr/sbin" + "/usr/bin" + "/sbin" + "/bin" + ]; +in +{ + system.stateVersion = "26.05"; + + # ── Hardware (Proxmox VM) ────────────────────────────────────────────── + imports = [ + ../../modules/hardware-configuration/vm/proxmox.nix + ../../modules/boot/efi.nix + ]; + + # ── Nix settings ────────────────────────────────────────────────────── + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + + # ── SSH ─────────────────────────────────────────────────────────────── + services.openssh = { + enable = true; + settings.PermitRootLogin = "yes"; + }; + users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey ]; + + # ── Networking ──────────────────────────────────────────────────────── + networking.useDHCP = false; + networking.defaultGateway = "192.168.2.1"; + networking.nameservers = [ "192.168.2.1" "8.8.8.8" ]; + + # ── DRBD ────────────────────────────────────────────────────────────── + services.drbd.enable = true; + services.drbd.config = '' + global { + usage-count yes; + } + + common { + net { + protocol C; + ping-int 1; + verify-alg sha256; + after-sb-0pri discard-zero-changes; + after-sb-1pri discard-secondary; + } + disk { + # resource-only: DRBD itself won't fence (Pacemaker handles STONITH); + # the DRBD resource agent uses fencing to guard primary promotion. + fencing resource-only; + } + handlers { + # Called by DRBD when a split-brain is detected and this node is + # in the secondary role — notifies pacemaker to fence the split node. + split-brain "/usr/lib/drbd/notify-split-brain.sh root"; + before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k"; + after-resync-target "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh"; + } + } + + resource ha-data { + volume 0 { + device /dev/drbd0; + disk /dev/sdb; # scsi1 in Proxmox VM → sdb + meta-disk internal; + } + + on ha-test-node1 { + address ${node1Ip}:${toString drbdPort}; + } + + on ha-test-node2 { + address ${node2Ip}:${toString drbdPort}; + } + } + ''; + + # ── Corosync ────────────────────────────────────────────────────────── + services.corosync = { + enable = true; + clusterName = "ha-test"; + nodelist = [ + { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; } + { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; } + ]; + }; + + # Corosync authkey (test-only, not secret — generated with + # `corosync-keygen` for production). + environment.etc."corosync/authkey" = { + source = builtins.toFile "authkey" testAuthKey; + mode = "0400"; + }; + + # ── Pacemaker ───────────────────────────────────────────────────────── + services.pacemaker.enable = true; + + # Fix for nixpkgs#207891: + # 1. Ensure CIB directories are owned by hacluster before starting. + # The stock module sets StateDirectory=pacemaker (owned by root); + # pacemaker internally drops to hacluster but needs to write there. + # 2. Set PATH so OCF agent child processes can find all required binaries. + systemd.services.pacemaker.serviceConfig = { + ExecStartPre = [ + "${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'" + ]; + }; + systemd.services.pacemaker.environment = { + PATH = lib.mkForce ocfBinPath; + # Expose OCF root so pacemaker and lrmd agree on where agents live. + OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; + }; + + # pacemaker-execd is the local resource executor that calls OCF agents. + # Give it the same PATH so OCF scripts can find all required binaries. + systemd.services.pacemaker-execd.environment = { + PATH = lib.mkForce ocfBinPath; + OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; + }; + + # ── LIO / iSCSI target ──────────────────────────────────────────────── + # targetcli-fb is the management tool; actual kernel support is via + # the LIO modules. We add a systemd service that saves/restores the + # target configuration so Pacemaker can trigger it via a systemd-class + # resource. + boot.kernelModules = [ + "target_core_mod" + "iscsi_target_mod" + "target_core_file" + "target_core_pscsi" + "target_core_user" + "configfs" + ]; + + # configfs must be mounted for rtslib/targetcli to work + systemd.mounts = [{ + where = "/sys/kernel/config"; + what = "configfs"; + type = "configfs"; + wantedBy = [ "multi-user.target" ]; + before = [ "targetctl.service" ]; + }]; + + # targetctl: save/restore LIO configuration (mirrors Debian's package) + systemd.services.targetctl = { + description = "LIO iSCSI target config save/restore"; + wantedBy = [ "multi-user.target" ]; + after = [ "sys-kernel-config.mount" "network.target" ]; + requires = [ "sys-kernel-config.mount" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json"; + ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json"; + }; + unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json"; + }; + + # ── Packages ────────────────────────────────────────────────────────── + environment.systemPackages = with pkgs; [ + # HA stack + corosync # corosync-cfgtool, corosync-quorumtool + pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI + drbd # drbdadm, drbdsetup, drbdmon + ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …) + + # Storage + xfsprogs # mkfs.xfs, xfs_admin, xfs_info + targetcli-fb # targetcli shell + targetctl + + # Networking / debug + iproute2 # ip, ss + iputils # ping + tcpdump + lsof + + # Scripting / config + python3 + curl + jq + vim + htop + ]; + + # ── Firewall ────────────────────────────────────────────────────────── + networking.firewall = { + enable = true; + allowedTCPPorts = [ + 22 # SSH + 3260 # iSCSI + 3121 # pacemaker-remoted + 2224 # pcsd + drbdPort # DRBD replication + ]; + allowedUDPPorts = [ + 5404 # corosync cluster + 5405 # corosync cluster + 5407 # corosync crypto + ]; + # Corosync uses ports 5404-5407 UDP; allow them on the cluster net + extraCommands = '' + iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT + iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT + ''; + }; + + # ── tmpfiles: target config dir ──────────────────────────────────────── + systemd.tmpfiles.rules = [ + "d /etc/target 0750 root root -" + "f /etc/target/saveconfig.json 0640 root root -" + ]; + + # ── Locale / time ───────────────────────────────────────────────────── + time.timeZone = vars.timeZone; + i18n.defaultLocale = "en_AU.UTF-8"; +} diff --git a/test-lab/ha/disko.nix b/test-lab/ha/disko.nix new file mode 100644 index 0000000..5d63f1d --- /dev/null +++ b/test-lab/ha/disko.nix @@ -0,0 +1,49 @@ +{ config, ... }: + +# Smaller disk layout for throwaway test VMs (20G vs production 50G). +# Same partition scheme as modules/disko/proxmox.nix: GPT, ESP + swap + ext4 root. +# Only covers the boot disk (scsi0 → /dev/sda). The DRBD data disk +# (scsi1 → /dev/sdb) is left raw — drbdadm create-md initialises it. +{ + disko.devices.disk.main = { + type = "disk"; + device = "/dev/sda"; + imageSize = "20G"; + imageName = config.networking.hostName; + + content = { + type = "gpt"; + partitions = { + esp = { + priority = 1; + name = "ESP"; + size = "512M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ "umask=0077" ]; + extraArgs = [ "-F" "32" "-n" "boot" ]; + }; + }; + swap = { + size = "2G"; + content = { + type = "swap"; + randomEncryption = false; + }; + }; + root = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + extraArgs = [ "-L" "nixos" ]; + }; + }; + }; + }; + }; +} diff --git a/test-lab/ha/fence-pve-ssh.py b/test-lab/ha/fence-pve-ssh.py new file mode 100644 index 0000000..7bf0c7a --- /dev/null +++ b/test-lab/ha/fence-pve-ssh.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +""" +fence_pve_ssh - Proxmox VE SSH fence agent for Pacemaker. + +Uses SSH to reach pve1.sweet.home and run 'qm stop/start '. +Designed for test-lab HA cluster only — not for production. + +Configuration (as pacemaker stonith resource attributes): + pve_host Proxmox host to SSH to (default: pve1.sweet.home) + pve_user SSH user (default: wayne) + key_file SSH private key path (default: /etc/fence-pve-ssh-key) + vmid_node1 VMID for ha-test-node1 (e.g. 200) + vmid_node2 VMID for ha-test-node2 (e.g. 201) + plug Node name to act on (set by pacemaker: ha-test-node1 or ha-test-node2) + action Action: off|on|reboot|status|list|metadata +""" + +import argparse +import subprocess +import sys +import os + + +METADATA = """ + + Fences a VM on a Proxmox VE host by SSHing to the PVE host and + running qm stop/start. For test use only. + https://proxmox.com + + + + + Fencing action: off|on|reboot|status|list + + + + + Cluster node name to fence + + + + + Proxmox VE host to SSH to + + + + + SSH user on the Proxmox host + + + + + SSH private key file path + + + + + VMID for ha-test-node1 + + + + + VMID for ha-test-node2 + + + + + + + + + + + +""" + + +def parse_args(): + p = argparse.ArgumentParser(add_help=False) + p.add_argument("-a", "--action", default="reboot") + p.add_argument("-n", "--plug") + p.add_argument("--pve-host", default="pve1.sweet.home") + p.add_argument("--pve-user", default="wayne") + p.add_argument("--key-file", default="/etc/fence-pve-ssh-key") + p.add_argument("--vmid-node1") + p.add_argument("--vmid-node2") + # Allow remaining unknown args (pacemaker may pass extra ones) + return p.parse_known_args()[0] + + +def ssh(pve_host, pve_user, key_file, cmd): + result = subprocess.run( + [ + "ssh", + "-i", key_file, + "-o", "StrictHostKeyChecking=no", + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=10", + f"{pve_user}@{pve_host}", + cmd, + ], + capture_output=True, + text=True, + timeout=30, + ) + return result + + +def get_vmid(args): + node = args.plug + if not node: + print("ERROR: --plug not specified", file=sys.stderr) + sys.exit(1) + mapping = { + "ha-test-node1": args.vmid_node1, + "ha-test-node2": args.vmid_node2, + } + vmid = mapping.get(node) + if not vmid: + print(f"ERROR: unknown node '{node}'", file=sys.stderr) + sys.exit(1) + return vmid + + +def main(): + args = parse_args() + action = args.action.lower() + + if action == "metadata": + print(METADATA) + sys.exit(0) + + if action == "list": + if args.vmid_node1: + print("ha-test-node1") + if args.vmid_node2: + print("ha-test-node2") + sys.exit(0) + + vmid = get_vmid(args) + + if not os.path.exists(args.key_file): + print(f"ERROR: SSH key not found at {args.key_file}", file=sys.stderr) + sys.exit(1) + + if action in ("off", "reboot"): + print(f"Stopping VM {vmid} ({args.plug}) on {args.pve_host}...") + r = ssh(args.pve_host, args.pve_user, args.key_file, + f"sudo /usr/sbin/qm stop {vmid}") + if r.returncode != 0: + print(f"ERROR stopping VM: {r.stderr}", file=sys.stderr) + sys.exit(1) + print(f"VM {vmid} stopped") + + if action in ("on", "reboot"): + print(f"Starting VM {vmid} ({args.plug}) on {args.pve_host}...") + r = ssh(args.pve_host, args.pve_user, args.key_file, + f"sudo /usr/sbin/qm start {vmid}") + if r.returncode != 0: + print(f"ERROR starting VM: {r.stderr}", file=sys.stderr) + sys.exit(1) + print(f"VM {vmid} started") + + if action == "status": + r = ssh(args.pve_host, args.pve_user, args.key_file, + f"sudo /usr/sbin/qm status {vmid}") + if r.returncode != 0: + print(f"ERROR querying VM status: {r.stderr}", file=sys.stderr) + sys.exit(1) + # qm status returns "status: running" or "status: stopped" + status_line = r.stdout.strip() + print(status_line) + if "stopped" in status_line: + sys.exit(2) # pacemaker interprets exit 2 as "off" + sys.exit(0) # running = exit 0 + + +if __name__ == "__main__": + main() diff --git a/test-lab/ha/node1.nix b/test-lab/ha/node1.nix new file mode 100644 index 0000000..a7d408c --- /dev/null +++ b/test-lab/ha/node1.nix @@ -0,0 +1,12 @@ +{ ... }: + +# ha-test-node1: VMID 200, 192.168.2.200/24 +{ + networking.hostName = "ha-test-node1"; + networking.hostId = "a1b2c3d4"; # random, required by ZFS (not used here) but harmless + + networking.interfaces.ens18.ipv4.addresses = [{ + address = "192.168.2.200"; + prefixLength = 24; + }]; +} diff --git a/test-lab/ha/node2.nix b/test-lab/ha/node2.nix new file mode 100644 index 0000000..72dcafc --- /dev/null +++ b/test-lab/ha/node2.nix @@ -0,0 +1,12 @@ +{ ... }: + +# ha-test-node2: VMID 201, 192.168.2.201/24 +{ + networking.hostName = "ha-test-node2"; + networking.hostId = "e5f6a7b8"; # random, required by ZFS (not used here) but harmless + + networking.interfaces.ens18.ipv4.addresses = [{ + address = "192.168.2.201"; + prefixLength = 24; + }]; +} From dc36a47ac9bad278a110a1bae6d955f67ca02075 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 27 Jul 2026 06:06:53 +1000 Subject: [PATCH 2/7] test-lab/ha: add session key + enable qemu-guest-agent Add nixos@nixos session key so the Claude Code session can SSH into test VMs directly. Also enable services.qemuGuest.enable so qm guest exec works as a fallback for key injection. Co-Authored-By: Claude Sonnet 4.6 --- test-lab/ha/common.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test-lab/ha/common.nix b/test-lab/ha/common.nix index 01659e0..1cc4f00 100644 --- a/test-lab/ha/common.nix +++ b/test-lab/ha/common.nix @@ -62,7 +62,14 @@ in enable = true; settings.PermitRootLogin = "yes"; }; - users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey ]; + users.users.root.openssh.authorizedKeys.keys = [ + vars.adminSshKey + # Claude Code session key (this machine) — test-lab only + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos" + ]; + + # Allow QEMU guest exec for key injection fallback + services.qemuGuest.enable = true; # ── Networking ──────────────────────────────────────────────────────── networking.useDHCP = false; From 34c55f27ca68e9f742eb9c0b836695ffc3676270 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 27 Jul 2026 06:18:42 +1000 Subject: [PATCH 3/7] test-lab/ha: nixpkgs-fmt formatting Co-Authored-By: Claude Sonnet 4.6 --- test-lab/ha/common.nix | 30 +++++++++++++++--------------- test-lab/ha/node1.nix | 2 +- test-lab/ha/node2.nix | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/test-lab/ha/common.nix b/test-lab/ha/common.nix index 1cc4f00..9b6a00e 100644 --- a/test-lab/ha/common.nix +++ b/test-lab/ha/common.nix @@ -206,18 +206,18 @@ in # ── Packages ────────────────────────────────────────────────────────── environment.systemPackages = with pkgs; [ # HA stack - corosync # corosync-cfgtool, corosync-quorumtool - pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI - drbd # drbdadm, drbdsetup, drbdmon - ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …) + corosync # corosync-cfgtool, corosync-quorumtool + pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI + drbd # drbdadm, drbdsetup, drbdmon + ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …) # Storage - xfsprogs # mkfs.xfs, xfs_admin, xfs_info - targetcli-fb # targetcli shell + targetctl + xfsprogs # mkfs.xfs, xfs_admin, xfs_info + targetcli-fb # targetcli shell + targetctl # Networking / debug - iproute2 # ip, ss - iputils # ping + iproute2 # ip, ss + iputils # ping tcpdump lsof @@ -233,16 +233,16 @@ in networking.firewall = { enable = true; allowedTCPPorts = [ - 22 # SSH - 3260 # iSCSI - 3121 # pacemaker-remoted - 2224 # pcsd + 22 # SSH + 3260 # iSCSI + 3121 # pacemaker-remoted + 2224 # pcsd drbdPort # DRBD replication ]; allowedUDPPorts = [ - 5404 # corosync cluster - 5405 # corosync cluster - 5407 # corosync crypto + 5404 # corosync cluster + 5405 # corosync cluster + 5407 # corosync crypto ]; # Corosync uses ports 5404-5407 UDP; allow them on the cluster net extraCommands = '' diff --git a/test-lab/ha/node1.nix b/test-lab/ha/node1.nix index a7d408c..6baf756 100644 --- a/test-lab/ha/node1.nix +++ b/test-lab/ha/node1.nix @@ -3,7 +3,7 @@ # ha-test-node1: VMID 200, 192.168.2.200/24 { networking.hostName = "ha-test-node1"; - networking.hostId = "a1b2c3d4"; # random, required by ZFS (not used here) but harmless + networking.hostId = "a1b2c3d4"; # random, required by ZFS (not used here) but harmless networking.interfaces.ens18.ipv4.addresses = [{ address = "192.168.2.200"; diff --git a/test-lab/ha/node2.nix b/test-lab/ha/node2.nix index 72dcafc..bf49d4b 100644 --- a/test-lab/ha/node2.nix +++ b/test-lab/ha/node2.nix @@ -3,7 +3,7 @@ # ha-test-node2: VMID 201, 192.168.2.201/24 { networking.hostName = "ha-test-node2"; - networking.hostId = "e5f6a7b8"; # random, required by ZFS (not used here) but harmless + networking.hostId = "e5f6a7b8"; # random, required by ZFS (not used here) but harmless networking.interfaces.ens18.ipv4.addresses = [{ address = "192.168.2.201"; From 23634134f008406e540cd6eafb4e0523136d1d46 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 27 Jul 2026 06:40:57 +1000 Subject: [PATCH 4/7] fix(ha-test): remove Debian LVM handlers and fix corosync authkey length - Remove before/after-resync-target LVM handlers (snapshot-resync-target-lvm.sh doesn't exist in NixOS; exit code 127 caused DRBD to drop connections on sync) - Remove split-brain handler pointing to /usr/lib/drbd/ (Debian path) - Fix testAuthKey from 126 to 128 bytes (corosync minimum is 1024 bits) - Fix cluster-init.sh quorum check: corosync-quorumtool has no -q flag; use -s | grep 'Quorate: Yes' instead Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8 --- test-lab/ha/cluster-init.sh | 2 +- test-lab/ha/common.nix | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/test-lab/ha/cluster-init.sh b/test-lab/ha/cluster-init.sh index 4274740..597017b 100644 --- a/test-lab/ha/cluster-init.sh +++ b/test-lab/ha/cluster-init.sh @@ -39,7 +39,7 @@ die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; } # ── 1. Wait for corosync quorum ────────────────────────────────────────── log "Waiting for corosync quorum..." for i in $(seq 1 30); do - if corosync-quorumtool -q &>/dev/null; then + if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then log "Quorum established" break fi diff --git a/test-lab/ha/common.nix b/test-lab/ha/common.nix index 9b6a00e..8b75d7b 100644 --- a/test-lab/ha/common.nix +++ b/test-lab/ha/common.nix @@ -8,9 +8,9 @@ let node2Ip = "192.168.2.201"; drbdPort = 7789; - # Test-only corosync authkey (128 bytes minimum). + # Test-only corosync authkey (128 bytes = 1024 bits minimum for corosync). # Not secret — this is a disposable test cluster, not production. - testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; + testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; # OCF agent path (from pacemaker's --with-ocfdir). # We expose this on PATH so custom scripts can find each other. @@ -97,11 +97,9 @@ in fencing resource-only; } handlers { - # Called by DRBD when a split-brain is detected and this node is - # in the secondary role — notifies pacemaker to fence the split node. - split-brain "/usr/lib/drbd/notify-split-brain.sh root"; - before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k"; - after-resync-target "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh"; + # NOTE: LVM-specific before/after-resync-target handlers omitted + # (raw /dev/sdb, no LVM). split-brain handler also omitted — + # Pacemaker STONITH manages split-brain fencing for the test cluster. } } From 724d9a45af1cb6a0b680064daeb7e2de7dc4b698 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 27 Jul 2026 09:13:55 +1000 Subject: [PATCH 5/7] feat(ha): add NixOS modules for DRBD+XFS+LIO+Corosync+Pacemaker HA stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 7 acceptance tests pass on live NixOS 25.11 VMs (VMIDs 200/201 on pve1). Failover completes in ~5 s with data integrity verified. modules/ha/pacemaker-stack.nix — fixes four NixOS-specific breakages: - systemd StateDirectory resets /var/lib/pacemaker to root:root; removed and replaced with ExecStartPre to create/chown dirs as hacluster - HA_SBIN_DIR points to a non-existent Nix store path; overridden to /run/current-system/sw/bin so crm_master resolves correctly - OCF agents need an explicit broad PATH (iproute2, util-linux, xfsprogs, drbd, bash, etc.) — NixOS services have no implicit PATH - FUSER=true bypasses the psmisc fuser check_binary call in the Filesystem OCF agent (psmisc not installed on minimal hosts) modules/ha/iscsi-target.nix — LIO iSCSI target via targetctl with a Python/rtslib_fb ExecStop that explicitly clears the kernel LIO state (not just saves JSON), so the XFS backing store's file descriptor is released before umount — preventing EBUSY stop timeouts on failover. Includes an empty-config guard so the secondary node never overwrites the primary's saveconfig.json with an empty one. test-lab/ha/common.nix — updated to import both modules, use fencing dont-care (no STONITH in test lab), omit LVM handlers (non-existent on NixOS paths), and merge repeated services/networking attr sets to satisfy statix W20. test-lab/ha/acceptance-tests.sh — final v4 with crm_standby fix (pacemaker 3.x API). Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8 --- modules/ha/iscsi-target.nix | 99 ++++++++++++ modules/ha/pacemaker-stack.nix | 94 +++++++++++ test-lab/ha/acceptance-tests.sh | 171 ++++++++++++++++++++ test-lab/ha/common.nix | 278 +++++++++++--------------------- 4 files changed, 454 insertions(+), 188 deletions(-) create mode 100644 modules/ha/iscsi-target.nix create mode 100644 modules/ha/pacemaker-stack.nix create mode 100755 test-lab/ha/acceptance-tests.sh diff --git a/modules/ha/iscsi-target.nix b/modules/ha/iscsi-target.nix new file mode 100644 index 0000000..c39ec88 --- /dev/null +++ b/modules/ha/iscsi-target.nix @@ -0,0 +1,99 @@ +# LIO iSCSI target service (targetctl) for NixOS HA clusters. +# +# Provides the targetctl.service that saves/restores LIO configuration from +# /etc/target/saveconfig.json. Pacemaker manages this service via its +# systemd resource agent (class="systemd" type="targetctl"). +# +# Why ExecStop is not simply "targetctl save": +# targetctl save writes the LIO config to JSON but does NOT remove the LIO +# target from the kernel's configfs. As a result, any fileio backing store +# that LIO has open (e.g. iscsi-lun.img on an XFS-over-DRBD filesystem) +# stays referenced in the kernel. The subsequent XFS umount from the +# Filesystem OCF resource then returns EBUSY and either hangs for the full +# op-stop timeout or fails outright, blocking the entire failover. +# +# The ExecStop script here additionally tears down the kernel LIO state +# via rtslib_fb after saving, so the backing-store file descriptor is +# released and umount succeeds immediately. +# +# Empty-config guard: +# The save step is skipped when no iSCSI targets are currently active. +# This prevents the secondary node (where LIO was never started) from +# overwriting a valid saveconfig.json with an empty one when Pacemaker +# stops the iscsi-target resource as part of a failover or cleanup. +{ pkgs, ... }: + +let + python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]); + targetctl = "${pkgs.targetcli-fb}/bin/targetctl"; + + targetctlStop = pkgs.writeScript "targetctl-stop" '' + #!${python3}/bin/python3 + import subprocess, sys + import rtslib_fb + + root = rtslib_fb.RTSRoot() + targets = list(root.targets) + if targets: + subprocess.run( + ["${targetctl}", "save", "/etc/target/saveconfig.json"], + capture_output=True, + ) + print(f"saved {len(targets)} iSCSI target(s)") + else: + print("no active LIO targets — saveconfig.json unchanged") + + for target in targets: + try: + for tpg in list(target.tpgs): + tpg.enable = False + target.delete() + except Exception as e: + print(f"warn (target): {e}", file=sys.stderr) + for so in list(root.storage_objects): + try: + so.delete() + except Exception as e: + print(f"warn (backstore): {e}", file=sys.stderr) + print("LIO kernel target cleared") + ''; +in +{ + boot.kernelModules = [ + "target_core_mod" + "iscsi_target_mod" + "target_core_file" + "target_core_pscsi" + "target_core_user" + "configfs" + ]; + + systemd = { + mounts = [{ + where = "/sys/kernel/config"; + what = "configfs"; + type = "configfs"; + wantedBy = [ "multi-user.target" ]; + before = [ "targetctl.service" ]; + }]; + services.targetctl = { + description = "LIO iSCSI target config save/restore"; + wantedBy = [ "multi-user.target" ]; + after = [ "sys-kernel-config.mount" "network.target" ]; + requires = [ "sys-kernel-config.mount" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${targetctl} restore /etc/target/saveconfig.json"; + ExecStop = "${targetctlStop}"; + }; + unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json"; + }; + tmpfiles.rules = [ + "d /etc/target 0750 root root -" + "f /etc/target/saveconfig.json 0640 root root -" + ]; + }; + + environment.systemPackages = [ pkgs.targetcli-fb ]; +} diff --git a/modules/ha/pacemaker-stack.nix b/modules/ha/pacemaker-stack.nix new file mode 100644 index 0000000..8fcab2c --- /dev/null +++ b/modules/ha/pacemaker-stack.nix @@ -0,0 +1,94 @@ +# Pacemaker + Corosync HA stack for NixOS with known-good workarounds. +# +# Issues fixed here (confirmed through live testing on NixOS 25.11): +# +# 1. StateDirectory ownership reset: systemd's StateDirectory=pacemaker +# creates /var/lib/pacemaker owned root:root. pacemaker-based (the CIB +# daemon) runs as the hacluster user and calls pcmk__daemon_can_write, +# which requires the CIB directory to be owned by hacluster or be +# group-writable by haclient. Workaround: remove StateDirectory and let +# ExecStartPre create every required subdirectory with correct ownership. +# +# 2. HA_SBIN_DIR wrong path: ocf-shellfuncs sets HA_SBIN_DIR to the Nix +# store path of the resource-agents derivation's /sbin, which doesn't +# exist. The DRBD OCF agent uses ${HA_SBIN_DIR}/crm_master, so it exits +# 127 without this override. Fix: export HA_SBIN_DIR=/run/current-system/sw/bin. +# +# 3. Broad PATH for OCF agents: the resource executor (pacemaker-execd) runs +# OCF agent scripts as children. NixOS provides no implicit PATH for +# system services; without an explicit PATH the agents can't find ip, ss, +# mount, umount, drbdadm, etc. +# +# 4. FUSER=true: the Filesystem OCF agent calls check_binary $FUSER (default: +# fuser from psmisc), which is not installed. Setting FUSER=true makes +# check_binary succeed (true is always in PATH) and the subsequent +# "$FUSER -km $mountpoint" becomes a no-op. Pair with force_unmount=false +# on each Filesystem resource unless you want lazy unmount behaviour. +{ lib, pkgs, ... }: + +let + ocfBinPath = lib.concatStringsSep ":" [ + "${pkgs.iproute2}/bin" + "${pkgs.iproute2}/sbin" + "${pkgs.iputils}/bin" + "${pkgs.util-linux}/bin" + "${pkgs.util-linux}/sbin" + "${pkgs.gawk}/bin" + "${pkgs.gnugrep}/bin" + "${pkgs.gnused}/bin" + "${pkgs.coreutils}/bin" + "${pkgs.bash}/bin" + "${pkgs.procps}/bin" + "${pkgs.xfsprogs}/bin" + "${pkgs.drbd}/bin" + "${pkgs.python3}/bin" + "/run/current-system/sw/bin" + "/run/current-system/sw/sbin" + "/usr/local/sbin" + "/usr/local/bin" + "/usr/sbin" + "/usr/bin" + "/sbin" + "/bin" + ]; + + # Single pre-start script: schemas symlink + directory ownership. + # Runs before pacemakerd so pacemaker-based finds hacluster-owned dirs. + preStartCmd = "${pkgs.bash}/bin/bash -c '" + + "ln -sfn ${pkgs.pacemaker}/share/pacemaker /var/lib/pacemaker/schemas; " + + "for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores " + + "/var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox " + + "/var/lib/pacemaker/hostcache; do " + + "mkdir -p \"\\$d\" && chown hacluster:pacemaker \"\\$d\" && chmod 2770 \"\\$d\"; " + + "done'"; + + ocfEnv = { + PATH = lib.mkForce ocfBinPath; + OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; + HA_SBIN_DIR = "/run/current-system/sw/bin"; + FUSER = "true"; + }; +in +{ + users.groups.haclient = { }; + + services.corosync.enable = true; + services.pacemaker.enable = true; + + systemd.services = { + pacemaker = { + serviceConfig = { + StateDirectory = lib.mkForce ""; + ExecStartPre = lib.mkBefore [ preStartCmd ]; + }; + environment = ocfEnv; + }; + pacemaker-execd.environment = ocfEnv; + }; + + environment.systemPackages = with pkgs; [ + corosync + pacemaker + ocf-resource-agents + ]; +} diff --git a/test-lab/ha/acceptance-tests.sh b/test-lab/ha/acceptance-tests.sh new file mode 100755 index 0000000..651f1e9 --- /dev/null +++ b/test-lab/ha/acceptance-tests.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# HA File Server Acceptance Tests T1-T7 +# Failover (T5) uses pacemaker standby mode to gracefully move resources, +# simulating what STONITH + node-restart does in production. +# Note: A production cluster requires real STONITH (fence agent for the hypervisor). +set -euo pipefail + +NODE2="root@192.168.2.201" +VIP="192.168.2.202" +IQN="iqn.2026-01.local.ha-test:storage" +MOUNT="/srv/ha-data" +PACEMAKERD="/nix/store/3v9sb74cg2qmpcyzb4h6fq0z8bvp5gw1-pacemaker-3.0.1/sbin/pacemakerd" +MYNODE=$(hostname) + +SSH="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15" +PASS=0; FAIL=0 + +pass() { echo "[PASS] $1"; ((PASS++)) || true; } +fail() { echo "[FAIL] $1"; ((FAIL++)) || true; } +info() { echo "[INFO] $1"; } + +echo "=== HA File Server Acceptance Tests ===" +echo "Node: $MYNODE Date: $(date)" +echo "" + +# T1: Corosync 2-node cluster with quorum +echo "--- T1: Corosync cluster quorum ---" +QUORATE=$(corosync-quorumtool -s 2>/dev/null | grep 'Quorate:' | awk '{print $2}') +NODE_COUNT=$(corosync-quorumtool -s 2>/dev/null | grep '^Nodes:' | awk '{print $2}') +if [ "$QUORATE" = "Yes" ] && [ "$NODE_COUNT" = "2" ]; then + pass "T1: Corosync quorate with 2 nodes" +else + fail "T1: quorate=$QUORATE nodes=$NODE_COUNT" +fi + +# T2: DRBD both UpToDate, replication Established +echo "--- T2: DRBD replication healthy ---" +DRBD_STATUS=$(drbdadm status ha-data 2>/dev/null) +N1_ROLE=$(echo "$DRBD_STATUS" | grep '^ha-data role:' | awk -F: '{print $2}') +N1_DISK=$(echo "$DRBD_STATUS" | grep -oP 'disk:\K\S+' | head -1) +N2_DISK=$(echo "$DRBD_STATUS" | grep -oP 'peer-disk:\K\S+' | head -1) +REPL=$(echo "$DRBD_STATUS" | grep -oP 'replication:\K\S+' | head -1) +info "DRBD: role=$N1_ROLE local_disk=$N1_DISK peer_disk=$N2_DISK replication=$REPL" +if [ "$N1_DISK" = "UpToDate" ] && [ "$N2_DISK" = "UpToDate" ] && [ "$REPL" = "Established" ]; then + pass "T2: DRBD both UpToDate, Established (role=$N1_ROLE)" +else + fail "T2: DRBD issue: local=$N1_DISK peer=$N2_DISK replication=$REPL" +fi + +# T3: XFS mounted on primary +echo "--- T3: XFS mount on primary ---" +if mountpoint -q $MOUNT && df -t xfs $MOUNT &>/dev/null; then + FSINFO=$(df -h $MOUNT | tail -1) + pass "T3: XFS mounted at $MOUNT: $FSINFO" +else + fail "T3: XFS not mounted at $MOUNT" +fi + +# T4: iSCSI active on primary, VIP responds on port 3260 +echo "--- T4: iSCSI target active ---" +ACTIVE_IQN=$(ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep iqn | head -1) +if [ "$ACTIVE_IQN" = "$IQN" ] && nc -w3 $VIP 3260 < /dev/null 2>/dev/null; then + pass "T4: iSCSI $IQN active, port 3260 open on VIP $VIP" +elif [ "$ACTIVE_IQN" = "$IQN" ]; then + fail "T4: iSCSI IQN active but port 3260 not reachable on VIP" +else + fail "T4: iSCSI not active (got '$ACTIVE_IQN')" +fi + +# Pre-T5: write test file for data integrity check +echo "--- Pre-T5: writing test data ---" +TESTFILE="$MOUNT/failover-test.txt" +TESTDATA="FAILOVER_INTEGRITY_$(date +%s)" +echo "$TESTDATA" > "$TESTFILE" +sync +info "Wrote: $TESTFILE (data: $TESTDATA)" + +# T5: Failover — put this node into pacemaker standby, forcing resource migration +# (In production this is triggered by real STONITH; standby simulates the result.) +echo "--- T5: Failover (pacemaker standby + node isolation) ---" +info "Putting $MYNODE into standby mode (triggers resource migration to node2)..." +crm_standby -N "$MYNODE" -v on 2>&1 || true + +FAILOVER_OK=false +info "Waiting up to 90s for node2 failover..." +for i in $(seq 1 18); do + sleep 5 + STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "UNREACHABLE") + if echo "$STATUS" | grep -q "Started ha-test-node2"; then + FAILOVER_OK=true + info "Failover complete at $((i*5))s" + echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted' + break + fi +done + +if $FAILOVER_OK; then + N2_DRBD=$($SSH $NODE2 "drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print \$2}'" 2>/dev/null || echo "unknown") + N2_MOUNT=$($SSH $NODE2 "mountpoint -q $MOUNT && echo 'mounted' || echo 'not-mounted'" 2>/dev/null || echo "unknown") + N2_ISCSI=$($SSH $NODE2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" 2>/dev/null || echo "0") + N2_VIP=$($SSH $NODE2 "ip addr show | grep -c '$VIP'" 2>/dev/null || echo "0") + info "Node2: DRBD=$N2_DRBD mount=$N2_MOUNT iSCSI_IQNs=$N2_ISCSI VIP=$N2_VIP" + + FAILS=0 + [ "$N2_DRBD" = "Primary" ] || { info "FAIL: DRBD not Primary on node2"; ((FAILS++)) || true; } + [ "$N2_MOUNT" = "mounted" ] || { info "FAIL: XFS not mounted on node2"; ((FAILS++)) || true; } + [ "$N2_ISCSI" -ge "1" ] 2>/dev/null || { info "FAIL: iSCSI not active on node2"; ((FAILS++)) || true; } + [ "$N2_VIP" -ge "1" ] 2>/dev/null || { info "FAIL: VIP not on node2"; ((FAILS++)) || true; } + + if [ $FAILS -eq 0 ]; then + pass "T5: Failover complete — DRBD Primary, XFS, iSCSI, VIP all on node2" + else + fail "T5: Partial failover ($FAILS sub-checks failed)" + fi +else + fail "T5: No failover detected within 90s" +fi + +# T7: Data integrity — test file readable on node2 after failover +echo "--- T7: Data integrity after failover ---" +if $SSH $NODE2 "grep -q '$TESTDATA' $TESTFILE 2>/dev/null"; then + pass "T7: Test data intact on node2 after failover" +else + ACTUAL=$($SSH $NODE2 "cat $TESTFILE 2>/dev/null || echo FILE_MISSING" 2>/dev/null || echo "SSH_FAIL") + fail "T7: Data integrity check failed (expected '$TESTDATA', got '$ACTUAL')" +fi + +# T6: Node rejoin — take node out of standby +echo "--- T6: Node rejoin ---" +info "Taking $MYNODE out of standby..." +crm_standby -N "$MYNODE" -v off 2>&1 || true + +REJOIN_OK=false +for i in $(seq 1 12); do + sleep 5 + # Check if this node is back online (no longer standby) + STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "") + if echo "$STATUS" | grep -q "Online:.*$MYNODE"; then + REJOIN_OK=true + info "Rejoined at $((i*5))s" + echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted' + break + fi +done + +if $REJOIN_OK; then + sleep 5 + N1_DRBD=$(drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print $2}') + N1_DISK=$(drbdadm status ha-data 2>/dev/null | grep -oP 'disk:\K\S+' | head -1) + info "Node1 DRBD after rejoin: role=$N1_DRBD disk=$N1_DISK" + pass "T6: Node rejoined cluster (DRBD role=$N1_DRBD, disk=$N1_DISK)" +else + fail "T6: Node did not rejoin within 60s" +fi + +echo "" +echo "========================================" +echo "RESULTS: $PASS passed, $FAIL failed" +echo "========================================" +echo "" +echo "NOTES:" +echo " T5 uses pacemaker standby to simulate failover (production needs STONITH" +echo " fence agent, e.g. fence_pve_ssh, to crash the VM — same requirement on Debian)" +echo "" +if [ $FAIL -eq 0 ]; then + echo "VERDICT: ALL TESTS PASSED → Deliverable A (NixOS modules)" + exit 0 +else + echo "VERDICT: $FAIL TEST(S) FAILED → review above" + exit 1 +fi diff --git a/test-lab/ha/common.nix b/test-lab/ha/common.nix index 8b75d7b..c984e1c 100644 --- a/test-lab/ha/common.nix +++ b/test-lab/ha/common.nix @@ -11,39 +11,6 @@ let # Test-only corosync authkey (128 bytes = 1024 bits minimum for corosync). # Not secret — this is a disposable test cluster, not production. testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; - - # OCF agent path (from pacemaker's --with-ocfdir). - # We expose this on PATH so custom scripts can find each other. - ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d"; - - # Concatenated PATH that includes all binaries OCF agents and pacemaker - # lrmd children need. This is the workaround for nixpkgs#207891 (PATH - # not set correctly for OCF agent child processes). - ocfBinPath = lib.concatStringsSep ":" [ - "${pkgs.iproute2}/bin" - "${pkgs.iproute2}/sbin" - "${pkgs.iputils}/bin" - "${pkgs.util-linux}/bin" - "${pkgs.util-linux}/sbin" - "${pkgs.gawk}/bin" - "${pkgs.gnugrep}/bin" - "${pkgs.gnused}/bin" - "${pkgs.coreutils}/bin" - "${pkgs.bash}/bin" - "${pkgs.procps}/bin" - "${pkgs.xfsprogs}/bin" - "${pkgs.drbd}/bin" - "${pkgs.targetcli-fb}/bin" - "${pkgs.python3}/bin" - "/run/current-system/sw/bin" - "/run/current-system/sw/sbin" - "/usr/local/sbin" - "/usr/local/bin" - "/usr/sbin" - "/usr/bin" - "/sbin" - "/bin" - ]; in { system.stateVersion = "26.05"; @@ -52,82 +19,86 @@ in imports = [ ../../modules/hardware-configuration/vm/proxmox.nix ../../modules/boot/efi.nix + ../../modules/ha/pacemaker-stack.nix + ../../modules/ha/iscsi-target.nix ]; # ── Nix settings ────────────────────────────────────────────────────── nix.settings.experimental-features = [ "nix-command" "flakes" ]; - # ── SSH ─────────────────────────────────────────────────────────────── - services.openssh = { - enable = true; - settings.PermitRootLogin = "yes"; - }; users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey # Claude Code session key (this machine) — test-lab only "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos" ]; - # Allow QEMU guest exec for key injection fallback - services.qemuGuest.enable = true; + # ── Services ────────────────────────────────────────────────────────── + services = { + openssh = { + enable = true; + settings.PermitRootLogin = "yes"; + }; - # ── Networking ──────────────────────────────────────────────────────── - networking.useDHCP = false; - networking.defaultGateway = "192.168.2.1"; - networking.nameservers = [ "192.168.2.1" "8.8.8.8" ]; + # Allow QEMU guest exec for key injection fallback + qemuGuest.enable = true; - # ── DRBD ────────────────────────────────────────────────────────────── - services.drbd.enable = true; - services.drbd.config = '' - global { - usage-count yes; - } + drbd = { + enable = true; + config = '' + global { + usage-count yes; + } - common { - net { - protocol C; - ping-int 1; - verify-alg sha256; - after-sb-0pri discard-zero-changes; - after-sb-1pri discard-secondary; - } - disk { - # resource-only: DRBD itself won't fence (Pacemaker handles STONITH); - # the DRBD resource agent uses fencing to guard primary promotion. - fencing resource-only; - } - handlers { - # NOTE: LVM-specific before/after-resync-target handlers omitted - # (raw /dev/sdb, no LVM). split-brain handler also omitted — - # Pacemaker STONITH manages split-brain fencing for the test cluster. - } - } + common { + net { + protocol C; + ping-int 1; + verify-alg sha256; + after-sb-0pri discard-zero-changes; + after-sb-1pri discard-secondary; + } + disk { + # dont-care: DRBD itself won't fence before promoting. Production + # clusters should use resource-only here and configure a STONITH + # fence agent (e.g. fence_pve_ssh) in Pacemaker so DRBD can safely + # protect against split-brain without risking dual-Primary. + # For this test cluster (no fence device) dont-care lets promotion + # proceed; the DRBD kernel module still refuses dual-Primary without + # allow-two-primaries in net {}. + fencing dont-care; + # LVM before/after-resync-target handlers omitted: the LVM snapshot + # scripts (/usr/lib/drbd/snapshot-resync-target-lvm.sh) don't exist + # on NixOS paths. If present, DRBD calls them on resync and exits 127, + # dropping the peer connection and leaving the secondary Outdated. + } + } - resource ha-data { - volume 0 { - device /dev/drbd0; - disk /dev/sdb; # scsi1 in Proxmox VM → sdb - meta-disk internal; - } + resource ha-data { + volume 0 { + device /dev/drbd0; + disk /dev/sdb; # scsi1 in Proxmox VM → sdb + meta-disk internal; + } - on ha-test-node1 { - address ${node1Ip}:${toString drbdPort}; - } + on ha-test-node1 { + address ${node1Ip}:${toString drbdPort}; + } - on ha-test-node2 { - address ${node2Ip}:${toString drbdPort}; - } - } - ''; + on ha-test-node2 { + address ${node2Ip}:${toString drbdPort}; + } + } + ''; + }; - # ── Corosync ────────────────────────────────────────────────────────── - services.corosync = { - enable = true; - clusterName = "ha-test"; - nodelist = [ - { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; } - { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; } - ]; + # services.corosync.enable = true is set by modules/ha/pacemaker-stack.nix + corosync = { + clusterName = "ha-test"; + nodelist = [ + { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; } + { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; } + ]; + }; }; # Corosync authkey (test-only, not secret — generated with @@ -137,81 +108,13 @@ in mode = "0400"; }; - # ── Pacemaker ───────────────────────────────────────────────────────── - services.pacemaker.enable = true; - - # Fix for nixpkgs#207891: - # 1. Ensure CIB directories are owned by hacluster before starting. - # The stock module sets StateDirectory=pacemaker (owned by root); - # pacemaker internally drops to hacluster but needs to write there. - # 2. Set PATH so OCF agent child processes can find all required binaries. - systemd.services.pacemaker.serviceConfig = { - ExecStartPre = [ - "${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'" - ]; - }; - systemd.services.pacemaker.environment = { - PATH = lib.mkForce ocfBinPath; - # Expose OCF root so pacemaker and lrmd agree on where agents live. - OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; - }; - - # pacemaker-execd is the local resource executor that calls OCF agents. - # Give it the same PATH so OCF scripts can find all required binaries. - systemd.services.pacemaker-execd.environment = { - PATH = lib.mkForce ocfBinPath; - OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; - }; - - # ── LIO / iSCSI target ──────────────────────────────────────────────── - # targetcli-fb is the management tool; actual kernel support is via - # the LIO modules. We add a systemd service that saves/restores the - # target configuration so Pacemaker can trigger it via a systemd-class - # resource. - boot.kernelModules = [ - "target_core_mod" - "iscsi_target_mod" - "target_core_file" - "target_core_pscsi" - "target_core_user" - "configfs" - ]; - - # configfs must be mounted for rtslib/targetcli to work - systemd.mounts = [{ - where = "/sys/kernel/config"; - what = "configfs"; - type = "configfs"; - wantedBy = [ "multi-user.target" ]; - before = [ "targetctl.service" ]; - }]; - - # targetctl: save/restore LIO configuration (mirrors Debian's package) - systemd.services.targetctl = { - description = "LIO iSCSI target config save/restore"; - wantedBy = [ "multi-user.target" ]; - after = [ "sys-kernel-config.mount" "network.target" ]; - requires = [ "sys-kernel-config.mount" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json"; - ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json"; - }; - unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json"; - }; - # ── Packages ────────────────────────────────────────────────────────── + # corosync, pacemaker, ocf-resource-agents, targetcli-fb already added + # by the ha/ modules; add the remaining stack-specific tools here. environment.systemPackages = with pkgs; [ - # HA stack - corosync # corosync-cfgtool, corosync-quorumtool - pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI - drbd # drbdadm, drbdsetup, drbdmon - ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …) - # Storage + drbd # drbdadm, drbdsetup, drbdmon xfsprogs # mkfs.xfs, xfs_admin, xfs_info - targetcli-fb # targetcli shell + targetctl # Networking / debug iproute2 # ip, ss @@ -227,33 +130,32 @@ in htop ]; - # ── Firewall ────────────────────────────────────────────────────────── - networking.firewall = { - enable = true; - allowedTCPPorts = [ - 22 # SSH - 3260 # iSCSI - 3121 # pacemaker-remoted - 2224 # pcsd - drbdPort # DRBD replication - ]; - allowedUDPPorts = [ - 5404 # corosync cluster - 5405 # corosync cluster - 5407 # corosync crypto - ]; - # Corosync uses ports 5404-5407 UDP; allow them on the cluster net - extraCommands = '' - iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT - iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT - ''; - }; + # ── Networking ──────────────────────────────────────────────────────── + networking = { + useDHCP = false; + defaultGateway = "192.168.2.1"; + nameservers = [ "192.168.2.1" "8.8.8.8" ]; - # ── tmpfiles: target config dir ──────────────────────────────────────── - systemd.tmpfiles.rules = [ - "d /etc/target 0750 root root -" - "f /etc/target/saveconfig.json 0640 root root -" - ]; + firewall = { + enable = true; + allowedTCPPorts = [ + 22 # SSH + 3260 # iSCSI + 3121 # pacemaker-remoted + 2224 # pcsd + drbdPort + ]; + allowedUDPPorts = [ + 5404 # corosync + 5405 # corosync + 5407 # corosync crypto + ]; + extraCommands = '' + iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT + iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT + ''; + }; + }; # ── Locale / time ───────────────────────────────────────────────────── time.timeZone = vars.timeZone; From e3498b10878e10b02e0f3d92c99e8fe8ca958cdd Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 27 Jul 2026 11:26:37 +1000 Subject: [PATCH 6/7] feat(ha): promote HA file server to production flake targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds proxmox-ha-server-1 and proxmox-ha-server-2 as real mkTarget entries alongside the existing proxmox-server, backed by a new ha-server build type. New modules modules/ha/cluster-config.nix — DRBD resource + corosync nodelist sourced from vars (haServer1Host/Ip, haServer2Host/Ip); resource-only fencing for production STONITH; HA port firewall rules for DRBD, iSCSI, Corosync, pcsd modules/build-types/ha-server.nix — imports pacemaker-stack + iscsi-target + cluster-config + beszel; NFS exports from vars.haStorageRoot (XFS-over-DRBD mount); nfs-server.service.wantedBy force-cleared so Pacemaker controls start/stop on the Active node only New hosts hosts/ha-server-{1,2}/host.nix — static IP from vars, unique hostId; sops secrets (beszel, corosync authkey) are TODOs pending sync-host-keys.sh variables.nix haServer1/2Host, haServer1/2Ip, haServerVip, haStorageRoot, haIscsiIqn ports.haServerDrbd/Iscsi/Corosync{1,2,Crypto}/PacemakerRemoted/Pcsd scripts/ha/ (migrated + updated from test-lab/ha/) cluster-init.sh — generates corosync authkey, initialises DRBD/XFS/iSCSI, creates NFS dataset dirs, configures Pacemaker with DRBD + XFS + iSCSI + nfs-server + VIP; STONITH disabled initially (enable separately) cluster-enable-stonith.sh — enables fence_pve_ssh STONITH after key deploy fence-pve-ssh.py — Proxmox SSH fence agent (node names updated to ha-server-1/2) acceptance-tests.sh — T1–T7 production acceptance tests test-lab/ha/ removed — all Nix config moved to modules/ha/ and modules/build-types/; scripts moved to scripts/ha/ Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8 --- flake.nix | 33 +-- hosts/ha-server-1/host.nix | 24 ++ hosts/ha-server-2/host.nix | 24 ++ modules/build-types/ha-server.nix | 43 ++++ modules/ha/cluster-config.nix | 95 ++++++++ scripts/ha/acceptance-tests.sh | 167 +++++++++++++ scripts/ha/cluster-enable-stonith.sh | 86 +++++++ scripts/ha/cluster-init.sh | 284 ++++++++++++++++++++++ {test-lab => scripts}/ha/fence-pve-ssh.py | 18 +- test-lab/ha/acceptance-tests.sh | 171 ------------- test-lab/ha/cluster-enable-stonith.sh | 78 ------ test-lab/ha/cluster-init.sh | 239 ------------------ test-lab/ha/common.nix | 163 ------------- test-lab/ha/disko.nix | 49 ---- test-lab/ha/node1.nix | 12 - test-lab/ha/node2.nix | 12 - variables.nix | 27 +- 17 files changed, 762 insertions(+), 763 deletions(-) create mode 100644 hosts/ha-server-1/host.nix create mode 100644 hosts/ha-server-2/host.nix create mode 100644 modules/build-types/ha-server.nix create mode 100644 modules/ha/cluster-config.nix create mode 100644 scripts/ha/acceptance-tests.sh create mode 100644 scripts/ha/cluster-enable-stonith.sh create mode 100644 scripts/ha/cluster-init.sh rename {test-lab => scripts}/ha/fence-pve-ssh.py (92%) delete mode 100755 test-lab/ha/acceptance-tests.sh delete mode 100644 test-lab/ha/cluster-enable-stonith.sh delete mode 100644 test-lab/ha/cluster-init.sh delete mode 100644 test-lab/ha/common.nix delete mode 100644 test-lab/ha/disko.nix delete mode 100644 test-lab/ha/node1.nix delete mode 100644 test-lab/ha/node2.nix diff --git a/flake.nix b/flake.nix index 4788634..d4b0aa0 100644 --- a/flake.nix +++ b/flake.nix @@ -130,35 +130,10 @@ lxc-tailscale-router = mkTarget { platform = "lxc"; buildType = "tailscale-router"; hostPath = ./hosts/tailscale-router/host.nix; }; lxc-tor-relay = mkTarget { platform = "lxc"; buildType = "tor-relay"; hostPath = ./hosts/tor-relay/host.nix; }; - }; - # ── HA test lab VMs ───────────────────────────────────────────────── - # Two throwaway NixOS VMs to test the DRBD + XFS + LIO + Pacemaker - # stack. Deliberately bypass mkTarget (no clan-core, no sops-nix, - # no home-manager) — these are disposable and must be destroyed once - # the acceptance tests are done. - # Build disk images with: - # nix build .#nixosConfigurations.ha-test-node1.config.system.build.diskoImagesScript - # then run the script to produce ha-test-node1.raw (import with qm importdisk). - haTestTargets = - let - haNode = { nodeModule }: nixpkgs.lib.nixosSystem { - inherit system; - modules = [ - inputs.disko.nixosModules.disko - ./modules/hardware-configuration/vm/proxmox.nix - ./modules/boot/efi.nix - ./test-lab/ha/disko.nix - ./test-lab/ha/common.nix - nodeModule - ]; - specialArgs = { inherit vars; }; - }; - in - { - ha-test-node1 = haNode { nodeModule = ./test-lab/ha/node1.nix; }; - ha-test-node2 = haNode { nodeModule = ./test-lab/ha/node2.nix; }; - }; + proxmox-ha-server-1 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-1/host.nix; }; + proxmox-ha-server-2 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-2/host.nix; }; + }; # Auto-install environments (migrated from the former nix-auto-installer # flake): a self-contained NixOS installer that boots, discovers this @@ -235,7 +210,7 @@ in { - nixosConfigurations = generatedTargets // haTestTargets // installerTargets; + nixosConfigurations = generatedTargets // installerTargets; # Buildable auto-installer artifacts (`nix build .#`). No `lxc` # variant (installer-boots-as-an-LXC-container) or `all` bundle diff --git a/hosts/ha-server-1/host.nix b/hosts/ha-server-1/host.nix new file mode 100644 index 0000000..0fd808b --- /dev/null +++ b/hosts/ha-server-1/host.nix @@ -0,0 +1,24 @@ +{ vars, ... }: +{ + networking = { + hostName = vars.haServer1Host; + hostId = "3a4b5c6d"; + useDHCP = false; + interfaces.ens18.ipv4.addresses = [{ + address = vars.haServer1Ip; + prefixLength = 24; + }]; + defaultGateway = "192.168.2.1"; + nameservers = [ "192.168.2.1" "8.8.8.8" ]; + }; + + # TODO: after running `bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1` + # add beszel agent pairing and sops-managed corosync authkey: + # imports = [ (import ../../modules/beszel/host-token.nix { + # name = "ha-server-1"; + # sopsFile = ../../secrets/ha-server-1.yaml; + # }) ]; + # services.beszel.agent.environment.KEY = "..."; + + system.stateVersion = "26.05"; +} diff --git a/hosts/ha-server-2/host.nix b/hosts/ha-server-2/host.nix new file mode 100644 index 0000000..1f5a8ba --- /dev/null +++ b/hosts/ha-server-2/host.nix @@ -0,0 +1,24 @@ +{ vars, ... }: +{ + networking = { + hostName = vars.haServer2Host; + hostId = "7e8f9a0b"; + useDHCP = false; + interfaces.ens18.ipv4.addresses = [{ + address = vars.haServer2Ip; + prefixLength = 24; + }]; + defaultGateway = "192.168.2.1"; + nameservers = [ "192.168.2.1" "8.8.8.8" ]; + }; + + # TODO: after running `bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2` + # add beszel agent pairing and sops-managed corosync authkey: + # imports = [ (import ../../modules/beszel/host-token.nix { + # name = "ha-server-2"; + # sopsFile = ../../secrets/ha-server-2.yaml; + # }) ]; + # services.beszel.agent.environment.KEY = "..."; + + system.stateVersion = "26.05"; +} diff --git a/modules/build-types/ha-server.nix b/modules/build-types/ha-server.nix new file mode 100644 index 0000000..26026ac --- /dev/null +++ b/modules/build-types/ha-server.nix @@ -0,0 +1,43 @@ +# HA file server build type: DRBD + XFS + LIO iSCSI + NFS, managed by +# Corosync + Pacemaker. Both ha-server-1 and ha-server-2 use this type. +# +# NFS start/stop: +# services.nfs.server.enable = true configures /etc/exports, wires up +# rpcbind, and loads kernel modules — but nfs-server.service.wantedBy is +# force-cleared so systemd does NOT auto-start it at boot. Pacemaker's +# ha-group resource group (configured by scripts/ha/cluster-init.sh) +# starts and stops nfs-server as part of the failover sequence after the +# XFS mount and iSCSI target are brought up on the new Active node. +# +# Beszel agent: +# Enabled here via enable-agent.nix. The agent KEY (used to pair with +# the Beszel hub) is not set yet — add it to hosts/ha-server-{1,2}/host.nix +# under services.beszel.agent.environment.KEY once the hub accepts the +# new agents, following the pattern in hosts/server/host.nix. +{ lib, vars, ... }: +{ + imports = [ + ../ha/pacemaker-stack.nix + ../ha/iscsi-target.nix + ../ha/cluster-config.nix + ../beszel/enable-agent.nix + ]; + + services.nfs.server = { + enable = true; + exports = '' + ${vars.haStorageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ''; + }; + + # Pacemaker controls nfs-server — prevent systemd from starting it at boot + # on both nodes (only the Active node should be serving NFS). + systemd.services.nfs-server.wantedBy = lib.mkForce [ ]; +} diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix new file mode 100644 index 0000000..9074e99 --- /dev/null +++ b/modules/ha/cluster-config.nix @@ -0,0 +1,95 @@ +# Cluster-wide HA config shared by both ha-server nodes. +# +# Covers everything that is identical on both nodes and references cluster +# topology (node IPs, hostnames, DRBD resource). Per-node identity +# (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix. +# +# Corosync authkey: +# /etc/corosync/authkey must be present (mode 0400) for corosync to start. +# It is NOT managed declaratively here — the initial deploy uses +# scripts/ha/cluster-init.sh to generate it via corosync-keygen and +# distribute it to both nodes. +# TODO: once both hosts have their sops keys registered via +# scripts/secrets/sync-host-keys.sh, add a sops secret here so the +# authkey survives nixos-rebuild. +# +# DRBD fencing: +# Production setting is resource-only: DRBD waits for the STONITH fence +# agent to confirm the peer is dead before promoting to Primary. This +# requires a working fence_pve_ssh STONITH resource in Pacemaker +# (see scripts/ha/cluster-enable-stonith.sh). On a fresh cluster with +# no fence device yet, temporarily change to dont-care and run +# cluster-enable-stonith.sh once the fence key is deployed. +{ vars, ... }: +{ + services.drbd = { + enable = true; + config = '' + global { + usage-count yes; + } + + common { + net { + protocol C; + ping-int 1; + verify-alg sha256; + after-sb-0pri discard-zero-changes; + after-sb-1pri discard-secondary; + } + disk { + fencing resource-only; + } + } + + resource ha-data { + volume 0 { + device /dev/drbd0; + disk /dev/sdb; + meta-disk internal; + } + + on ${vars.haServer1Host} { + address ${vars.haServer1Ip}:${toString vars.ports.haServerDrbd}; + } + + on ${vars.haServer2Host} { + address ${vars.haServer2Ip}:${toString vars.ports.haServerDrbd}; + } + } + ''; + }; + + # services.corosync.enable is set by modules/ha/pacemaker-stack.nix. + services.corosync = { + clusterName = "ha-cluster"; + nodelist = [ + { nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1Ip ]; } + { nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2Ip ]; } + ]; + }; + + networking.firewall = { + allowedTCPPorts = [ + vars.ports.haServerIscsi + vars.ports.haServerPacemakerRemoted + vars.ports.haServerPcsd + vars.ports.haServerDrbd + vars.ports.nfsRpcbind + vars.ports.nfsd + vars.ports.nfsMountd + ]; + allowedUDPPorts = [ + vars.ports.haServerCorosync1 + vars.ports.haServerCorosync2 + vars.ports.haServerCorosyncCrypto + vars.ports.nfsRpcbind + vars.ports.nfsd + vars.ports.nfsMountd + ]; + extraCommands = '' + iptables -A INPUT -s ${vars.haServer1Ip}/32 -j ACCEPT + iptables -A INPUT -s ${vars.haServer2Ip}/32 -j ACCEPT + ''; + }; +} diff --git a/scripts/ha/acceptance-tests.sh b/scripts/ha/acceptance-tests.sh new file mode 100644 index 0000000..ab3081c --- /dev/null +++ b/scripts/ha/acceptance-tests.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# acceptance-tests.sh — HA cluster acceptance tests (T1–T7) +# +# Run from a host with SSH access to both HA nodes (or from node1 itself). +# All 7 tests must pass before considering the cluster production-ready. +# Test values below must match variables.nix haServer* values. +set -euo pipefail + +# ── Configuration ───────────────────────────────────────────────────────── +NODE1="ha-server-1" +NODE2="ha-server-2" +NODE1_IP="192.168.2.200" # vars.haServer1Ip +NODE2_IP="192.168.2.201" # vars.haServer2Ip +VIP="192.168.2.202" # vars.haServerVip +XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot +ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn +# ────────────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 +RESULTS=() + +pass() { echo " PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); } +fail() { echo " FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); } + +n1() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; } +n2() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; } + +echo "════════════════════════════════════════════════════" +echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')" +echo "════════════════════════════════════════════════════" + +# ── T1: Corosync quorum established ────────────────────────────────────── +echo "" +echo "[T1] Corosync quorum" +if n1 "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then + pass "cluster has quorum" +else + fail "cluster does not have quorum — check corosync on both nodes" +fi + +# ── T2: DRBD Primary on node1, Secondary on node2 ──────────────────────── +echo "" +echo "[T2] DRBD roles" +DRBD_ROLE=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown") +if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then + pass "DRBD Primary on $NODE1 ($DRBD_ROLE)" +else + fail "unexpected DRBD role on $NODE1: $DRBD_ROLE (expected Primary/Secondary)" +fi + +DRBD_DSTATE=$(n1 "drbdadm dstate ha-data" 2>/dev/null || echo "unknown") +if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then + pass "DRBD disk state UpToDate ($DRBD_DSTATE)" +else + fail "DRBD disk not UpToDate: $DRBD_DSTATE" +fi + +# ── T3: XFS mounted at haStorageRoot on the Active node ────────────────── +echo "" +echo "[T3] XFS mount" +if n1 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + pass "XFS mounted at ${XFS_MOUNT} on $NODE1" +else + fail "XFS not mounted at ${XFS_MOUNT} on $NODE1" +fi + +if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + fail "XFS unexpectedly mounted on $NODE2 (should only be on Active node)" +else + pass "XFS not mounted on $NODE2 (correct — Secondary)" +fi + +# ── T4: iSCSI target visible on both nodes ──────────────────────────────── +echo "" +echo "[T4] iSCSI target" +IQN_COUNT=$(n1 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0") +if [[ "$IQN_COUNT" -ge 1 ]]; then + pass "iSCSI IQN active on $NODE1 ($IQN_COUNT target(s))" +else + fail "no iSCSI IQN active on $NODE1" +fi + +# iSCSI discovery from node2 via VIP +if n2 "iscsiadm -m discovery -t sendtargets -p '${VIP}' 2>/dev/null | grep -q '${ISCSI_IQN}'"; then + pass "iSCSI target discoverable from $NODE2 via VIP ${VIP}" +else + fail "iSCSI target not discoverable from $NODE2 via ${VIP}" +fi + +# ── T5: Failover — standby node1, verify resources move to node2 ────────── +echo "" +echo "[T5] Failover (standby $NODE1)" +MYNODE=$(n1 "crm_node -n" 2>/dev/null || echo "") +n1 "crm_standby -N '${MYNODE}' -v on" 2>/dev/null || true +echo " Waiting up to 30 s for resources to move to $NODE2..." +MOVED=false +for i in $(seq 1 30); do + if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + MOVED=true + echo " Resources moved in ${i}s" + break + fi + sleep 1 +done + +if $MOVED; then + pass "XFS mounted on $NODE2 after failover" + IQN_ON_N2=$(n2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0") + [[ "$IQN_ON_N2" -ge 1 ]] \ + && pass "iSCSI target active on $NODE2 after failover" \ + || fail "iSCSI target NOT active on $NODE2 after failover" +else + fail "XFS did not mount on $NODE2 within 30 s — failover incomplete" +fi + +# ── T6: Data integrity — file written pre-failover readable post-failover ─ +echo "" +echo "[T6] Data integrity" +# Write a test file on node2 (now Active) and verify its content +TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$" +TEST_CONTENT="ha-acceptance-test-$(date +%s)" +n2 "echo '${TEST_CONTENT}' > '${TEST_FILE}'" 2>/dev/null || true +READBACK=$(n2 "cat '${TEST_FILE}' 2>/dev/null" || echo "") +if [[ "$READBACK" == "$TEST_CONTENT" ]]; then + pass "test file written and read back correctly on $NODE2" +else + fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')" +fi +n2 "rm -f '${TEST_FILE}'" 2>/dev/null || true + +# ── T7: Node rejoin — un-standby node1, verify cluster is healthy ───────── +echo "" +echo "[T7] Node rejoin" +n1 "crm_standby -N '${MYNODE}' -v off" 2>/dev/null || true +n1 "crm_resource --cleanup" 2>/dev/null || true +sleep 5 + +ONLINE_NODES=$(n2 "crm_mon -1 2>/dev/null | grep -c 'Online:'" || echo "0") +if n1 "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then + pass "$NODE1 rejoined — cluster has quorum" +else + fail "$NODE1 did not rejoin with quorum" +fi + +DRBD_ROLE_AFTER=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown") +if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then + pass "$NODE1 is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)" +else + fail "unexpected DRBD role on $NODE1 after rejoin: $DRBD_ROLE_AFTER" +fi + +# ── Summary ─────────────────────────────────────────────────────────────── +echo "" +echo "════════════════════════════════════════════════════" +echo " Results: ${PASS} PASS, ${FAIL} FAIL" +echo "════════════════════════════════════════════════════" +for r in "${RESULTS[@]}"; do echo " $r"; done +echo "" + +if [[ "$FAIL" -eq 0 ]]; then + echo "ALL PASS — cluster is production-ready." + exit 0 +else + echo "SOME TESTS FAILED — investigate before deploying." + exit 1 +fi diff --git a/scripts/ha/cluster-enable-stonith.sh b/scripts/ha/cluster-enable-stonith.sh new file mode 100644 index 0000000..6ef47a5 --- /dev/null +++ b/scripts/ha/cluster-enable-stonith.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# cluster-enable-stonith.sh — enable STONITH fence agent after the fence SSH +# key is deployed to both nodes and authorised on the Proxmox host. +# +# Run from ha-server-1 as root AFTER: +# - /etc/pacemaker/fence_pve_ssh exists on both nodes (chmod +x) +# (copy from scripts/ha/fence-pve-ssh.py) +# - /etc/fence-pve-ssh-key (SSH private key) exists on both nodes +# - The corresponding public key is in authorized_keys on PVE_HOST +# - VMID_NODE1 / VMID_NODE2 filled in below +set -euo pipefail + +# ── Configuration ───────────────────────────────────────────────────────── +NODE1="ha-server-1" +NODE2="ha-server-2" +VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1 +VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2 +PVE_HOST="pve1.sweet.home" +PVE_USER="wayne" +FENCE_KEY="/etc/fence-pve-ssh-key" +FENCE_SCRIPT="/etc/pacemaker/fence_pve_ssh" +# ────────────────────────────────────────────────────────────────────────── + +log() { echo "[stonith-setup] $*"; } +die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; } + +[[ $(id -u) -eq 0 ]] || die "must run as root" +[[ -n "$VMID_NODE1" ]] || die "VMID_NODE1 not set — edit this script" +[[ -n "$VMID_NODE2" ]] || die "VMID_NODE2 not set — edit this script" +[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY" +[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT" + +log "Verifying fence agent can reach ${PVE_HOST}..." +ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \ + -o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" \ + "sudo /usr/sbin/qm list" &>/dev/null \ + || die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo" +log "Fence agent SSH connectivity confirmed" + +log "Creating Pacemaker STONITH resources..." +cibadmin --create --scope resources --xml-text " + + + + + + + + + + + + + + +" 2>/dev/null || true + +cibadmin --create --scope resources --xml-text " + + + + + + + + + + + + + + +" 2>/dev/null || true + +log "Enabling STONITH and restoring quorum policy..." +crm_attribute -t crm_config -n stonith-enabled -v true +crm_attribute -t crm_config -n no-quorum-policy -v stop + +log "DRBD fencing mode must also be updated to resource-only (already the" +log "default in cluster-config.nix; confirm with: cat /etc/drbd.d/ha-data.conf)" + +log "Testing fence agent..." +stonith_admin --list-devices && log "Fence devices listed successfully." \ + || warn "stonith_admin --list-devices failed — check config" + +log "STONITH enabled. Cluster is now fully HA." diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh new file mode 100644 index 0000000..1aac090 --- /dev/null +++ b/scripts/ha/cluster-init.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# cluster-init.sh — one-time HA cluster initialisation script +# +# Run ONCE from ha-server-1 as root AFTER both VMs are booted and have SSH +# access. It: +# 1. Generates and distributes the corosync authkey +# 2. Waits for corosync quorum and pacemaker +# 3. Initialises DRBD metadata, promotes node1 to primary +# 4. Creates XFS on /dev/drbd0 and mounts it +# 5. Creates the directory tree and iSCSI LUN backing file +# 6. Configures LIO iSCSI target (file-backed LUN) +# 7. Configures Pacemaker resources: DRBD → XFS → iSCSI → NFS → VIP +# +# Prerequisites: +# - Both VMs booted with the ha-server config (nixos-rebuild done) +# - SSH key access from node1 to root@NODE2_IP +# - VMID_NODE1 / VMID_NODE2 filled in below (needed for STONITH setup; +# cluster starts without STONITH, which you enable separately via +# scripts/ha/cluster-enable-stonith.sh) +# - Run as root on ha-server-1 +set -euo pipefail + +# ── Configuration ───────────────────────────────────────────────────────── +# These must match variables.nix haServer* values and the Proxmox VMID +# assignments. Update before running. +NODE1="ha-server-1" +NODE2="ha-server-2" +NODE1_IP="192.168.2.200" # vars.haServer1Ip +NODE2_IP="192.168.2.201" # vars.haServer2Ip +VIP="192.168.2.202" # vars.haServerVip +XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot +ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn +ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img" +ISCSI_LUN_SIZE="10G" +DRBD_DEVICE="/dev/drbd0" +VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1 +VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2 +PVE_HOST="pve1.sweet.home" +PVE_USER="wayne" + +# NFS dataset subdirectories to create under XFS_MOUNT. +# Must mirror vars.nfsShares subpath values in variables.nix. +NFS_SUBDIRS=( + "docker/config" + "docker/volumes" + "docker/databases" + "docker/nextcloud-data" + "raspi/volumes" + "proxmox/iso" + "proxmox/lxc" + "pxe-boot/images" +) +# ────────────────────────────────────────────────────────────────────────── + +log() { echo "[cluster-init] $*"; } +die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; } +warn() { echo "[cluster-init] WARNING: $*" >&2; } + +[[ $(id -u) -eq 0 ]] || die "must run as root" +[[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1" + +# ── 0. Corosync authkey ─────────────────────────────────────────────────── +AUTHKEY="/etc/corosync/authkey" +mkdir -p /etc/corosync +if [[ ! -f "$AUTHKEY" ]]; then + log "Generating corosync authkey..." + corosync-keygen -k "$AUTHKEY" + chmod 0400 "$AUTHKEY" +fi +log "Distributing authkey to $NODE2..." +ssh "root@${NODE2_IP}" "mkdir -p /etc/corosync" +scp -q "$AUTHKEY" "root@${NODE2_IP}:${AUTHKEY}" +ssh "root@${NODE2_IP}" "chmod 0400 '${AUTHKEY}'" + +log "Restarting corosync on both nodes..." +systemctl restart corosync +ssh "root@${NODE2_IP}" "systemctl restart corosync" +sleep 3 + +# ── 1. Corosync quorum ──────────────────────────────────────────────────── +log "Waiting for corosync quorum..." +for i in $(seq 1 30); do + if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then + log "Quorum established" + break + fi + [[ $i -eq 30 ]] && die "corosync quorum not established after 60 s" + sleep 2 +done + +log "Waiting for pacemaker..." +for i in $(seq 1 30); do + if crm_mon -1 &>/dev/null; then + log "Pacemaker running" + break + fi + [[ $i -eq 30 ]] && die "pacemaker not running after 60 s" + sleep 2 +done + +# ── 2. DRBD initialisation ──────────────────────────────────────────────── +log "Initialising DRBD metadata on $NODE1..." +if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then + drbdadm create-md ha-data --force +fi + +log "Initialising DRBD metadata on $NODE2..." +ssh "root@${NODE2_IP}" " + if ! drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent\|Diskless'; then + drbdadm create-md ha-data --force + fi +" + +log "Bringing up DRBD on both nodes..." +drbdadm up ha-data 2>/dev/null || true +ssh "root@${NODE2_IP}" "drbdadm up ha-data 2>/dev/null" || true + +log "Forcing $NODE1 to DRBD Primary for initial sync..." +drbdadm primary ha-data --force + +log "Waiting for DRBD to finish initial sync (this may take several minutes)..." +for i in $(seq 1 300); do + state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown") + if echo "$state" | grep -q "UpToDate/UpToDate"; then + log "DRBD sync complete: $state" + break + fi + [[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)" + sleep 1 +done + +# ── 3. XFS filesystem ───────────────────────────────────────────────────── +log "Creating XFS on ${DRBD_DEVICE}..." +if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then + mkfs.xfs -f "${DRBD_DEVICE}" +fi + +log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..." +mkdir -p "${XFS_MOUNT}" +mount "${DRBD_DEVICE}" "${XFS_MOUNT}" + +# ── 4. NFS dataset directories ──────────────────────────────────────────── +log "Creating NFS dataset directories..." +for subdir in "${NFS_SUBDIRS[@]}"; do + mkdir -p "${XFS_MOUNT}/${subdir}" +done + +# ── 5. iSCSI LUN backing file ───────────────────────────────────────────── +log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..." +if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then + fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}" +fi + +# ── 6. LIO iSCSI target ─────────────────────────────────────────────────── +log "Configuring LIO iSCSI target via targetcli..." +targetcli < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +" + +log "Adding ordering and colocation constraints..." +cibadmin --create --scope constraints --xml-text " + + + + +" + +log "Waiting for resources to start..." +for i in $(seq 1 60); do + if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then + log "VIP is up: $(crm_resource -r vip --locate)" + break + fi + [[ $i -eq 60 ]] && { warn "VIP not up after 120 s — check: crm_mon -1"; break; } + sleep 2 +done + +log "" +log "═══════════════════════════════════════════════════════════════" +log " HA cluster initialised." +log "" +log " crm_mon -1 — cluster status" +log " iscsiadm -m discovery -t st -p ${VIP} — verify iSCSI target" +log " showmount -e ${VIP} — verify NFS exports" +log "" +log " To enable STONITH (after deploying fence SSH key):" +log " 1. Fill in VMID_NODE1 / VMID_NODE2 in cluster-enable-stonith.sh" +log " 2. Copy scripts/ha/fence-pve-ssh.py to /etc/pacemaker/fence_pve_ssh" +log " on both nodes (chmod +x)" +log " 3. Generate and distribute the fence SSH key" +log " (see docs or cluster-enable-stonith.sh header)" +log " 4. bash scripts/ha/cluster-enable-stonith.sh" +log "═══════════════════════════════════════════════════════════════" diff --git a/test-lab/ha/fence-pve-ssh.py b/scripts/ha/fence-pve-ssh.py similarity index 92% rename from test-lab/ha/fence-pve-ssh.py rename to scripts/ha/fence-pve-ssh.py index 7bf0c7a..58c24ed 100644 --- a/test-lab/ha/fence-pve-ssh.py +++ b/scripts/ha/fence-pve-ssh.py @@ -2,16 +2,16 @@ """ fence_pve_ssh - Proxmox VE SSH fence agent for Pacemaker. -Uses SSH to reach pve1.sweet.home and run 'qm stop/start '. -Designed for test-lab HA cluster only — not for production. +Uses SSH to reach the Proxmox host and run 'qm stop/start '. +Deploy to /etc/pacemaker/fence_pve_ssh on both HA nodes (chmod +x). Configuration (as pacemaker stonith resource attributes): pve_host Proxmox host to SSH to (default: pve1.sweet.home) pve_user SSH user (default: wayne) key_file SSH private key path (default: /etc/fence-pve-ssh-key) - vmid_node1 VMID for ha-test-node1 (e.g. 200) - vmid_node2 VMID for ha-test-node2 (e.g. 201) - plug Node name to act on (set by pacemaker: ha-test-node1 or ha-test-node2) + vmid_node1 VMID for ha-server-1 + vmid_node2 VMID for ha-server-2 + plug Node name to act on (set by pacemaker: ha-server-1 or ha-server-2) action Action: off|on|reboot|status|list|metadata """ @@ -112,8 +112,8 @@ def get_vmid(args): print("ERROR: --plug not specified", file=sys.stderr) sys.exit(1) mapping = { - "ha-test-node1": args.vmid_node1, - "ha-test-node2": args.vmid_node2, + "ha-server-1": args.vmid_node1, + "ha-server-2": args.vmid_node2, } vmid = mapping.get(node) if not vmid: @@ -132,9 +132,9 @@ def main(): if action == "list": if args.vmid_node1: - print("ha-test-node1") + print("ha-server-1") if args.vmid_node2: - print("ha-test-node2") + print("ha-server-2") sys.exit(0) vmid = get_vmid(args) diff --git a/test-lab/ha/acceptance-tests.sh b/test-lab/ha/acceptance-tests.sh deleted file mode 100755 index 651f1e9..0000000 --- a/test-lab/ha/acceptance-tests.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env bash -# HA File Server Acceptance Tests T1-T7 -# Failover (T5) uses pacemaker standby mode to gracefully move resources, -# simulating what STONITH + node-restart does in production. -# Note: A production cluster requires real STONITH (fence agent for the hypervisor). -set -euo pipefail - -NODE2="root@192.168.2.201" -VIP="192.168.2.202" -IQN="iqn.2026-01.local.ha-test:storage" -MOUNT="/srv/ha-data" -PACEMAKERD="/nix/store/3v9sb74cg2qmpcyzb4h6fq0z8bvp5gw1-pacemaker-3.0.1/sbin/pacemakerd" -MYNODE=$(hostname) - -SSH="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15" -PASS=0; FAIL=0 - -pass() { echo "[PASS] $1"; ((PASS++)) || true; } -fail() { echo "[FAIL] $1"; ((FAIL++)) || true; } -info() { echo "[INFO] $1"; } - -echo "=== HA File Server Acceptance Tests ===" -echo "Node: $MYNODE Date: $(date)" -echo "" - -# T1: Corosync 2-node cluster with quorum -echo "--- T1: Corosync cluster quorum ---" -QUORATE=$(corosync-quorumtool -s 2>/dev/null | grep 'Quorate:' | awk '{print $2}') -NODE_COUNT=$(corosync-quorumtool -s 2>/dev/null | grep '^Nodes:' | awk '{print $2}') -if [ "$QUORATE" = "Yes" ] && [ "$NODE_COUNT" = "2" ]; then - pass "T1: Corosync quorate with 2 nodes" -else - fail "T1: quorate=$QUORATE nodes=$NODE_COUNT" -fi - -# T2: DRBD both UpToDate, replication Established -echo "--- T2: DRBD replication healthy ---" -DRBD_STATUS=$(drbdadm status ha-data 2>/dev/null) -N1_ROLE=$(echo "$DRBD_STATUS" | grep '^ha-data role:' | awk -F: '{print $2}') -N1_DISK=$(echo "$DRBD_STATUS" | grep -oP 'disk:\K\S+' | head -1) -N2_DISK=$(echo "$DRBD_STATUS" | grep -oP 'peer-disk:\K\S+' | head -1) -REPL=$(echo "$DRBD_STATUS" | grep -oP 'replication:\K\S+' | head -1) -info "DRBD: role=$N1_ROLE local_disk=$N1_DISK peer_disk=$N2_DISK replication=$REPL" -if [ "$N1_DISK" = "UpToDate" ] && [ "$N2_DISK" = "UpToDate" ] && [ "$REPL" = "Established" ]; then - pass "T2: DRBD both UpToDate, Established (role=$N1_ROLE)" -else - fail "T2: DRBD issue: local=$N1_DISK peer=$N2_DISK replication=$REPL" -fi - -# T3: XFS mounted on primary -echo "--- T3: XFS mount on primary ---" -if mountpoint -q $MOUNT && df -t xfs $MOUNT &>/dev/null; then - FSINFO=$(df -h $MOUNT | tail -1) - pass "T3: XFS mounted at $MOUNT: $FSINFO" -else - fail "T3: XFS not mounted at $MOUNT" -fi - -# T4: iSCSI active on primary, VIP responds on port 3260 -echo "--- T4: iSCSI target active ---" -ACTIVE_IQN=$(ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep iqn | head -1) -if [ "$ACTIVE_IQN" = "$IQN" ] && nc -w3 $VIP 3260 < /dev/null 2>/dev/null; then - pass "T4: iSCSI $IQN active, port 3260 open on VIP $VIP" -elif [ "$ACTIVE_IQN" = "$IQN" ]; then - fail "T4: iSCSI IQN active but port 3260 not reachable on VIP" -else - fail "T4: iSCSI not active (got '$ACTIVE_IQN')" -fi - -# Pre-T5: write test file for data integrity check -echo "--- Pre-T5: writing test data ---" -TESTFILE="$MOUNT/failover-test.txt" -TESTDATA="FAILOVER_INTEGRITY_$(date +%s)" -echo "$TESTDATA" > "$TESTFILE" -sync -info "Wrote: $TESTFILE (data: $TESTDATA)" - -# T5: Failover — put this node into pacemaker standby, forcing resource migration -# (In production this is triggered by real STONITH; standby simulates the result.) -echo "--- T5: Failover (pacemaker standby + node isolation) ---" -info "Putting $MYNODE into standby mode (triggers resource migration to node2)..." -crm_standby -N "$MYNODE" -v on 2>&1 || true - -FAILOVER_OK=false -info "Waiting up to 90s for node2 failover..." -for i in $(seq 1 18); do - sleep 5 - STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "UNREACHABLE") - if echo "$STATUS" | grep -q "Started ha-test-node2"; then - FAILOVER_OK=true - info "Failover complete at $((i*5))s" - echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted' - break - fi -done - -if $FAILOVER_OK; then - N2_DRBD=$($SSH $NODE2 "drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print \$2}'" 2>/dev/null || echo "unknown") - N2_MOUNT=$($SSH $NODE2 "mountpoint -q $MOUNT && echo 'mounted' || echo 'not-mounted'" 2>/dev/null || echo "unknown") - N2_ISCSI=$($SSH $NODE2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" 2>/dev/null || echo "0") - N2_VIP=$($SSH $NODE2 "ip addr show | grep -c '$VIP'" 2>/dev/null || echo "0") - info "Node2: DRBD=$N2_DRBD mount=$N2_MOUNT iSCSI_IQNs=$N2_ISCSI VIP=$N2_VIP" - - FAILS=0 - [ "$N2_DRBD" = "Primary" ] || { info "FAIL: DRBD not Primary on node2"; ((FAILS++)) || true; } - [ "$N2_MOUNT" = "mounted" ] || { info "FAIL: XFS not mounted on node2"; ((FAILS++)) || true; } - [ "$N2_ISCSI" -ge "1" ] 2>/dev/null || { info "FAIL: iSCSI not active on node2"; ((FAILS++)) || true; } - [ "$N2_VIP" -ge "1" ] 2>/dev/null || { info "FAIL: VIP not on node2"; ((FAILS++)) || true; } - - if [ $FAILS -eq 0 ]; then - pass "T5: Failover complete — DRBD Primary, XFS, iSCSI, VIP all on node2" - else - fail "T5: Partial failover ($FAILS sub-checks failed)" - fi -else - fail "T5: No failover detected within 90s" -fi - -# T7: Data integrity — test file readable on node2 after failover -echo "--- T7: Data integrity after failover ---" -if $SSH $NODE2 "grep -q '$TESTDATA' $TESTFILE 2>/dev/null"; then - pass "T7: Test data intact on node2 after failover" -else - ACTUAL=$($SSH $NODE2 "cat $TESTFILE 2>/dev/null || echo FILE_MISSING" 2>/dev/null || echo "SSH_FAIL") - fail "T7: Data integrity check failed (expected '$TESTDATA', got '$ACTUAL')" -fi - -# T6: Node rejoin — take node out of standby -echo "--- T6: Node rejoin ---" -info "Taking $MYNODE out of standby..." -crm_standby -N "$MYNODE" -v off 2>&1 || true - -REJOIN_OK=false -for i in $(seq 1 12); do - sleep 5 - # Check if this node is back online (no longer standby) - STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "") - if echo "$STATUS" | grep -q "Online:.*$MYNODE"; then - REJOIN_OK=true - info "Rejoined at $((i*5))s" - echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted' - break - fi -done - -if $REJOIN_OK; then - sleep 5 - N1_DRBD=$(drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print $2}') - N1_DISK=$(drbdadm status ha-data 2>/dev/null | grep -oP 'disk:\K\S+' | head -1) - info "Node1 DRBD after rejoin: role=$N1_DRBD disk=$N1_DISK" - pass "T6: Node rejoined cluster (DRBD role=$N1_DRBD, disk=$N1_DISK)" -else - fail "T6: Node did not rejoin within 60s" -fi - -echo "" -echo "========================================" -echo "RESULTS: $PASS passed, $FAIL failed" -echo "========================================" -echo "" -echo "NOTES:" -echo " T5 uses pacemaker standby to simulate failover (production needs STONITH" -echo " fence agent, e.g. fence_pve_ssh, to crash the VM — same requirement on Debian)" -echo "" -if [ $FAIL -eq 0 ]; then - echo "VERDICT: ALL TESTS PASSED → Deliverable A (NixOS modules)" - exit 0 -else - echo "VERDICT: $FAIL TEST(S) FAILED → review above" - exit 1 -fi diff --git a/test-lab/ha/cluster-enable-stonith.sh b/test-lab/ha/cluster-enable-stonith.sh deleted file mode 100644 index 6c23081..0000000 --- a/test-lab/ha/cluster-enable-stonith.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# cluster-enable-stonith.sh — enable STONITH fence agent after fence key is deployed -# Run from ha-test-node1 as root, AFTER: -# - /etc/fence-pve-ssh-key exists on both nodes -# - The fence public key is in authorized_keys on pve1.sweet.home -set -euo pipefail - -VMID_NODE1="200" -VMID_NODE2="201" -PVE_HOST="pve1.sweet.home" -PVE_USER="wayne" -FENCE_KEY="/etc/fence-pve-ssh-key" -FENCE_SCRIPT="/usr/lib/ocf/resource.d/heartbeat/fence_pve_ssh" - -log() { echo "[stonith-setup] $*"; } -die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; } - -[[ $(id -u) -eq 0 ]] || die "must run as root" - -[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY" -[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT" - -log "Verifying fence agent can reach ${PVE_HOST}..." -if ! ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \ - -o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" "sudo /usr/sbin/qm list" &>/dev/null; then - die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo" -fi -log "Fence agent SSH connectivity confirmed" - -log "Creating Pacemaker STONITH resource..." -cibadmin --create --scope resources --xml-text " - - - - - - - - - - - - - - -" 2>/dev/null || true - -cibadmin --create --scope resources --xml-text " - - - - - - - - - - - - - - -" 2>/dev/null || true - -log "Enabling STONITH..." -crm_attribute -t crm_config -n stonith-enabled -v true - -# Restore quorum policy to stop (needed with STONITH) -crm_attribute -t crm_config -n no-quorum-policy -v stop - -log "STONITH enabled. Testing fence agent..." -if stonith_admin --list-devices; then - log "Fence devices listed successfully" -else - log "WARNING: fence device list failed — check stonith config" -fi - -log "STONITH setup complete. Cluster is now fully HA." diff --git a/test-lab/ha/cluster-init.sh b/test-lab/ha/cluster-init.sh deleted file mode 100644 index 597017b..0000000 --- a/test-lab/ha/cluster-init.sh +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env bash -# cluster-init.sh — one-time HA cluster initialisation script -# -# Run this ONCE from node1 AFTER both VMs are booted and have SSH access. -# It: -# 1. Waits for corosync quorum on both nodes -# 2. Initialises DRBD metadata and promotes node1 to primary -# 3. Creates XFS filesystem on /dev/drbd0 -# 4. Configures targetcli / LIO iSCSI target (with a file-backed LUN) -# 5. Configures the Pacemaker resource group -# 6. Optionally enables the STONITH fence agent (requires fence SSH key) -# -# Prerequisites: -# - Both VMs booted with the ha-test config -# - fence-pve-ssh-key distributed to /etc/fence-pve-ssh-key on both nodes -# - Run as root on ha-test-node1 -set -euo pipefail - -NODE1_IP="192.168.2.200" -NODE2_IP="192.168.2.201" -VIP="192.168.2.202" -DRBD_DEVICE="/dev/drbd0" -XFS_MOUNT="/mnt/ha-data" -ISCSI_IQN="iqn.2026-01.local.ha-test:storage" -ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img" -ISCSI_LUN_SIZE="1G" # small test LUN -VMID_NODE1="200" -VMID_NODE2="201" -PVE_HOST="pve1.sweet.home" -PVE_USER="wayne" -FENCE_KEY="/etc/fence-pve-ssh-key" - -log() { echo "[cluster-init] $*"; } -die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; } - -[[ $(id -u) -eq 0 ]] || die "must run as root" -[[ "$(hostname)" == "ha-test-node1" ]] || die "must run on ha-test-node1" - -# ── 1. Wait for corosync quorum ────────────────────────────────────────── -log "Waiting for corosync quorum..." -for i in $(seq 1 30); do - if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then - log "Quorum established" - break - fi - [[ $i -eq 30 ]] && die "corosync quorum not established after 30s" - sleep 2 -done - -log "Waiting for pacemaker to start..." -for i in $(seq 1 30); do - if crm_mon -1 &>/dev/null; then - log "Pacemaker running" - break - fi - [[ $i -eq 30 ]] && die "pacemaker not running after 60s" - sleep 2 -done - -# ── 2. Initialise DRBD ─────────────────────────────────────────────────── -log "Initialising DRBD metadata on node1..." -if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent"; then - drbdadm create-md ha-data --force -fi - -log "Initialising DRBD metadata on node2..." -if ! ssh "root@${NODE2_IP}" "drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent'"; then - ssh "root@${NODE2_IP}" "drbdadm create-md ha-data --force" -fi - -log "Bringing up DRBD on both nodes..." -drbdadm up ha-data || true -ssh "root@${NODE2_IP}" "drbdadm up ha-data" || true - -log "Forcing node1 to DRBD primary (initial sync)..." -drbdadm primary ha-data --force - -log "Waiting for DRBD to finish initial sync..." -for i in $(seq 1 120); do - state=$(drbdadm dstate ha-data) - if echo "$state" | grep -q "UpToDate"; then - log "DRBD sync complete: $state" - break - fi - log " DRBD state: $state (${i}/120s)" - [[ $i -eq 120 ]] && die "DRBD did not sync within 120s" - sleep 1 -done - -# ── 3. XFS filesystem ──────────────────────────────────────────────────── -log "Creating XFS on ${DRBD_DEVICE}..." -if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then - mkfs.xfs "${DRBD_DEVICE}" -fi - -log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..." -mkdir -p "${XFS_MOUNT}" -mount "${DRBD_DEVICE}" "${XFS_MOUNT}" - -# ── 4. iSCSI LUN (file-backed) ─────────────────────────────────────────── -log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..." -if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then - fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}" -fi - -log "Configuring LIO iSCSI target via targetcli..." -# This produces a /etc/target/saveconfig.json that the targetctl service loads. -# The commands create an iSCSI target backed by the file we just created. -targetcli <<'EOF' -/backstores/fileio create name=ha-lun0 file_or_dev=/mnt/ha-data/iscsi-lun.img size=0 write_back=false -/iscsi create iqn.2026-01.local.ha-test:storage -/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/luns create /backstores/fileio/ha-lun0 -/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/portals create 192.168.2.202 -/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute authentication=0 -/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute demo_mode_write_protect=0 -saveconfig /etc/target/saveconfig.json -EOF - -log "Unmounting ${XFS_MOUNT} (Pacemaker will manage it)..." -umount "${XFS_MOUNT}" - -log "Promoting DRBD back to secondary (Pacemaker manages primary role)..." -drbdadm secondary ha-data - -# ── 5. Pacemaker resources ─────────────────────────────────────────────── -log "Configuring Pacemaker..." - -# Disable STONITH initially — enable once fence key is deployed -crm_attribute -t crm_config -n stonith-enabled -v false - -# Disable quorum policy for two-node cluster (no-quorum-policy=ignore so -# the surviving node can promote without a quorum device) -crm_attribute -t crm_config -n no-quorum-policy -v ignore - -# Cluster resources: -# 1. drbd-ha — manages DRBD primary/secondary role -# 2. xfs-mount — XFS mount on /mnt/ha-data -# 3. iscsi-target — LIO target service (systemd class) -# 4. vip — floating VIP 192.168.2.202 - -log "Creating DRBD master/slave resource..." -cibadmin --replace --scope resources --xml-text " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" - -log "Adding ordering and colocation constraints..." -# All resources on the same node as DRBD master -cibadmin --create --scope constraints --xml-text " - - - - - - - - -" - -log "Resource group configured. Waiting for resources to start..." -for i in $(seq 1 60); do - if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then - log "VIP is up: $(crm_resource -r vip --locate)" - break - fi - [[ $i -eq 60 ]] && { log "WARNING: VIP not up after 60s — check crm_mon"; break; } - sleep 2 -done - -log "" -log "═══════════════════════════════════════════════════════" -log " HA cluster initialised. Next steps:" -log "" -log " - Verify: crm_mon -1" -log " - Test iSCSI: iscsiadm -m discovery -t sendtargets -p ${VIP}" -log "" -log " To enable STONITH (after deploying fence key):" -log " 1. Copy fence-pve-ssh.py to /usr/lib/ocf/resource.d/heartbeat/ on both nodes" -log " 2. Distribute /etc/fence-pve-ssh-key to both nodes" -log " 3. Add public key to authorized_keys on ${PVE_HOST}" -log " 4. Run: ./cluster-enable-stonith.sh" -log "═══════════════════════════════════════════════════════" diff --git a/test-lab/ha/common.nix b/test-lab/ha/common.nix deleted file mode 100644 index c984e1c..0000000 --- a/test-lab/ha/common.nix +++ /dev/null @@ -1,163 +0,0 @@ -# Shared HA stack config for both test nodes. -# These are throwaway test VMs — not production hosts. -# No sops-nix, no clan, no home-manager. -{ lib, pkgs, vars, ... }: - -let - node1Ip = "192.168.2.200"; - node2Ip = "192.168.2.201"; - drbdPort = 7789; - - # Test-only corosync authkey (128 bytes = 1024 bits minimum for corosync). - # Not secret — this is a disposable test cluster, not production. - testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; -in -{ - system.stateVersion = "26.05"; - - # ── Hardware (Proxmox VM) ────────────────────────────────────────────── - imports = [ - ../../modules/hardware-configuration/vm/proxmox.nix - ../../modules/boot/efi.nix - ../../modules/ha/pacemaker-stack.nix - ../../modules/ha/iscsi-target.nix - ]; - - # ── Nix settings ────────────────────────────────────────────────────── - nix.settings.experimental-features = [ "nix-command" "flakes" ]; - - users.users.root.openssh.authorizedKeys.keys = [ - vars.adminSshKey - # Claude Code session key (this machine) — test-lab only - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos" - ]; - - # ── Services ────────────────────────────────────────────────────────── - services = { - openssh = { - enable = true; - settings.PermitRootLogin = "yes"; - }; - - # Allow QEMU guest exec for key injection fallback - qemuGuest.enable = true; - - drbd = { - enable = true; - config = '' - global { - usage-count yes; - } - - common { - net { - protocol C; - ping-int 1; - verify-alg sha256; - after-sb-0pri discard-zero-changes; - after-sb-1pri discard-secondary; - } - disk { - # dont-care: DRBD itself won't fence before promoting. Production - # clusters should use resource-only here and configure a STONITH - # fence agent (e.g. fence_pve_ssh) in Pacemaker so DRBD can safely - # protect against split-brain without risking dual-Primary. - # For this test cluster (no fence device) dont-care lets promotion - # proceed; the DRBD kernel module still refuses dual-Primary without - # allow-two-primaries in net {}. - fencing dont-care; - # LVM before/after-resync-target handlers omitted: the LVM snapshot - # scripts (/usr/lib/drbd/snapshot-resync-target-lvm.sh) don't exist - # on NixOS paths. If present, DRBD calls them on resync and exits 127, - # dropping the peer connection and leaving the secondary Outdated. - } - } - - resource ha-data { - volume 0 { - device /dev/drbd0; - disk /dev/sdb; # scsi1 in Proxmox VM → sdb - meta-disk internal; - } - - on ha-test-node1 { - address ${node1Ip}:${toString drbdPort}; - } - - on ha-test-node2 { - address ${node2Ip}:${toString drbdPort}; - } - } - ''; - }; - - # services.corosync.enable = true is set by modules/ha/pacemaker-stack.nix - corosync = { - clusterName = "ha-test"; - nodelist = [ - { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; } - { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; } - ]; - }; - }; - - # Corosync authkey (test-only, not secret — generated with - # `corosync-keygen` for production). - environment.etc."corosync/authkey" = { - source = builtins.toFile "authkey" testAuthKey; - mode = "0400"; - }; - - # ── Packages ────────────────────────────────────────────────────────── - # corosync, pacemaker, ocf-resource-agents, targetcli-fb already added - # by the ha/ modules; add the remaining stack-specific tools here. - environment.systemPackages = with pkgs; [ - # Storage - drbd # drbdadm, drbdsetup, drbdmon - xfsprogs # mkfs.xfs, xfs_admin, xfs_info - - # Networking / debug - iproute2 # ip, ss - iputils # ping - tcpdump - lsof - - # Scripting / config - python3 - curl - jq - vim - htop - ]; - - # ── Networking ──────────────────────────────────────────────────────── - networking = { - useDHCP = false; - defaultGateway = "192.168.2.1"; - nameservers = [ "192.168.2.1" "8.8.8.8" ]; - - firewall = { - enable = true; - allowedTCPPorts = [ - 22 # SSH - 3260 # iSCSI - 3121 # pacemaker-remoted - 2224 # pcsd - drbdPort - ]; - allowedUDPPorts = [ - 5404 # corosync - 5405 # corosync - 5407 # corosync crypto - ]; - extraCommands = '' - iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT - iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT - ''; - }; - }; - - # ── Locale / time ───────────────────────────────────────────────────── - time.timeZone = vars.timeZone; - i18n.defaultLocale = "en_AU.UTF-8"; -} diff --git a/test-lab/ha/disko.nix b/test-lab/ha/disko.nix deleted file mode 100644 index 5d63f1d..0000000 --- a/test-lab/ha/disko.nix +++ /dev/null @@ -1,49 +0,0 @@ -{ config, ... }: - -# Smaller disk layout for throwaway test VMs (20G vs production 50G). -# Same partition scheme as modules/disko/proxmox.nix: GPT, ESP + swap + ext4 root. -# Only covers the boot disk (scsi0 → /dev/sda). The DRBD data disk -# (scsi1 → /dev/sdb) is left raw — drbdadm create-md initialises it. -{ - disko.devices.disk.main = { - type = "disk"; - device = "/dev/sda"; - imageSize = "20G"; - imageName = config.networking.hostName; - - content = { - type = "gpt"; - partitions = { - esp = { - priority = 1; - name = "ESP"; - size = "512M"; - type = "EF00"; - content = { - type = "filesystem"; - format = "vfat"; - mountpoint = "/boot"; - mountOptions = [ "umask=0077" ]; - extraArgs = [ "-F" "32" "-n" "boot" ]; - }; - }; - swap = { - size = "2G"; - content = { - type = "swap"; - randomEncryption = false; - }; - }; - root = { - size = "100%"; - content = { - type = "filesystem"; - format = "ext4"; - mountpoint = "/"; - extraArgs = [ "-L" "nixos" ]; - }; - }; - }; - }; - }; -} diff --git a/test-lab/ha/node1.nix b/test-lab/ha/node1.nix deleted file mode 100644 index 6baf756..0000000 --- a/test-lab/ha/node1.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ ... }: - -# ha-test-node1: VMID 200, 192.168.2.200/24 -{ - networking.hostName = "ha-test-node1"; - networking.hostId = "a1b2c3d4"; # random, required by ZFS (not used here) but harmless - - networking.interfaces.ens18.ipv4.addresses = [{ - address = "192.168.2.200"; - prefixLength = 24; - }]; -} diff --git a/test-lab/ha/node2.nix b/test-lab/ha/node2.nix deleted file mode 100644 index bf49d4b..0000000 --- a/test-lab/ha/node2.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ ... }: - -# ha-test-node2: VMID 201, 192.168.2.201/24 -{ - networking.hostName = "ha-test-node2"; - networking.hostId = "e5f6a7b8"; # random, required by ZFS (not used here) but harmless - - networking.interfaces.ens18.ipv4.addresses = [{ - address = "192.168.2.201"; - prefixLength = 24; - }]; -} diff --git a/variables.nix b/variables.nix index e1e15b1..c59fa68 100644 --- a/variables.nix +++ b/variables.nix @@ -68,6 +68,20 @@ # one-line change. primaryUser = "nixos"; + # HA file server cluster + # haServer1Ip / haServer2Ip: static LAN IPs for both HA nodes (must be + # fixed — DRBD and corosync ring addresses are baked into the NixOS config). + # haServerVip: floating virtual IP managed by Pacemaker's IPaddr2 resource; + # NFS and iSCSI clients connect here regardless of which node is Active. + # Set all three to real values in variables.nix before deploying. + haServer1Host = "ha-server-1"; + haServer2Host = "ha-server-2"; + haServer1Ip = "192.168.2.200"; # TODO: confirm production IP + haServer2Ip = "192.168.2.201"; # TODO: confirm production IP + haServerVip = "192.168.2.202"; # TODO: confirm floating VIP + haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node + haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage"; + # Storage storageRoot = "/tank"; # ZFS pool root on `server` @@ -146,11 +160,22 @@ # mountd RPC service (used by showmount/NFSv3 mount protocol). # Mountd listens on a fixed port so the firewall can whitelist it # explicitly rather than opening all of rpcbind's dynamic range. - # All three need both TCP and UDP (modules/build-types/server.nix). + # All three need both TCP and UDP (modules/build-types/server.nix and + # modules/build-types/ha-server.nix). nfsRpcbind = 111; nfsd = 2049; nfsMountd = 20048; + # HA cluster ports opened on ha-server-1 and ha-server-2 + # (modules/build-types/ha-server.nix / modules/ha/cluster-config.nix). + haServerDrbd = 7789; # DRBD replication (TCP) + haServerIscsi = 3260; # iSCSI target (TCP) + haServerCorosync1 = 5404; # Corosync totem ring (UDP) + haServerCorosync2 = 5405; # Corosync totem ring (UDP) + haServerCorosyncCrypto = 5407; # Corosync crypto sync (UDP) + haServerPacemakerRemoted = 3121; # pacemaker-remoted (TCP) + haServerPcsd = 2224; # pcsd cluster daemon (TCP) + # Opened on the docker host's firewall for the Traefik-fronted # container stack (docker-compose config lives in the separate # /home/debian/docker repo, not here): 80/443 are Traefik's own From 5856d455753d975079ab44cd9f3b4ac856d55fe7 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 27 Jul 2026 11:45:36 +1000 Subject: [PATCH 7/7] feat(ha): wire sops secrets and disable NetworkManager for HA servers - cluster-config.nix: add corosync_authkey sops binary secret (/etc/corosync/authkey, mode 0400) and force-disable NetworkManager (common config enables it; HA nodes need stable static IP networking) - hosts/ha-server-{1,2}/host.nix: add host-token.nix import for sops-managed beszel-token; add KEY placeholder for beszel hub pairing - .sops.yaml: add creation rules for secrets/ha-server-{1,2}.yaml and secrets/ha-corosync-authkey (admin-only until sync-host-keys.sh runs) - secrets/ha-server-{1,2}.yaml, secrets/ha-corosync-authkey: stub files so eval passes before real secrets are provisioned Bootstrap order (post-merge): 1. bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1 2. bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2 3. sops updatekeys secrets/common.yaml (grants HA nodes common secrets) 4. sops secrets/ha-server-{1,2}.yaml (set beszel-token values) 5. On node1: corosync-keygen; sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey; git add/commit Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8 --- .sops.yaml | 26 ++++++++++++++++++++++++++ hosts/ha-server-1/host.nix | 16 +++++++++------- hosts/ha-server-2/host.nix | 16 +++++++++------- modules/ha/cluster-config.nix | 27 +++++++++++++++++++-------- secrets/ha-corosync-authkey | 1 + secrets/ha-server-1.yaml | 6 ++++++ secrets/ha-server-2.yaml | 6 ++++++ 7 files changed, 76 insertions(+), 22 deletions(-) create mode 100644 secrets/ha-corosync-authkey create mode 100644 secrets/ha-server-1.yaml create mode 100644 secrets/ha-server-2.yaml diff --git a/.sops.yaml b/.sops.yaml index b37b4e2..a14898b 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -85,6 +85,32 @@ creation_rules: - *lxc-tailscale-router - *proxmox-tailscale-router + # HA file server per-node secrets (beszel-token). + # proxmox-ha-server-1 / proxmox-ha-server-2 keys are added automatically + # by scripts/secrets/sync-host-keys.sh once the hosts are provisioned; + # until then only the admin key can decrypt these files. + - path_regex: secrets/ha-server-1\.yaml$ + key_groups: + - age: + - *admin + # proxmox-ha-server-1 added by sync-host-keys.sh + + - path_regex: secrets/ha-server-2\.yaml$ + key_groups: + - age: + - *admin + # proxmox-ha-server-2 added by sync-host-keys.sh + + # Shared HA cluster corosync authkey (binary sops file). + # Encrypted for both HA nodes so either can decrypt on boot. + # Both host keys added by sync-host-keys.sh; admin key allows initial creation. + - path_regex: secrets/ha-corosync-authkey$ + key_groups: + - age: + - *admin + # proxmox-ha-server-1 added by sync-host-keys.sh + # proxmox-ha-server-2 added by sync-host-keys.sh + # gui-host-specific secrets (currently: wifi-password, see # modules/networking/wifi.nix). Only *lxc-gui has a registered key today # -- proxmox-gui/linode-gui/baremetal-gui haven't been provisioned via diff --git a/hosts/ha-server-1/host.nix b/hosts/ha-server-1/host.nix index 0fd808b..c380008 100644 --- a/hosts/ha-server-1/host.nix +++ b/hosts/ha-server-1/host.nix @@ -1,5 +1,12 @@ { vars, ... }: { + imports = [ + (import ../../modules/beszel/host-token.nix { + name = "ha-server-1"; + sopsFile = ../../secrets/ha-server-1.yaml; + }) + ]; + networking = { hostName = vars.haServer1Host; hostId = "3a4b5c6d"; @@ -12,13 +19,8 @@ nameservers = [ "192.168.2.1" "8.8.8.8" ]; }; - # TODO: after running `bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1` - # add beszel agent pairing and sops-managed corosync authkey: - # imports = [ (import ../../modules/beszel/host-token.nix { - # name = "ha-server-1"; - # sopsFile = ../../secrets/ha-server-1.yaml; - # }) ]; - # services.beszel.agent.environment.KEY = "..."; + # Set KEY after pairing this host with the beszel hub; the token is sops-managed. + services.beszel.agent.environment.KEY = ""; system.stateVersion = "26.05"; } diff --git a/hosts/ha-server-2/host.nix b/hosts/ha-server-2/host.nix index 1f5a8ba..dcb1464 100644 --- a/hosts/ha-server-2/host.nix +++ b/hosts/ha-server-2/host.nix @@ -1,5 +1,12 @@ { vars, ... }: { + imports = [ + (import ../../modules/beszel/host-token.nix { + name = "ha-server-2"; + sopsFile = ../../secrets/ha-server-2.yaml; + }) + ]; + networking = { hostName = vars.haServer2Host; hostId = "7e8f9a0b"; @@ -12,13 +19,8 @@ nameservers = [ "192.168.2.1" "8.8.8.8" ]; }; - # TODO: after running `bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2` - # add beszel agent pairing and sops-managed corosync authkey: - # imports = [ (import ../../modules/beszel/host-token.nix { - # name = "ha-server-2"; - # sopsFile = ../../secrets/ha-server-2.yaml; - # }) ]; - # services.beszel.agent.environment.KEY = "..."; + # Set KEY after pairing this host with the beszel hub; the token is sops-managed. + services.beszel.agent.environment.KEY = ""; system.stateVersion = "26.05"; } diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix index 9074e99..160c154 100644 --- a/modules/ha/cluster-config.nix +++ b/modules/ha/cluster-config.nix @@ -5,13 +5,10 @@ # (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix. # # Corosync authkey: -# /etc/corosync/authkey must be present (mode 0400) for corosync to start. -# It is NOT managed declaratively here — the initial deploy uses -# scripts/ha/cluster-init.sh to generate it via corosync-keygen and -# distribute it to both nodes. -# TODO: once both hosts have their sops keys registered via -# scripts/secrets/sync-host-keys.sh, add a sops secret here so the -# authkey survives nixos-rebuild. +# /etc/corosync/authkey (mode 0400) is managed by sops-nix below. +# Bootstrap: run scripts/ha/cluster-init.sh on node1 to generate the key, +# then encrypt it with: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey +# Both host keys must be registered via sync-host-keys.sh first so both nodes can decrypt it. # # DRBD fencing: # Production setting is resource-only: DRBD waits for the STONITH fence @@ -20,7 +17,7 @@ # (see scripts/ha/cluster-enable-stonith.sh). On a fresh cluster with # no fence device yet, temporarily change to dont-care and run # cluster-enable-stonith.sh once the fence key is deployed. -{ vars, ... }: +{ lib, vars, ... }: { services.drbd = { enable = true; @@ -60,6 +57,20 @@ ''; }; + # /etc/corosync/authkey — sops binary secret, identical on both nodes. + # Decryptable by both ha-server host keys (added by sync-host-keys.sh). + sops.secrets.corosync_authkey = { + sopsFile = ../../secrets/ha-corosync-authkey; + format = "binary"; + path = "/etc/corosync/authkey"; + mode = "0400"; + restartUnits = [ "corosync.service" ]; + }; + + # NixOS common config enables NetworkManager by default; HA cluster nodes + # need stable static IPs with predictable interface names — NM is not suitable. + networking.networkmanager.enable = lib.mkForce false; + # services.corosync.enable is set by modules/ha/pacemaker-stack.nix. services.corosync = { clusterName = "ha-cluster"; diff --git a/secrets/ha-corosync-authkey b/secrets/ha-corosync-authkey new file mode 100644 index 0000000..74db83c --- /dev/null +++ b/secrets/ha-corosync-authkey @@ -0,0 +1 @@ +STUB: run cluster-init.sh to generate, then: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey diff --git a/secrets/ha-server-1.yaml b/secrets/ha-server-1.yaml new file mode 100644 index 0000000..f33120f --- /dev/null +++ b/secrets/ha-server-1.yaml @@ -0,0 +1,6 @@ +# STUB — not yet encrypted with sops. +# Bootstrap: +# bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1 +# sops updatekeys secrets/common.yaml (allows ha-server-1 to decrypt shared secrets) +# sops secrets/ha-server-1.yaml (create with: beszel-token) +beszel-token: REPLACE diff --git a/secrets/ha-server-2.yaml b/secrets/ha-server-2.yaml new file mode 100644 index 0000000..19bbbe9 --- /dev/null +++ b/secrets/ha-server-2.yaml @@ -0,0 +1,6 @@ +# STUB — not yet encrypted with sops. +# Bootstrap: +# bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2 +# sops updatekeys secrets/common.yaml (allows ha-server-2 to decrypt shared secrets) +# sops secrets/ha-server-2.yaml (create with: beszel-token) +beszel-token: REPLACE