From 750121e9dd19477bc4180fcc44b174a4d4664eba Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 27 Jul 2026 05:46:04 +1000 Subject: [PATCH] test-lab: add two-node HA file-server test cluster config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disposable test VMs (ha-test-node1 / ha-test-node2, VMIDs 200/201 on pve1) to evaluate whether the DRBD + XFS + LIO + Corosync + Pacemaker stack runs correctly on NixOS before deciding NixOS vs Debian for production. Includes: - test-lab/ha/disko.nix: 20G boot disk layout (smaller than production) - test-lab/ha/common.nix: shared HA stack (drbd, corosync, pacemaker, targetcli-fb, xfsprogs), OCF PATH workaround for nixpkgs#207891 - test-lab/ha/node1.nix / node2.nix: per-node hostname + static IP - test-lab/ha/fence-pve-ssh.py: Proxmox SSH fence agent for STONITH - test-lab/ha/cluster-init.sh: one-shot cluster bootstrap script - test-lab/ha/cluster-enable-stonith.sh: enables STONITH post-key-deploy - flake.nix: adds ha-test-node1 / ha-test-node2 nixosConfigurations (bypasses mkTarget / clan-core / sops-nix — test-only) These VMs must be destroyed once acceptance testing is complete. Co-Authored-By: Claude Sonnet 4.6 --- flake.nix | 30 ++- test-lab/ha/cluster-enable-stonith.sh | 78 ++++++++ test-lab/ha/cluster-init.sh | 239 ++++++++++++++++++++++++ test-lab/ha/common.nix | 256 ++++++++++++++++++++++++++ test-lab/ha/disko.nix | 49 +++++ test-lab/ha/fence-pve-ssh.py | 179 ++++++++++++++++++ test-lab/ha/node1.nix | 12 ++ test-lab/ha/node2.nix | 12 ++ 8 files changed, 854 insertions(+), 1 deletion(-) create mode 100644 test-lab/ha/cluster-enable-stonith.sh create mode 100644 test-lab/ha/cluster-init.sh create mode 100644 test-lab/ha/common.nix create mode 100644 test-lab/ha/disko.nix create mode 100644 test-lab/ha/fence-pve-ssh.py create mode 100644 test-lab/ha/node1.nix create mode 100644 test-lab/ha/node2.nix diff --git a/flake.nix b/flake.nix index 47ac295..4788634 100644 --- a/flake.nix +++ b/flake.nix @@ -132,6 +132,34 @@ lxc-tor-relay = mkTarget { platform = "lxc"; buildType = "tor-relay"; hostPath = ./hosts/tor-relay/host.nix; }; }; + # ── HA test lab VMs ───────────────────────────────────────────────── + # Two throwaway NixOS VMs to test the DRBD + XFS + LIO + Pacemaker + # stack. Deliberately bypass mkTarget (no clan-core, no sops-nix, + # no home-manager) — these are disposable and must be destroyed once + # the acceptance tests are done. + # Build disk images with: + # nix build .#nixosConfigurations.ha-test-node1.config.system.build.diskoImagesScript + # then run the script to produce ha-test-node1.raw (import with qm importdisk). + haTestTargets = + let + haNode = { nodeModule }: nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + inputs.disko.nixosModules.disko + ./modules/hardware-configuration/vm/proxmox.nix + ./modules/boot/efi.nix + ./test-lab/ha/disko.nix + ./test-lab/ha/common.nix + nodeModule + ]; + specialArgs = { inherit vars; }; + }; + in + { + ha-test-node1 = haNode { nodeModule = ./test-lab/ha/node1.nix; }; + ha-test-node2 = haNode { nodeModule = ./test-lab/ha/node2.nix; }; + }; + # Auto-install environments (migrated from the former nix-auto-installer # flake): a self-contained NixOS installer that boots, discovers this # flake's own nixosConfigurations over the network, and runs @@ -207,7 +235,7 @@ in { - nixosConfigurations = generatedTargets // installerTargets; + nixosConfigurations = generatedTargets // haTestTargets // installerTargets; # Buildable auto-installer artifacts (`nix build .#`). No `lxc` # variant (installer-boots-as-an-LXC-container) or `all` bundle diff --git a/test-lab/ha/cluster-enable-stonith.sh b/test-lab/ha/cluster-enable-stonith.sh new file mode 100644 index 0000000..6c23081 --- /dev/null +++ b/test-lab/ha/cluster-enable-stonith.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# cluster-enable-stonith.sh — enable STONITH fence agent after fence key is deployed +# Run from ha-test-node1 as root, AFTER: +# - /etc/fence-pve-ssh-key exists on both nodes +# - The fence public key is in authorized_keys on pve1.sweet.home +set -euo pipefail + +VMID_NODE1="200" +VMID_NODE2="201" +PVE_HOST="pve1.sweet.home" +PVE_USER="wayne" +FENCE_KEY="/etc/fence-pve-ssh-key" +FENCE_SCRIPT="/usr/lib/ocf/resource.d/heartbeat/fence_pve_ssh" + +log() { echo "[stonith-setup] $*"; } +die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; } + +[[ $(id -u) -eq 0 ]] || die "must run as root" + +[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY" +[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT" + +log "Verifying fence agent can reach ${PVE_HOST}..." +if ! ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \ + -o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" "sudo /usr/sbin/qm list" &>/dev/null; then + die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo" +fi +log "Fence agent SSH connectivity confirmed" + +log "Creating Pacemaker STONITH resource..." +cibadmin --create --scope resources --xml-text " + + + + + + + + + + + + + + +" 2>/dev/null || true + +cibadmin --create --scope resources --xml-text " + + + + + + + + + + + + + + +" 2>/dev/null || true + +log "Enabling STONITH..." +crm_attribute -t crm_config -n stonith-enabled -v true + +# Restore quorum policy to stop (needed with STONITH) +crm_attribute -t crm_config -n no-quorum-policy -v stop + +log "STONITH enabled. Testing fence agent..." +if stonith_admin --list-devices; then + log "Fence devices listed successfully" +else + log "WARNING: fence device list failed — check stonith config" +fi + +log "STONITH setup complete. Cluster is now fully HA." diff --git a/test-lab/ha/cluster-init.sh b/test-lab/ha/cluster-init.sh new file mode 100644 index 0000000..4274740 --- /dev/null +++ b/test-lab/ha/cluster-init.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +# cluster-init.sh — one-time HA cluster initialisation script +# +# Run this ONCE from node1 AFTER both VMs are booted and have SSH access. +# It: +# 1. Waits for corosync quorum on both nodes +# 2. Initialises DRBD metadata and promotes node1 to primary +# 3. Creates XFS filesystem on /dev/drbd0 +# 4. Configures targetcli / LIO iSCSI target (with a file-backed LUN) +# 5. Configures the Pacemaker resource group +# 6. Optionally enables the STONITH fence agent (requires fence SSH key) +# +# Prerequisites: +# - Both VMs booted with the ha-test config +# - fence-pve-ssh-key distributed to /etc/fence-pve-ssh-key on both nodes +# - Run as root on ha-test-node1 +set -euo pipefail + +NODE1_IP="192.168.2.200" +NODE2_IP="192.168.2.201" +VIP="192.168.2.202" +DRBD_DEVICE="/dev/drbd0" +XFS_MOUNT="/mnt/ha-data" +ISCSI_IQN="iqn.2026-01.local.ha-test:storage" +ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img" +ISCSI_LUN_SIZE="1G" # small test LUN +VMID_NODE1="200" +VMID_NODE2="201" +PVE_HOST="pve1.sweet.home" +PVE_USER="wayne" +FENCE_KEY="/etc/fence-pve-ssh-key" + +log() { echo "[cluster-init] $*"; } +die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; } + +[[ $(id -u) -eq 0 ]] || die "must run as root" +[[ "$(hostname)" == "ha-test-node1" ]] || die "must run on ha-test-node1" + +# ── 1. Wait for corosync quorum ────────────────────────────────────────── +log "Waiting for corosync quorum..." +for i in $(seq 1 30); do + if corosync-quorumtool -q &>/dev/null; then + log "Quorum established" + break + fi + [[ $i -eq 30 ]] && die "corosync quorum not established after 30s" + sleep 2 +done + +log "Waiting for pacemaker to start..." +for i in $(seq 1 30); do + if crm_mon -1 &>/dev/null; then + log "Pacemaker running" + break + fi + [[ $i -eq 30 ]] && die "pacemaker not running after 60s" + sleep 2 +done + +# ── 2. Initialise DRBD ─────────────────────────────────────────────────── +log "Initialising DRBD metadata on node1..." +if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent"; then + drbdadm create-md ha-data --force +fi + +log "Initialising DRBD metadata on node2..." +if ! ssh "root@${NODE2_IP}" "drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent'"; then + ssh "root@${NODE2_IP}" "drbdadm create-md ha-data --force" +fi + +log "Bringing up DRBD on both nodes..." +drbdadm up ha-data || true +ssh "root@${NODE2_IP}" "drbdadm up ha-data" || true + +log "Forcing node1 to DRBD primary (initial sync)..." +drbdadm primary ha-data --force + +log "Waiting for DRBD to finish initial sync..." +for i in $(seq 1 120); do + state=$(drbdadm dstate ha-data) + if echo "$state" | grep -q "UpToDate"; then + log "DRBD sync complete: $state" + break + fi + log " DRBD state: $state (${i}/120s)" + [[ $i -eq 120 ]] && die "DRBD did not sync within 120s" + sleep 1 +done + +# ── 3. XFS filesystem ──────────────────────────────────────────────────── +log "Creating XFS on ${DRBD_DEVICE}..." +if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then + mkfs.xfs "${DRBD_DEVICE}" +fi + +log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..." +mkdir -p "${XFS_MOUNT}" +mount "${DRBD_DEVICE}" "${XFS_MOUNT}" + +# ── 4. iSCSI LUN (file-backed) ─────────────────────────────────────────── +log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..." +if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then + fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}" +fi + +log "Configuring LIO iSCSI target via targetcli..." +# This produces a /etc/target/saveconfig.json that the targetctl service loads. +# The commands create an iSCSI target backed by the file we just created. +targetcli <<'EOF' +/backstores/fileio create name=ha-lun0 file_or_dev=/mnt/ha-data/iscsi-lun.img size=0 write_back=false +/iscsi create iqn.2026-01.local.ha-test:storage +/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/luns create /backstores/fileio/ha-lun0 +/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/portals create 192.168.2.202 +/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute authentication=0 +/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute demo_mode_write_protect=0 +saveconfig /etc/target/saveconfig.json +EOF + +log "Unmounting ${XFS_MOUNT} (Pacemaker will manage it)..." +umount "${XFS_MOUNT}" + +log "Promoting DRBD back to secondary (Pacemaker manages primary role)..." +drbdadm secondary ha-data + +# ── 5. Pacemaker resources ─────────────────────────────────────────────── +log "Configuring Pacemaker..." + +# Disable STONITH initially — enable once fence key is deployed +crm_attribute -t crm_config -n stonith-enabled -v false + +# Disable quorum policy for two-node cluster (no-quorum-policy=ignore so +# the surviving node can promote without a quorum device) +crm_attribute -t crm_config -n no-quorum-policy -v ignore + +# Cluster resources: +# 1. drbd-ha — manages DRBD primary/secondary role +# 2. xfs-mount — XFS mount on /mnt/ha-data +# 3. iscsi-target — LIO target service (systemd class) +# 4. vip — floating VIP 192.168.2.202 + +log "Creating DRBD master/slave resource..." +cibadmin --replace --scope resources --xml-text " + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +" + +log "Adding ordering and colocation constraints..." +# All resources on the same node as DRBD master +cibadmin --create --scope constraints --xml-text " + + + + + + + + +" + +log "Resource group configured. Waiting for resources to start..." +for i in $(seq 1 60); do + if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then + log "VIP is up: $(crm_resource -r vip --locate)" + break + fi + [[ $i -eq 60 ]] && { log "WARNING: VIP not up after 60s — check crm_mon"; break; } + sleep 2 +done + +log "" +log "═══════════════════════════════════════════════════════" +log " HA cluster initialised. Next steps:" +log "" +log " - Verify: crm_mon -1" +log " - Test iSCSI: iscsiadm -m discovery -t sendtargets -p ${VIP}" +log "" +log " To enable STONITH (after deploying fence key):" +log " 1. Copy fence-pve-ssh.py to /usr/lib/ocf/resource.d/heartbeat/ on both nodes" +log " 2. Distribute /etc/fence-pve-ssh-key to both nodes" +log " 3. Add public key to authorized_keys on ${PVE_HOST}" +log " 4. Run: ./cluster-enable-stonith.sh" +log "═══════════════════════════════════════════════════════" diff --git a/test-lab/ha/common.nix b/test-lab/ha/common.nix new file mode 100644 index 0000000..01659e0 --- /dev/null +++ b/test-lab/ha/common.nix @@ -0,0 +1,256 @@ +# Shared HA stack config for both test nodes. +# These are throwaway test VMs — not production hosts. +# No sops-nix, no clan, no home-manager. +{ lib, pkgs, vars, ... }: + +let + node1Ip = "192.168.2.200"; + node2Ip = "192.168.2.201"; + drbdPort = 7789; + + # Test-only corosync authkey (128 bytes minimum). + # Not secret — this is a disposable test cluster, not production. + testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; + + # OCF agent path (from pacemaker's --with-ocfdir). + # We expose this on PATH so custom scripts can find each other. + ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d"; + + # Concatenated PATH that includes all binaries OCF agents and pacemaker + # lrmd children need. This is the workaround for nixpkgs#207891 (PATH + # not set correctly for OCF agent child processes). + ocfBinPath = lib.concatStringsSep ":" [ + "${pkgs.iproute2}/bin" + "${pkgs.iproute2}/sbin" + "${pkgs.iputils}/bin" + "${pkgs.util-linux}/bin" + "${pkgs.util-linux}/sbin" + "${pkgs.gawk}/bin" + "${pkgs.gnugrep}/bin" + "${pkgs.gnused}/bin" + "${pkgs.coreutils}/bin" + "${pkgs.bash}/bin" + "${pkgs.procps}/bin" + "${pkgs.xfsprogs}/bin" + "${pkgs.drbd}/bin" + "${pkgs.targetcli-fb}/bin" + "${pkgs.python3}/bin" + "/run/current-system/sw/bin" + "/run/current-system/sw/sbin" + "/usr/local/sbin" + "/usr/local/bin" + "/usr/sbin" + "/usr/bin" + "/sbin" + "/bin" + ]; +in +{ + system.stateVersion = "26.05"; + + # ── Hardware (Proxmox VM) ────────────────────────────────────────────── + imports = [ + ../../modules/hardware-configuration/vm/proxmox.nix + ../../modules/boot/efi.nix + ]; + + # ── Nix settings ────────────────────────────────────────────────────── + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + + # ── SSH ─────────────────────────────────────────────────────────────── + services.openssh = { + enable = true; + settings.PermitRootLogin = "yes"; + }; + users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey ]; + + # ── Networking ──────────────────────────────────────────────────────── + networking.useDHCP = false; + networking.defaultGateway = "192.168.2.1"; + networking.nameservers = [ "192.168.2.1" "8.8.8.8" ]; + + # ── DRBD ────────────────────────────────────────────────────────────── + services.drbd.enable = true; + services.drbd.config = '' + global { + usage-count yes; + } + + common { + net { + protocol C; + ping-int 1; + verify-alg sha256; + after-sb-0pri discard-zero-changes; + after-sb-1pri discard-secondary; + } + disk { + # resource-only: DRBD itself won't fence (Pacemaker handles STONITH); + # the DRBD resource agent uses fencing to guard primary promotion. + fencing resource-only; + } + handlers { + # Called by DRBD when a split-brain is detected and this node is + # in the secondary role — notifies pacemaker to fence the split node. + split-brain "/usr/lib/drbd/notify-split-brain.sh root"; + before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k"; + after-resync-target "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh"; + } + } + + resource ha-data { + volume 0 { + device /dev/drbd0; + disk /dev/sdb; # scsi1 in Proxmox VM → sdb + meta-disk internal; + } + + on ha-test-node1 { + address ${node1Ip}:${toString drbdPort}; + } + + on ha-test-node2 { + address ${node2Ip}:${toString drbdPort}; + } + } + ''; + + # ── Corosync ────────────────────────────────────────────────────────── + services.corosync = { + enable = true; + clusterName = "ha-test"; + nodelist = [ + { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; } + { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; } + ]; + }; + + # Corosync authkey (test-only, not secret — generated with + # `corosync-keygen` for production). + environment.etc."corosync/authkey" = { + source = builtins.toFile "authkey" testAuthKey; + mode = "0400"; + }; + + # ── Pacemaker ───────────────────────────────────────────────────────── + services.pacemaker.enable = true; + + # Fix for nixpkgs#207891: + # 1. Ensure CIB directories are owned by hacluster before starting. + # The stock module sets StateDirectory=pacemaker (owned by root); + # pacemaker internally drops to hacluster but needs to write there. + # 2. Set PATH so OCF agent child processes can find all required binaries. + systemd.services.pacemaker.serviceConfig = { + ExecStartPre = [ + "${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'" + ]; + }; + systemd.services.pacemaker.environment = { + PATH = lib.mkForce ocfBinPath; + # Expose OCF root so pacemaker and lrmd agree on where agents live. + OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; + }; + + # pacemaker-execd is the local resource executor that calls OCF agents. + # Give it the same PATH so OCF scripts can find all required binaries. + systemd.services.pacemaker-execd.environment = { + PATH = lib.mkForce ocfBinPath; + OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; + }; + + # ── LIO / iSCSI target ──────────────────────────────────────────────── + # targetcli-fb is the management tool; actual kernel support is via + # the LIO modules. We add a systemd service that saves/restores the + # target configuration so Pacemaker can trigger it via a systemd-class + # resource. + boot.kernelModules = [ + "target_core_mod" + "iscsi_target_mod" + "target_core_file" + "target_core_pscsi" + "target_core_user" + "configfs" + ]; + + # configfs must be mounted for rtslib/targetcli to work + systemd.mounts = [{ + where = "/sys/kernel/config"; + what = "configfs"; + type = "configfs"; + wantedBy = [ "multi-user.target" ]; + before = [ "targetctl.service" ]; + }]; + + # targetctl: save/restore LIO configuration (mirrors Debian's package) + systemd.services.targetctl = { + description = "LIO iSCSI target config save/restore"; + wantedBy = [ "multi-user.target" ]; + after = [ "sys-kernel-config.mount" "network.target" ]; + requires = [ "sys-kernel-config.mount" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json"; + ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json"; + }; + unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json"; + }; + + # ── Packages ────────────────────────────────────────────────────────── + environment.systemPackages = with pkgs; [ + # HA stack + corosync # corosync-cfgtool, corosync-quorumtool + pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI + drbd # drbdadm, drbdsetup, drbdmon + ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …) + + # Storage + xfsprogs # mkfs.xfs, xfs_admin, xfs_info + targetcli-fb # targetcli shell + targetctl + + # Networking / debug + iproute2 # ip, ss + iputils # ping + tcpdump + lsof + + # Scripting / config + python3 + curl + jq + vim + htop + ]; + + # ── Firewall ────────────────────────────────────────────────────────── + networking.firewall = { + enable = true; + allowedTCPPorts = [ + 22 # SSH + 3260 # iSCSI + 3121 # pacemaker-remoted + 2224 # pcsd + drbdPort # DRBD replication + ]; + allowedUDPPorts = [ + 5404 # corosync cluster + 5405 # corosync cluster + 5407 # corosync crypto + ]; + # Corosync uses ports 5404-5407 UDP; allow them on the cluster net + extraCommands = '' + iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT + iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT + ''; + }; + + # ── tmpfiles: target config dir ──────────────────────────────────────── + systemd.tmpfiles.rules = [ + "d /etc/target 0750 root root -" + "f /etc/target/saveconfig.json 0640 root root -" + ]; + + # ── Locale / time ───────────────────────────────────────────────────── + time.timeZone = vars.timeZone; + i18n.defaultLocale = "en_AU.UTF-8"; +} diff --git a/test-lab/ha/disko.nix b/test-lab/ha/disko.nix new file mode 100644 index 0000000..5d63f1d --- /dev/null +++ b/test-lab/ha/disko.nix @@ -0,0 +1,49 @@ +{ config, ... }: + +# Smaller disk layout for throwaway test VMs (20G vs production 50G). +# Same partition scheme as modules/disko/proxmox.nix: GPT, ESP + swap + ext4 root. +# Only covers the boot disk (scsi0 → /dev/sda). The DRBD data disk +# (scsi1 → /dev/sdb) is left raw — drbdadm create-md initialises it. +{ + disko.devices.disk.main = { + type = "disk"; + device = "/dev/sda"; + imageSize = "20G"; + imageName = config.networking.hostName; + + content = { + type = "gpt"; + partitions = { + esp = { + priority = 1; + name = "ESP"; + size = "512M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ "umask=0077" ]; + extraArgs = [ "-F" "32" "-n" "boot" ]; + }; + }; + swap = { + size = "2G"; + content = { + type = "swap"; + randomEncryption = false; + }; + }; + root = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + extraArgs = [ "-L" "nixos" ]; + }; + }; + }; + }; + }; +} diff --git a/test-lab/ha/fence-pve-ssh.py b/test-lab/ha/fence-pve-ssh.py new file mode 100644 index 0000000..7bf0c7a --- /dev/null +++ b/test-lab/ha/fence-pve-ssh.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +""" +fence_pve_ssh - Proxmox VE SSH fence agent for Pacemaker. + +Uses SSH to reach pve1.sweet.home and run 'qm stop/start '. +Designed for test-lab HA cluster only — not for production. + +Configuration (as pacemaker stonith resource attributes): + pve_host Proxmox host to SSH to (default: pve1.sweet.home) + pve_user SSH user (default: wayne) + key_file SSH private key path (default: /etc/fence-pve-ssh-key) + vmid_node1 VMID for ha-test-node1 (e.g. 200) + vmid_node2 VMID for ha-test-node2 (e.g. 201) + plug Node name to act on (set by pacemaker: ha-test-node1 or ha-test-node2) + action Action: off|on|reboot|status|list|metadata +""" + +import argparse +import subprocess +import sys +import os + + +METADATA = """ + + Fences a VM on a Proxmox VE host by SSHing to the PVE host and + running qm stop/start. For test use only. + https://proxmox.com + + + + + Fencing action: off|on|reboot|status|list + + + + + Cluster node name to fence + + + + + Proxmox VE host to SSH to + + + + + SSH user on the Proxmox host + + + + + SSH private key file path + + + + + VMID for ha-test-node1 + + + + + VMID for ha-test-node2 + + + + + + + + + + + +""" + + +def parse_args(): + p = argparse.ArgumentParser(add_help=False) + p.add_argument("-a", "--action", default="reboot") + p.add_argument("-n", "--plug") + p.add_argument("--pve-host", default="pve1.sweet.home") + p.add_argument("--pve-user", default="wayne") + p.add_argument("--key-file", default="/etc/fence-pve-ssh-key") + p.add_argument("--vmid-node1") + p.add_argument("--vmid-node2") + # Allow remaining unknown args (pacemaker may pass extra ones) + return p.parse_known_args()[0] + + +def ssh(pve_host, pve_user, key_file, cmd): + result = subprocess.run( + [ + "ssh", + "-i", key_file, + "-o", "StrictHostKeyChecking=no", + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=10", + f"{pve_user}@{pve_host}", + cmd, + ], + capture_output=True, + text=True, + timeout=30, + ) + return result + + +def get_vmid(args): + node = args.plug + if not node: + print("ERROR: --plug not specified", file=sys.stderr) + sys.exit(1) + mapping = { + "ha-test-node1": args.vmid_node1, + "ha-test-node2": args.vmid_node2, + } + vmid = mapping.get(node) + if not vmid: + print(f"ERROR: unknown node '{node}'", file=sys.stderr) + sys.exit(1) + return vmid + + +def main(): + args = parse_args() + action = args.action.lower() + + if action == "metadata": + print(METADATA) + sys.exit(0) + + if action == "list": + if args.vmid_node1: + print("ha-test-node1") + if args.vmid_node2: + print("ha-test-node2") + sys.exit(0) + + vmid = get_vmid(args) + + if not os.path.exists(args.key_file): + print(f"ERROR: SSH key not found at {args.key_file}", file=sys.stderr) + sys.exit(1) + + if action in ("off", "reboot"): + print(f"Stopping VM {vmid} ({args.plug}) on {args.pve_host}...") + r = ssh(args.pve_host, args.pve_user, args.key_file, + f"sudo /usr/sbin/qm stop {vmid}") + if r.returncode != 0: + print(f"ERROR stopping VM: {r.stderr}", file=sys.stderr) + sys.exit(1) + print(f"VM {vmid} stopped") + + if action in ("on", "reboot"): + print(f"Starting VM {vmid} ({args.plug}) on {args.pve_host}...") + r = ssh(args.pve_host, args.pve_user, args.key_file, + f"sudo /usr/sbin/qm start {vmid}") + if r.returncode != 0: + print(f"ERROR starting VM: {r.stderr}", file=sys.stderr) + sys.exit(1) + print(f"VM {vmid} started") + + if action == "status": + r = ssh(args.pve_host, args.pve_user, args.key_file, + f"sudo /usr/sbin/qm status {vmid}") + if r.returncode != 0: + print(f"ERROR querying VM status: {r.stderr}", file=sys.stderr) + sys.exit(1) + # qm status returns "status: running" or "status: stopped" + status_line = r.stdout.strip() + print(status_line) + if "stopped" in status_line: + sys.exit(2) # pacemaker interprets exit 2 as "off" + sys.exit(0) # running = exit 0 + + +if __name__ == "__main__": + main() diff --git a/test-lab/ha/node1.nix b/test-lab/ha/node1.nix new file mode 100644 index 0000000..a7d408c --- /dev/null +++ b/test-lab/ha/node1.nix @@ -0,0 +1,12 @@ +{ ... }: + +# ha-test-node1: VMID 200, 192.168.2.200/24 +{ + networking.hostName = "ha-test-node1"; + networking.hostId = "a1b2c3d4"; # random, required by ZFS (not used here) but harmless + + networking.interfaces.ens18.ipv4.addresses = [{ + address = "192.168.2.200"; + prefixLength = 24; + }]; +} diff --git a/test-lab/ha/node2.nix b/test-lab/ha/node2.nix new file mode 100644 index 0000000..72dcafc --- /dev/null +++ b/test-lab/ha/node2.nix @@ -0,0 +1,12 @@ +{ ... }: + +# ha-test-node2: VMID 201, 192.168.2.201/24 +{ + networking.hostName = "ha-test-node2"; + networking.hostId = "e5f6a7b8"; # random, required by ZFS (not used here) but harmless + + networking.interfaces.ens18.ipv4.addresses = [{ + address = "192.168.2.201"; + prefixLength = 24; + }]; +}