diff --git a/flake.nix b/flake.nix index 4788634..d4b0aa0 100644 --- a/flake.nix +++ b/flake.nix @@ -130,35 +130,10 @@ lxc-tailscale-router = mkTarget { platform = "lxc"; buildType = "tailscale-router"; hostPath = ./hosts/tailscale-router/host.nix; }; lxc-tor-relay = mkTarget { platform = "lxc"; buildType = "tor-relay"; hostPath = ./hosts/tor-relay/host.nix; }; - }; - # ── HA test lab VMs ───────────────────────────────────────────────── - # Two throwaway NixOS VMs to test the DRBD + XFS + LIO + Pacemaker - # stack. Deliberately bypass mkTarget (no clan-core, no sops-nix, - # no home-manager) — these are disposable and must be destroyed once - # the acceptance tests are done. - # Build disk images with: - # nix build .#nixosConfigurations.ha-test-node1.config.system.build.diskoImagesScript - # then run the script to produce ha-test-node1.raw (import with qm importdisk). - haTestTargets = - let - haNode = { nodeModule }: nixpkgs.lib.nixosSystem { - inherit system; - modules = [ - inputs.disko.nixosModules.disko - ./modules/hardware-configuration/vm/proxmox.nix - ./modules/boot/efi.nix - ./test-lab/ha/disko.nix - ./test-lab/ha/common.nix - nodeModule - ]; - specialArgs = { inherit vars; }; - }; - in - { - ha-test-node1 = haNode { nodeModule = ./test-lab/ha/node1.nix; }; - ha-test-node2 = haNode { nodeModule = ./test-lab/ha/node2.nix; }; - }; + proxmox-ha-server-1 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-1/host.nix; }; + proxmox-ha-server-2 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-2/host.nix; }; + }; # Auto-install environments (migrated from the former nix-auto-installer # flake): a self-contained NixOS installer that boots, discovers this @@ -235,7 +210,7 @@ in { - nixosConfigurations = generatedTargets // haTestTargets // installerTargets; + nixosConfigurations = generatedTargets // installerTargets; # Buildable auto-installer artifacts (`nix build .#`). No `lxc` # variant (installer-boots-as-an-LXC-container) or `all` bundle diff --git a/hosts/ha-server-1/host.nix b/hosts/ha-server-1/host.nix new file mode 100644 index 0000000..0fd808b --- /dev/null +++ b/hosts/ha-server-1/host.nix @@ -0,0 +1,24 @@ +{ vars, ... }: +{ + networking = { + hostName = vars.haServer1Host; + hostId = "3a4b5c6d"; + useDHCP = false; + interfaces.ens18.ipv4.addresses = [{ + address = vars.haServer1Ip; + prefixLength = 24; + }]; + defaultGateway = "192.168.2.1"; + nameservers = [ "192.168.2.1" "8.8.8.8" ]; + }; + + # TODO: after running `bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1` + # add beszel agent pairing and sops-managed corosync authkey: + # imports = [ (import ../../modules/beszel/host-token.nix { + # name = "ha-server-1"; + # sopsFile = ../../secrets/ha-server-1.yaml; + # }) ]; + # services.beszel.agent.environment.KEY = "..."; + + system.stateVersion = "26.05"; +} diff --git a/hosts/ha-server-2/host.nix b/hosts/ha-server-2/host.nix new file mode 100644 index 0000000..1f5a8ba --- /dev/null +++ b/hosts/ha-server-2/host.nix @@ -0,0 +1,24 @@ +{ vars, ... }: +{ + networking = { + hostName = vars.haServer2Host; + hostId = "7e8f9a0b"; + useDHCP = false; + interfaces.ens18.ipv4.addresses = [{ + address = vars.haServer2Ip; + prefixLength = 24; + }]; + defaultGateway = "192.168.2.1"; + nameservers = [ "192.168.2.1" "8.8.8.8" ]; + }; + + # TODO: after running `bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2` + # add beszel agent pairing and sops-managed corosync authkey: + # imports = [ (import ../../modules/beszel/host-token.nix { + # name = "ha-server-2"; + # sopsFile = ../../secrets/ha-server-2.yaml; + # }) ]; + # services.beszel.agent.environment.KEY = "..."; + + system.stateVersion = "26.05"; +} diff --git a/modules/build-types/ha-server.nix b/modules/build-types/ha-server.nix new file mode 100644 index 0000000..26026ac --- /dev/null +++ b/modules/build-types/ha-server.nix @@ -0,0 +1,43 @@ +# HA file server build type: DRBD + XFS + LIO iSCSI + NFS, managed by +# Corosync + Pacemaker. Both ha-server-1 and ha-server-2 use this type. +# +# NFS start/stop: +# services.nfs.server.enable = true configures /etc/exports, wires up +# rpcbind, and loads kernel modules — but nfs-server.service.wantedBy is +# force-cleared so systemd does NOT auto-start it at boot. Pacemaker's +# ha-group resource group (configured by scripts/ha/cluster-init.sh) +# starts and stops nfs-server as part of the failover sequence after the +# XFS mount and iSCSI target are brought up on the new Active node. +# +# Beszel agent: +# Enabled here via enable-agent.nix. The agent KEY (used to pair with +# the Beszel hub) is not set yet — add it to hosts/ha-server-{1,2}/host.nix +# under services.beszel.agent.environment.KEY once the hub accepts the +# new agents, following the pattern in hosts/server/host.nix. +{ lib, vars, ... }: +{ + imports = [ + ../ha/pacemaker-stack.nix + ../ha/iscsi-target.nix + ../ha/cluster-config.nix + ../beszel/enable-agent.nix + ]; + + services.nfs.server = { + enable = true; + exports = '' + ${vars.haStorageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ''; + }; + + # Pacemaker controls nfs-server — prevent systemd from starting it at boot + # on both nodes (only the Active node should be serving NFS). + systemd.services.nfs-server.wantedBy = lib.mkForce [ ]; +} diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix new file mode 100644 index 0000000..9074e99 --- /dev/null +++ b/modules/ha/cluster-config.nix @@ -0,0 +1,95 @@ +# Cluster-wide HA config shared by both ha-server nodes. +# +# Covers everything that is identical on both nodes and references cluster +# topology (node IPs, hostnames, DRBD resource). Per-node identity +# (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix. +# +# Corosync authkey: +# /etc/corosync/authkey must be present (mode 0400) for corosync to start. +# It is NOT managed declaratively here — the initial deploy uses +# scripts/ha/cluster-init.sh to generate it via corosync-keygen and +# distribute it to both nodes. +# TODO: once both hosts have their sops keys registered via +# scripts/secrets/sync-host-keys.sh, add a sops secret here so the +# authkey survives nixos-rebuild. +# +# DRBD fencing: +# Production setting is resource-only: DRBD waits for the STONITH fence +# agent to confirm the peer is dead before promoting to Primary. This +# requires a working fence_pve_ssh STONITH resource in Pacemaker +# (see scripts/ha/cluster-enable-stonith.sh). On a fresh cluster with +# no fence device yet, temporarily change to dont-care and run +# cluster-enable-stonith.sh once the fence key is deployed. +{ vars, ... }: +{ + services.drbd = { + enable = true; + config = '' + global { + usage-count yes; + } + + common { + net { + protocol C; + ping-int 1; + verify-alg sha256; + after-sb-0pri discard-zero-changes; + after-sb-1pri discard-secondary; + } + disk { + fencing resource-only; + } + } + + resource ha-data { + volume 0 { + device /dev/drbd0; + disk /dev/sdb; + meta-disk internal; + } + + on ${vars.haServer1Host} { + address ${vars.haServer1Ip}:${toString vars.ports.haServerDrbd}; + } + + on ${vars.haServer2Host} { + address ${vars.haServer2Ip}:${toString vars.ports.haServerDrbd}; + } + } + ''; + }; + + # services.corosync.enable is set by modules/ha/pacemaker-stack.nix. + services.corosync = { + clusterName = "ha-cluster"; + nodelist = [ + { nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1Ip ]; } + { nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2Ip ]; } + ]; + }; + + networking.firewall = { + allowedTCPPorts = [ + vars.ports.haServerIscsi + vars.ports.haServerPacemakerRemoted + vars.ports.haServerPcsd + vars.ports.haServerDrbd + vars.ports.nfsRpcbind + vars.ports.nfsd + vars.ports.nfsMountd + ]; + allowedUDPPorts = [ + vars.ports.haServerCorosync1 + vars.ports.haServerCorosync2 + vars.ports.haServerCorosyncCrypto + vars.ports.nfsRpcbind + vars.ports.nfsd + vars.ports.nfsMountd + ]; + extraCommands = '' + iptables -A INPUT -s ${vars.haServer1Ip}/32 -j ACCEPT + iptables -A INPUT -s ${vars.haServer2Ip}/32 -j ACCEPT + ''; + }; +} diff --git a/scripts/ha/acceptance-tests.sh b/scripts/ha/acceptance-tests.sh new file mode 100644 index 0000000..ab3081c --- /dev/null +++ b/scripts/ha/acceptance-tests.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# acceptance-tests.sh — HA cluster acceptance tests (T1–T7) +# +# Run from a host with SSH access to both HA nodes (or from node1 itself). +# All 7 tests must pass before considering the cluster production-ready. +# Test values below must match variables.nix haServer* values. +set -euo pipefail + +# ── Configuration ───────────────────────────────────────────────────────── +NODE1="ha-server-1" +NODE2="ha-server-2" +NODE1_IP="192.168.2.200" # vars.haServer1Ip +NODE2_IP="192.168.2.201" # vars.haServer2Ip +VIP="192.168.2.202" # vars.haServerVip +XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot +ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn +# ────────────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 +RESULTS=() + +pass() { echo " PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); } +fail() { echo " FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); } + +n1() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; } +n2() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; } + +echo "════════════════════════════════════════════════════" +echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')" +echo "════════════════════════════════════════════════════" + +# ── T1: Corosync quorum established ────────────────────────────────────── +echo "" +echo "[T1] Corosync quorum" +if n1 "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then + pass "cluster has quorum" +else + fail "cluster does not have quorum — check corosync on both nodes" +fi + +# ── T2: DRBD Primary on node1, Secondary on node2 ──────────────────────── +echo "" +echo "[T2] DRBD roles" +DRBD_ROLE=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown") +if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then + pass "DRBD Primary on $NODE1 ($DRBD_ROLE)" +else + fail "unexpected DRBD role on $NODE1: $DRBD_ROLE (expected Primary/Secondary)" +fi + +DRBD_DSTATE=$(n1 "drbdadm dstate ha-data" 2>/dev/null || echo "unknown") +if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then + pass "DRBD disk state UpToDate ($DRBD_DSTATE)" +else + fail "DRBD disk not UpToDate: $DRBD_DSTATE" +fi + +# ── T3: XFS mounted at haStorageRoot on the Active node ────────────────── +echo "" +echo "[T3] XFS mount" +if n1 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + pass "XFS mounted at ${XFS_MOUNT} on $NODE1" +else + fail "XFS not mounted at ${XFS_MOUNT} on $NODE1" +fi + +if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + fail "XFS unexpectedly mounted on $NODE2 (should only be on Active node)" +else + pass "XFS not mounted on $NODE2 (correct — Secondary)" +fi + +# ── T4: iSCSI target visible on both nodes ──────────────────────────────── +echo "" +echo "[T4] iSCSI target" +IQN_COUNT=$(n1 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0") +if [[ "$IQN_COUNT" -ge 1 ]]; then + pass "iSCSI IQN active on $NODE1 ($IQN_COUNT target(s))" +else + fail "no iSCSI IQN active on $NODE1" +fi + +# iSCSI discovery from node2 via VIP +if n2 "iscsiadm -m discovery -t sendtargets -p '${VIP}' 2>/dev/null | grep -q '${ISCSI_IQN}'"; then + pass "iSCSI target discoverable from $NODE2 via VIP ${VIP}" +else + fail "iSCSI target not discoverable from $NODE2 via ${VIP}" +fi + +# ── T5: Failover — standby node1, verify resources move to node2 ────────── +echo "" +echo "[T5] Failover (standby $NODE1)" +MYNODE=$(n1 "crm_node -n" 2>/dev/null || echo "") +n1 "crm_standby -N '${MYNODE}' -v on" 2>/dev/null || true +echo " Waiting up to 30 s for resources to move to $NODE2..." +MOVED=false +for i in $(seq 1 30); do + if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + MOVED=true + echo " Resources moved in ${i}s" + break + fi + sleep 1 +done + +if $MOVED; then + pass "XFS mounted on $NODE2 after failover" + IQN_ON_N2=$(n2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0") + [[ "$IQN_ON_N2" -ge 1 ]] \ + && pass "iSCSI target active on $NODE2 after failover" \ + || fail "iSCSI target NOT active on $NODE2 after failover" +else + fail "XFS did not mount on $NODE2 within 30 s — failover incomplete" +fi + +# ── T6: Data integrity — file written pre-failover readable post-failover ─ +echo "" +echo "[T6] Data integrity" +# Write a test file on node2 (now Active) and verify its content +TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$" +TEST_CONTENT="ha-acceptance-test-$(date +%s)" +n2 "echo '${TEST_CONTENT}' > '${TEST_FILE}'" 2>/dev/null || true +READBACK=$(n2 "cat '${TEST_FILE}' 2>/dev/null" || echo "") +if [[ "$READBACK" == "$TEST_CONTENT" ]]; then + pass "test file written and read back correctly on $NODE2" +else + fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')" +fi +n2 "rm -f '${TEST_FILE}'" 2>/dev/null || true + +# ── T7: Node rejoin — un-standby node1, verify cluster is healthy ───────── +echo "" +echo "[T7] Node rejoin" +n1 "crm_standby -N '${MYNODE}' -v off" 2>/dev/null || true +n1 "crm_resource --cleanup" 2>/dev/null || true +sleep 5 + +ONLINE_NODES=$(n2 "crm_mon -1 2>/dev/null | grep -c 'Online:'" || echo "0") +if n1 "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then + pass "$NODE1 rejoined — cluster has quorum" +else + fail "$NODE1 did not rejoin with quorum" +fi + +DRBD_ROLE_AFTER=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown") +if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then + pass "$NODE1 is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)" +else + fail "unexpected DRBD role on $NODE1 after rejoin: $DRBD_ROLE_AFTER" +fi + +# ── Summary ─────────────────────────────────────────────────────────────── +echo "" +echo "════════════════════════════════════════════════════" +echo " Results: ${PASS} PASS, ${FAIL} FAIL" +echo "════════════════════════════════════════════════════" +for r in "${RESULTS[@]}"; do echo " $r"; done +echo "" + +if [[ "$FAIL" -eq 0 ]]; then + echo "ALL PASS — cluster is production-ready." + exit 0 +else + echo "SOME TESTS FAILED — investigate before deploying." + exit 1 +fi diff --git a/scripts/ha/cluster-enable-stonith.sh b/scripts/ha/cluster-enable-stonith.sh new file mode 100644 index 0000000..6ef47a5 --- /dev/null +++ b/scripts/ha/cluster-enable-stonith.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# cluster-enable-stonith.sh — enable STONITH fence agent after the fence SSH +# key is deployed to both nodes and authorised on the Proxmox host. +# +# Run from ha-server-1 as root AFTER: +# - /etc/pacemaker/fence_pve_ssh exists on both nodes (chmod +x) +# (copy from scripts/ha/fence-pve-ssh.py) +# - /etc/fence-pve-ssh-key (SSH private key) exists on both nodes +# - The corresponding public key is in authorized_keys on PVE_HOST +# - VMID_NODE1 / VMID_NODE2 filled in below +set -euo pipefail + +# ── Configuration ───────────────────────────────────────────────────────── +NODE1="ha-server-1" +NODE2="ha-server-2" +VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1 +VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2 +PVE_HOST="pve1.sweet.home" +PVE_USER="wayne" +FENCE_KEY="/etc/fence-pve-ssh-key" +FENCE_SCRIPT="/etc/pacemaker/fence_pve_ssh" +# ────────────────────────────────────────────────────────────────────────── + +log() { echo "[stonith-setup] $*"; } +die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; } + +[[ $(id -u) -eq 0 ]] || die "must run as root" +[[ -n "$VMID_NODE1" ]] || die "VMID_NODE1 not set — edit this script" +[[ -n "$VMID_NODE2" ]] || die "VMID_NODE2 not set — edit this script" +[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY" +[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT" + +log "Verifying fence agent can reach ${PVE_HOST}..." +ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \ + -o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" \ + "sudo /usr/sbin/qm list" &>/dev/null \ + || die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo" +log "Fence agent SSH connectivity confirmed" + +log "Creating Pacemaker STONITH resources..." +cibadmin --create --scope resources --xml-text " + + + + + + + + + + + + + + +" 2>/dev/null || true + +cibadmin --create --scope resources --xml-text " + + + + + + + + + + + + + + +" 2>/dev/null || true + +log "Enabling STONITH and restoring quorum policy..." +crm_attribute -t crm_config -n stonith-enabled -v true +crm_attribute -t crm_config -n no-quorum-policy -v stop + +log "DRBD fencing mode must also be updated to resource-only (already the" +log "default in cluster-config.nix; confirm with: cat /etc/drbd.d/ha-data.conf)" + +log "Testing fence agent..." +stonith_admin --list-devices && log "Fence devices listed successfully." \ + || warn "stonith_admin --list-devices failed — check config" + +log "STONITH enabled. Cluster is now fully HA." diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh new file mode 100644 index 0000000..1aac090 --- /dev/null +++ b/scripts/ha/cluster-init.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# cluster-init.sh — one-time HA cluster initialisation script +# +# Run ONCE from ha-server-1 as root AFTER both VMs are booted and have SSH +# access. It: +# 1. Generates and distributes the corosync authkey +# 2. Waits for corosync quorum and pacemaker +# 3. Initialises DRBD metadata, promotes node1 to primary +# 4. Creates XFS on /dev/drbd0 and mounts it +# 5. Creates the directory tree and iSCSI LUN backing file +# 6. Configures LIO iSCSI target (file-backed LUN) +# 7. Configures Pacemaker resources: DRBD → XFS → iSCSI → NFS → VIP +# +# Prerequisites: +# - Both VMs booted with the ha-server config (nixos-rebuild done) +# - SSH key access from node1 to root@NODE2_IP +# - VMID_NODE1 / VMID_NODE2 filled in below (needed for STONITH setup; +# cluster starts without STONITH, which you enable separately via +# scripts/ha/cluster-enable-stonith.sh) +# - Run as root on ha-server-1 +set -euo pipefail + +# ── Configuration ───────────────────────────────────────────────────────── +# These must match variables.nix haServer* values and the Proxmox VMID +# assignments. Update before running. +NODE1="ha-server-1" +NODE2="ha-server-2" +NODE1_IP="192.168.2.200" # vars.haServer1Ip +NODE2_IP="192.168.2.201" # vars.haServer2Ip +VIP="192.168.2.202" # vars.haServerVip +XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot +ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn +ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img" +ISCSI_LUN_SIZE="10G" +DRBD_DEVICE="/dev/drbd0" +VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1 +VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2 +PVE_HOST="pve1.sweet.home" +PVE_USER="wayne" + +# NFS dataset subdirectories to create under XFS_MOUNT. +# Must mirror vars.nfsShares subpath values in variables.nix. +NFS_SUBDIRS=( + "docker/config" + "docker/volumes" + "docker/databases" + "docker/nextcloud-data" + "raspi/volumes" + "proxmox/iso" + "proxmox/lxc" + "pxe-boot/images" +) +# ────────────────────────────────────────────────────────────────────────── + +log() { echo "[cluster-init] $*"; } +die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; } +warn() { echo "[cluster-init] WARNING: $*" >&2; } + +[[ $(id -u) -eq 0 ]] || die "must run as root" +[[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1" + +# ── 0. Corosync authkey ─────────────────────────────────────────────────── +AUTHKEY="/etc/corosync/authkey" +mkdir -p /etc/corosync +if [[ ! -f "$AUTHKEY" ]]; then + log "Generating corosync authkey..." + corosync-keygen -k "$AUTHKEY" + chmod 0400 "$AUTHKEY" +fi +log "Distributing authkey to $NODE2..." +ssh "root@${NODE2_IP}" "mkdir -p /etc/corosync" +scp -q "$AUTHKEY" "root@${NODE2_IP}:${AUTHKEY}" +ssh "root@${NODE2_IP}" "chmod 0400 '${AUTHKEY}'" + +log "Restarting corosync on both nodes..." +systemctl restart corosync +ssh "root@${NODE2_IP}" "systemctl restart corosync" +sleep 3 + +# ── 1. Corosync quorum ──────────────────────────────────────────────────── +log "Waiting for corosync quorum..." +for i in $(seq 1 30); do + if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then + log "Quorum established" + break + fi + [[ $i -eq 30 ]] && die "corosync quorum not established after 60 s" + sleep 2 +done + +log "Waiting for pacemaker..." +for i in $(seq 1 30); do + if crm_mon -1 &>/dev/null; then + log "Pacemaker running" + break + fi + [[ $i -eq 30 ]] && die "pacemaker not running after 60 s" + sleep 2 +done + +# ── 2. DRBD initialisation ──────────────────────────────────────────────── +log "Initialising DRBD metadata on $NODE1..." +if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then + drbdadm create-md ha-data --force +fi + +log "Initialising DRBD metadata on $NODE2..." +ssh "root@${NODE2_IP}" " + if ! drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent\|Diskless'; then + drbdadm create-md ha-data --force + fi +" + +log "Bringing up DRBD on both nodes..." +drbdadm up ha-data 2>/dev/null || true +ssh "root@${NODE2_IP}" "drbdadm up ha-data 2>/dev/null" || true + +log "Forcing $NODE1 to DRBD Primary for initial sync..." +drbdadm primary ha-data --force + +log "Waiting for DRBD to finish initial sync (this may take several minutes)..." +for i in $(seq 1 300); do + state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown") + if echo "$state" | grep -q "UpToDate/UpToDate"; then + log "DRBD sync complete: $state" + break + fi + [[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)" + sleep 1 +done + +# ── 3. XFS filesystem ───────────────────────────────────────────────────── +log "Creating XFS on ${DRBD_DEVICE}..." +if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then + mkfs.xfs -f "${DRBD_DEVICE}" +fi + +log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..." +mkdir -p "${XFS_MOUNT}" +mount "${DRBD_DEVICE}" "${XFS_MOUNT}" + +# ── 4. NFS dataset directories ──────────────────────────────────────────── +log "Creating NFS dataset directories..." +for subdir in "${NFS_SUBDIRS[@]}"; do + mkdir -p "${XFS_MOUNT}/${subdir}" +done + +# ── 5. iSCSI LUN backing file ───────────────────────────────────────────── +log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..." +if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then + fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}" +fi + +# ── 6. LIO iSCSI target ─────────────────────────────────────────────────── +log "Configuring LIO iSCSI target via targetcli..." +targetcli < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +" + +log "Adding ordering and colocation constraints..." +cibadmin --create --scope constraints --xml-text " + + + + +" + +log "Waiting for resources to start..." +for i in $(seq 1 60); do + if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then + log "VIP is up: $(crm_resource -r vip --locate)" + break + fi + [[ $i -eq 60 ]] && { warn "VIP not up after 120 s — check: crm_mon -1"; break; } + sleep 2 +done + +log "" +log "═══════════════════════════════════════════════════════════════" +log " HA cluster initialised." +log "" +log " crm_mon -1 — cluster status" +log " iscsiadm -m discovery -t st -p ${VIP} — verify iSCSI target" +log " showmount -e ${VIP} — verify NFS exports" +log "" +log " To enable STONITH (after deploying fence SSH key):" +log " 1. Fill in VMID_NODE1 / VMID_NODE2 in cluster-enable-stonith.sh" +log " 2. Copy scripts/ha/fence-pve-ssh.py to /etc/pacemaker/fence_pve_ssh" +log " on both nodes (chmod +x)" +log " 3. Generate and distribute the fence SSH key" +log " (see docs or cluster-enable-stonith.sh header)" +log " 4. bash scripts/ha/cluster-enable-stonith.sh" +log "═══════════════════════════════════════════════════════════════" diff --git a/test-lab/ha/fence-pve-ssh.py b/scripts/ha/fence-pve-ssh.py similarity index 92% rename from test-lab/ha/fence-pve-ssh.py rename to scripts/ha/fence-pve-ssh.py index 7bf0c7a..58c24ed 100644 --- a/test-lab/ha/fence-pve-ssh.py +++ b/scripts/ha/fence-pve-ssh.py @@ -2,16 +2,16 @@ """ fence_pve_ssh - Proxmox VE SSH fence agent for Pacemaker. -Uses SSH to reach pve1.sweet.home and run 'qm stop/start '. -Designed for test-lab HA cluster only — not for production. +Uses SSH to reach the Proxmox host and run 'qm stop/start '. +Deploy to /etc/pacemaker/fence_pve_ssh on both HA nodes (chmod +x). Configuration (as pacemaker stonith resource attributes): pve_host Proxmox host to SSH to (default: pve1.sweet.home) pve_user SSH user (default: wayne) key_file SSH private key path (default: /etc/fence-pve-ssh-key) - vmid_node1 VMID for ha-test-node1 (e.g. 200) - vmid_node2 VMID for ha-test-node2 (e.g. 201) - plug Node name to act on (set by pacemaker: ha-test-node1 or ha-test-node2) + vmid_node1 VMID for ha-server-1 + vmid_node2 VMID for ha-server-2 + plug Node name to act on (set by pacemaker: ha-server-1 or ha-server-2) action Action: off|on|reboot|status|list|metadata """ @@ -112,8 +112,8 @@ def get_vmid(args): print("ERROR: --plug not specified", file=sys.stderr) sys.exit(1) mapping = { - "ha-test-node1": args.vmid_node1, - "ha-test-node2": args.vmid_node2, + "ha-server-1": args.vmid_node1, + "ha-server-2": args.vmid_node2, } vmid = mapping.get(node) if not vmid: @@ -132,9 +132,9 @@ def main(): if action == "list": if args.vmid_node1: - print("ha-test-node1") + print("ha-server-1") if args.vmid_node2: - print("ha-test-node2") + print("ha-server-2") sys.exit(0) vmid = get_vmid(args) diff --git a/test-lab/ha/acceptance-tests.sh b/test-lab/ha/acceptance-tests.sh deleted file mode 100755 index 651f1e9..0000000 --- a/test-lab/ha/acceptance-tests.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env bash -# HA File Server Acceptance Tests T1-T7 -# Failover (T5) uses pacemaker standby mode to gracefully move resources, -# simulating what STONITH + node-restart does in production. -# Note: A production cluster requires real STONITH (fence agent for the hypervisor). -set -euo pipefail - -NODE2="root@192.168.2.201" -VIP="192.168.2.202" -IQN="iqn.2026-01.local.ha-test:storage" -MOUNT="/srv/ha-data" -PACEMAKERD="/nix/store/3v9sb74cg2qmpcyzb4h6fq0z8bvp5gw1-pacemaker-3.0.1/sbin/pacemakerd" -MYNODE=$(hostname) - -SSH="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15" -PASS=0; FAIL=0 - -pass() { echo "[PASS] $1"; ((PASS++)) || true; } -fail() { echo "[FAIL] $1"; ((FAIL++)) || true; } -info() { echo "[INFO] $1"; } - -echo "=== HA File Server Acceptance Tests ===" -echo "Node: $MYNODE Date: $(date)" -echo "" - -# T1: Corosync 2-node cluster with quorum -echo "--- T1: Corosync cluster quorum ---" -QUORATE=$(corosync-quorumtool -s 2>/dev/null | grep 'Quorate:' | awk '{print $2}') -NODE_COUNT=$(corosync-quorumtool -s 2>/dev/null | grep '^Nodes:' | awk '{print $2}') -if [ "$QUORATE" = "Yes" ] && [ "$NODE_COUNT" = "2" ]; then - pass "T1: Corosync quorate with 2 nodes" -else - fail "T1: quorate=$QUORATE nodes=$NODE_COUNT" -fi - -# T2: DRBD both UpToDate, replication Established -echo "--- T2: DRBD replication healthy ---" -DRBD_STATUS=$(drbdadm status ha-data 2>/dev/null) -N1_ROLE=$(echo "$DRBD_STATUS" | grep '^ha-data role:' | awk -F: '{print $2}') -N1_DISK=$(echo "$DRBD_STATUS" | grep -oP 'disk:\K\S+' | head -1) -N2_DISK=$(echo "$DRBD_STATUS" | grep -oP 'peer-disk:\K\S+' | head -1) -REPL=$(echo "$DRBD_STATUS" | grep -oP 'replication:\K\S+' | head -1) -info "DRBD: role=$N1_ROLE local_disk=$N1_DISK peer_disk=$N2_DISK replication=$REPL" -if [ "$N1_DISK" = "UpToDate" ] && [ "$N2_DISK" = "UpToDate" ] && [ "$REPL" = "Established" ]; then - pass "T2: DRBD both UpToDate, Established (role=$N1_ROLE)" -else - fail "T2: DRBD issue: local=$N1_DISK peer=$N2_DISK replication=$REPL" -fi - -# T3: XFS mounted on primary -echo "--- T3: XFS mount on primary ---" -if mountpoint -q $MOUNT && df -t xfs $MOUNT &>/dev/null; then - FSINFO=$(df -h $MOUNT | tail -1) - pass "T3: XFS mounted at $MOUNT: $FSINFO" -else - fail "T3: XFS not mounted at $MOUNT" -fi - -# T4: iSCSI active on primary, VIP responds on port 3260 -echo "--- T4: iSCSI target active ---" -ACTIVE_IQN=$(ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep iqn | head -1) -if [ "$ACTIVE_IQN" = "$IQN" ] && nc -w3 $VIP 3260 < /dev/null 2>/dev/null; then - pass "T4: iSCSI $IQN active, port 3260 open on VIP $VIP" -elif [ "$ACTIVE_IQN" = "$IQN" ]; then - fail "T4: iSCSI IQN active but port 3260 not reachable on VIP" -else - fail "T4: iSCSI not active (got '$ACTIVE_IQN')" -fi - -# Pre-T5: write test file for data integrity check -echo "--- Pre-T5: writing test data ---" -TESTFILE="$MOUNT/failover-test.txt" -TESTDATA="FAILOVER_INTEGRITY_$(date +%s)" -echo "$TESTDATA" > "$TESTFILE" -sync -info "Wrote: $TESTFILE (data: $TESTDATA)" - -# T5: Failover — put this node into pacemaker standby, forcing resource migration -# (In production this is triggered by real STONITH; standby simulates the result.) -echo "--- T5: Failover (pacemaker standby + node isolation) ---" -info "Putting $MYNODE into standby mode (triggers resource migration to node2)..." -crm_standby -N "$MYNODE" -v on 2>&1 || true - -FAILOVER_OK=false -info "Waiting up to 90s for node2 failover..." -for i in $(seq 1 18); do - sleep 5 - STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "UNREACHABLE") - if echo "$STATUS" | grep -q "Started ha-test-node2"; then - FAILOVER_OK=true - info "Failover complete at $((i*5))s" - echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted' - break - fi -done - -if $FAILOVER_OK; then - N2_DRBD=$($SSH $NODE2 "drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print \$2}'" 2>/dev/null || echo "unknown") - N2_MOUNT=$($SSH $NODE2 "mountpoint -q $MOUNT && echo 'mounted' || echo 'not-mounted'" 2>/dev/null || echo "unknown") - N2_ISCSI=$($SSH $NODE2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" 2>/dev/null || echo "0") - N2_VIP=$($SSH $NODE2 "ip addr show | grep -c '$VIP'" 2>/dev/null || echo "0") - info "Node2: DRBD=$N2_DRBD mount=$N2_MOUNT iSCSI_IQNs=$N2_ISCSI VIP=$N2_VIP" - - FAILS=0 - [ "$N2_DRBD" = "Primary" ] || { info "FAIL: DRBD not Primary on node2"; ((FAILS++)) || true; } - [ "$N2_MOUNT" = "mounted" ] || { info "FAIL: XFS not mounted on node2"; ((FAILS++)) || true; } - [ "$N2_ISCSI" -ge "1" ] 2>/dev/null || { info "FAIL: iSCSI not active on node2"; ((FAILS++)) || true; } - [ "$N2_VIP" -ge "1" ] 2>/dev/null || { info "FAIL: VIP not on node2"; ((FAILS++)) || true; } - - if [ $FAILS -eq 0 ]; then - pass "T5: Failover complete — DRBD Primary, XFS, iSCSI, VIP all on node2" - else - fail "T5: Partial failover ($FAILS sub-checks failed)" - fi -else - fail "T5: No failover detected within 90s" -fi - -# T7: Data integrity — test file readable on node2 after failover -echo "--- T7: Data integrity after failover ---" -if $SSH $NODE2 "grep -q '$TESTDATA' $TESTFILE 2>/dev/null"; then - pass "T7: Test data intact on node2 after failover" -else - ACTUAL=$($SSH $NODE2 "cat $TESTFILE 2>/dev/null || echo FILE_MISSING" 2>/dev/null || echo "SSH_FAIL") - fail "T7: Data integrity check failed (expected '$TESTDATA', got '$ACTUAL')" -fi - -# T6: Node rejoin — take node out of standby -echo "--- T6: Node rejoin ---" -info "Taking $MYNODE out of standby..." -crm_standby -N "$MYNODE" -v off 2>&1 || true - -REJOIN_OK=false -for i in $(seq 1 12); do - sleep 5 - # Check if this node is back online (no longer standby) - STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "") - if echo "$STATUS" | grep -q "Online:.*$MYNODE"; then - REJOIN_OK=true - info "Rejoined at $((i*5))s" - echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted' - break - fi -done - -if $REJOIN_OK; then - sleep 5 - N1_DRBD=$(drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print $2}') - N1_DISK=$(drbdadm status ha-data 2>/dev/null | grep -oP 'disk:\K\S+' | head -1) - info "Node1 DRBD after rejoin: role=$N1_DRBD disk=$N1_DISK" - pass "T6: Node rejoined cluster (DRBD role=$N1_DRBD, disk=$N1_DISK)" -else - fail "T6: Node did not rejoin within 60s" -fi - -echo "" -echo "========================================" -echo "RESULTS: $PASS passed, $FAIL failed" -echo "========================================" -echo "" -echo "NOTES:" -echo " T5 uses pacemaker standby to simulate failover (production needs STONITH" -echo " fence agent, e.g. fence_pve_ssh, to crash the VM — same requirement on Debian)" -echo "" -if [ $FAIL -eq 0 ]; then - echo "VERDICT: ALL TESTS PASSED → Deliverable A (NixOS modules)" - exit 0 -else - echo "VERDICT: $FAIL TEST(S) FAILED → review above" - exit 1 -fi diff --git a/test-lab/ha/cluster-enable-stonith.sh b/test-lab/ha/cluster-enable-stonith.sh deleted file mode 100644 index 6c23081..0000000 --- a/test-lab/ha/cluster-enable-stonith.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# cluster-enable-stonith.sh — enable STONITH fence agent after fence key is deployed -# Run from ha-test-node1 as root, AFTER: -# - /etc/fence-pve-ssh-key exists on both nodes -# - The fence public key is in authorized_keys on pve1.sweet.home -set -euo pipefail - -VMID_NODE1="200" -VMID_NODE2="201" -PVE_HOST="pve1.sweet.home" -PVE_USER="wayne" -FENCE_KEY="/etc/fence-pve-ssh-key" -FENCE_SCRIPT="/usr/lib/ocf/resource.d/heartbeat/fence_pve_ssh" - -log() { echo "[stonith-setup] $*"; } -die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; } - -[[ $(id -u) -eq 0 ]] || die "must run as root" - -[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY" -[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT" - -log "Verifying fence agent can reach ${PVE_HOST}..." -if ! ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \ - -o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" "sudo /usr/sbin/qm list" &>/dev/null; then - die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo" -fi -log "Fence agent SSH connectivity confirmed" - -log "Creating Pacemaker STONITH resource..." -cibadmin --create --scope resources --xml-text " - - - - - - - - - - - - - - -" 2>/dev/null || true - -cibadmin --create --scope resources --xml-text " - - - - - - - - - - - - - - -" 2>/dev/null || true - -log "Enabling STONITH..." -crm_attribute -t crm_config -n stonith-enabled -v true - -# Restore quorum policy to stop (needed with STONITH) -crm_attribute -t crm_config -n no-quorum-policy -v stop - -log "STONITH enabled. Testing fence agent..." -if stonith_admin --list-devices; then - log "Fence devices listed successfully" -else - log "WARNING: fence device list failed — check stonith config" -fi - -log "STONITH setup complete. Cluster is now fully HA." diff --git a/test-lab/ha/cluster-init.sh b/test-lab/ha/cluster-init.sh deleted file mode 100644 index 597017b..0000000 --- a/test-lab/ha/cluster-init.sh +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env bash -# cluster-init.sh — one-time HA cluster initialisation script -# -# Run this ONCE from node1 AFTER both VMs are booted and have SSH access. -# It: -# 1. Waits for corosync quorum on both nodes -# 2. Initialises DRBD metadata and promotes node1 to primary -# 3. Creates XFS filesystem on /dev/drbd0 -# 4. Configures targetcli / LIO iSCSI target (with a file-backed LUN) -# 5. Configures the Pacemaker resource group -# 6. Optionally enables the STONITH fence agent (requires fence SSH key) -# -# Prerequisites: -# - Both VMs booted with the ha-test config -# - fence-pve-ssh-key distributed to /etc/fence-pve-ssh-key on both nodes -# - Run as root on ha-test-node1 -set -euo pipefail - -NODE1_IP="192.168.2.200" -NODE2_IP="192.168.2.201" -VIP="192.168.2.202" -DRBD_DEVICE="/dev/drbd0" -XFS_MOUNT="/mnt/ha-data" -ISCSI_IQN="iqn.2026-01.local.ha-test:storage" -ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img" -ISCSI_LUN_SIZE="1G" # small test LUN -VMID_NODE1="200" -VMID_NODE2="201" -PVE_HOST="pve1.sweet.home" -PVE_USER="wayne" -FENCE_KEY="/etc/fence-pve-ssh-key" - -log() { echo "[cluster-init] $*"; } -die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; } - -[[ $(id -u) -eq 0 ]] || die "must run as root" -[[ "$(hostname)" == "ha-test-node1" ]] || die "must run on ha-test-node1" - -# ── 1. Wait for corosync quorum ────────────────────────────────────────── -log "Waiting for corosync quorum..." -for i in $(seq 1 30); do - if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then - log "Quorum established" - break - fi - [[ $i -eq 30 ]] && die "corosync quorum not established after 30s" - sleep 2 -done - -log "Waiting for pacemaker to start..." -for i in $(seq 1 30); do - if crm_mon -1 &>/dev/null; then - log "Pacemaker running" - break - fi - [[ $i -eq 30 ]] && die "pacemaker not running after 60s" - sleep 2 -done - -# ── 2. Initialise DRBD ─────────────────────────────────────────────────── -log "Initialising DRBD metadata on node1..." -if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent"; then - drbdadm create-md ha-data --force -fi - -log "Initialising DRBD metadata on node2..." -if ! ssh "root@${NODE2_IP}" "drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent'"; then - ssh "root@${NODE2_IP}" "drbdadm create-md ha-data --force" -fi - -log "Bringing up DRBD on both nodes..." -drbdadm up ha-data || true -ssh "root@${NODE2_IP}" "drbdadm up ha-data" || true - -log "Forcing node1 to DRBD primary (initial sync)..." -drbdadm primary ha-data --force - -log "Waiting for DRBD to finish initial sync..." -for i in $(seq 1 120); do - state=$(drbdadm dstate ha-data) - if echo "$state" | grep -q "UpToDate"; then - log "DRBD sync complete: $state" - break - fi - log " DRBD state: $state (${i}/120s)" - [[ $i -eq 120 ]] && die "DRBD did not sync within 120s" - sleep 1 -done - -# ── 3. XFS filesystem ──────────────────────────────────────────────────── -log "Creating XFS on ${DRBD_DEVICE}..." -if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then - mkfs.xfs "${DRBD_DEVICE}" -fi - -log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..." -mkdir -p "${XFS_MOUNT}" -mount "${DRBD_DEVICE}" "${XFS_MOUNT}" - -# ── 4. iSCSI LUN (file-backed) ─────────────────────────────────────────── -log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..." -if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then - fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}" -fi - -log "Configuring LIO iSCSI target via targetcli..." -# This produces a /etc/target/saveconfig.json that the targetctl service loads. -# The commands create an iSCSI target backed by the file we just created. -targetcli <<'EOF' -/backstores/fileio create name=ha-lun0 file_or_dev=/mnt/ha-data/iscsi-lun.img size=0 write_back=false -/iscsi create iqn.2026-01.local.ha-test:storage -/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/luns create /backstores/fileio/ha-lun0 -/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/portals create 192.168.2.202 -/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute authentication=0 -/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute demo_mode_write_protect=0 -saveconfig /etc/target/saveconfig.json -EOF - -log "Unmounting ${XFS_MOUNT} (Pacemaker will manage it)..." -umount "${XFS_MOUNT}" - -log "Promoting DRBD back to secondary (Pacemaker manages primary role)..." -drbdadm secondary ha-data - -# ── 5. Pacemaker resources ─────────────────────────────────────────────── -log "Configuring Pacemaker..." - -# Disable STONITH initially — enable once fence key is deployed -crm_attribute -t crm_config -n stonith-enabled -v false - -# Disable quorum policy for two-node cluster (no-quorum-policy=ignore so -# the surviving node can promote without a quorum device) -crm_attribute -t crm_config -n no-quorum-policy -v ignore - -# Cluster resources: -# 1. drbd-ha — manages DRBD primary/secondary role -# 2. xfs-mount — XFS mount on /mnt/ha-data -# 3. iscsi-target — LIO target service (systemd class) -# 4. vip — floating VIP 192.168.2.202 - -log "Creating DRBD master/slave resource..." -cibadmin --replace --scope resources --xml-text " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" - -log "Adding ordering and colocation constraints..." -# All resources on the same node as DRBD master -cibadmin --create --scope constraints --xml-text " - - - - - - - - -" - -log "Resource group configured. Waiting for resources to start..." -for i in $(seq 1 60); do - if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then - log "VIP is up: $(crm_resource -r vip --locate)" - break - fi - [[ $i -eq 60 ]] && { log "WARNING: VIP not up after 60s — check crm_mon"; break; } - sleep 2 -done - -log "" -log "═══════════════════════════════════════════════════════" -log " HA cluster initialised. Next steps:" -log "" -log " - Verify: crm_mon -1" -log " - Test iSCSI: iscsiadm -m discovery -t sendtargets -p ${VIP}" -log "" -log " To enable STONITH (after deploying fence key):" -log " 1. Copy fence-pve-ssh.py to /usr/lib/ocf/resource.d/heartbeat/ on both nodes" -log " 2. Distribute /etc/fence-pve-ssh-key to both nodes" -log " 3. Add public key to authorized_keys on ${PVE_HOST}" -log " 4. Run: ./cluster-enable-stonith.sh" -log "═══════════════════════════════════════════════════════" diff --git a/test-lab/ha/common.nix b/test-lab/ha/common.nix deleted file mode 100644 index c984e1c..0000000 --- a/test-lab/ha/common.nix +++ /dev/null @@ -1,163 +0,0 @@ -# Shared HA stack config for both test nodes. -# These are throwaway test VMs — not production hosts. -# No sops-nix, no clan, no home-manager. -{ lib, pkgs, vars, ... }: - -let - node1Ip = "192.168.2.200"; - node2Ip = "192.168.2.201"; - drbdPort = 7789; - - # Test-only corosync authkey (128 bytes = 1024 bits minimum for corosync). - # Not secret — this is a disposable test cluster, not production. - testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; -in -{ - system.stateVersion = "26.05"; - - # ── Hardware (Proxmox VM) ────────────────────────────────────────────── - imports = [ - ../../modules/hardware-configuration/vm/proxmox.nix - ../../modules/boot/efi.nix - ../../modules/ha/pacemaker-stack.nix - ../../modules/ha/iscsi-target.nix - ]; - - # ── Nix settings ────────────────────────────────────────────────────── - nix.settings.experimental-features = [ "nix-command" "flakes" ]; - - users.users.root.openssh.authorizedKeys.keys = [ - vars.adminSshKey - # Claude Code session key (this machine) — test-lab only - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos" - ]; - - # ── Services ────────────────────────────────────────────────────────── - services = { - openssh = { - enable = true; - settings.PermitRootLogin = "yes"; - }; - - # Allow QEMU guest exec for key injection fallback - qemuGuest.enable = true; - - drbd = { - enable = true; - config = '' - global { - usage-count yes; - } - - common { - net { - protocol C; - ping-int 1; - verify-alg sha256; - after-sb-0pri discard-zero-changes; - after-sb-1pri discard-secondary; - } - disk { - # dont-care: DRBD itself won't fence before promoting. Production - # clusters should use resource-only here and configure a STONITH - # fence agent (e.g. fence_pve_ssh) in Pacemaker so DRBD can safely - # protect against split-brain without risking dual-Primary. - # For this test cluster (no fence device) dont-care lets promotion - # proceed; the DRBD kernel module still refuses dual-Primary without - # allow-two-primaries in net {}. - fencing dont-care; - # LVM before/after-resync-target handlers omitted: the LVM snapshot - # scripts (/usr/lib/drbd/snapshot-resync-target-lvm.sh) don't exist - # on NixOS paths. If present, DRBD calls them on resync and exits 127, - # dropping the peer connection and leaving the secondary Outdated. - } - } - - resource ha-data { - volume 0 { - device /dev/drbd0; - disk /dev/sdb; # scsi1 in Proxmox VM → sdb - meta-disk internal; - } - - on ha-test-node1 { - address ${node1Ip}:${toString drbdPort}; - } - - on ha-test-node2 { - address ${node2Ip}:${toString drbdPort}; - } - } - ''; - }; - - # services.corosync.enable = true is set by modules/ha/pacemaker-stack.nix - corosync = { - clusterName = "ha-test"; - nodelist = [ - { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; } - { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; } - ]; - }; - }; - - # Corosync authkey (test-only, not secret — generated with - # `corosync-keygen` for production). - environment.etc."corosync/authkey" = { - source = builtins.toFile "authkey" testAuthKey; - mode = "0400"; - }; - - # ── Packages ────────────────────────────────────────────────────────── - # corosync, pacemaker, ocf-resource-agents, targetcli-fb already added - # by the ha/ modules; add the remaining stack-specific tools here. - environment.systemPackages = with pkgs; [ - # Storage - drbd # drbdadm, drbdsetup, drbdmon - xfsprogs # mkfs.xfs, xfs_admin, xfs_info - - # Networking / debug - iproute2 # ip, ss - iputils # ping - tcpdump - lsof - - # Scripting / config - python3 - curl - jq - vim - htop - ]; - - # ── Networking ──────────────────────────────────────────────────────── - networking = { - useDHCP = false; - defaultGateway = "192.168.2.1"; - nameservers = [ "192.168.2.1" "8.8.8.8" ]; - - firewall = { - enable = true; - allowedTCPPorts = [ - 22 # SSH - 3260 # iSCSI - 3121 # pacemaker-remoted - 2224 # pcsd - drbdPort - ]; - allowedUDPPorts = [ - 5404 # corosync - 5405 # corosync - 5407 # corosync crypto - ]; - extraCommands = '' - iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT - iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT - ''; - }; - }; - - # ── Locale / time ───────────────────────────────────────────────────── - time.timeZone = vars.timeZone; - i18n.defaultLocale = "en_AU.UTF-8"; -} diff --git a/test-lab/ha/disko.nix b/test-lab/ha/disko.nix deleted file mode 100644 index 5d63f1d..0000000 --- a/test-lab/ha/disko.nix +++ /dev/null @@ -1,49 +0,0 @@ -{ config, ... }: - -# Smaller disk layout for throwaway test VMs (20G vs production 50G). -# Same partition scheme as modules/disko/proxmox.nix: GPT, ESP + swap + ext4 root. -# Only covers the boot disk (scsi0 → /dev/sda). The DRBD data disk -# (scsi1 → /dev/sdb) is left raw — drbdadm create-md initialises it. -{ - disko.devices.disk.main = { - type = "disk"; - device = "/dev/sda"; - imageSize = "20G"; - imageName = config.networking.hostName; - - content = { - type = "gpt"; - partitions = { - esp = { - priority = 1; - name = "ESP"; - size = "512M"; - type = "EF00"; - content = { - type = "filesystem"; - format = "vfat"; - mountpoint = "/boot"; - mountOptions = [ "umask=0077" ]; - extraArgs = [ "-F" "32" "-n" "boot" ]; - }; - }; - swap = { - size = "2G"; - content = { - type = "swap"; - randomEncryption = false; - }; - }; - root = { - size = "100%"; - content = { - type = "filesystem"; - format = "ext4"; - mountpoint = "/"; - extraArgs = [ "-L" "nixos" ]; - }; - }; - }; - }; - }; -} diff --git a/test-lab/ha/node1.nix b/test-lab/ha/node1.nix deleted file mode 100644 index 6baf756..0000000 --- a/test-lab/ha/node1.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ ... }: - -# ha-test-node1: VMID 200, 192.168.2.200/24 -{ - networking.hostName = "ha-test-node1"; - networking.hostId = "a1b2c3d4"; # random, required by ZFS (not used here) but harmless - - networking.interfaces.ens18.ipv4.addresses = [{ - address = "192.168.2.200"; - prefixLength = 24; - }]; -} diff --git a/test-lab/ha/node2.nix b/test-lab/ha/node2.nix deleted file mode 100644 index bf49d4b..0000000 --- a/test-lab/ha/node2.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ ... }: - -# ha-test-node2: VMID 201, 192.168.2.201/24 -{ - networking.hostName = "ha-test-node2"; - networking.hostId = "e5f6a7b8"; # random, required by ZFS (not used here) but harmless - - networking.interfaces.ens18.ipv4.addresses = [{ - address = "192.168.2.201"; - prefixLength = 24; - }]; -} diff --git a/variables.nix b/variables.nix index e1e15b1..c59fa68 100644 --- a/variables.nix +++ b/variables.nix @@ -68,6 +68,20 @@ # one-line change. primaryUser = "nixos"; + # HA file server cluster + # haServer1Ip / haServer2Ip: static LAN IPs for both HA nodes (must be + # fixed — DRBD and corosync ring addresses are baked into the NixOS config). + # haServerVip: floating virtual IP managed by Pacemaker's IPaddr2 resource; + # NFS and iSCSI clients connect here regardless of which node is Active. + # Set all three to real values in variables.nix before deploying. + haServer1Host = "ha-server-1"; + haServer2Host = "ha-server-2"; + haServer1Ip = "192.168.2.200"; # TODO: confirm production IP + haServer2Ip = "192.168.2.201"; # TODO: confirm production IP + haServerVip = "192.168.2.202"; # TODO: confirm floating VIP + haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node + haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage"; + # Storage storageRoot = "/tank"; # ZFS pool root on `server` @@ -146,11 +160,22 @@ # mountd RPC service (used by showmount/NFSv3 mount protocol). # Mountd listens on a fixed port so the firewall can whitelist it # explicitly rather than opening all of rpcbind's dynamic range. - # All three need both TCP and UDP (modules/build-types/server.nix). + # All three need both TCP and UDP (modules/build-types/server.nix and + # modules/build-types/ha-server.nix). nfsRpcbind = 111; nfsd = 2049; nfsMountd = 20048; + # HA cluster ports opened on ha-server-1 and ha-server-2 + # (modules/build-types/ha-server.nix / modules/ha/cluster-config.nix). + haServerDrbd = 7789; # DRBD replication (TCP) + haServerIscsi = 3260; # iSCSI target (TCP) + haServerCorosync1 = 5404; # Corosync totem ring (UDP) + haServerCorosync2 = 5405; # Corosync totem ring (UDP) + haServerCorosyncCrypto = 5407; # Corosync crypto sync (UDP) + haServerPacemakerRemoted = 3121; # pacemaker-remoted (TCP) + haServerPcsd = 2224; # pcsd cluster daemon (TCP) + # Opened on the docker host's firewall for the Traefik-fronted # container stack (docker-compose config lives in the separate # /home/debian/docker repo, not here): 80/443 are Traefik's own