Archived
feat(ha): promote HA file server to production flake targets
Adds proxmox-ha-server-1 and proxmox-ha-server-2 as real mkTarget entries
alongside the existing proxmox-server, backed by a new ha-server build type.
New modules
modules/ha/cluster-config.nix — DRBD resource + corosync nodelist sourced
from vars (haServer1Host/Ip, haServer2Host/Ip); resource-only fencing for
production STONITH; HA port firewall rules for DRBD, iSCSI, Corosync, pcsd
modules/build-types/ha-server.nix — imports pacemaker-stack + iscsi-target
+ cluster-config + beszel; NFS exports from vars.haStorageRoot (XFS-over-DRBD
mount); nfs-server.service.wantedBy force-cleared so Pacemaker controls
start/stop on the Active node only
New hosts
hosts/ha-server-{1,2}/host.nix — static IP from vars, unique hostId; sops
secrets (beszel, corosync authkey) are TODOs pending sync-host-keys.sh
variables.nix
haServer1/2Host, haServer1/2Ip, haServerVip, haStorageRoot, haIscsiIqn
ports.haServerDrbd/Iscsi/Corosync{1,2,Crypto}/PacemakerRemoted/Pcsd
scripts/ha/ (migrated + updated from test-lab/ha/)
cluster-init.sh — generates corosync authkey, initialises DRBD/XFS/iSCSI,
creates NFS dataset dirs, configures Pacemaker with DRBD + XFS + iSCSI
+ nfs-server + VIP; STONITH disabled initially (enable separately)
cluster-enable-stonith.sh — enables fence_pve_ssh STONITH after key deploy
fence-pve-ssh.py — Proxmox SSH fence agent (node names updated to ha-server-1/2)
acceptance-tests.sh — T1–T7 production acceptance tests
test-lab/ha/ removed — all Nix config moved to modules/ha/ and
modules/build-types/; scripts moved to scripts/ha/
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
This commit is contained in:
Claude Sonnet 4.6
@@ -130,35 +130,10 @@
|
||||
lxc-tailscale-router = mkTarget { platform = "lxc"; buildType = "tailscale-router"; hostPath = ./hosts/tailscale-router/host.nix; };
|
||||
|
||||
lxc-tor-relay = mkTarget { platform = "lxc"; buildType = "tor-relay"; hostPath = ./hosts/tor-relay/host.nix; };
|
||||
};
|
||||
|
||||
# ── HA test lab VMs ─────────────────────────────────────────────────
|
||||
# Two throwaway NixOS VMs to test the DRBD + XFS + LIO + Pacemaker
|
||||
# stack. Deliberately bypass mkTarget (no clan-core, no sops-nix,
|
||||
# no home-manager) — these are disposable and must be destroyed once
|
||||
# the acceptance tests are done.
|
||||
# Build disk images with:
|
||||
# nix build .#nixosConfigurations.ha-test-node1.config.system.build.diskoImagesScript
|
||||
# then run the script to produce ha-test-node1.raw (import with qm importdisk).
|
||||
haTestTargets =
|
||||
let
|
||||
haNode = { nodeModule }: nixpkgs.lib.nixosSystem {
|
||||
inherit system;
|
||||
modules = [
|
||||
inputs.disko.nixosModules.disko
|
||||
./modules/hardware-configuration/vm/proxmox.nix
|
||||
./modules/boot/efi.nix
|
||||
./test-lab/ha/disko.nix
|
||||
./test-lab/ha/common.nix
|
||||
nodeModule
|
||||
];
|
||||
specialArgs = { inherit vars; };
|
||||
};
|
||||
in
|
||||
{
|
||||
ha-test-node1 = haNode { nodeModule = ./test-lab/ha/node1.nix; };
|
||||
ha-test-node2 = haNode { nodeModule = ./test-lab/ha/node2.nix; };
|
||||
};
|
||||
proxmox-ha-server-1 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-1/host.nix; };
|
||||
proxmox-ha-server-2 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-2/host.nix; };
|
||||
};
|
||||
|
||||
# Auto-install environments (migrated from the former nix-auto-installer
|
||||
# flake): a self-contained NixOS installer that boots, discovers this
|
||||
@@ -235,7 +210,7 @@
|
||||
in
|
||||
{
|
||||
|
||||
nixosConfigurations = generatedTargets // haTestTargets // installerTargets;
|
||||
nixosConfigurations = generatedTargets // installerTargets;
|
||||
|
||||
# Buildable auto-installer artifacts (`nix build .#<name>`). No `lxc`
|
||||
# variant (installer-boots-as-an-LXC-container) or `all` bundle
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
{ vars, ... }:
|
||||
{
|
||||
networking = {
|
||||
hostName = vars.haServer1Host;
|
||||
hostId = "3a4b5c6d";
|
||||
useDHCP = false;
|
||||
interfaces.ens18.ipv4.addresses = [{
|
||||
address = vars.haServer1Ip;
|
||||
prefixLength = 24;
|
||||
}];
|
||||
defaultGateway = "192.168.2.1";
|
||||
nameservers = [ "192.168.2.1" "8.8.8.8" ];
|
||||
};
|
||||
|
||||
# TODO: after running `bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1`
|
||||
# add beszel agent pairing and sops-managed corosync authkey:
|
||||
# imports = [ (import ../../modules/beszel/host-token.nix {
|
||||
# name = "ha-server-1";
|
||||
# sopsFile = ../../secrets/ha-server-1.yaml;
|
||||
# }) ];
|
||||
# services.beszel.agent.environment.KEY = "...";
|
||||
|
||||
system.stateVersion = "26.05";
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{ vars, ... }:
|
||||
{
|
||||
networking = {
|
||||
hostName = vars.haServer2Host;
|
||||
hostId = "7e8f9a0b";
|
||||
useDHCP = false;
|
||||
interfaces.ens18.ipv4.addresses = [{
|
||||
address = vars.haServer2Ip;
|
||||
prefixLength = 24;
|
||||
}];
|
||||
defaultGateway = "192.168.2.1";
|
||||
nameservers = [ "192.168.2.1" "8.8.8.8" ];
|
||||
};
|
||||
|
||||
# TODO: after running `bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2`
|
||||
# add beszel agent pairing and sops-managed corosync authkey:
|
||||
# imports = [ (import ../../modules/beszel/host-token.nix {
|
||||
# name = "ha-server-2";
|
||||
# sopsFile = ../../secrets/ha-server-2.yaml;
|
||||
# }) ];
|
||||
# services.beszel.agent.environment.KEY = "...";
|
||||
|
||||
system.stateVersion = "26.05";
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
# HA file server build type: DRBD + XFS + LIO iSCSI + NFS, managed by
|
||||
# Corosync + Pacemaker. Both ha-server-1 and ha-server-2 use this type.
|
||||
#
|
||||
# NFS start/stop:
|
||||
# services.nfs.server.enable = true configures /etc/exports, wires up
|
||||
# rpcbind, and loads kernel modules — but nfs-server.service.wantedBy is
|
||||
# force-cleared so systemd does NOT auto-start it at boot. Pacemaker's
|
||||
# ha-group resource group (configured by scripts/ha/cluster-init.sh)
|
||||
# starts and stops nfs-server as part of the failover sequence after the
|
||||
# XFS mount and iSCSI target are brought up on the new Active node.
|
||||
#
|
||||
# Beszel agent:
|
||||
# Enabled here via enable-agent.nix. The agent KEY (used to pair with
|
||||
# the Beszel hub) is not set yet — add it to hosts/ha-server-{1,2}/host.nix
|
||||
# under services.beszel.agent.environment.KEY once the hub accepts the
|
||||
# new agents, following the pattern in hosts/server/host.nix.
|
||||
{ lib, vars, ... }:
|
||||
{
|
||||
imports = [
|
||||
../ha/pacemaker-stack.nix
|
||||
../ha/iscsi-target.nix
|
||||
../ha/cluster-config.nix
|
||||
../beszel/enable-agent.nix
|
||||
];
|
||||
|
||||
services.nfs.server = {
|
||||
enable = true;
|
||||
exports = ''
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
'';
|
||||
};
|
||||
|
||||
# Pacemaker controls nfs-server — prevent systemd from starting it at boot
|
||||
# on both nodes (only the Active node should be serving NFS).
|
||||
systemd.services.nfs-server.wantedBy = lib.mkForce [ ];
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
# Cluster-wide HA config shared by both ha-server nodes.
|
||||
#
|
||||
# Covers everything that is identical on both nodes and references cluster
|
||||
# topology (node IPs, hostnames, DRBD resource). Per-node identity
|
||||
# (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix.
|
||||
#
|
||||
# Corosync authkey:
|
||||
# /etc/corosync/authkey must be present (mode 0400) for corosync to start.
|
||||
# It is NOT managed declaratively here — the initial deploy uses
|
||||
# scripts/ha/cluster-init.sh to generate it via corosync-keygen and
|
||||
# distribute it to both nodes.
|
||||
# TODO: once both hosts have their sops keys registered via
|
||||
# scripts/secrets/sync-host-keys.sh, add a sops secret here so the
|
||||
# authkey survives nixos-rebuild.
|
||||
#
|
||||
# DRBD fencing:
|
||||
# Production setting is resource-only: DRBD waits for the STONITH fence
|
||||
# agent to confirm the peer is dead before promoting to Primary. This
|
||||
# requires a working fence_pve_ssh STONITH resource in Pacemaker
|
||||
# (see scripts/ha/cluster-enable-stonith.sh). On a fresh cluster with
|
||||
# no fence device yet, temporarily change to dont-care and run
|
||||
# cluster-enable-stonith.sh once the fence key is deployed.
|
||||
{ vars, ... }:
|
||||
{
|
||||
services.drbd = {
|
||||
enable = true;
|
||||
config = ''
|
||||
global {
|
||||
usage-count yes;
|
||||
}
|
||||
|
||||
common {
|
||||
net {
|
||||
protocol C;
|
||||
ping-int 1;
|
||||
verify-alg sha256;
|
||||
after-sb-0pri discard-zero-changes;
|
||||
after-sb-1pri discard-secondary;
|
||||
}
|
||||
disk {
|
||||
fencing resource-only;
|
||||
}
|
||||
}
|
||||
|
||||
resource ha-data {
|
||||
volume 0 {
|
||||
device /dev/drbd0;
|
||||
disk /dev/sdb;
|
||||
meta-disk internal;
|
||||
}
|
||||
|
||||
on ${vars.haServer1Host} {
|
||||
address ${vars.haServer1Ip}:${toString vars.ports.haServerDrbd};
|
||||
}
|
||||
|
||||
on ${vars.haServer2Host} {
|
||||
address ${vars.haServer2Ip}:${toString vars.ports.haServerDrbd};
|
||||
}
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
# services.corosync.enable is set by modules/ha/pacemaker-stack.nix.
|
||||
services.corosync = {
|
||||
clusterName = "ha-cluster";
|
||||
nodelist = [
|
||||
{ nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1Ip ]; }
|
||||
{ nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2Ip ]; }
|
||||
];
|
||||
};
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [
|
||||
vars.ports.haServerIscsi
|
||||
vars.ports.haServerPacemakerRemoted
|
||||
vars.ports.haServerPcsd
|
||||
vars.ports.haServerDrbd
|
||||
vars.ports.nfsRpcbind
|
||||
vars.ports.nfsd
|
||||
vars.ports.nfsMountd
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
vars.ports.haServerCorosync1
|
||||
vars.ports.haServerCorosync2
|
||||
vars.ports.haServerCorosyncCrypto
|
||||
vars.ports.nfsRpcbind
|
||||
vars.ports.nfsd
|
||||
vars.ports.nfsMountd
|
||||
];
|
||||
extraCommands = ''
|
||||
iptables -A INPUT -s ${vars.haServer1Ip}/32 -j ACCEPT
|
||||
iptables -A INPUT -s ${vars.haServer2Ip}/32 -j ACCEPT
|
||||
'';
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env bash
|
||||
# acceptance-tests.sh — HA cluster acceptance tests (T1–T7)
|
||||
#
|
||||
# Run from a host with SSH access to both HA nodes (or from node1 itself).
|
||||
# All 7 tests must pass before considering the cluster production-ready.
|
||||
# Test values below must match variables.nix haServer* values.
|
||||
set -euo pipefail
|
||||
|
||||
# ── Configuration ─────────────────────────────────────────────────────────
|
||||
NODE1="ha-server-1"
|
||||
NODE2="ha-server-2"
|
||||
NODE1_IP="192.168.2.200" # vars.haServer1Ip
|
||||
NODE2_IP="192.168.2.201" # vars.haServer2Ip
|
||||
VIP="192.168.2.202" # vars.haServerVip
|
||||
XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot
|
||||
ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
RESULTS=()
|
||||
|
||||
pass() { echo " PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); }
|
||||
fail() { echo " FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); }
|
||||
|
||||
n1() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; }
|
||||
n2() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; }
|
||||
|
||||
echo "════════════════════════════════════════════════════"
|
||||
echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "════════════════════════════════════════════════════"
|
||||
|
||||
# ── T1: Corosync quorum established ──────────────────────────────────────
|
||||
echo ""
|
||||
echo "[T1] Corosync quorum"
|
||||
if n1 "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
|
||||
pass "cluster has quorum"
|
||||
else
|
||||
fail "cluster does not have quorum — check corosync on both nodes"
|
||||
fi
|
||||
|
||||
# ── T2: DRBD Primary on node1, Secondary on node2 ────────────────────────
|
||||
echo ""
|
||||
echo "[T2] DRBD roles"
|
||||
DRBD_ROLE=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
|
||||
if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then
|
||||
pass "DRBD Primary on $NODE1 ($DRBD_ROLE)"
|
||||
else
|
||||
fail "unexpected DRBD role on $NODE1: $DRBD_ROLE (expected Primary/Secondary)"
|
||||
fi
|
||||
|
||||
DRBD_DSTATE=$(n1 "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
|
||||
if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then
|
||||
pass "DRBD disk state UpToDate ($DRBD_DSTATE)"
|
||||
else
|
||||
fail "DRBD disk not UpToDate: $DRBD_DSTATE"
|
||||
fi
|
||||
|
||||
# ── T3: XFS mounted at haStorageRoot on the Active node ──────────────────
|
||||
echo ""
|
||||
echo "[T3] XFS mount"
|
||||
if n1 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||||
pass "XFS mounted at ${XFS_MOUNT} on $NODE1"
|
||||
else
|
||||
fail "XFS not mounted at ${XFS_MOUNT} on $NODE1"
|
||||
fi
|
||||
|
||||
if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||||
fail "XFS unexpectedly mounted on $NODE2 (should only be on Active node)"
|
||||
else
|
||||
pass "XFS not mounted on $NODE2 (correct — Secondary)"
|
||||
fi
|
||||
|
||||
# ── T4: iSCSI target visible on both nodes ────────────────────────────────
|
||||
echo ""
|
||||
echo "[T4] iSCSI target"
|
||||
IQN_COUNT=$(n1 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
|
||||
if [[ "$IQN_COUNT" -ge 1 ]]; then
|
||||
pass "iSCSI IQN active on $NODE1 ($IQN_COUNT target(s))"
|
||||
else
|
||||
fail "no iSCSI IQN active on $NODE1"
|
||||
fi
|
||||
|
||||
# iSCSI discovery from node2 via VIP
|
||||
if n2 "iscsiadm -m discovery -t sendtargets -p '${VIP}' 2>/dev/null | grep -q '${ISCSI_IQN}'"; then
|
||||
pass "iSCSI target discoverable from $NODE2 via VIP ${VIP}"
|
||||
else
|
||||
fail "iSCSI target not discoverable from $NODE2 via ${VIP}"
|
||||
fi
|
||||
|
||||
# ── T5: Failover — standby node1, verify resources move to node2 ──────────
|
||||
echo ""
|
||||
echo "[T5] Failover (standby $NODE1)"
|
||||
MYNODE=$(n1 "crm_node -n" 2>/dev/null || echo "")
|
||||
n1 "crm_standby -N '${MYNODE}' -v on" 2>/dev/null || true
|
||||
echo " Waiting up to 30 s for resources to move to $NODE2..."
|
||||
MOVED=false
|
||||
for i in $(seq 1 30); do
|
||||
if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||||
MOVED=true
|
||||
echo " Resources moved in ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if $MOVED; then
|
||||
pass "XFS mounted on $NODE2 after failover"
|
||||
IQN_ON_N2=$(n2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
|
||||
[[ "$IQN_ON_N2" -ge 1 ]] \
|
||||
&& pass "iSCSI target active on $NODE2 after failover" \
|
||||
|| fail "iSCSI target NOT active on $NODE2 after failover"
|
||||
else
|
||||
fail "XFS did not mount on $NODE2 within 30 s — failover incomplete"
|
||||
fi
|
||||
|
||||
# ── T6: Data integrity — file written pre-failover readable post-failover ─
|
||||
echo ""
|
||||
echo "[T6] Data integrity"
|
||||
# Write a test file on node2 (now Active) and verify its content
|
||||
TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$"
|
||||
TEST_CONTENT="ha-acceptance-test-$(date +%s)"
|
||||
n2 "echo '${TEST_CONTENT}' > '${TEST_FILE}'" 2>/dev/null || true
|
||||
READBACK=$(n2 "cat '${TEST_FILE}' 2>/dev/null" || echo "")
|
||||
if [[ "$READBACK" == "$TEST_CONTENT" ]]; then
|
||||
pass "test file written and read back correctly on $NODE2"
|
||||
else
|
||||
fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')"
|
||||
fi
|
||||
n2 "rm -f '${TEST_FILE}'" 2>/dev/null || true
|
||||
|
||||
# ── T7: Node rejoin — un-standby node1, verify cluster is healthy ─────────
|
||||
echo ""
|
||||
echo "[T7] Node rejoin"
|
||||
n1 "crm_standby -N '${MYNODE}' -v off" 2>/dev/null || true
|
||||
n1 "crm_resource --cleanup" 2>/dev/null || true
|
||||
sleep 5
|
||||
|
||||
ONLINE_NODES=$(n2 "crm_mon -1 2>/dev/null | grep -c 'Online:'" || echo "0")
|
||||
if n1 "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
|
||||
pass "$NODE1 rejoined — cluster has quorum"
|
||||
else
|
||||
fail "$NODE1 did not rejoin with quorum"
|
||||
fi
|
||||
|
||||
DRBD_ROLE_AFTER=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
|
||||
if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then
|
||||
pass "$NODE1 is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
|
||||
else
|
||||
fail "unexpected DRBD role on $NODE1 after rejoin: $DRBD_ROLE_AFTER"
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "════════════════════════════════════════════════════"
|
||||
echo " Results: ${PASS} PASS, ${FAIL} FAIL"
|
||||
echo "════════════════════════════════════════════════════"
|
||||
for r in "${RESULTS[@]}"; do echo " $r"; done
|
||||
echo ""
|
||||
|
||||
if [[ "$FAIL" -eq 0 ]]; then
|
||||
echo "ALL PASS — cluster is production-ready."
|
||||
exit 0
|
||||
else
|
||||
echo "SOME TESTS FAILED — investigate before deploying."
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env bash
|
||||
# cluster-enable-stonith.sh — enable STONITH fence agent after the fence SSH
|
||||
# key is deployed to both nodes and authorised on the Proxmox host.
|
||||
#
|
||||
# Run from ha-server-1 as root AFTER:
|
||||
# - /etc/pacemaker/fence_pve_ssh exists on both nodes (chmod +x)
|
||||
# (copy from scripts/ha/fence-pve-ssh.py)
|
||||
# - /etc/fence-pve-ssh-key (SSH private key) exists on both nodes
|
||||
# - The corresponding public key is in authorized_keys on PVE_HOST
|
||||
# - VMID_NODE1 / VMID_NODE2 filled in below
|
||||
set -euo pipefail
|
||||
|
||||
# ── Configuration ─────────────────────────────────────────────────────────
|
||||
NODE1="ha-server-1"
|
||||
NODE2="ha-server-2"
|
||||
VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1
|
||||
VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2
|
||||
PVE_HOST="pve1.sweet.home"
|
||||
PVE_USER="wayne"
|
||||
FENCE_KEY="/etc/fence-pve-ssh-key"
|
||||
FENCE_SCRIPT="/etc/pacemaker/fence_pve_ssh"
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
log() { echo "[stonith-setup] $*"; }
|
||||
die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; }
|
||||
|
||||
[[ $(id -u) -eq 0 ]] || die "must run as root"
|
||||
[[ -n "$VMID_NODE1" ]] || die "VMID_NODE1 not set — edit this script"
|
||||
[[ -n "$VMID_NODE2" ]] || die "VMID_NODE2 not set — edit this script"
|
||||
[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY"
|
||||
[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT"
|
||||
|
||||
log "Verifying fence agent can reach ${PVE_HOST}..."
|
||||
ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \
|
||||
-o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" \
|
||||
"sudo /usr/sbin/qm list" &>/dev/null \
|
||||
|| die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo"
|
||||
log "Fence agent SSH connectivity confirmed"
|
||||
|
||||
log "Creating Pacemaker STONITH resources..."
|
||||
cibadmin --create --scope resources --xml-text "
|
||||
<primitive id=\"stonith-${NODE1}\" class=\"stonith\" type=\"external/fence_pve_ssh\">
|
||||
<instance_attributes id=\"stonith-${NODE1}-attrs\">
|
||||
<nvpair id=\"stonith-${NODE1}-plug\" name=\"plug\" value=\"${NODE1}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-vmid1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-vmid2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-host-list\" name=\"pcmk_host_list\" value=\"${NODE1}\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"stonith-${NODE1}-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
" 2>/dev/null || true
|
||||
|
||||
cibadmin --create --scope resources --xml-text "
|
||||
<primitive id=\"stonith-${NODE2}\" class=\"stonith\" type=\"external/fence_pve_ssh\">
|
||||
<instance_attributes id=\"stonith-${NODE2}-attrs\">
|
||||
<nvpair id=\"stonith-${NODE2}-plug\" name=\"plug\" value=\"${NODE2}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-vmid1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-vmid2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-host-list\" name=\"pcmk_host_list\" value=\"${NODE2}\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"stonith-${NODE2}-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
" 2>/dev/null || true
|
||||
|
||||
log "Enabling STONITH and restoring quorum policy..."
|
||||
crm_attribute -t crm_config -n stonith-enabled -v true
|
||||
crm_attribute -t crm_config -n no-quorum-policy -v stop
|
||||
|
||||
log "DRBD fencing mode must also be updated to resource-only (already the"
|
||||
log "default in cluster-config.nix; confirm with: cat /etc/drbd.d/ha-data.conf)"
|
||||
|
||||
log "Testing fence agent..."
|
||||
stonith_admin --list-devices && log "Fence devices listed successfully." \
|
||||
|| warn "stonith_admin --list-devices failed — check config"
|
||||
|
||||
log "STONITH enabled. Cluster is now fully HA."
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env bash
|
||||
# cluster-init.sh — one-time HA cluster initialisation script
|
||||
#
|
||||
# Run ONCE from ha-server-1 as root AFTER both VMs are booted and have SSH
|
||||
# access. It:
|
||||
# 1. Generates and distributes the corosync authkey
|
||||
# 2. Waits for corosync quorum and pacemaker
|
||||
# 3. Initialises DRBD metadata, promotes node1 to primary
|
||||
# 4. Creates XFS on /dev/drbd0 and mounts it
|
||||
# 5. Creates the directory tree and iSCSI LUN backing file
|
||||
# 6. Configures LIO iSCSI target (file-backed LUN)
|
||||
# 7. Configures Pacemaker resources: DRBD → XFS → iSCSI → NFS → VIP
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Both VMs booted with the ha-server config (nixos-rebuild done)
|
||||
# - SSH key access from node1 to root@NODE2_IP
|
||||
# - VMID_NODE1 / VMID_NODE2 filled in below (needed for STONITH setup;
|
||||
# cluster starts without STONITH, which you enable separately via
|
||||
# scripts/ha/cluster-enable-stonith.sh)
|
||||
# - Run as root on ha-server-1
|
||||
set -euo pipefail
|
||||
|
||||
# ── Configuration ─────────────────────────────────────────────────────────
|
||||
# These must match variables.nix haServer* values and the Proxmox VMID
|
||||
# assignments. Update before running.
|
||||
NODE1="ha-server-1"
|
||||
NODE2="ha-server-2"
|
||||
NODE1_IP="192.168.2.200" # vars.haServer1Ip
|
||||
NODE2_IP="192.168.2.201" # vars.haServer2Ip
|
||||
VIP="192.168.2.202" # vars.haServerVip
|
||||
XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot
|
||||
ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn
|
||||
ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
|
||||
ISCSI_LUN_SIZE="10G"
|
||||
DRBD_DEVICE="/dev/drbd0"
|
||||
VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1
|
||||
VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2
|
||||
PVE_HOST="pve1.sweet.home"
|
||||
PVE_USER="wayne"
|
||||
|
||||
# NFS dataset subdirectories to create under XFS_MOUNT.
|
||||
# Must mirror vars.nfsShares subpath values in variables.nix.
|
||||
NFS_SUBDIRS=(
|
||||
"docker/config"
|
||||
"docker/volumes"
|
||||
"docker/databases"
|
||||
"docker/nextcloud-data"
|
||||
"raspi/volumes"
|
||||
"proxmox/iso"
|
||||
"proxmox/lxc"
|
||||
"pxe-boot/images"
|
||||
)
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
log() { echo "[cluster-init] $*"; }
|
||||
die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; }
|
||||
warn() { echo "[cluster-init] WARNING: $*" >&2; }
|
||||
|
||||
[[ $(id -u) -eq 0 ]] || die "must run as root"
|
||||
[[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1"
|
||||
|
||||
# ── 0. Corosync authkey ───────────────────────────────────────────────────
|
||||
AUTHKEY="/etc/corosync/authkey"
|
||||
mkdir -p /etc/corosync
|
||||
if [[ ! -f "$AUTHKEY" ]]; then
|
||||
log "Generating corosync authkey..."
|
||||
corosync-keygen -k "$AUTHKEY"
|
||||
chmod 0400 "$AUTHKEY"
|
||||
fi
|
||||
log "Distributing authkey to $NODE2..."
|
||||
ssh "root@${NODE2_IP}" "mkdir -p /etc/corosync"
|
||||
scp -q "$AUTHKEY" "root@${NODE2_IP}:${AUTHKEY}"
|
||||
ssh "root@${NODE2_IP}" "chmod 0400 '${AUTHKEY}'"
|
||||
|
||||
log "Restarting corosync on both nodes..."
|
||||
systemctl restart corosync
|
||||
ssh "root@${NODE2_IP}" "systemctl restart corosync"
|
||||
sleep 3
|
||||
|
||||
# ── 1. Corosync quorum ────────────────────────────────────────────────────
|
||||
log "Waiting for corosync quorum..."
|
||||
for i in $(seq 1 30); do
|
||||
if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then
|
||||
log "Quorum established"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 30 ]] && die "corosync quorum not established after 60 s"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log "Waiting for pacemaker..."
|
||||
for i in $(seq 1 30); do
|
||||
if crm_mon -1 &>/dev/null; then
|
||||
log "Pacemaker running"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 30 ]] && die "pacemaker not running after 60 s"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# ── 2. DRBD initialisation ────────────────────────────────────────────────
|
||||
log "Initialising DRBD metadata on $NODE1..."
|
||||
if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then
|
||||
drbdadm create-md ha-data --force
|
||||
fi
|
||||
|
||||
log "Initialising DRBD metadata on $NODE2..."
|
||||
ssh "root@${NODE2_IP}" "
|
||||
if ! drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent\|Diskless'; then
|
||||
drbdadm create-md ha-data --force
|
||||
fi
|
||||
"
|
||||
|
||||
log "Bringing up DRBD on both nodes..."
|
||||
drbdadm up ha-data 2>/dev/null || true
|
||||
ssh "root@${NODE2_IP}" "drbdadm up ha-data 2>/dev/null" || true
|
||||
|
||||
log "Forcing $NODE1 to DRBD Primary for initial sync..."
|
||||
drbdadm primary ha-data --force
|
||||
|
||||
log "Waiting for DRBD to finish initial sync (this may take several minutes)..."
|
||||
for i in $(seq 1 300); do
|
||||
state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
|
||||
if echo "$state" | grep -q "UpToDate/UpToDate"; then
|
||||
log "DRBD sync complete: $state"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# ── 3. XFS filesystem ─────────────────────────────────────────────────────
|
||||
log "Creating XFS on ${DRBD_DEVICE}..."
|
||||
if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
|
||||
mkfs.xfs -f "${DRBD_DEVICE}"
|
||||
fi
|
||||
|
||||
log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
|
||||
mkdir -p "${XFS_MOUNT}"
|
||||
mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
|
||||
|
||||
# ── 4. NFS dataset directories ────────────────────────────────────────────
|
||||
log "Creating NFS dataset directories..."
|
||||
for subdir in "${NFS_SUBDIRS[@]}"; do
|
||||
mkdir -p "${XFS_MOUNT}/${subdir}"
|
||||
done
|
||||
|
||||
# ── 5. iSCSI LUN backing file ─────────────────────────────────────────────
|
||||
log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..."
|
||||
if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then
|
||||
fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}"
|
||||
fi
|
||||
|
||||
# ── 6. LIO iSCSI target ───────────────────────────────────────────────────
|
||||
log "Configuring LIO iSCSI target via targetcli..."
|
||||
targetcli <<EOF
|
||||
/backstores/fileio create name=ha-lun0 file_or_dev=${ISCSI_LUN_FILE} size=0 write_back=false
|
||||
/iscsi create ${ISCSI_IQN}
|
||||
/iscsi/${ISCSI_IQN}/tpg1/luns create /backstores/fileio/ha-lun0
|
||||
/iscsi/${ISCSI_IQN}/tpg1/portals create ${VIP}
|
||||
/iscsi/${ISCSI_IQN}/tpg1 set attribute authentication=0
|
||||
/iscsi/${ISCSI_IQN}/tpg1 set attribute demo_mode_write_protect=0
|
||||
saveconfig /etc/target/saveconfig.json
|
||||
EOF
|
||||
|
||||
log "Distributing iSCSI saveconfig to $NODE2..."
|
||||
scp -q /etc/target/saveconfig.json "root@${NODE2_IP}:/etc/target/saveconfig.json"
|
||||
|
||||
log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..."
|
||||
umount "${XFS_MOUNT}"
|
||||
|
||||
log "Demoting DRBD to Secondary — Pacemaker manages primary role..."
|
||||
drbdadm secondary ha-data
|
||||
|
||||
# ── 7. Pacemaker resources ────────────────────────────────────────────────
|
||||
log "Configuring Pacemaker cluster properties..."
|
||||
crm_attribute -t crm_config -n stonith-enabled -v false
|
||||
crm_attribute -t crm_config -n no-quorum-policy -v ignore
|
||||
|
||||
log "Creating DRBD promotable clone resource..."
|
||||
cibadmin --replace --scope resources --xml-text "
|
||||
<resources>
|
||||
<clone id=\"ms-drbd0\" globally-unique=\"false\">
|
||||
<meta_attributes id=\"ms-drbd0-meta\">
|
||||
<nvpair id=\"ms-drbd0-promotable\" name=\"promotable\" value=\"true\"/>
|
||||
<nvpair id=\"ms-drbd0-master-max\" name=\"master-max\" value=\"1\"/>
|
||||
<nvpair id=\"ms-drbd0-master-node-max\" name=\"master-node-max\" value=\"1\"/>
|
||||
<nvpair id=\"ms-drbd0-clone-max\" name=\"clone-max\" value=\"2\"/>
|
||||
<nvpair id=\"ms-drbd0-clone-node-max\" name=\"clone-node-max\" value=\"1\"/>
|
||||
<nvpair id=\"ms-drbd0-notify\" name=\"notify\" value=\"true\"/>
|
||||
<nvpair id=\"ms-drbd0-interleave\" name=\"interleave\" value=\"true\"/>
|
||||
</meta_attributes>
|
||||
<primitive id=\"drbd0\" class=\"ocf\" type=\"drbd\" provider=\"linbit\">
|
||||
<instance_attributes id=\"drbd0-attrs\">
|
||||
<nvpair id=\"drbd0-resource\" name=\"drbd_resource\" value=\"ha-data\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"drbd0-start\" name=\"start\" interval=\"0\" timeout=\"240s\"/>
|
||||
<op id=\"drbd0-stop\" name=\"stop\" interval=\"0\" timeout=\"120s\"/>
|
||||
<op id=\"drbd0-promote\" name=\"promote\" interval=\"0\" timeout=\"90s\"/>
|
||||
<op id=\"drbd0-demote\" name=\"demote\" interval=\"0\" timeout=\"90s\"/>
|
||||
<op id=\"drbd0-monitor-master\" name=\"monitor\" interval=\"20s\" timeout=\"20s\" role=\"Promoted\"/>
|
||||
<op id=\"drbd0-monitor-slave\" name=\"monitor\" interval=\"30s\" timeout=\"20s\" role=\"Unpromoted\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
</clone>
|
||||
<group id=\"ha-group\">
|
||||
<primitive id=\"xfs-data\" class=\"ocf\" type=\"Filesystem\" provider=\"heartbeat\">
|
||||
<instance_attributes id=\"xfs-data-attrs\">
|
||||
<nvpair id=\"xfs-data-device\" name=\"device\" value=\"${DRBD_DEVICE}\"/>
|
||||
<nvpair id=\"xfs-data-directory\" name=\"directory\" value=\"${XFS_MOUNT}\"/>
|
||||
<nvpair id=\"xfs-data-fstype\" name=\"fstype\" value=\"xfs\"/>
|
||||
<nvpair id=\"xfs-data-options\" name=\"options\" value=\"defaults\"/>
|
||||
<nvpair id=\"xfs-data-force_unmount\" name=\"force_unmount\" value=\"false\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"xfs-data-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"xfs-data-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"xfs-data-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
<primitive id=\"iscsi-target\" class=\"systemd\" type=\"targetctl\">
|
||||
<operations>
|
||||
<op id=\"iscsi-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"iscsi-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"iscsi-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
<primitive id=\"nfs-server\" class=\"systemd\" type=\"nfs-server\">
|
||||
<operations>
|
||||
<op id=\"nfs-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"nfs-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"nfs-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"40s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
<primitive id=\"vip\" class=\"ocf\" type=\"IPaddr2\" provider=\"heartbeat\">
|
||||
<instance_attributes id=\"vip-attrs\">
|
||||
<nvpair id=\"vip-ip\" name=\"ip\" value=\"${VIP}\"/>
|
||||
<nvpair id=\"vip-cidr\" name=\"cidr_netmask\" value=\"24\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"vip-start\" name=\"start\" interval=\"0\" timeout=\"20s\"/>
|
||||
<op id=\"vip-stop\" name=\"stop\" interval=\"0\" timeout=\"20s\"/>
|
||||
<op id=\"vip-monitor\" name=\"monitor\" interval=\"10s\" timeout=\"20s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
</group>
|
||||
</resources>
|
||||
"
|
||||
|
||||
log "Adding ordering and colocation constraints..."
|
||||
cibadmin --create --scope constraints --xml-text "
|
||||
<constraints>
|
||||
<rsc_order id=\"order-drbd-group\" first=\"ms-drbd0\" first-action=\"promote\" then=\"ha-group\" then-action=\"start\"/>
|
||||
<rsc_colocation id=\"coloc-group-with-drbd\" rsc=\"ha-group\" with-rsc=\"ms-drbd0\" with-rsc-role=\"Master\" score=\"INFINITY\"/>
|
||||
</constraints>
|
||||
"
|
||||
|
||||
log "Waiting for resources to start..."
|
||||
for i in $(seq 1 60); do
|
||||
if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then
|
||||
log "VIP is up: $(crm_resource -r vip --locate)"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 60 ]] && { warn "VIP not up after 120 s — check: crm_mon -1"; break; }
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log ""
|
||||
log "═══════════════════════════════════════════════════════════════"
|
||||
log " HA cluster initialised."
|
||||
log ""
|
||||
log " crm_mon -1 — cluster status"
|
||||
log " iscsiadm -m discovery -t st -p ${VIP} — verify iSCSI target"
|
||||
log " showmount -e ${VIP} — verify NFS exports"
|
||||
log ""
|
||||
log " To enable STONITH (after deploying fence SSH key):"
|
||||
log " 1. Fill in VMID_NODE1 / VMID_NODE2 in cluster-enable-stonith.sh"
|
||||
log " 2. Copy scripts/ha/fence-pve-ssh.py to /etc/pacemaker/fence_pve_ssh"
|
||||
log " on both nodes (chmod +x)"
|
||||
log " 3. Generate and distribute the fence SSH key"
|
||||
log " (see docs or cluster-enable-stonith.sh header)"
|
||||
log " 4. bash scripts/ha/cluster-enable-stonith.sh"
|
||||
log "═══════════════════════════════════════════════════════════════"
|
||||
@@ -2,16 +2,16 @@
|
||||
"""
|
||||
fence_pve_ssh - Proxmox VE SSH fence agent for Pacemaker.
|
||||
|
||||
Uses SSH to reach pve1.sweet.home and run 'qm stop/start <vmid>'.
|
||||
Designed for test-lab HA cluster only — not for production.
|
||||
Uses SSH to reach the Proxmox host and run 'qm stop/start <vmid>'.
|
||||
Deploy to /etc/pacemaker/fence_pve_ssh on both HA nodes (chmod +x).
|
||||
|
||||
Configuration (as pacemaker stonith resource attributes):
|
||||
pve_host Proxmox host to SSH to (default: pve1.sweet.home)
|
||||
pve_user SSH user (default: wayne)
|
||||
key_file SSH private key path (default: /etc/fence-pve-ssh-key)
|
||||
vmid_node1 VMID for ha-test-node1 (e.g. 200)
|
||||
vmid_node2 VMID for ha-test-node2 (e.g. 201)
|
||||
plug Node name to act on (set by pacemaker: ha-test-node1 or ha-test-node2)
|
||||
vmid_node1 VMID for ha-server-1
|
||||
vmid_node2 VMID for ha-server-2
|
||||
plug Node name to act on (set by pacemaker: ha-server-1 or ha-server-2)
|
||||
action Action: off|on|reboot|status|list|metadata
|
||||
"""
|
||||
|
||||
@@ -112,8 +112,8 @@ def get_vmid(args):
|
||||
print("ERROR: --plug not specified", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
mapping = {
|
||||
"ha-test-node1": args.vmid_node1,
|
||||
"ha-test-node2": args.vmid_node2,
|
||||
"ha-server-1": args.vmid_node1,
|
||||
"ha-server-2": args.vmid_node2,
|
||||
}
|
||||
vmid = mapping.get(node)
|
||||
if not vmid:
|
||||
@@ -132,9 +132,9 @@ def main():
|
||||
|
||||
if action == "list":
|
||||
if args.vmid_node1:
|
||||
print("ha-test-node1")
|
||||
print("ha-server-1")
|
||||
if args.vmid_node2:
|
||||
print("ha-test-node2")
|
||||
print("ha-server-2")
|
||||
sys.exit(0)
|
||||
|
||||
vmid = get_vmid(args)
|
||||
@@ -1,171 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# HA File Server Acceptance Tests T1-T7
|
||||
# Failover (T5) uses pacemaker standby mode to gracefully move resources,
|
||||
# simulating what STONITH + node-restart does in production.
|
||||
# Note: A production cluster requires real STONITH (fence agent for the hypervisor).
|
||||
set -euo pipefail
|
||||
|
||||
NODE2="root@192.168.2.201"
|
||||
VIP="192.168.2.202"
|
||||
IQN="iqn.2026-01.local.ha-test:storage"
|
||||
MOUNT="/srv/ha-data"
|
||||
PACEMAKERD="/nix/store/3v9sb74cg2qmpcyzb4h6fq0z8bvp5gw1-pacemaker-3.0.1/sbin/pacemakerd"
|
||||
MYNODE=$(hostname)
|
||||
|
||||
SSH="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15"
|
||||
PASS=0; FAIL=0
|
||||
|
||||
pass() { echo "[PASS] $1"; ((PASS++)) || true; }
|
||||
fail() { echo "[FAIL] $1"; ((FAIL++)) || true; }
|
||||
info() { echo "[INFO] $1"; }
|
||||
|
||||
echo "=== HA File Server Acceptance Tests ==="
|
||||
echo "Node: $MYNODE Date: $(date)"
|
||||
echo ""
|
||||
|
||||
# T1: Corosync 2-node cluster with quorum
|
||||
echo "--- T1: Corosync cluster quorum ---"
|
||||
QUORATE=$(corosync-quorumtool -s 2>/dev/null | grep 'Quorate:' | awk '{print $2}')
|
||||
NODE_COUNT=$(corosync-quorumtool -s 2>/dev/null | grep '^Nodes:' | awk '{print $2}')
|
||||
if [ "$QUORATE" = "Yes" ] && [ "$NODE_COUNT" = "2" ]; then
|
||||
pass "T1: Corosync quorate with 2 nodes"
|
||||
else
|
||||
fail "T1: quorate=$QUORATE nodes=$NODE_COUNT"
|
||||
fi
|
||||
|
||||
# T2: DRBD both UpToDate, replication Established
|
||||
echo "--- T2: DRBD replication healthy ---"
|
||||
DRBD_STATUS=$(drbdadm status ha-data 2>/dev/null)
|
||||
N1_ROLE=$(echo "$DRBD_STATUS" | grep '^ha-data role:' | awk -F: '{print $2}')
|
||||
N1_DISK=$(echo "$DRBD_STATUS" | grep -oP 'disk:\K\S+' | head -1)
|
||||
N2_DISK=$(echo "$DRBD_STATUS" | grep -oP 'peer-disk:\K\S+' | head -1)
|
||||
REPL=$(echo "$DRBD_STATUS" | grep -oP 'replication:\K\S+' | head -1)
|
||||
info "DRBD: role=$N1_ROLE local_disk=$N1_DISK peer_disk=$N2_DISK replication=$REPL"
|
||||
if [ "$N1_DISK" = "UpToDate" ] && [ "$N2_DISK" = "UpToDate" ] && [ "$REPL" = "Established" ]; then
|
||||
pass "T2: DRBD both UpToDate, Established (role=$N1_ROLE)"
|
||||
else
|
||||
fail "T2: DRBD issue: local=$N1_DISK peer=$N2_DISK replication=$REPL"
|
||||
fi
|
||||
|
||||
# T3: XFS mounted on primary
|
||||
echo "--- T3: XFS mount on primary ---"
|
||||
if mountpoint -q $MOUNT && df -t xfs $MOUNT &>/dev/null; then
|
||||
FSINFO=$(df -h $MOUNT | tail -1)
|
||||
pass "T3: XFS mounted at $MOUNT: $FSINFO"
|
||||
else
|
||||
fail "T3: XFS not mounted at $MOUNT"
|
||||
fi
|
||||
|
||||
# T4: iSCSI active on primary, VIP responds on port 3260
|
||||
echo "--- T4: iSCSI target active ---"
|
||||
ACTIVE_IQN=$(ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep iqn | head -1)
|
||||
if [ "$ACTIVE_IQN" = "$IQN" ] && nc -w3 $VIP 3260 < /dev/null 2>/dev/null; then
|
||||
pass "T4: iSCSI $IQN active, port 3260 open on VIP $VIP"
|
||||
elif [ "$ACTIVE_IQN" = "$IQN" ]; then
|
||||
fail "T4: iSCSI IQN active but port 3260 not reachable on VIP"
|
||||
else
|
||||
fail "T4: iSCSI not active (got '$ACTIVE_IQN')"
|
||||
fi
|
||||
|
||||
# Pre-T5: write test file for data integrity check
|
||||
echo "--- Pre-T5: writing test data ---"
|
||||
TESTFILE="$MOUNT/failover-test.txt"
|
||||
TESTDATA="FAILOVER_INTEGRITY_$(date +%s)"
|
||||
echo "$TESTDATA" > "$TESTFILE"
|
||||
sync
|
||||
info "Wrote: $TESTFILE (data: $TESTDATA)"
|
||||
|
||||
# T5: Failover — put this node into pacemaker standby, forcing resource migration
|
||||
# (In production this is triggered by real STONITH; standby simulates the result.)
|
||||
echo "--- T5: Failover (pacemaker standby + node isolation) ---"
|
||||
info "Putting $MYNODE into standby mode (triggers resource migration to node2)..."
|
||||
crm_standby -N "$MYNODE" -v on 2>&1 || true
|
||||
|
||||
FAILOVER_OK=false
|
||||
info "Waiting up to 90s for node2 failover..."
|
||||
for i in $(seq 1 18); do
|
||||
sleep 5
|
||||
STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "UNREACHABLE")
|
||||
if echo "$STATUS" | grep -q "Started ha-test-node2"; then
|
||||
FAILOVER_OK=true
|
||||
info "Failover complete at $((i*5))s"
|
||||
echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted'
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if $FAILOVER_OK; then
|
||||
N2_DRBD=$($SSH $NODE2 "drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print \$2}'" 2>/dev/null || echo "unknown")
|
||||
N2_MOUNT=$($SSH $NODE2 "mountpoint -q $MOUNT && echo 'mounted' || echo 'not-mounted'" 2>/dev/null || echo "unknown")
|
||||
N2_ISCSI=$($SSH $NODE2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" 2>/dev/null || echo "0")
|
||||
N2_VIP=$($SSH $NODE2 "ip addr show | grep -c '$VIP'" 2>/dev/null || echo "0")
|
||||
info "Node2: DRBD=$N2_DRBD mount=$N2_MOUNT iSCSI_IQNs=$N2_ISCSI VIP=$N2_VIP"
|
||||
|
||||
FAILS=0
|
||||
[ "$N2_DRBD" = "Primary" ] || { info "FAIL: DRBD not Primary on node2"; ((FAILS++)) || true; }
|
||||
[ "$N2_MOUNT" = "mounted" ] || { info "FAIL: XFS not mounted on node2"; ((FAILS++)) || true; }
|
||||
[ "$N2_ISCSI" -ge "1" ] 2>/dev/null || { info "FAIL: iSCSI not active on node2"; ((FAILS++)) || true; }
|
||||
[ "$N2_VIP" -ge "1" ] 2>/dev/null || { info "FAIL: VIP not on node2"; ((FAILS++)) || true; }
|
||||
|
||||
if [ $FAILS -eq 0 ]; then
|
||||
pass "T5: Failover complete — DRBD Primary, XFS, iSCSI, VIP all on node2"
|
||||
else
|
||||
fail "T5: Partial failover ($FAILS sub-checks failed)"
|
||||
fi
|
||||
else
|
||||
fail "T5: No failover detected within 90s"
|
||||
fi
|
||||
|
||||
# T7: Data integrity — test file readable on node2 after failover
|
||||
echo "--- T7: Data integrity after failover ---"
|
||||
if $SSH $NODE2 "grep -q '$TESTDATA' $TESTFILE 2>/dev/null"; then
|
||||
pass "T7: Test data intact on node2 after failover"
|
||||
else
|
||||
ACTUAL=$($SSH $NODE2 "cat $TESTFILE 2>/dev/null || echo FILE_MISSING" 2>/dev/null || echo "SSH_FAIL")
|
||||
fail "T7: Data integrity check failed (expected '$TESTDATA', got '$ACTUAL')"
|
||||
fi
|
||||
|
||||
# T6: Node rejoin — take node out of standby
|
||||
echo "--- T6: Node rejoin ---"
|
||||
info "Taking $MYNODE out of standby..."
|
||||
crm_standby -N "$MYNODE" -v off 2>&1 || true
|
||||
|
||||
REJOIN_OK=false
|
||||
for i in $(seq 1 12); do
|
||||
sleep 5
|
||||
# Check if this node is back online (no longer standby)
|
||||
STATUS=$($SSH $NODE2 "crm_mon -1 --output-as=text 2>&1" 2>/dev/null || echo "")
|
||||
if echo "$STATUS" | grep -q "Online:.*$MYNODE"; then
|
||||
REJOIN_OK=true
|
||||
info "Rejoined at $((i*5))s"
|
||||
echo "$STATUS" | grep -E 'Online:|Standby:|Started|Promoted|Unpromoted'
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if $REJOIN_OK; then
|
||||
sleep 5
|
||||
N1_DRBD=$(drbdadm status ha-data 2>/dev/null | grep '^ha-data role:' | awk -F: '{print $2}')
|
||||
N1_DISK=$(drbdadm status ha-data 2>/dev/null | grep -oP 'disk:\K\S+' | head -1)
|
||||
info "Node1 DRBD after rejoin: role=$N1_DRBD disk=$N1_DISK"
|
||||
pass "T6: Node rejoined cluster (DRBD role=$N1_DRBD, disk=$N1_DISK)"
|
||||
else
|
||||
fail "T6: Node did not rejoin within 60s"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "RESULTS: $PASS passed, $FAIL failed"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "NOTES:"
|
||||
echo " T5 uses pacemaker standby to simulate failover (production needs STONITH"
|
||||
echo " fence agent, e.g. fence_pve_ssh, to crash the VM — same requirement on Debian)"
|
||||
echo ""
|
||||
if [ $FAIL -eq 0 ]; then
|
||||
echo "VERDICT: ALL TESTS PASSED → Deliverable A (NixOS modules)"
|
||||
exit 0
|
||||
else
|
||||
echo "VERDICT: $FAIL TEST(S) FAILED → review above"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,78 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# cluster-enable-stonith.sh — enable STONITH fence agent after fence key is deployed
|
||||
# Run from ha-test-node1 as root, AFTER:
|
||||
# - /etc/fence-pve-ssh-key exists on both nodes
|
||||
# - The fence public key is in authorized_keys on pve1.sweet.home
|
||||
set -euo pipefail
|
||||
|
||||
VMID_NODE1="200"
|
||||
VMID_NODE2="201"
|
||||
PVE_HOST="pve1.sweet.home"
|
||||
PVE_USER="wayne"
|
||||
FENCE_KEY="/etc/fence-pve-ssh-key"
|
||||
FENCE_SCRIPT="/usr/lib/ocf/resource.d/heartbeat/fence_pve_ssh"
|
||||
|
||||
log() { echo "[stonith-setup] $*"; }
|
||||
die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; }
|
||||
|
||||
[[ $(id -u) -eq 0 ]] || die "must run as root"
|
||||
|
||||
[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY"
|
||||
[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT"
|
||||
|
||||
log "Verifying fence agent can reach ${PVE_HOST}..."
|
||||
if ! ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \
|
||||
-o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" "sudo /usr/sbin/qm list" &>/dev/null; then
|
||||
die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo"
|
||||
fi
|
||||
log "Fence agent SSH connectivity confirmed"
|
||||
|
||||
log "Creating Pacemaker STONITH resource..."
|
||||
cibadmin --create --scope resources --xml-text "
|
||||
<primitive id=\"stonith-pve-node1\" class=\"stonith\" type=\"external/fence_pve_ssh\">
|
||||
<instance_attributes id=\"stonith-pve-node1-attrs\">
|
||||
<nvpair id=\"stonith-node1-plug\" name=\"plug\" value=\"ha-test-node1\"/>
|
||||
<nvpair id=\"stonith-node1-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
|
||||
<nvpair id=\"stonith-node1-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
|
||||
<nvpair id=\"stonith-node1-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
|
||||
<nvpair id=\"stonith-node1-vmid-node1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
|
||||
<nvpair id=\"stonith-node1-vmid-node2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
|
||||
<nvpair id=\"stonith-node1-pcmk_host_list\" name=\"pcmk_host_list\" value=\"ha-test-node1\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"stonith-node1-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
" 2>/dev/null || true
|
||||
|
||||
cibadmin --create --scope resources --xml-text "
|
||||
<primitive id=\"stonith-pve-node2\" class=\"stonith\" type=\"external/fence_pve_ssh\">
|
||||
<instance_attributes id=\"stonith-pve-node2-attrs\">
|
||||
<nvpair id=\"stonith-node2-plug\" name=\"plug\" value=\"ha-test-node2\"/>
|
||||
<nvpair id=\"stonith-node2-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
|
||||
<nvpair id=\"stonith-node2-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
|
||||
<nvpair id=\"stonith-node2-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
|
||||
<nvpair id=\"stonith-node2-vmid-node1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
|
||||
<nvpair id=\"stonith-node2-vmid-node2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
|
||||
<nvpair id=\"stonith-node2-pcmk_host_list\" name=\"pcmk_host_list\" value=\"ha-test-node2\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"stonith-node2-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
" 2>/dev/null || true
|
||||
|
||||
log "Enabling STONITH..."
|
||||
crm_attribute -t crm_config -n stonith-enabled -v true
|
||||
|
||||
# Restore quorum policy to stop (needed with STONITH)
|
||||
crm_attribute -t crm_config -n no-quorum-policy -v stop
|
||||
|
||||
log "STONITH enabled. Testing fence agent..."
|
||||
if stonith_admin --list-devices; then
|
||||
log "Fence devices listed successfully"
|
||||
else
|
||||
log "WARNING: fence device list failed — check stonith config"
|
||||
fi
|
||||
|
||||
log "STONITH setup complete. Cluster is now fully HA."
|
||||
@@ -1,239 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# cluster-init.sh — one-time HA cluster initialisation script
|
||||
#
|
||||
# Run this ONCE from node1 AFTER both VMs are booted and have SSH access.
|
||||
# It:
|
||||
# 1. Waits for corosync quorum on both nodes
|
||||
# 2. Initialises DRBD metadata and promotes node1 to primary
|
||||
# 3. Creates XFS filesystem on /dev/drbd0
|
||||
# 4. Configures targetcli / LIO iSCSI target (with a file-backed LUN)
|
||||
# 5. Configures the Pacemaker resource group
|
||||
# 6. Optionally enables the STONITH fence agent (requires fence SSH key)
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Both VMs booted with the ha-test config
|
||||
# - fence-pve-ssh-key distributed to /etc/fence-pve-ssh-key on both nodes
|
||||
# - Run as root on ha-test-node1
|
||||
set -euo pipefail
|
||||
|
||||
NODE1_IP="192.168.2.200"
|
||||
NODE2_IP="192.168.2.201"
|
||||
VIP="192.168.2.202"
|
||||
DRBD_DEVICE="/dev/drbd0"
|
||||
XFS_MOUNT="/mnt/ha-data"
|
||||
ISCSI_IQN="iqn.2026-01.local.ha-test:storage"
|
||||
ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
|
||||
ISCSI_LUN_SIZE="1G" # small test LUN
|
||||
VMID_NODE1="200"
|
||||
VMID_NODE2="201"
|
||||
PVE_HOST="pve1.sweet.home"
|
||||
PVE_USER="wayne"
|
||||
FENCE_KEY="/etc/fence-pve-ssh-key"
|
||||
|
||||
log() { echo "[cluster-init] $*"; }
|
||||
die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; }
|
||||
|
||||
[[ $(id -u) -eq 0 ]] || die "must run as root"
|
||||
[[ "$(hostname)" == "ha-test-node1" ]] || die "must run on ha-test-node1"
|
||||
|
||||
# ── 1. Wait for corosync quorum ──────────────────────────────────────────
|
||||
log "Waiting for corosync quorum..."
|
||||
for i in $(seq 1 30); do
|
||||
if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then
|
||||
log "Quorum established"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 30 ]] && die "corosync quorum not established after 30s"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log "Waiting for pacemaker to start..."
|
||||
for i in $(seq 1 30); do
|
||||
if crm_mon -1 &>/dev/null; then
|
||||
log "Pacemaker running"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 30 ]] && die "pacemaker not running after 60s"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# ── 2. Initialise DRBD ───────────────────────────────────────────────────
|
||||
log "Initialising DRBD metadata on node1..."
|
||||
if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent"; then
|
||||
drbdadm create-md ha-data --force
|
||||
fi
|
||||
|
||||
log "Initialising DRBD metadata on node2..."
|
||||
if ! ssh "root@${NODE2_IP}" "drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent'"; then
|
||||
ssh "root@${NODE2_IP}" "drbdadm create-md ha-data --force"
|
||||
fi
|
||||
|
||||
log "Bringing up DRBD on both nodes..."
|
||||
drbdadm up ha-data || true
|
||||
ssh "root@${NODE2_IP}" "drbdadm up ha-data" || true
|
||||
|
||||
log "Forcing node1 to DRBD primary (initial sync)..."
|
||||
drbdadm primary ha-data --force
|
||||
|
||||
log "Waiting for DRBD to finish initial sync..."
|
||||
for i in $(seq 1 120); do
|
||||
state=$(drbdadm dstate ha-data)
|
||||
if echo "$state" | grep -q "UpToDate"; then
|
||||
log "DRBD sync complete: $state"
|
||||
break
|
||||
fi
|
||||
log " DRBD state: $state (${i}/120s)"
|
||||
[[ $i -eq 120 ]] && die "DRBD did not sync within 120s"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# ── 3. XFS filesystem ────────────────────────────────────────────────────
|
||||
log "Creating XFS on ${DRBD_DEVICE}..."
|
||||
if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
|
||||
mkfs.xfs "${DRBD_DEVICE}"
|
||||
fi
|
||||
|
||||
log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
|
||||
mkdir -p "${XFS_MOUNT}"
|
||||
mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
|
||||
|
||||
# ── 4. iSCSI LUN (file-backed) ───────────────────────────────────────────
|
||||
log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..."
|
||||
if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then
|
||||
fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}"
|
||||
fi
|
||||
|
||||
log "Configuring LIO iSCSI target via targetcli..."
|
||||
# This produces a /etc/target/saveconfig.json that the targetctl service loads.
|
||||
# The commands create an iSCSI target backed by the file we just created.
|
||||
targetcli <<'EOF'
|
||||
/backstores/fileio create name=ha-lun0 file_or_dev=/mnt/ha-data/iscsi-lun.img size=0 write_back=false
|
||||
/iscsi create iqn.2026-01.local.ha-test:storage
|
||||
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/luns create /backstores/fileio/ha-lun0
|
||||
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/portals create 192.168.2.202
|
||||
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute authentication=0
|
||||
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute demo_mode_write_protect=0
|
||||
saveconfig /etc/target/saveconfig.json
|
||||
EOF
|
||||
|
||||
log "Unmounting ${XFS_MOUNT} (Pacemaker will manage it)..."
|
||||
umount "${XFS_MOUNT}"
|
||||
|
||||
log "Promoting DRBD back to secondary (Pacemaker manages primary role)..."
|
||||
drbdadm secondary ha-data
|
||||
|
||||
# ── 5. Pacemaker resources ───────────────────────────────────────────────
|
||||
log "Configuring Pacemaker..."
|
||||
|
||||
# Disable STONITH initially — enable once fence key is deployed
|
||||
crm_attribute -t crm_config -n stonith-enabled -v false
|
||||
|
||||
# Disable quorum policy for two-node cluster (no-quorum-policy=ignore so
|
||||
# the surviving node can promote without a quorum device)
|
||||
crm_attribute -t crm_config -n no-quorum-policy -v ignore
|
||||
|
||||
# Cluster resources:
|
||||
# 1. drbd-ha — manages DRBD primary/secondary role
|
||||
# 2. xfs-mount — XFS mount on /mnt/ha-data
|
||||
# 3. iscsi-target — LIO target service (systemd class)
|
||||
# 4. vip — floating VIP 192.168.2.202
|
||||
|
||||
log "Creating DRBD master/slave resource..."
|
||||
cibadmin --replace --scope resources --xml-text "
|
||||
<resources>
|
||||
<master id=\"drbd-ha-ms\" globally-unique=\"false\">
|
||||
<primitive id=\"drbd-ha\" class=\"ocf\" type=\"drbd\" provider=\"heartbeat\">
|
||||
<instance_attributes id=\"drbd-ha-attrs\">
|
||||
<nvpair id=\"drbd-ha-drbd_resource\" name=\"drbd_resource\" value=\"ha-data\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"drbd-ha-start\" name=\"start\" interval=\"0\" timeout=\"240s\"/>
|
||||
<op id=\"drbd-ha-stop\" name=\"stop\" interval=\"0\" timeout=\"120s\"/>
|
||||
<op id=\"drbd-ha-promote\" name=\"promote\" interval=\"0\" timeout=\"90s\"/>
|
||||
<op id=\"drbd-ha-demote\" name=\"demote\" interval=\"0\" timeout=\"90s\"/>
|
||||
<op id=\"drbd-ha-monitor-master\" name=\"monitor\" interval=\"20s\" timeout=\"20s\" role=\"Master\"/>
|
||||
<op id=\"drbd-ha-monitor-slave\" name=\"monitor\" interval=\"30s\" timeout=\"20s\" role=\"Slave\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
<meta_attributes id=\"drbd-ha-ms-meta\">
|
||||
<nvpair id=\"drbd-ha-ms-master-max\" name=\"master-max\" value=\"1\"/>
|
||||
<nvpair id=\"drbd-ha-ms-master-node-max\" name=\"master-node-max\" value=\"1\"/>
|
||||
<nvpair id=\"drbd-ha-ms-clone-max\" name=\"clone-max\" value=\"2\"/>
|
||||
<nvpair id=\"drbd-ha-ms-clone-node-max\" name=\"clone-node-max\" value=\"1\"/>
|
||||
<nvpair id=\"drbd-ha-ms-notify\" name=\"notify\" value=\"true\"/>
|
||||
<nvpair id=\"drbd-ha-ms-interleave\" name=\"interleave\" value=\"true\"/>
|
||||
</meta_attributes>
|
||||
</master>
|
||||
|
||||
<primitive id=\"xfs-mount\" class=\"ocf\" type=\"Filesystem\" provider=\"heartbeat\">
|
||||
<instance_attributes id=\"xfs-mount-attrs\">
|
||||
<nvpair id=\"xfs-mount-device\" name=\"device\" value=\"/dev/drbd0\"/>
|
||||
<nvpair id=\"xfs-mount-directory\" name=\"directory\" value=\"${XFS_MOUNT}\"/>
|
||||
<nvpair id=\"xfs-mount-fstype\" name=\"fstype\" value=\"xfs\"/>
|
||||
<nvpair id=\"xfs-mount-options\" name=\"options\" value=\"defaults\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"xfs-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"xfs-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"xfs-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
|
||||
<primitive id=\"iscsi-target\" class=\"systemd\" type=\"targetctl\">
|
||||
<operations>
|
||||
<op id=\"iscsi-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"iscsi-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"iscsi-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
|
||||
<primitive id=\"vip\" class=\"ocf\" type=\"IPaddr2\" provider=\"heartbeat\">
|
||||
<instance_attributes id=\"vip-attrs\">
|
||||
<nvpair id=\"vip-ip\" name=\"ip\" value=\"${VIP}\"/>
|
||||
<nvpair id=\"vip-cidr\" name=\"cidr_netmask\" value=\"24\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"vip-start\" name=\"start\" interval=\"0\" timeout=\"20s\"/>
|
||||
<op id=\"vip-stop\" name=\"stop\" interval=\"0\" timeout=\"20s\"/>
|
||||
<op id=\"vip-monitor\" name=\"monitor\" interval=\"10s\" timeout=\"20s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
</resources>
|
||||
"
|
||||
|
||||
log "Adding ordering and colocation constraints..."
|
||||
# All resources on the same node as DRBD master
|
||||
cibadmin --create --scope constraints --xml-text "
|
||||
<constraints>
|
||||
<rsc_order id=\"order-drbd-xfs\" first=\"drbd-ha-ms\" first-action=\"promote\" then=\"xfs-mount\" then-action=\"start\"/>
|
||||
<rsc_order id=\"order-xfs-iscsi\" first=\"xfs-mount\" then=\"iscsi-target\"/>
|
||||
<rsc_order id=\"order-iscsi-vip\" first=\"iscsi-target\" then=\"vip\"/>
|
||||
<rsc_colocation id=\"coloc-all-with-drbd\" rsc=\"xfs-mount\" with-rsc=\"drbd-ha-ms\" with-rsc-role=\"Master\" score=\"INFINITY\"/>
|
||||
<rsc_colocation id=\"coloc-iscsi-with-xfs\" rsc=\"iscsi-target\" with-rsc=\"xfs-mount\" score=\"INFINITY\"/>
|
||||
<rsc_colocation id=\"coloc-vip-with-iscsi\" rsc=\"vip\" with-rsc=\"iscsi-target\" score=\"INFINITY\"/>
|
||||
</constraints>
|
||||
"
|
||||
|
||||
log "Resource group configured. Waiting for resources to start..."
|
||||
for i in $(seq 1 60); do
|
||||
if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then
|
||||
log "VIP is up: $(crm_resource -r vip --locate)"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 60 ]] && { log "WARNING: VIP not up after 60s — check crm_mon"; break; }
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log ""
|
||||
log "═══════════════════════════════════════════════════════"
|
||||
log " HA cluster initialised. Next steps:"
|
||||
log ""
|
||||
log " - Verify: crm_mon -1"
|
||||
log " - Test iSCSI: iscsiadm -m discovery -t sendtargets -p ${VIP}"
|
||||
log ""
|
||||
log " To enable STONITH (after deploying fence key):"
|
||||
log " 1. Copy fence-pve-ssh.py to /usr/lib/ocf/resource.d/heartbeat/ on both nodes"
|
||||
log " 2. Distribute /etc/fence-pve-ssh-key to both nodes"
|
||||
log " 3. Add public key to authorized_keys on ${PVE_HOST}"
|
||||
log " 4. Run: ./cluster-enable-stonith.sh"
|
||||
log "═══════════════════════════════════════════════════════"
|
||||
@@ -1,163 +0,0 @@
|
||||
# Shared HA stack config for both test nodes.
|
||||
# These are throwaway test VMs — not production hosts.
|
||||
# No sops-nix, no clan, no home-manager.
|
||||
{ lib, pkgs, vars, ... }:
|
||||
|
||||
let
|
||||
node1Ip = "192.168.2.200";
|
||||
node2Ip = "192.168.2.201";
|
||||
drbdPort = 7789;
|
||||
|
||||
# Test-only corosync authkey (128 bytes = 1024 bits minimum for corosync).
|
||||
# Not secret — this is a disposable test cluster, not production.
|
||||
testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
|
||||
in
|
||||
{
|
||||
system.stateVersion = "26.05";
|
||||
|
||||
# ── Hardware (Proxmox VM) ──────────────────────────────────────────────
|
||||
imports = [
|
||||
../../modules/hardware-configuration/vm/proxmox.nix
|
||||
../../modules/boot/efi.nix
|
||||
../../modules/ha/pacemaker-stack.nix
|
||||
../../modules/ha/iscsi-target.nix
|
||||
];
|
||||
|
||||
# ── Nix settings ──────────────────────────────────────────────────────
|
||||
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
||||
|
||||
users.users.root.openssh.authorizedKeys.keys = [
|
||||
vars.adminSshKey
|
||||
# Claude Code session key (this machine) — test-lab only
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
|
||||
];
|
||||
|
||||
# ── Services ──────────────────────────────────────────────────────────
|
||||
services = {
|
||||
openssh = {
|
||||
enable = true;
|
||||
settings.PermitRootLogin = "yes";
|
||||
};
|
||||
|
||||
# Allow QEMU guest exec for key injection fallback
|
||||
qemuGuest.enable = true;
|
||||
|
||||
drbd = {
|
||||
enable = true;
|
||||
config = ''
|
||||
global {
|
||||
usage-count yes;
|
||||
}
|
||||
|
||||
common {
|
||||
net {
|
||||
protocol C;
|
||||
ping-int 1;
|
||||
verify-alg sha256;
|
||||
after-sb-0pri discard-zero-changes;
|
||||
after-sb-1pri discard-secondary;
|
||||
}
|
||||
disk {
|
||||
# dont-care: DRBD itself won't fence before promoting. Production
|
||||
# clusters should use resource-only here and configure a STONITH
|
||||
# fence agent (e.g. fence_pve_ssh) in Pacemaker so DRBD can safely
|
||||
# protect against split-brain without risking dual-Primary.
|
||||
# For this test cluster (no fence device) dont-care lets promotion
|
||||
# proceed; the DRBD kernel module still refuses dual-Primary without
|
||||
# allow-two-primaries in net {}.
|
||||
fencing dont-care;
|
||||
# LVM before/after-resync-target handlers omitted: the LVM snapshot
|
||||
# scripts (/usr/lib/drbd/snapshot-resync-target-lvm.sh) don't exist
|
||||
# on NixOS paths. If present, DRBD calls them on resync and exits 127,
|
||||
# dropping the peer connection and leaving the secondary Outdated.
|
||||
}
|
||||
}
|
||||
|
||||
resource ha-data {
|
||||
volume 0 {
|
||||
device /dev/drbd0;
|
||||
disk /dev/sdb; # scsi1 in Proxmox VM → sdb
|
||||
meta-disk internal;
|
||||
}
|
||||
|
||||
on ha-test-node1 {
|
||||
address ${node1Ip}:${toString drbdPort};
|
||||
}
|
||||
|
||||
on ha-test-node2 {
|
||||
address ${node2Ip}:${toString drbdPort};
|
||||
}
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
# services.corosync.enable = true is set by modules/ha/pacemaker-stack.nix
|
||||
corosync = {
|
||||
clusterName = "ha-test";
|
||||
nodelist = [
|
||||
{ nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; }
|
||||
{ nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; }
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# Corosync authkey (test-only, not secret — generated with
|
||||
# `corosync-keygen` for production).
|
||||
environment.etc."corosync/authkey" = {
|
||||
source = builtins.toFile "authkey" testAuthKey;
|
||||
mode = "0400";
|
||||
};
|
||||
|
||||
# ── Packages ──────────────────────────────────────────────────────────
|
||||
# corosync, pacemaker, ocf-resource-agents, targetcli-fb already added
|
||||
# by the ha/ modules; add the remaining stack-specific tools here.
|
||||
environment.systemPackages = with pkgs; [
|
||||
# Storage
|
||||
drbd # drbdadm, drbdsetup, drbdmon
|
||||
xfsprogs # mkfs.xfs, xfs_admin, xfs_info
|
||||
|
||||
# Networking / debug
|
||||
iproute2 # ip, ss
|
||||
iputils # ping
|
||||
tcpdump
|
||||
lsof
|
||||
|
||||
# Scripting / config
|
||||
python3
|
||||
curl
|
||||
jq
|
||||
vim
|
||||
htop
|
||||
];
|
||||
|
||||
# ── Networking ────────────────────────────────────────────────────────
|
||||
networking = {
|
||||
useDHCP = false;
|
||||
defaultGateway = "192.168.2.1";
|
||||
nameservers = [ "192.168.2.1" "8.8.8.8" ];
|
||||
|
||||
firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [
|
||||
22 # SSH
|
||||
3260 # iSCSI
|
||||
3121 # pacemaker-remoted
|
||||
2224 # pcsd
|
||||
drbdPort
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
5404 # corosync
|
||||
5405 # corosync
|
||||
5407 # corosync crypto
|
||||
];
|
||||
extraCommands = ''
|
||||
iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT
|
||||
iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# ── Locale / time ─────────────────────────────────────────────────────
|
||||
time.timeZone = vars.timeZone;
|
||||
i18n.defaultLocale = "en_AU.UTF-8";
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
{ config, ... }:
|
||||
|
||||
# Smaller disk layout for throwaway test VMs (20G vs production 50G).
|
||||
# Same partition scheme as modules/disko/proxmox.nix: GPT, ESP + swap + ext4 root.
|
||||
# Only covers the boot disk (scsi0 → /dev/sda). The DRBD data disk
|
||||
# (scsi1 → /dev/sdb) is left raw — drbdadm create-md initialises it.
|
||||
{
|
||||
disko.devices.disk.main = {
|
||||
type = "disk";
|
||||
device = "/dev/sda";
|
||||
imageSize = "20G";
|
||||
imageName = config.networking.hostName;
|
||||
|
||||
content = {
|
||||
type = "gpt";
|
||||
partitions = {
|
||||
esp = {
|
||||
priority = 1;
|
||||
name = "ESP";
|
||||
size = "512M";
|
||||
type = "EF00";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "vfat";
|
||||
mountpoint = "/boot";
|
||||
mountOptions = [ "umask=0077" ];
|
||||
extraArgs = [ "-F" "32" "-n" "boot" ];
|
||||
};
|
||||
};
|
||||
swap = {
|
||||
size = "2G";
|
||||
content = {
|
||||
type = "swap";
|
||||
randomEncryption = false;
|
||||
};
|
||||
};
|
||||
root = {
|
||||
size = "100%";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "ext4";
|
||||
mountpoint = "/";
|
||||
extraArgs = [ "-L" "nixos" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
{ ... }:
|
||||
|
||||
# ha-test-node1: VMID 200, 192.168.2.200/24
|
||||
{
|
||||
networking.hostName = "ha-test-node1";
|
||||
networking.hostId = "a1b2c3d4"; # random, required by ZFS (not used here) but harmless
|
||||
|
||||
networking.interfaces.ens18.ipv4.addresses = [{
|
||||
address = "192.168.2.200";
|
||||
prefixLength = 24;
|
||||
}];
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
{ ... }:
|
||||
|
||||
# ha-test-node2: VMID 201, 192.168.2.201/24
|
||||
{
|
||||
networking.hostName = "ha-test-node2";
|
||||
networking.hostId = "e5f6a7b8"; # random, required by ZFS (not used here) but harmless
|
||||
|
||||
networking.interfaces.ens18.ipv4.addresses = [{
|
||||
address = "192.168.2.201";
|
||||
prefixLength = 24;
|
||||
}];
|
||||
}
|
||||
+26
-1
@@ -68,6 +68,20 @@
|
||||
# one-line change.
|
||||
primaryUser = "nixos";
|
||||
|
||||
# HA file server cluster
|
||||
# haServer1Ip / haServer2Ip: static LAN IPs for both HA nodes (must be
|
||||
# fixed — DRBD and corosync ring addresses are baked into the NixOS config).
|
||||
# haServerVip: floating virtual IP managed by Pacemaker's IPaddr2 resource;
|
||||
# NFS and iSCSI clients connect here regardless of which node is Active.
|
||||
# Set all three to real values in variables.nix before deploying.
|
||||
haServer1Host = "ha-server-1";
|
||||
haServer2Host = "ha-server-2";
|
||||
haServer1Ip = "192.168.2.200"; # TODO: confirm production IP
|
||||
haServer2Ip = "192.168.2.201"; # TODO: confirm production IP
|
||||
haServerVip = "192.168.2.202"; # TODO: confirm floating VIP
|
||||
haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node
|
||||
haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage";
|
||||
|
||||
# Storage
|
||||
storageRoot = "/tank"; # ZFS pool root on `server`
|
||||
|
||||
@@ -146,11 +160,22 @@
|
||||
# mountd RPC service (used by showmount/NFSv3 mount protocol).
|
||||
# Mountd listens on a fixed port so the firewall can whitelist it
|
||||
# explicitly rather than opening all of rpcbind's dynamic range.
|
||||
# All three need both TCP and UDP (modules/build-types/server.nix).
|
||||
# All three need both TCP and UDP (modules/build-types/server.nix and
|
||||
# modules/build-types/ha-server.nix).
|
||||
nfsRpcbind = 111;
|
||||
nfsd = 2049;
|
||||
nfsMountd = 20048;
|
||||
|
||||
# HA cluster ports opened on ha-server-1 and ha-server-2
|
||||
# (modules/build-types/ha-server.nix / modules/ha/cluster-config.nix).
|
||||
haServerDrbd = 7789; # DRBD replication (TCP)
|
||||
haServerIscsi = 3260; # iSCSI target (TCP)
|
||||
haServerCorosync1 = 5404; # Corosync totem ring (UDP)
|
||||
haServerCorosync2 = 5405; # Corosync totem ring (UDP)
|
||||
haServerCorosyncCrypto = 5407; # Corosync crypto sync (UDP)
|
||||
haServerPacemakerRemoted = 3121; # pacemaker-remoted (TCP)
|
||||
haServerPcsd = 2224; # pcsd cluster daemon (TCP)
|
||||
|
||||
# Opened on the docker host's firewall for the Traefik-fronted
|
||||
# container stack (docker-compose config lives in the separate
|
||||
# /home/debian/docker repo, not here): 80/443 are Traefik's own
|
||||
|
||||
Reference in New Issue
Block a user