Archived
test-lab: add two-node HA file-server test cluster config
Disposable test VMs (ha-test-node1 / ha-test-node2, VMIDs 200/201 on pve1) to evaluate whether the DRBD + XFS + LIO + Corosync + Pacemaker stack runs correctly on NixOS before deciding NixOS vs Debian for production. Includes: - test-lab/ha/disko.nix: 20G boot disk layout (smaller than production) - test-lab/ha/common.nix: shared HA stack (drbd, corosync, pacemaker, targetcli-fb, xfsprogs), OCF PATH workaround for nixpkgs#207891 - test-lab/ha/node1.nix / node2.nix: per-node hostname + static IP - test-lab/ha/fence-pve-ssh.py: Proxmox SSH fence agent for STONITH - test-lab/ha/cluster-init.sh: one-shot cluster bootstrap script - test-lab/ha/cluster-enable-stonith.sh: enables STONITH post-key-deploy - flake.nix: adds ha-test-node1 / ha-test-node2 nixosConfigurations (bypasses mkTarget / clan-core / sops-nix — test-only) These VMs must be destroyed once acceptance testing is complete. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env bash
|
||||
# cluster-init.sh — one-time HA cluster initialisation script
|
||||
#
|
||||
# Run this ONCE from node1 AFTER both VMs are booted and have SSH access.
|
||||
# It:
|
||||
# 1. Waits for corosync quorum on both nodes
|
||||
# 2. Initialises DRBD metadata and promotes node1 to primary
|
||||
# 3. Creates XFS filesystem on /dev/drbd0
|
||||
# 4. Configures targetcli / LIO iSCSI target (with a file-backed LUN)
|
||||
# 5. Configures the Pacemaker resource group
|
||||
# 6. Optionally enables the STONITH fence agent (requires fence SSH key)
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Both VMs booted with the ha-test config
|
||||
# - fence-pve-ssh-key distributed to /etc/fence-pve-ssh-key on both nodes
|
||||
# - Run as root on ha-test-node1
|
||||
set -euo pipefail
|
||||
|
||||
NODE1_IP="192.168.2.200"
|
||||
NODE2_IP="192.168.2.201"
|
||||
VIP="192.168.2.202"
|
||||
DRBD_DEVICE="/dev/drbd0"
|
||||
XFS_MOUNT="/mnt/ha-data"
|
||||
ISCSI_IQN="iqn.2026-01.local.ha-test:storage"
|
||||
ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
|
||||
ISCSI_LUN_SIZE="1G" # small test LUN
|
||||
VMID_NODE1="200"
|
||||
VMID_NODE2="201"
|
||||
PVE_HOST="pve1.sweet.home"
|
||||
PVE_USER="wayne"
|
||||
FENCE_KEY="/etc/fence-pve-ssh-key"
|
||||
|
||||
log() { echo "[cluster-init] $*"; }
|
||||
die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; }
|
||||
|
||||
[[ $(id -u) -eq 0 ]] || die "must run as root"
|
||||
[[ "$(hostname)" == "ha-test-node1" ]] || die "must run on ha-test-node1"
|
||||
|
||||
# ── 1. Wait for corosync quorum ──────────────────────────────────────────
|
||||
log "Waiting for corosync quorum..."
|
||||
for i in $(seq 1 30); do
|
||||
if corosync-quorumtool -q &>/dev/null; then
|
||||
log "Quorum established"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 30 ]] && die "corosync quorum not established after 30s"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log "Waiting for pacemaker to start..."
|
||||
for i in $(seq 1 30); do
|
||||
if crm_mon -1 &>/dev/null; then
|
||||
log "Pacemaker running"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 30 ]] && die "pacemaker not running after 60s"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# ── 2. Initialise DRBD ───────────────────────────────────────────────────
|
||||
log "Initialising DRBD metadata on node1..."
|
||||
if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent"; then
|
||||
drbdadm create-md ha-data --force
|
||||
fi
|
||||
|
||||
log "Initialising DRBD metadata on node2..."
|
||||
if ! ssh "root@${NODE2_IP}" "drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent'"; then
|
||||
ssh "root@${NODE2_IP}" "drbdadm create-md ha-data --force"
|
||||
fi
|
||||
|
||||
log "Bringing up DRBD on both nodes..."
|
||||
drbdadm up ha-data || true
|
||||
ssh "root@${NODE2_IP}" "drbdadm up ha-data" || true
|
||||
|
||||
log "Forcing node1 to DRBD primary (initial sync)..."
|
||||
drbdadm primary ha-data --force
|
||||
|
||||
log "Waiting for DRBD to finish initial sync..."
|
||||
for i in $(seq 1 120); do
|
||||
state=$(drbdadm dstate ha-data)
|
||||
if echo "$state" | grep -q "UpToDate"; then
|
||||
log "DRBD sync complete: $state"
|
||||
break
|
||||
fi
|
||||
log " DRBD state: $state (${i}/120s)"
|
||||
[[ $i -eq 120 ]] && die "DRBD did not sync within 120s"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# ── 3. XFS filesystem ────────────────────────────────────────────────────
|
||||
log "Creating XFS on ${DRBD_DEVICE}..."
|
||||
if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
|
||||
mkfs.xfs "${DRBD_DEVICE}"
|
||||
fi
|
||||
|
||||
log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
|
||||
mkdir -p "${XFS_MOUNT}"
|
||||
mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
|
||||
|
||||
# ── 4. iSCSI LUN (file-backed) ───────────────────────────────────────────
|
||||
log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..."
|
||||
if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then
|
||||
fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}"
|
||||
fi
|
||||
|
||||
log "Configuring LIO iSCSI target via targetcli..."
|
||||
# This produces a /etc/target/saveconfig.json that the targetctl service loads.
|
||||
# The commands create an iSCSI target backed by the file we just created.
|
||||
targetcli <<'EOF'
|
||||
/backstores/fileio create name=ha-lun0 file_or_dev=/mnt/ha-data/iscsi-lun.img size=0 write_back=false
|
||||
/iscsi create iqn.2026-01.local.ha-test:storage
|
||||
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/luns create /backstores/fileio/ha-lun0
|
||||
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/portals create 192.168.2.202
|
||||
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute authentication=0
|
||||
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute demo_mode_write_protect=0
|
||||
saveconfig /etc/target/saveconfig.json
|
||||
EOF
|
||||
|
||||
log "Unmounting ${XFS_MOUNT} (Pacemaker will manage it)..."
|
||||
umount "${XFS_MOUNT}"
|
||||
|
||||
log "Promoting DRBD back to secondary (Pacemaker manages primary role)..."
|
||||
drbdadm secondary ha-data
|
||||
|
||||
# ── 5. Pacemaker resources ───────────────────────────────────────────────
|
||||
log "Configuring Pacemaker..."
|
||||
|
||||
# Disable STONITH initially — enable once fence key is deployed
|
||||
crm_attribute -t crm_config -n stonith-enabled -v false
|
||||
|
||||
# Disable quorum policy for two-node cluster (no-quorum-policy=ignore so
|
||||
# the surviving node can promote without a quorum device)
|
||||
crm_attribute -t crm_config -n no-quorum-policy -v ignore
|
||||
|
||||
# Cluster resources:
|
||||
# 1. drbd-ha — manages DRBD primary/secondary role
|
||||
# 2. xfs-mount — XFS mount on /mnt/ha-data
|
||||
# 3. iscsi-target — LIO target service (systemd class)
|
||||
# 4. vip — floating VIP 192.168.2.202
|
||||
|
||||
log "Creating DRBD master/slave resource..."
|
||||
cibadmin --replace --scope resources --xml-text "
|
||||
<resources>
|
||||
<master id=\"drbd-ha-ms\" globally-unique=\"false\">
|
||||
<primitive id=\"drbd-ha\" class=\"ocf\" type=\"drbd\" provider=\"heartbeat\">
|
||||
<instance_attributes id=\"drbd-ha-attrs\">
|
||||
<nvpair id=\"drbd-ha-drbd_resource\" name=\"drbd_resource\" value=\"ha-data\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"drbd-ha-start\" name=\"start\" interval=\"0\" timeout=\"240s\"/>
|
||||
<op id=\"drbd-ha-stop\" name=\"stop\" interval=\"0\" timeout=\"120s\"/>
|
||||
<op id=\"drbd-ha-promote\" name=\"promote\" interval=\"0\" timeout=\"90s\"/>
|
||||
<op id=\"drbd-ha-demote\" name=\"demote\" interval=\"0\" timeout=\"90s\"/>
|
||||
<op id=\"drbd-ha-monitor-master\" name=\"monitor\" interval=\"20s\" timeout=\"20s\" role=\"Master\"/>
|
||||
<op id=\"drbd-ha-monitor-slave\" name=\"monitor\" interval=\"30s\" timeout=\"20s\" role=\"Slave\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
<meta_attributes id=\"drbd-ha-ms-meta\">
|
||||
<nvpair id=\"drbd-ha-ms-master-max\" name=\"master-max\" value=\"1\"/>
|
||||
<nvpair id=\"drbd-ha-ms-master-node-max\" name=\"master-node-max\" value=\"1\"/>
|
||||
<nvpair id=\"drbd-ha-ms-clone-max\" name=\"clone-max\" value=\"2\"/>
|
||||
<nvpair id=\"drbd-ha-ms-clone-node-max\" name=\"clone-node-max\" value=\"1\"/>
|
||||
<nvpair id=\"drbd-ha-ms-notify\" name=\"notify\" value=\"true\"/>
|
||||
<nvpair id=\"drbd-ha-ms-interleave\" name=\"interleave\" value=\"true\"/>
|
||||
</meta_attributes>
|
||||
</master>
|
||||
|
||||
<primitive id=\"xfs-mount\" class=\"ocf\" type=\"Filesystem\" provider=\"heartbeat\">
|
||||
<instance_attributes id=\"xfs-mount-attrs\">
|
||||
<nvpair id=\"xfs-mount-device\" name=\"device\" value=\"/dev/drbd0\"/>
|
||||
<nvpair id=\"xfs-mount-directory\" name=\"directory\" value=\"${XFS_MOUNT}\"/>
|
||||
<nvpair id=\"xfs-mount-fstype\" name=\"fstype\" value=\"xfs\"/>
|
||||
<nvpair id=\"xfs-mount-options\" name=\"options\" value=\"defaults\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"xfs-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"xfs-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"xfs-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
|
||||
<primitive id=\"iscsi-target\" class=\"systemd\" type=\"targetctl\">
|
||||
<operations>
|
||||
<op id=\"iscsi-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"iscsi-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"iscsi-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
|
||||
<primitive id=\"vip\" class=\"ocf\" type=\"IPaddr2\" provider=\"heartbeat\">
|
||||
<instance_attributes id=\"vip-attrs\">
|
||||
<nvpair id=\"vip-ip\" name=\"ip\" value=\"${VIP}\"/>
|
||||
<nvpair id=\"vip-cidr\" name=\"cidr_netmask\" value=\"24\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"vip-start\" name=\"start\" interval=\"0\" timeout=\"20s\"/>
|
||||
<op id=\"vip-stop\" name=\"stop\" interval=\"0\" timeout=\"20s\"/>
|
||||
<op id=\"vip-monitor\" name=\"monitor\" interval=\"10s\" timeout=\"20s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
</resources>
|
||||
"
|
||||
|
||||
log "Adding ordering and colocation constraints..."
|
||||
# All resources on the same node as DRBD master
|
||||
cibadmin --create --scope constraints --xml-text "
|
||||
<constraints>
|
||||
<rsc_order id=\"order-drbd-xfs\" first=\"drbd-ha-ms\" first-action=\"promote\" then=\"xfs-mount\" then-action=\"start\"/>
|
||||
<rsc_order id=\"order-xfs-iscsi\" first=\"xfs-mount\" then=\"iscsi-target\"/>
|
||||
<rsc_order id=\"order-iscsi-vip\" first=\"iscsi-target\" then=\"vip\"/>
|
||||
<rsc_colocation id=\"coloc-all-with-drbd\" rsc=\"xfs-mount\" with-rsc=\"drbd-ha-ms\" with-rsc-role=\"Master\" score=\"INFINITY\"/>
|
||||
<rsc_colocation id=\"coloc-iscsi-with-xfs\" rsc=\"iscsi-target\" with-rsc=\"xfs-mount\" score=\"INFINITY\"/>
|
||||
<rsc_colocation id=\"coloc-vip-with-iscsi\" rsc=\"vip\" with-rsc=\"iscsi-target\" score=\"INFINITY\"/>
|
||||
</constraints>
|
||||
"
|
||||
|
||||
log "Resource group configured. Waiting for resources to start..."
|
||||
for i in $(seq 1 60); do
|
||||
if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then
|
||||
log "VIP is up: $(crm_resource -r vip --locate)"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 60 ]] && { log "WARNING: VIP not up after 60s — check crm_mon"; break; }
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log ""
|
||||
log "═══════════════════════════════════════════════════════"
|
||||
log " HA cluster initialised. Next steps:"
|
||||
log ""
|
||||
log " - Verify: crm_mon -1"
|
||||
log " - Test iSCSI: iscsiadm -m discovery -t sendtargets -p ${VIP}"
|
||||
log ""
|
||||
log " To enable STONITH (after deploying fence key):"
|
||||
log " 1. Copy fence-pve-ssh.py to /usr/lib/ocf/resource.d/heartbeat/ on both nodes"
|
||||
log " 2. Distribute /etc/fence-pve-ssh-key to both nodes"
|
||||
log " 3. Add public key to authorized_keys on ${PVE_HOST}"
|
||||
log " 4. Run: ./cluster-enable-stonith.sh"
|
||||
log "═══════════════════════════════════════════════════════"
|
||||
Reference in New Issue
Block a user