Archived
beatzaplentyandClaude Sonnet 4.6
750121e9dd
Disposable test VMs (ha-test-node1 / ha-test-node2, VMIDs 200/201 on pve1) to evaluate whether the DRBD + XFS + LIO + Corosync + Pacemaker stack runs correctly on NixOS before deciding NixOS vs Debian for production. Includes: - test-lab/ha/disko.nix: 20G boot disk layout (smaller than production) - test-lab/ha/common.nix: shared HA stack (drbd, corosync, pacemaker, targetcli-fb, xfsprogs), OCF PATH workaround for nixpkgs#207891 - test-lab/ha/node1.nix / node2.nix: per-node hostname + static IP - test-lab/ha/fence-pve-ssh.py: Proxmox SSH fence agent for STONITH - test-lab/ha/cluster-init.sh: one-shot cluster bootstrap script - test-lab/ha/cluster-enable-stonith.sh: enables STONITH post-key-deploy - flake.nix: adds ha-test-node1 / ha-test-node2 nixosConfigurations (bypasses mkTarget / clan-core / sops-nix — test-only) These VMs must be destroyed once acceptance testing is complete. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
240 lines
10 KiB
Bash
240 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
# cluster-init.sh — one-time HA cluster initialisation script
|
|
#
|
|
# Run this ONCE from node1 AFTER both VMs are booted and have SSH access.
|
|
# It:
|
|
# 1. Waits for corosync quorum on both nodes
|
|
# 2. Initialises DRBD metadata and promotes node1 to primary
|
|
# 3. Creates XFS filesystem on /dev/drbd0
|
|
# 4. Configures targetcli / LIO iSCSI target (with a file-backed LUN)
|
|
# 5. Configures the Pacemaker resource group
|
|
# 6. Optionally enables the STONITH fence agent (requires fence SSH key)
|
|
#
|
|
# Prerequisites:
|
|
# - Both VMs booted with the ha-test config
|
|
# - fence-pve-ssh-key distributed to /etc/fence-pve-ssh-key on both nodes
|
|
# - Run as root on ha-test-node1
|
|
set -euo pipefail
|
|
|
|
NODE1_IP="192.168.2.200"
|
|
NODE2_IP="192.168.2.201"
|
|
VIP="192.168.2.202"
|
|
DRBD_DEVICE="/dev/drbd0"
|
|
XFS_MOUNT="/mnt/ha-data"
|
|
ISCSI_IQN="iqn.2026-01.local.ha-test:storage"
|
|
ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
|
|
ISCSI_LUN_SIZE="1G" # small test LUN
|
|
VMID_NODE1="200"
|
|
VMID_NODE2="201"
|
|
PVE_HOST="pve1.sweet.home"
|
|
PVE_USER="wayne"
|
|
FENCE_KEY="/etc/fence-pve-ssh-key"
|
|
|
|
log() { echo "[cluster-init] $*"; }
|
|
die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; }
|
|
|
|
[[ $(id -u) -eq 0 ]] || die "must run as root"
|
|
[[ "$(hostname)" == "ha-test-node1" ]] || die "must run on ha-test-node1"
|
|
|
|
# ── 1. Wait for corosync quorum ──────────────────────────────────────────
|
|
log "Waiting for corosync quorum..."
|
|
for i in $(seq 1 30); do
|
|
if corosync-quorumtool -q &>/dev/null; then
|
|
log "Quorum established"
|
|
break
|
|
fi
|
|
[[ $i -eq 30 ]] && die "corosync quorum not established after 30s"
|
|
sleep 2
|
|
done
|
|
|
|
log "Waiting for pacemaker to start..."
|
|
for i in $(seq 1 30); do
|
|
if crm_mon -1 &>/dev/null; then
|
|
log "Pacemaker running"
|
|
break
|
|
fi
|
|
[[ $i -eq 30 ]] && die "pacemaker not running after 60s"
|
|
sleep 2
|
|
done
|
|
|
|
# ── 2. Initialise DRBD ───────────────────────────────────────────────────
|
|
log "Initialising DRBD metadata on node1..."
|
|
if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent"; then
|
|
drbdadm create-md ha-data --force
|
|
fi
|
|
|
|
log "Initialising DRBD metadata on node2..."
|
|
if ! ssh "root@${NODE2_IP}" "drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent'"; then
|
|
ssh "root@${NODE2_IP}" "drbdadm create-md ha-data --force"
|
|
fi
|
|
|
|
log "Bringing up DRBD on both nodes..."
|
|
drbdadm up ha-data || true
|
|
ssh "root@${NODE2_IP}" "drbdadm up ha-data" || true
|
|
|
|
log "Forcing node1 to DRBD primary (initial sync)..."
|
|
drbdadm primary ha-data --force
|
|
|
|
log "Waiting for DRBD to finish initial sync..."
|
|
for i in $(seq 1 120); do
|
|
state=$(drbdadm dstate ha-data)
|
|
if echo "$state" | grep -q "UpToDate"; then
|
|
log "DRBD sync complete: $state"
|
|
break
|
|
fi
|
|
log " DRBD state: $state (${i}/120s)"
|
|
[[ $i -eq 120 ]] && die "DRBD did not sync within 120s"
|
|
sleep 1
|
|
done
|
|
|
|
# ── 3. XFS filesystem ────────────────────────────────────────────────────
|
|
log "Creating XFS on ${DRBD_DEVICE}..."
|
|
if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
|
|
mkfs.xfs "${DRBD_DEVICE}"
|
|
fi
|
|
|
|
log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
|
|
mkdir -p "${XFS_MOUNT}"
|
|
mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
|
|
|
|
# ── 4. iSCSI LUN (file-backed) ───────────────────────────────────────────
|
|
log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..."
|
|
if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then
|
|
fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}"
|
|
fi
|
|
|
|
log "Configuring LIO iSCSI target via targetcli..."
|
|
# This produces a /etc/target/saveconfig.json that the targetctl service loads.
|
|
# The commands create an iSCSI target backed by the file we just created.
|
|
targetcli <<'EOF'
|
|
/backstores/fileio create name=ha-lun0 file_or_dev=/mnt/ha-data/iscsi-lun.img size=0 write_back=false
|
|
/iscsi create iqn.2026-01.local.ha-test:storage
|
|
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/luns create /backstores/fileio/ha-lun0
|
|
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/portals create 192.168.2.202
|
|
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute authentication=0
|
|
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute demo_mode_write_protect=0
|
|
saveconfig /etc/target/saveconfig.json
|
|
EOF
|
|
|
|
log "Unmounting ${XFS_MOUNT} (Pacemaker will manage it)..."
|
|
umount "${XFS_MOUNT}"
|
|
|
|
log "Promoting DRBD back to secondary (Pacemaker manages primary role)..."
|
|
drbdadm secondary ha-data
|
|
|
|
# ── 5. Pacemaker resources ───────────────────────────────────────────────
|
|
log "Configuring Pacemaker..."
|
|
|
|
# Disable STONITH initially — enable once fence key is deployed
|
|
crm_attribute -t crm_config -n stonith-enabled -v false
|
|
|
|
# Disable quorum policy for two-node cluster (no-quorum-policy=ignore so
|
|
# the surviving node can promote without a quorum device)
|
|
crm_attribute -t crm_config -n no-quorum-policy -v ignore
|
|
|
|
# Cluster resources:
|
|
# 1. drbd-ha — manages DRBD primary/secondary role
|
|
# 2. xfs-mount — XFS mount on /mnt/ha-data
|
|
# 3. iscsi-target — LIO target service (systemd class)
|
|
# 4. vip — floating VIP 192.168.2.202
|
|
|
|
log "Creating DRBD master/slave resource..."
|
|
cibadmin --replace --scope resources --xml-text "
|
|
<resources>
|
|
<master id=\"drbd-ha-ms\" globally-unique=\"false\">
|
|
<primitive id=\"drbd-ha\" class=\"ocf\" type=\"drbd\" provider=\"heartbeat\">
|
|
<instance_attributes id=\"drbd-ha-attrs\">
|
|
<nvpair id=\"drbd-ha-drbd_resource\" name=\"drbd_resource\" value=\"ha-data\"/>
|
|
</instance_attributes>
|
|
<operations>
|
|
<op id=\"drbd-ha-start\" name=\"start\" interval=\"0\" timeout=\"240s\"/>
|
|
<op id=\"drbd-ha-stop\" name=\"stop\" interval=\"0\" timeout=\"120s\"/>
|
|
<op id=\"drbd-ha-promote\" name=\"promote\" interval=\"0\" timeout=\"90s\"/>
|
|
<op id=\"drbd-ha-demote\" name=\"demote\" interval=\"0\" timeout=\"90s\"/>
|
|
<op id=\"drbd-ha-monitor-master\" name=\"monitor\" interval=\"20s\" timeout=\"20s\" role=\"Master\"/>
|
|
<op id=\"drbd-ha-monitor-slave\" name=\"monitor\" interval=\"30s\" timeout=\"20s\" role=\"Slave\"/>
|
|
</operations>
|
|
</primitive>
|
|
<meta_attributes id=\"drbd-ha-ms-meta\">
|
|
<nvpair id=\"drbd-ha-ms-master-max\" name=\"master-max\" value=\"1\"/>
|
|
<nvpair id=\"drbd-ha-ms-master-node-max\" name=\"master-node-max\" value=\"1\"/>
|
|
<nvpair id=\"drbd-ha-ms-clone-max\" name=\"clone-max\" value=\"2\"/>
|
|
<nvpair id=\"drbd-ha-ms-clone-node-max\" name=\"clone-node-max\" value=\"1\"/>
|
|
<nvpair id=\"drbd-ha-ms-notify\" name=\"notify\" value=\"true\"/>
|
|
<nvpair id=\"drbd-ha-ms-interleave\" name=\"interleave\" value=\"true\"/>
|
|
</meta_attributes>
|
|
</master>
|
|
|
|
<primitive id=\"xfs-mount\" class=\"ocf\" type=\"Filesystem\" provider=\"heartbeat\">
|
|
<instance_attributes id=\"xfs-mount-attrs\">
|
|
<nvpair id=\"xfs-mount-device\" name=\"device\" value=\"/dev/drbd0\"/>
|
|
<nvpair id=\"xfs-mount-directory\" name=\"directory\" value=\"${XFS_MOUNT}\"/>
|
|
<nvpair id=\"xfs-mount-fstype\" name=\"fstype\" value=\"xfs\"/>
|
|
<nvpair id=\"xfs-mount-options\" name=\"options\" value=\"defaults\"/>
|
|
</instance_attributes>
|
|
<operations>
|
|
<op id=\"xfs-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
|
<op id=\"xfs-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
|
<op id=\"xfs-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
|
|
</operations>
|
|
</primitive>
|
|
|
|
<primitive id=\"iscsi-target\" class=\"systemd\" type=\"targetctl\">
|
|
<operations>
|
|
<op id=\"iscsi-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
|
<op id=\"iscsi-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
|
<op id=\"iscsi-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
|
|
</operations>
|
|
</primitive>
|
|
|
|
<primitive id=\"vip\" class=\"ocf\" type=\"IPaddr2\" provider=\"heartbeat\">
|
|
<instance_attributes id=\"vip-attrs\">
|
|
<nvpair id=\"vip-ip\" name=\"ip\" value=\"${VIP}\"/>
|
|
<nvpair id=\"vip-cidr\" name=\"cidr_netmask\" value=\"24\"/>
|
|
</instance_attributes>
|
|
<operations>
|
|
<op id=\"vip-start\" name=\"start\" interval=\"0\" timeout=\"20s\"/>
|
|
<op id=\"vip-stop\" name=\"stop\" interval=\"0\" timeout=\"20s\"/>
|
|
<op id=\"vip-monitor\" name=\"monitor\" interval=\"10s\" timeout=\"20s\"/>
|
|
</operations>
|
|
</primitive>
|
|
</resources>
|
|
"
|
|
|
|
log "Adding ordering and colocation constraints..."
|
|
# All resources on the same node as DRBD master
|
|
cibadmin --create --scope constraints --xml-text "
|
|
<constraints>
|
|
<rsc_order id=\"order-drbd-xfs\" first=\"drbd-ha-ms\" first-action=\"promote\" then=\"xfs-mount\" then-action=\"start\"/>
|
|
<rsc_order id=\"order-xfs-iscsi\" first=\"xfs-mount\" then=\"iscsi-target\"/>
|
|
<rsc_order id=\"order-iscsi-vip\" first=\"iscsi-target\" then=\"vip\"/>
|
|
<rsc_colocation id=\"coloc-all-with-drbd\" rsc=\"xfs-mount\" with-rsc=\"drbd-ha-ms\" with-rsc-role=\"Master\" score=\"INFINITY\"/>
|
|
<rsc_colocation id=\"coloc-iscsi-with-xfs\" rsc=\"iscsi-target\" with-rsc=\"xfs-mount\" score=\"INFINITY\"/>
|
|
<rsc_colocation id=\"coloc-vip-with-iscsi\" rsc=\"vip\" with-rsc=\"iscsi-target\" score=\"INFINITY\"/>
|
|
</constraints>
|
|
"
|
|
|
|
log "Resource group configured. Waiting for resources to start..."
|
|
for i in $(seq 1 60); do
|
|
if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then
|
|
log "VIP is up: $(crm_resource -r vip --locate)"
|
|
break
|
|
fi
|
|
[[ $i -eq 60 ]] && { log "WARNING: VIP not up after 60s — check crm_mon"; break; }
|
|
sleep 2
|
|
done
|
|
|
|
log ""
|
|
log "═══════════════════════════════════════════════════════"
|
|
log " HA cluster initialised. Next steps:"
|
|
log ""
|
|
log " - Verify: crm_mon -1"
|
|
log " - Test iSCSI: iscsiadm -m discovery -t sendtargets -p ${VIP}"
|
|
log ""
|
|
log " To enable STONITH (after deploying fence key):"
|
|
log " 1. Copy fence-pve-ssh.py to /usr/lib/ocf/resource.d/heartbeat/ on both nodes"
|
|
log " 2. Distribute /etc/fence-pve-ssh-key to both nodes"
|
|
log " 3. Add public key to authorized_keys on ${PVE_HOST}"
|
|
log " 4. Run: ./cluster-enable-stonith.sh"
|
|
log "═══════════════════════════════════════════════════════"
|