Archived
beatzaplentyandClaude Sonnet 4.6
e3498b1087
Adds proxmox-ha-server-1 and proxmox-ha-server-2 as real mkTarget entries
alongside the existing proxmox-server, backed by a new ha-server build type.
New modules
modules/ha/cluster-config.nix — DRBD resource + corosync nodelist sourced
from vars (haServer1Host/Ip, haServer2Host/Ip); resource-only fencing for
production STONITH; HA port firewall rules for DRBD, iSCSI, Corosync, pcsd
modules/build-types/ha-server.nix — imports pacemaker-stack + iscsi-target
+ cluster-config + beszel; NFS exports from vars.haStorageRoot (XFS-over-DRBD
mount); nfs-server.service.wantedBy force-cleared so Pacemaker controls
start/stop on the Active node only
New hosts
hosts/ha-server-{1,2}/host.nix — static IP from vars, unique hostId; sops
secrets (beszel, corosync authkey) are TODOs pending sync-host-keys.sh
variables.nix
haServer1/2Host, haServer1/2Ip, haServerVip, haStorageRoot, haIscsiIqn
ports.haServerDrbd/Iscsi/Corosync{1,2,Crypto}/PacemakerRemoted/Pcsd
scripts/ha/ (migrated + updated from test-lab/ha/)
cluster-init.sh — generates corosync authkey, initialises DRBD/XFS/iSCSI,
creates NFS dataset dirs, configures Pacemaker with DRBD + XFS + iSCSI
+ nfs-server + VIP; STONITH disabled initially (enable separately)
cluster-enable-stonith.sh — enables fence_pve_ssh STONITH after key deploy
fence-pve-ssh.py — Proxmox SSH fence agent (node names updated to ha-server-1/2)
acceptance-tests.sh — T1–T7 production acceptance tests
test-lab/ha/ removed — all Nix config moved to modules/ha/ and
modules/build-types/; scripts moved to scripts/ha/
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
87 lines
4.4 KiB
Bash
87 lines
4.4 KiB
Bash
#!/usr/bin/env bash
|
|
# cluster-enable-stonith.sh — enable STONITH fence agent after the fence SSH
|
|
# key is deployed to both nodes and authorised on the Proxmox host.
|
|
#
|
|
# Run from ha-server-1 as root AFTER:
|
|
# - /etc/pacemaker/fence_pve_ssh exists on both nodes (chmod +x)
|
|
# (copy from scripts/ha/fence-pve-ssh.py)
|
|
# - /etc/fence-pve-ssh-key (SSH private key) exists on both nodes
|
|
# - The corresponding public key is in authorized_keys on PVE_HOST
|
|
# - VMID_NODE1 / VMID_NODE2 filled in below
|
|
set -euo pipefail
|
|
|
|
# ── Configuration ─────────────────────────────────────────────────────────
|
|
NODE1="ha-server-1"
|
|
NODE2="ha-server-2"
|
|
VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1
|
|
VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2
|
|
PVE_HOST="pve1.sweet.home"
|
|
PVE_USER="wayne"
|
|
FENCE_KEY="/etc/fence-pve-ssh-key"
|
|
FENCE_SCRIPT="/etc/pacemaker/fence_pve_ssh"
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
|
|
log() { echo "[stonith-setup] $*"; }
|
|
die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; }
|
|
|
|
[[ $(id -u) -eq 0 ]] || die "must run as root"
|
|
[[ -n "$VMID_NODE1" ]] || die "VMID_NODE1 not set — edit this script"
|
|
[[ -n "$VMID_NODE2" ]] || die "VMID_NODE2 not set — edit this script"
|
|
[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY"
|
|
[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT"
|
|
|
|
log "Verifying fence agent can reach ${PVE_HOST}..."
|
|
ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \
|
|
-o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" \
|
|
"sudo /usr/sbin/qm list" &>/dev/null \
|
|
|| die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo"
|
|
log "Fence agent SSH connectivity confirmed"
|
|
|
|
log "Creating Pacemaker STONITH resources..."
|
|
cibadmin --create --scope resources --xml-text "
|
|
<primitive id=\"stonith-${NODE1}\" class=\"stonith\" type=\"external/fence_pve_ssh\">
|
|
<instance_attributes id=\"stonith-${NODE1}-attrs\">
|
|
<nvpair id=\"stonith-${NODE1}-plug\" name=\"plug\" value=\"${NODE1}\"/>
|
|
<nvpair id=\"stonith-${NODE1}-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
|
|
<nvpair id=\"stonith-${NODE1}-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
|
|
<nvpair id=\"stonith-${NODE1}-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
|
|
<nvpair id=\"stonith-${NODE1}-vmid1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
|
|
<nvpair id=\"stonith-${NODE1}-vmid2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
|
|
<nvpair id=\"stonith-${NODE1}-host-list\" name=\"pcmk_host_list\" value=\"${NODE1}\"/>
|
|
</instance_attributes>
|
|
<operations>
|
|
<op id=\"stonith-${NODE1}-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
|
|
</operations>
|
|
</primitive>
|
|
" 2>/dev/null || true
|
|
|
|
cibadmin --create --scope resources --xml-text "
|
|
<primitive id=\"stonith-${NODE2}\" class=\"stonith\" type=\"external/fence_pve_ssh\">
|
|
<instance_attributes id=\"stonith-${NODE2}-attrs\">
|
|
<nvpair id=\"stonith-${NODE2}-plug\" name=\"plug\" value=\"${NODE2}\"/>
|
|
<nvpair id=\"stonith-${NODE2}-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
|
|
<nvpair id=\"stonith-${NODE2}-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
|
|
<nvpair id=\"stonith-${NODE2}-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
|
|
<nvpair id=\"stonith-${NODE2}-vmid1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
|
|
<nvpair id=\"stonith-${NODE2}-vmid2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
|
|
<nvpair id=\"stonith-${NODE2}-host-list\" name=\"pcmk_host_list\" value=\"${NODE2}\"/>
|
|
</instance_attributes>
|
|
<operations>
|
|
<op id=\"stonith-${NODE2}-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
|
|
</operations>
|
|
</primitive>
|
|
" 2>/dev/null || true
|
|
|
|
log "Enabling STONITH and restoring quorum policy..."
|
|
crm_attribute -t crm_config -n stonith-enabled -v true
|
|
crm_attribute -t crm_config -n no-quorum-policy -v stop
|
|
|
|
log "DRBD fencing mode must also be updated to resource-only (already the"
|
|
log "default in cluster-config.nix; confirm with: cat /etc/drbd.d/ha-data.conf)"
|
|
|
|
log "Testing fence agent..."
|
|
stonith_admin --list-devices && log "Fence devices listed successfully." \
|
|
|| warn "stonith_admin --list-devices failed — check config"
|
|
|
|
log "STONITH enabled. Cluster is now fully HA."
|