#!/usr/bin/env bash
# cluster-init.sh — one-time HA cluster initialisation script
#
# Run this ONCE from node1 AFTER both VMs are booted and have SSH access.
# It:
# 1. Waits for corosync quorum on both nodes
# 2. Initialises DRBD metadata and promotes node1 to primary
# 3. Creates XFS filesystem on /dev/drbd0
# 4. Configures targetcli / LIO iSCSI target (with a file-backed LUN)
# 5. Configures the Pacemaker resource group
# 6. Optionally enables the STONITH fence agent (requires fence SSH key)
#
# Prerequisites:
# - Both VMs booted with the ha-test config
# - fence-pve-ssh-key distributed to /etc/fence-pve-ssh-key on both nodes
# - Run as root on ha-test-node1
set -euo pipefail
NODE1_IP="192.168.2.200"
NODE2_IP="192.168.2.201"
VIP="192.168.2.202"
DRBD_DEVICE="/dev/drbd0"
XFS_MOUNT="/mnt/ha-data"
ISCSI_IQN="iqn.2026-01.local.ha-test:storage"
ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
ISCSI_LUN_SIZE="1G" # small test LUN
VMID_NODE1="200"
VMID_NODE2="201"
PVE_HOST="pve1.sweet.home"
PVE_USER="wayne"
FENCE_KEY="/etc/fence-pve-ssh-key"
log() { echo "[cluster-init] $*"; }
die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; }
[[ $(id -u) -eq 0 ]] || die "must run as root"
[[ "$(hostname)" == "ha-test-node1" ]] || die "must run on ha-test-node1"
# ── 1. Wait for corosync quorum ──────────────────────────────────────────
log "Waiting for corosync quorum..."
for i in $(seq 1 30); do
if corosync-quorumtool -q &>/dev/null; then
log "Quorum established"
break
fi
[[ $i -eq 30 ]] && die "corosync quorum not established after 30s"
sleep 2
done
log "Waiting for pacemaker to start..."
for i in $(seq 1 30); do
if crm_mon -1 &>/dev/null; then
log "Pacemaker running"
break
fi
[[ $i -eq 30 ]] && die "pacemaker not running after 60s"
sleep 2
done
# ── 2. Initialise DRBD ───────────────────────────────────────────────────
log "Initialising DRBD metadata on node1..."
if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent"; then
drbdadm create-md ha-data --force
fi
log "Initialising DRBD metadata on node2..."
if ! ssh "root@${NODE2_IP}" "drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent'"; then
ssh "root@${NODE2_IP}" "drbdadm create-md ha-data --force"
fi
log "Bringing up DRBD on both nodes..."
drbdadm up ha-data || true
ssh "root@${NODE2_IP}" "drbdadm up ha-data" || true
log "Forcing node1 to DRBD primary (initial sync)..."
drbdadm primary ha-data --force
log "Waiting for DRBD to finish initial sync..."
for i in $(seq 1 120); do
state=$(drbdadm dstate ha-data)
if echo "$state" | grep -q "UpToDate"; then
log "DRBD sync complete: $state"
break
fi
log " DRBD state: $state (${i}/120s)"
[[ $i -eq 120 ]] && die "DRBD did not sync within 120s"
sleep 1
done
# ── 3. XFS filesystem ────────────────────────────────────────────────────
log "Creating XFS on ${DRBD_DEVICE}..."
if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
mkfs.xfs "${DRBD_DEVICE}"
fi
log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
mkdir -p "${XFS_MOUNT}"
mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
# ── 4. iSCSI LUN (file-backed) ───────────────────────────────────────────
log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..."
if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then
fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}"
fi
log "Configuring LIO iSCSI target via targetcli..."
# This produces a /etc/target/saveconfig.json that the targetctl service loads.
# The commands create an iSCSI target backed by the file we just created.
targetcli <<'EOF'
/backstores/fileio create name=ha-lun0 file_or_dev=/mnt/ha-data/iscsi-lun.img size=0 write_back=false
/iscsi create iqn.2026-01.local.ha-test:storage
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/luns create /backstores/fileio/ha-lun0
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1/portals create 192.168.2.202
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute authentication=0
/iscsi/iqn.2026-01.local.ha-test:storage/tpg1 set attribute demo_mode_write_protect=0
saveconfig /etc/target/saveconfig.json
EOF
log "Unmounting ${XFS_MOUNT} (Pacemaker will manage it)..."
umount "${XFS_MOUNT}"
log "Promoting DRBD back to secondary (Pacemaker manages primary role)..."
drbdadm secondary ha-data
# ── 5. Pacemaker resources ───────────────────────────────────────────────
log "Configuring Pacemaker..."
# Disable STONITH initially — enable once fence key is deployed
crm_attribute -t crm_config -n stonith-enabled -v false
# Disable quorum policy for two-node cluster (no-quorum-policy=ignore so
# the surviving node can promote without a quorum device)
crm_attribute -t crm_config -n no-quorum-policy -v ignore
# Cluster resources:
# 1. drbd-ha — manages DRBD primary/secondary role
# 2. xfs-mount — XFS mount on /mnt/ha-data
# 3. iscsi-target — LIO target service (systemd class)
# 4. vip — floating VIP 192.168.2.202
log "Creating DRBD master/slave resource..."
cibadmin --replace --scope resources --xml-text "
"
log "Adding ordering and colocation constraints..."
# All resources on the same node as DRBD master
cibadmin --create --scope constraints --xml-text "
"
log "Resource group configured. Waiting for resources to start..."
for i in $(seq 1 60); do
if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then
log "VIP is up: $(crm_resource -r vip --locate)"
break
fi
[[ $i -eq 60 ]] && { log "WARNING: VIP not up after 60s — check crm_mon"; break; }
sleep 2
done
log ""
log "═══════════════════════════════════════════════════════"
log " HA cluster initialised. Next steps:"
log ""
log " - Verify: crm_mon -1"
log " - Test iSCSI: iscsiadm -m discovery -t sendtargets -p ${VIP}"
log ""
log " To enable STONITH (after deploying fence key):"
log " 1. Copy fence-pve-ssh.py to /usr/lib/ocf/resource.d/heartbeat/ on both nodes"
log " 2. Distribute /etc/fence-pve-ssh-key to both nodes"
log " 3. Add public key to authorized_keys on ${PVE_HOST}"
log " 4. Run: ./cluster-enable-stonith.sh"
log "═══════════════════════════════════════════════════════"