nixos/scripts/ha/cluster-init.sh

#!/usr/bin/env bash
# cluster-init.sh — one-time HA cluster initialisation script
#
# Run ONCE from ha-server-1 as root AFTER both VMs are booted and have SSH
# access.  It:
#   1. Generates and distributes the corosync authkey
#   2. Waits for corosync quorum and pacemaker
#   3. Initialises DRBD metadata, promotes node1 to primary
#   4. Creates XFS on /dev/drbd0 and mounts it
#   5. Creates the directory tree and iSCSI LUN backing file
#   6. Configures LIO iSCSI target (file-backed LUN)
#   7. Configures Pacemaker resources: DRBD → XFS → iSCSI → NFS → VIP
#
# Prerequisites:
#   - Both VMs booted with the ha-server config (nixos-rebuild done)
#   - SSH key access from node1 to root@NODE2_IP
#   - VMID_NODE1 / VMID_NODE2 filled in below (needed for STONITH setup;
#     cluster starts without STONITH, which you enable separately via
#     scripts/ha/cluster-enable-stonith.sh)
#   - Run as root on ha-server-1
set -euo pipefail

# ── Configuration ─────────────────────────────────────────────────────────
# All values override-able via environment variables; defaults match variables.nix.
NODE1="${NODE1:-ha-server-1}"
NODE2="${NODE2:-ha-server-2}"
NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}"  # vars.haIscsiIqn
ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
ISCSI_LUN_SIZE="10G"
DRBD_DEVICE="/dev/drbd0"
VMID_NODE1="${VMID_NODE1:-}"            # set by deploy.sh; needed for STONITH
VMID_NODE2="${VMID_NODE2:-}"
PVE_HOST="${PVE_HOST:-pve1.sweet.home}"
PVE_USER="${PVE_USER:-wayne}"
# Inter-node SSH: HA_USER is the user to SSH as on NODE2; HA_KEY is the private
# key to use.  Default is root-to-root (no key arg).  deploy.sh sets HA_USER=nixos
# and HA_KEY=/tmp/cluster-init-key so the script works even when root-to-root SSH
# is not available.
HA_USER="${HA_USER:-root}"
HA_KEY="${HA_KEY:-}"

# NFS dataset subdirectories to create under XFS_MOUNT.
# Must mirror vars.nfsShares subpath values in variables.nix.
NFS_SUBDIRS=(
  "docker/config"
  "docker/volumes"
  "docker/databases"
  "docker/nextcloud-data"
  "raspi/volumes"
  "proxmox/iso"
  "proxmox/lxc"
  "pxe-boot/images"
)
# ──────────────────────────────────────────────────────────────────────────

log()  { echo "[cluster-init] $*"; }
die()  { echo "[cluster-init] ERROR: $*" >&2; exit 1; }
warn() { echo "[cluster-init] WARNING: $*" >&2; }

[[ $(id -u) -eq 0 ]] || die "must run as root"
[[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1"

# NixOS may not include xfsprogs in root's PATH even when it's in the store.
# If mkfs.xfs is missing, search the Nix store for it.
if ! command -v mkfs.xfs &>/dev/null; then
  _xfs_bin=$(find /nix/store -maxdepth 3 -name mkfs.xfs 2>/dev/null | head -1 | xargs dirname 2>/dev/null || true)
  [[ -n "$_xfs_bin" ]] && export PATH="$_xfs_bin:$PATH" \
    || die "mkfs.xfs not found — add xfsprogs to ha-server.nix environment.systemPackages and rebuild"
fi

# Inter-node SSH/SCP helpers — abstract over root-to-root vs nixos+sudo.
_SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10"
[[ -n "$HA_KEY" ]] && _SSH_OPTS="-i $HA_KEY $_SSH_OPTS"
if [[ "$HA_USER" == "root" ]]; then
  n2_ssh() { ssh $_SSH_OPTS "root@${NODE2_IP}" "$@"; }
  n2_scp() { scp $_SSH_OPTS "$1" "root@${NODE2_IP}:$2"; }
else
  # Non-root user with passwordless sudo; wrap each command with sudo.
  n2_ssh() { ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@"; }
  n2_scp() {
    # SCP to a tmp path, then sudo-move to the real destination as the remote user.
    local src="$1" dst="$2"
    local tmp="/tmp/_cluster_init_scp_$$"
    scp $_SSH_OPTS "$src" "${HA_USER}@${NODE2_IP}:${tmp}"
    ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo mv "${tmp}" "${dst}"
  }
fi

# ── 0. Corosync authkey ───────────────────────────────────────────────────
AUTHKEY="/etc/corosync/authkey"
mkdir -p /etc/corosync
if [[ ! -f "$AUTHKEY" ]]; then
  log "Generating corosync authkey..."
  corosync-keygen -k "$AUTHKEY"
  chmod 0400 "$AUTHKEY"
fi
log "Distributing authkey to $NODE2..."
n2_ssh "mkdir -p /etc/corosync"
n2_scp "$AUTHKEY" "$AUTHKEY"
n2_ssh "chmod 0400 '${AUTHKEY}'"

log "Restarting corosync on both nodes..."
systemctl restart corosync
n2_ssh "systemctl restart corosync"
sleep 3

# ── 1. Corosync quorum ────────────────────────────────────────────────────
log "Waiting for corosync quorum..."
for i in $(seq 1 30); do
  if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then
    log "Quorum established"
    break
  fi
  [[ $i -eq 30 ]] && die "corosync quorum not established after 60 s"
  sleep 2
done

log "Waiting for pacemaker..."
for i in $(seq 1 30); do
  if crm_mon -1 &>/dev/null; then
    log "Pacemaker running"
    break
  fi
  [[ $i -eq 30 ]] && die "pacemaker not running after 60 s"
  sleep 2
done

# ── 2. DRBD initialisation ────────────────────────────────────────────────
log "Initialising DRBD metadata on $NODE1..."
if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then
  drbdadm create-md ha-data --force
fi

log "Initialising DRBD metadata on $NODE2..."
# Use grep -E for ERE alternation inside the remote bash -c string (avoids \| quoting issues).
n2_ssh "bash -c 'drbdadm dstate ha-data 2>/dev/null | grep -qE \"UpToDate|Inconsistent|Diskless\" || drbdadm create-md ha-data --force'"

log "Bringing up DRBD on both nodes..."
drbdadm up ha-data 2>/dev/null || true
n2_ssh "drbdadm up ha-data" 2>/dev/null || true

log "Forcing $NODE1 to DRBD Primary for initial sync..."
drbdadm primary ha-data --force

log "Waiting for DRBD to finish initial sync (this may take several minutes)..."
for i in $(seq 1 300); do
  state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
  if echo "$state" | grep -q "UpToDate/UpToDate"; then
    log "DRBD sync complete: $state"
    break
  fi
  [[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)"
  sleep 1
done

# ── 3. XFS filesystem ─────────────────────────────────────────────────────
log "Creating XFS on ${DRBD_DEVICE}..."
if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
  mkfs.xfs -f "${DRBD_DEVICE}"
fi

log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
mkdir -p "${XFS_MOUNT}"
mountpoint -q "${XFS_MOUNT}" || mount "${DRBD_DEVICE}" "${XFS_MOUNT}"

# ── 4. NFS dataset directories ────────────────────────────────────────────
log "Creating NFS dataset directories..."
for subdir in "${NFS_SUBDIRS[@]}"; do
  mkdir -p "${XFS_MOUNT}/${subdir}"
done

# ── 5. iSCSI LUN backing file ─────────────────────────────────────────────
log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..."
if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then
  fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}"
fi

# ── 6. LIO iSCSI target ───────────────────────────────────────────────────
log "Configuring LIO iSCSI target via targetcli..."
# Note: do NOT bind portal to ${VIP} here — the VIP isn't assigned yet (Pacemaker
# creates it).  The default portal (all IPs, port 3260) is correct; Pacemaker's
# VIP resource will make the target reachable at the VIP address.
#
# Clear any existing LIO state first (idempotent: re-run after a partial failure).
# Use specific delete commands — clearconfig does not reliably clear kernel state.
if ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -q "${ISCSI_IQN}"; then
  log "Clearing existing LIO target ${ISCSI_IQN} before reconfiguration..."
  targetcli "/iscsi delete ${ISCSI_IQN}" 2>/dev/null || true
fi
if ls /sys/kernel/config/target/core/ 2>/dev/null | grep -q "fileio"; then
  log "Clearing existing LIO backstore ha-lun0 before reconfiguration..."
  targetcli "/backstores/fileio delete ha-lun0" 2>/dev/null || true
fi
targetcli <<EOF
/backstores/fileio create name=ha-lun0 file_or_dev=${ISCSI_LUN_FILE} size=0 write_back=false
/iscsi create ${ISCSI_IQN}
/iscsi/${ISCSI_IQN}/tpg1/luns create /backstores/fileio/ha-lun0
/iscsi/${ISCSI_IQN}/tpg1 set attribute authentication=0
/iscsi/${ISCSI_IQN}/tpg1 set attribute demo_mode_write_protect=0
saveconfig /etc/target/saveconfig.json
EOF

log "Tearing down LIO kernel objects — Pacemaker will restore via targetctl on the Active node..."
# LIO holds the backing file open; clear kernel state now so the XFS unmount succeeds.
# Use specific delete commands (clearconfig does not reliably clear kernel configfs state).
targetcli "/iscsi delete ${ISCSI_IQN}" 2>/dev/null || warn "LIO iscsi delete failed — umount may fail"
targetcli "/backstores/fileio delete ha-lun0" 2>/dev/null || warn "LIO backstore delete failed"

log "Distributing iSCSI saveconfig to $NODE2..."
n2_scp /etc/target/saveconfig.json /etc/target/saveconfig.json

log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..."
umount "${XFS_MOUNT}" || { sync; umount -l "${XFS_MOUNT}"; }

log "Demoting DRBD to Secondary — Pacemaker manages primary role..."
[[ "$(drbdadm role ha-data 2>/dev/null)" == "Primary/Secondary" ]] && drbdadm secondary ha-data || true

# ── 7. Pacemaker resources ────────────────────────────────────────────────
log "Configuring Pacemaker cluster properties..."
crm_attribute -t crm_config -n stonith-enabled -v false
crm_attribute -t crm_config -n no-quorum-policy -v ignore

log "Creating Pacemaker resources via crm configure..."
# Use crm configure (schema-aware) instead of raw cibadmin XML to avoid
# pacemaker-4.0 schema incompatibilities with direct clone attributes.
# --force skips the interactive prompt; crm configure exits 0 on success.
crm configure <<'CRM_EOF'
primitive drbd0 ocf:linbit:drbd \
  params drbd_resource=ha-data \
  op start timeout=240s interval=0 \
  op stop timeout=120s interval=0 \
  op promote timeout=90s interval=0 \
  op demote timeout=90s interval=0 \
  op monitor interval=20s timeout=20s role=Promoted \
  op monitor interval=30s timeout=20s role=Unpromoted

clone ms-drbd0 drbd0 \
  meta promotable=true promoted-max=1 promoted-node-max=1 \
       clone-max=2 clone-node-max=1 \
       notify=true interleave=true globally-unique=false

primitive xfs-data ocf:heartbeat:Filesystem \
  params device=/dev/drbd0 directory=/srv/ha-data fstype=xfs options=defaults \
         force_unmount=false \
  op start timeout=60s interval=0 \
  op stop timeout=60s interval=0 \
  op monitor interval=20s timeout=40s

primitive iscsi-target systemd:targetctl \
  op start timeout=60s interval=0 \
  op stop timeout=60s interval=0 \
  op monitor interval=20s timeout=40s

primitive nfs-server systemd:nfs-server \
  op start timeout=60s interval=0 \
  op stop timeout=60s interval=0 \
  op monitor interval=30s timeout=40s

primitive vip ocf:heartbeat:IPaddr2 \
  params ip=192.168.2.229 cidr_netmask=24 \
  op start timeout=20s interval=0 \
  op stop timeout=20s interval=0 \
  op monitor interval=10s timeout=20s

group ha-group xfs-data iscsi-target nfs-server vip

order order-drbd-group Mandatory: ms-drbd0:promote ha-group:start
colocation coloc-group-with-drbd INFINITY: ha-group ms-drbd0:Promoted
commit
CRM_EOF

log "Waiting for resources to start..."
for i in $(seq 1 60); do
  if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then
    log "VIP is up: $(crm_resource -r vip --locate)"
    break
  fi
  [[ $i -eq 60 ]] && { warn "VIP not up after 120 s — check: crm_mon -1"; break; }
  sleep 2
done

log ""
log "═══════════════════════════════════════════════════════════════"
log " HA cluster initialised."
log ""
log "  crm_mon -1                         — cluster status"
log "  iscsiadm -m discovery -t st -p ${VIP}  — verify iSCSI target"
log "  showmount -e ${VIP}                    — verify NFS exports"
log ""
log "  To enable STONITH (after deploying fence SSH key):"
log "    1. Fill in VMID_NODE1 / VMID_NODE2 in cluster-enable-stonith.sh"
log "    2. Copy scripts/ha/fence-pve-ssh.py to /etc/pacemaker/fence_pve_ssh"
log "       on both nodes (chmod +x)"
log "    3. Generate and distribute the fence SSH key"
log "       (see docs or cluster-enable-stonith.sh header)"
log "    4. bash scripts/ha/cluster-enable-stonith.sh"
log "═══════════════════════════════════════════════════════════════"