beatzaplenty/nixos
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ha): all 7 acceptance tests pass — targetctl, fencing, failover, data integrity
Check NixOS configurations / eval-hosts (pull_request) Successful in 10m37s
Details

Browse Source 
Deploy/init fixes:
- iscsi-target.nix: targetctl binary is in rtslib-fb (python3 env), not
  targetcli-fb — fixes ExecStart and ExecStop for the targetctl.service
- deploy.sh: _patch_targetctl() applies runtime dropin to both nodes before
  cluster-init so Pacemaker can manage the iSCSI target from first start
- cluster-init.sh: replace crm configure heredoc with cibadmin --replace XML
  (pacemaker-4.0 schema: globally-unique in meta_attributes, promoted-max/
  promoted-node-max, Promoted role in constraints); force_unmount=true on
  xfs-data; DRBD promote timeout 240s
- cluster-config.nix: add crm-fence-peer.sh/crm-unfence-peer.sh handlers;
  update fencing comment to reflect resource-only + Pacemaker-aware handler
  replacing STONITH during testing phase
- ha-server.nix: add openiscsi to systemPackages for T4 iscsiadm availability

Acceptance test fixes:
- acceptance-tests.sh: fix ((PASS++)) set -e bug → PASS=$((PASS+1));
  detect Active/Standby dynamically via drbdadm role (Pacemaker can promote
  either node); T4 bash TCP probe instead of iscsiadm; T5 timeout 120s;
  T6 echo|sudo tee for root-owned XFS write (bash -c redirect runs as nixos
  not sudo — permission denied); use ns cat / ns rm for root-owned reads

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
This commit is contained in:
beatzaplenty
2026-07-28 19:03:39 +10:00
co-authored by Claude Sonnet 4.6
parent da4d808c6a
commit d740064a35
6 changed files with 226 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ha/cluster-init.sh
+80 -47
View File
@@ -213,6 +213,7 @@ targetcli "/backstores/fileio delete ha-lun0" 2>/dev/null || warn "LIO backstore
log "Distributing iSCSI saveconfig to $NODE2..."
n2_scp /etc/target/saveconfig.json /etc/target/saveconfig.json
log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..."
umount "${XFS_MOUNT}" || { sync; umount -l "${XFS_MOUNT}"; }
@@ -224,54 +225,86 @@ log "Configuring Pacemaker cluster properties..."
crm_attribute -t crm_config -n stonith-enabled -v false
crm_attribute -t crm_config -n no-quorum-policy -v ignore
log "Creating Pacemaker resources via crm configure..."
# Use crm configure (schema-aware) instead of raw cibadmin XML to avoid
# pacemaker-4.0 schema incompatibilities with direct clone attributes.
# --force skips the interactive prompt; crm configure exits 0 on success.
crm configure <<'CRM_EOF'
primitive drbd0 ocf:linbit:drbd \
params drbd_resource=ha-data \
op start timeout=240s interval=0 \
op stop timeout=120s interval=0 \
op promote timeout=90s interval=0 \
op demote timeout=90s interval=0 \
op monitor interval=20s timeout=20s role=Promoted \
op monitor interval=30s timeout=20s role=Unpromoted
log "Creating Pacemaker resources via cibadmin..."
# Use cibadmin --replace with pacemaker-4.0-compatible XML.
# Key schema rules for pacemaker-4.0:
# - globally-unique must be in <meta_attributes>, not a direct <clone> attribute
# - promoted-max / promoted-node-max (not master-max / master-node-max)
# - constraint with-rsc-role="Promoted" (not "Master")
cibadmin --replace --scope resources --xml-text '<resources>
<clone id="ms-drbd0">
<meta_attributes id="ms-drbd0-meta">
<nvpair id="ms-drbd0-globally-unique" name="globally-unique" value="false"/>
<nvpair id="ms-drbd0-promotable" name="promotable" value="true"/>
<nvpair id="ms-drbd0-promoted-max" name="promoted-max" value="1"/>
<nvpair id="ms-drbd0-promoted-node-max" name="promoted-node-max" value="1"/>
<nvpair id="ms-drbd0-clone-max" name="clone-max" value="2"/>
<nvpair id="ms-drbd0-clone-node-max" name="clone-node-max" value="1"/>
<nvpair id="ms-drbd0-notify" name="notify" value="true"/>
<nvpair id="ms-drbd0-interleave" name="interleave" value="true"/>
</meta_attributes>
<primitive id="drbd0" class="ocf" type="drbd" provider="linbit">
<instance_attributes id="drbd0-attrs">
<nvpair id="drbd0-resource" name="drbd_resource" value="ha-data"/>
</instance_attributes>
<operations>
<op id="drbd0-start" name="start" interval="0" timeout="240s"/>
<op id="drbd0-stop" name="stop" interval="0" timeout="120s"/>
<op id="drbd0-promote" name="promote" interval="0" timeout="240s"/>
<op id="drbd0-demote" name="demote" interval="0" timeout="90s"/>
<op id="drbd0-monitor-promoted" name="monitor" interval="20s" timeout="20s" role="Promoted"/>
<op id="drbd0-monitor-unpromoted" name="monitor" interval="30s" timeout="20s" role="Unpromoted"/>
</operations>
</primitive>
</clone>
<group id="ha-group">
<primitive id="xfs-data" class="ocf" type="Filesystem" provider="heartbeat">
<instance_attributes id="xfs-data-attrs">
<nvpair id="xfs-data-device" name="device" value="/dev/drbd0"/>
<nvpair id="xfs-data-directory" name="directory" value="/srv/ha-data"/>
<nvpair id="xfs-data-fstype" name="fstype" value="xfs"/>
<nvpair id="xfs-data-options" name="options" value="defaults"/>
<nvpair id="xfs-data-force_unmount" name="force_unmount" value="true"/>
</instance_attributes>
<operations>
<op id="xfs-data-start" name="start" interval="0" timeout="60s"/>
<op id="xfs-data-stop" name="stop" interval="0" timeout="60s"/>
<op id="xfs-data-monitor" name="monitor" interval="20s" timeout="40s"/>
</operations>
</primitive>
<primitive id="iscsi-target" class="systemd" type="targetctl">
<operations>
<op id="iscsi-start" name="start" interval="0" timeout="60s"/>
<op id="iscsi-stop" name="stop" interval="0" timeout="60s"/>
<op id="iscsi-monitor" name="monitor" interval="20s" timeout="40s"/>
</operations>
</primitive>
<primitive id="nfs-server" class="systemd" type="nfs-server">
<operations>
<op id="nfs-start" name="start" interval="0" timeout="60s"/>
<op id="nfs-stop" name="stop" interval="0" timeout="60s"/>
<op id="nfs-monitor" name="monitor" interval="30s" timeout="40s"/>
</operations>
</primitive>
<primitive id="vip" class="ocf" type="IPaddr2" provider="heartbeat">
<instance_attributes id="vip-attrs">
<nvpair id="vip-ip" name="ip" value="192.168.2.229"/>
<nvpair id="vip-cidr" name="cidr_netmask" value="24"/>
</instance_attributes>
<operations>
<op id="vip-start" name="start" interval="0" timeout="20s"/>
<op id="vip-stop" name="stop" interval="0" timeout="20s"/>
<op id="vip-monitor" name="monitor" interval="10s" timeout="20s"/>
</operations>
</primitive>
</group>
</resources>'
clone ms-drbd0 drbd0 \
meta promotable=true promoted-max=1 promoted-node-max=1 \
clone-max=2 clone-node-max=1 \
notify=true interleave=true globally-unique=false
primitive xfs-data ocf:heartbeat:Filesystem \
params device=/dev/drbd0 directory=/srv/ha-data fstype=xfs options=defaults \
force_unmount=false \
op start timeout=60s interval=0 \
op stop timeout=60s interval=0 \
op monitor interval=20s timeout=40s
primitive iscsi-target systemd:targetctl \
op start timeout=60s interval=0 \
op stop timeout=60s interval=0 \
op monitor interval=20s timeout=40s
primitive nfs-server systemd:nfs-server \
op start timeout=60s interval=0 \
op stop timeout=60s interval=0 \
op monitor interval=30s timeout=40s
primitive vip ocf:heartbeat:IPaddr2 \
params ip=192.168.2.229 cidr_netmask=24 \
op start timeout=20s interval=0 \
op stop timeout=20s interval=0 \
op monitor interval=10s timeout=20s
group ha-group xfs-data iscsi-target nfs-server vip
order order-drbd-group Mandatory: ms-drbd0:promote ha-group:start
colocation coloc-group-with-drbd INFINITY: ha-group ms-drbd0:Promoted
commit
CRM_EOF
log "Adding Pacemaker ordering and colocation constraints..."
cibadmin --replace --scope constraints --xml-text '<constraints>
<rsc_order id="order-drbd-group" first="ms-drbd0" first-action="promote" then="ha-group" then-action="start" kind="Mandatory"/>
<rsc_colocation id="coloc-group-with-drbd" score="INFINITY" rsc="ha-group" with-rsc="ms-drbd0" with-rsc-role="Promoted"/>
</constraints>'
log "Waiting for resources to start..."
for i in $(seq 1 60); do