fix(ha): add DRBD sync progress indicator and pre-flight sync gate
Check NixOS configurations / eval-hosts (push) Successful in 10m22s

cluster-init.sh: replace the fixed 300-iteration sync wait with an
indefinite loop that prints an in-place progress line (%done, ETA, speed)
from /proc/drbd every 3 s. Clears the line with printf \r before logging
completion, so the output stays clean alongside the [cluster-init] log
lines.

acceptance-tests.sh: add a pre-flight check that hard-exits if
drbdadm dstate is not UpToDate/UpToDate, with a hint to the watch
command for monitoring progress. Tests cannot give accurate results
while the initial full-sync is in progress.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-07-29 00:46:57 +10:00
co-authored by Claude Sonnet 4.6
parent c8d4440787
commit e085d4707c
2 changed files with 39 additions and 7 deletions
+19
View File
@@ -34,6 +34,25 @@ echo "════════════════════════
echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
echo "════════════════════════════════════════════════════"
# ── Pre-flight: DRBD sync must be complete ────────────────────────────────
# Tests that check disk state, XFS mount, and iSCSI will fail or give false
# results while the initial full sync is in progress. Block until done.
echo ""
echo "Pre-flight: verifying DRBD sync is complete..."
DRBD_PREFLIGHT=$(n1 "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
if ! echo "$DRBD_PREFLIGHT" | grep -q "^UpToDate/UpToDate$"; then
echo ""
echo " ERROR: DRBD initial sync not complete."
echo " Current dstate on $NODE1: $DRBD_PREFLIGHT"
echo ""
echo " Monitor progress:"
echo " ssh nixos@$NODE1_IP 'sudo watch -n3 cat /proc/drbd'"
echo ""
echo " Re-run this script once dstate shows UpToDate/UpToDate."
exit 1
fi
echo " dstate: $DRBD_PREFLIGHT — ready."
# ── Detect Active/Standby nodes ────────────────────────────────────────────
# Use crm_mon to detect which node holds the Promoted (Primary) DRBD resource.
# Pacemaker is authoritative; DRBD role can briefly read as Secondary while
+20 -7
View File
@@ -239,15 +239,28 @@ crm_standby -N "$NODE2" -v off 2>/dev/null || true
log "Forcing $NODE1 to DRBD Primary for initial sync..."
drbdadm primary ha-data --force
log "Waiting for DRBD to finish initial sync (this may take several minutes)..."
for i in $(seq 1 300); do
state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
if echo "$state" | grep -q "UpToDate/UpToDate"; then
log "DRBD sync complete: $state"
log "Waiting for DRBD initial sync to complete (32 GB may take 1020 min)..."
_sync_spin=0
_sync_chars=('|' '/' '-' $'\\')
while true; do
_dstate=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
if echo "$_dstate" | grep -q "UpToDate/UpToDate"; then
printf "\r%-80s\r" "" # clear progress line
log "DRBD initial sync complete (dstate: $_dstate)"
break
fi
[[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)"
sleep 1
_pct=$(grep -oE "sync'ed: [0-9.]+%" /proc/drbd 2>/dev/null | grep -oE "[0-9.]+" | head -1 || echo "")
_eta=$(grep -oE "finish: [0-9:]+" /proc/drbd 2>/dev/null | sed 's/finish: //' | head -1 || echo "")
_spd=$(grep -oE "speed: [0-9,]+" /proc/drbd 2>/dev/null | sed 's/speed: //' | head -1 || echo "")
_sc="${_sync_chars[$_sync_spin]}"
_sync_spin=$(( (_sync_spin + 1) % 4 ))
if [[ -n "$_pct" ]]; then
printf "\r [%s] syncing: %s%% done — ETA %s @ %s K/s " \
"$_sc" "$_pct" "${_eta:-??:??:??}" "${_spd:-?}"
else
printf "\r [%s] waiting for sync to start (dstate: %s) " "$_sc" "$_dstate"
fi
sleep 3
done
# ── 3. XFS filesystem ─────────────────────────────────────────────────────