Archived
fix(ha): add DRBD sync progress indicator and pre-flight sync gate
Check NixOS configurations / eval-hosts (push) Successful in 10m22s
Check NixOS configurations / eval-hosts (push) Successful in 10m22s
cluster-init.sh: replace the fixed 300-iteration sync wait with an indefinite loop that prints an in-place progress line (%done, ETA, speed) from /proc/drbd every 3 s. Clears the line with printf \r before logging completion, so the output stays clean alongside the [cluster-init] log lines. acceptance-tests.sh: add a pre-flight check that hard-exits if drbdadm dstate is not UpToDate/UpToDate, with a hint to the watch command for monitoring progress. Tests cannot give accurate results while the initial full-sync is in progress. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -34,6 +34,25 @@ echo "════════════════════════
|
||||
echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "════════════════════════════════════════════════════"
|
||||
|
||||
# ── Pre-flight: DRBD sync must be complete ────────────────────────────────
|
||||
# Tests that check disk state, XFS mount, and iSCSI will fail or give false
|
||||
# results while the initial full sync is in progress. Block until done.
|
||||
echo ""
|
||||
echo "Pre-flight: verifying DRBD sync is complete..."
|
||||
DRBD_PREFLIGHT=$(n1 "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
|
||||
if ! echo "$DRBD_PREFLIGHT" | grep -q "^UpToDate/UpToDate$"; then
|
||||
echo ""
|
||||
echo " ERROR: DRBD initial sync not complete."
|
||||
echo " Current dstate on $NODE1: $DRBD_PREFLIGHT"
|
||||
echo ""
|
||||
echo " Monitor progress:"
|
||||
echo " ssh nixos@$NODE1_IP 'sudo watch -n3 cat /proc/drbd'"
|
||||
echo ""
|
||||
echo " Re-run this script once dstate shows UpToDate/UpToDate."
|
||||
exit 1
|
||||
fi
|
||||
echo " dstate: $DRBD_PREFLIGHT — ready."
|
||||
|
||||
# ── Detect Active/Standby nodes ────────────────────────────────────────────
|
||||
# Use crm_mon to detect which node holds the Promoted (Primary) DRBD resource.
|
||||
# Pacemaker is authoritative; DRBD role can briefly read as Secondary while
|
||||
|
||||
@@ -239,15 +239,28 @@ crm_standby -N "$NODE2" -v off 2>/dev/null || true
|
||||
log "Forcing $NODE1 to DRBD Primary for initial sync..."
|
||||
drbdadm primary ha-data --force
|
||||
|
||||
log "Waiting for DRBD to finish initial sync (this may take several minutes)..."
|
||||
for i in $(seq 1 300); do
|
||||
state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
|
||||
if echo "$state" | grep -q "UpToDate/UpToDate"; then
|
||||
log "DRBD sync complete: $state"
|
||||
log "Waiting for DRBD initial sync to complete (32 GB may take 10–20 min)..."
|
||||
_sync_spin=0
|
||||
_sync_chars=('|' '/' '-' $'\\')
|
||||
while true; do
|
||||
_dstate=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
|
||||
if echo "$_dstate" | grep -q "UpToDate/UpToDate"; then
|
||||
printf "\r%-80s\r" "" # clear progress line
|
||||
log "DRBD initial sync complete (dstate: $_dstate)"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)"
|
||||
sleep 1
|
||||
_pct=$(grep -oE "sync'ed: [0-9.]+%" /proc/drbd 2>/dev/null | grep -oE "[0-9.]+" | head -1 || echo "")
|
||||
_eta=$(grep -oE "finish: [0-9:]+" /proc/drbd 2>/dev/null | sed 's/finish: //' | head -1 || echo "")
|
||||
_spd=$(grep -oE "speed: [0-9,]+" /proc/drbd 2>/dev/null | sed 's/speed: //' | head -1 || echo "")
|
||||
_sc="${_sync_chars[$_sync_spin]}"
|
||||
_sync_spin=$(( (_sync_spin + 1) % 4 ))
|
||||
if [[ -n "$_pct" ]]; then
|
||||
printf "\r [%s] syncing: %s%% done — ETA %s @ %s K/s " \
|
||||
"$_sc" "$_pct" "${_eta:-??:??:??}" "${_spd:-?}"
|
||||
else
|
||||
printf "\r [%s] waiting for sync to start (dstate: %s) " "$_sc" "$_dstate"
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# ── 3. XFS filesystem ─────────────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user