diff --git a/scripts/ha/acceptance-tests.sh b/scripts/ha/acceptance-tests.sh index 6b10d3c..c405038 100755 --- a/scripts/ha/acceptance-tests.sh +++ b/scripts/ha/acceptance-tests.sh @@ -34,6 +34,25 @@ echo "════════════════════════ echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')" echo "════════════════════════════════════════════════════" +# ── Pre-flight: DRBD sync must be complete ──────────────────────────────── +# Tests that check disk state, XFS mount, and iSCSI will fail or give false +# results while the initial full sync is in progress. Block until done. +echo "" +echo "Pre-flight: verifying DRBD sync is complete..." +DRBD_PREFLIGHT=$(n1 "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown") +if ! echo "$DRBD_PREFLIGHT" | grep -q "^UpToDate/UpToDate$"; then + echo "" + echo " ERROR: DRBD initial sync not complete." + echo " Current dstate on $NODE1: $DRBD_PREFLIGHT" + echo "" + echo " Monitor progress:" + echo " ssh nixos@$NODE1_IP 'sudo watch -n3 cat /proc/drbd'" + echo "" + echo " Re-run this script once dstate shows UpToDate/UpToDate." + exit 1 +fi +echo " dstate: $DRBD_PREFLIGHT — ready." + # ── Detect Active/Standby nodes ──────────────────────────────────────────── # Use crm_mon to detect which node holds the Promoted (Primary) DRBD resource. # Pacemaker is authoritative; DRBD role can briefly read as Secondary while diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh index cb0130d..f222b8f 100755 --- a/scripts/ha/cluster-init.sh +++ b/scripts/ha/cluster-init.sh @@ -239,15 +239,28 @@ crm_standby -N "$NODE2" -v off 2>/dev/null || true log "Forcing $NODE1 to DRBD Primary for initial sync..." drbdadm primary ha-data --force -log "Waiting for DRBD to finish initial sync (this may take several minutes)..." -for i in $(seq 1 300); do - state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown") - if echo "$state" | grep -q "UpToDate/UpToDate"; then - log "DRBD sync complete: $state" +log "Waiting for DRBD initial sync to complete (32 GB may take 10–20 min)..." +_sync_spin=0 +_sync_chars=('|' '/' '-' $'\\') +while true; do + _dstate=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown") + if echo "$_dstate" | grep -q "UpToDate/UpToDate"; then + printf "\r%-80s\r" "" # clear progress line + log "DRBD initial sync complete (dstate: $_dstate)" break fi - [[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)" - sleep 1 + _pct=$(grep -oE "sync'ed: [0-9.]+%" /proc/drbd 2>/dev/null | grep -oE "[0-9.]+" | head -1 || echo "") + _eta=$(grep -oE "finish: [0-9:]+" /proc/drbd 2>/dev/null | sed 's/finish: //' | head -1 || echo "") + _spd=$(grep -oE "speed: [0-9,]+" /proc/drbd 2>/dev/null | sed 's/speed: //' | head -1 || echo "") + _sc="${_sync_chars[$_sync_spin]}" + _sync_spin=$(( (_sync_spin + 1) % 4 )) + if [[ -n "$_pct" ]]; then + printf "\r [%s] syncing: %s%% done — ETA %s @ %s K/s " \ + "$_sc" "$_pct" "${_eta:-??:??:??}" "${_spd:-?}" + else + printf "\r [%s] waiting for sync to start (dstate: %s) " "$_sc" "$_dstate" + fi + sleep 3 done # ── 3. XFS filesystem ─────────────────────────────────────────────────────