fix(ha): resolve data disk device via by-id symlink even in dry-run

The by-id lookup is read-only so it's safe to run in dry-run mode.
Previously it was gated behind `if ! $DRY_RUN`, which always triggered
the sdb fallback warning in dry-run — making it look like the device
path wasn't reliable when the symlink actually exists on both servers.

Now the lookup always runs and the script errors out with a clear message
if the by-id symlink is genuinely missing, instead of silently falling
back to a guessed /dev/sd* name.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scripts/gc-hosts.sh

@@ -14,8 +14,8 @@
 # "nix-collect-garbage -d" if sudo needs a password — still collects
 # unreferenced store paths and old nixos-user profile generations, but leaves
 # old system generations in place.
-# pve1: runs "bash -l -c nix-collect-garbage -d" as the login user so
-# /etc/profile is sourced and the Nix daemon's PATH is set up automatically.
+# pve1: runs "nix-collect-garbage -d" as the login user (no system generations
+# on a non-NixOS host).
 #
 # Usage (from repo root):
 #   bash scripts/gc-hosts.sh [--dry-run]
@@ -30,12 +30,7 @@ source scripts/lib/nix-eval.sh 2>/dev/null || true
 : "${MAX_JOBS:=8}"
 : "${NIXOS_USER:=nixos}"
 : "${PVE1_SSH_USER:=${PROXMOX_SSH_USER:-wayne}}"
-
-# GC connections use BatchMode — no interactive prompts, just succeed or fail.
 SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10)
-# Discovery connections do NOT use BatchMode so that sudo can prompt if needed
-# (pct/qm list require root access on Proxmox).
-SSH_QUERY_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=10)
 
 DRY_RUN=0
 for arg in "$@"; do
@@ -49,7 +44,7 @@ done
 
 # ORDERED_HOSTS: names in display/execution order.
 # HOST_TARGET[name]: SSH target string (user@host).
-# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (login-shell gc).
+# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (user gc only).
 declare -a  ORDERED_HOSTS=()
 declare -A  HOST_TARGET=()
 declare -A  HOST_TYPE=()
@@ -83,10 +78,10 @@ if ! hostname_map="$(
   echo "  warning: flake eval failed — skipping dynamic host discovery" >&2
 fi
 
-# SSH_QUERY_OPTS (no BatchMode) so sudo can prompt if wayne's sudo needs a password.
-if ssh "${SSH_QUERY_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
+# Get names of all currently running guests from pve1.
+if ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
   running_guests="$(
-    ssh "${SSH_QUERY_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash <<'REMOTE'
+    ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash <<'REMOTE'
       { sudo pct list 2>/dev/null | awk 'NR>1 && $2=="running" { print $NF }';
         sudo qm  list 2>/dev/null | awk 'NR>1 && $3=="running" { print $2  }'; } | sort -u
 REMOTE
@@ -100,7 +95,7 @@ REMOTE
       | jq -r --arg g "$guest" '.[$g] // empty' 2>/dev/null || true)"
     [[ -z "$hostname" ]] && continue
 
-    # Exclude nix-cache and any hostname already in the list.
+    # Exclude nix-cache and any target whose hostname is already in our list.
     case "$hostname" in nix-cache) continue ;; esac
     if [[ -n "${_SEEN_HOSTNAMES[$hostname]+_}" ]]; then continue; fi
 
@@ -126,7 +121,7 @@ if [[ "$DRY_RUN" -eq 1 ]]; then
       echo "  ssh ${SSH_OPTS[*]} $target 'sudo -n nix-collect-garbage -d'"
       echo "  # fallback: ssh ... $target 'nix-collect-garbage -d'"
     else
-      echo "  ssh ${SSH_OPTS[*]} $target 'bash -l -c nix-collect-garbage -d'"
+      echo "  ssh ${SSH_OPTS[*]} $target '. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d'"
     fi
   done
   exit 0
@@ -152,10 +147,10 @@ gc_one() {
       echo "ok(user)"; return
     fi
   else
-    # Non-NixOS node: use a login shell so /etc/profile is sourced and the
-    # Nix daemon's bin dir is on PATH (set up by /etc/profile.d/nix-daemon.sh
-    # which the Nix installer adds to /etc/profile).
-    if ssh "${SSH_OPTS[@]}" "$target" "bash -l -c 'nix-collect-garbage -d'" \
+    # Non-NixOS node: BatchMode SSH doesn't source the Nix daemon profile, so
+    # nix-collect-garbage won't be on PATH unless we source it explicitly.
+    local nix_profile='. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null || true'
+    if ssh "${SSH_OPTS[@]}" "$target" "$nix_profile && nix-collect-garbage -d" \
         >>"$logfile" 2>>"$logfile"; then
       echo "ok"; return
     fi

scripts/ha/resize-data-disk.sh

@@ -30,6 +30,8 @@ DATA_DISK_SLOT="${DATA_DISK_SLOT:-scsi1}"   # Proxmox disk name (scsi1 = data di
 HA_USER="${HA_USER:-nixos}"
 PVE_HOST="${PVE_HOST:-${PVE1_HOST}}"
 PVE_SSH_USER="${PVE_SSH_USER:-${PROXMOX_SSH_USER}}"
+PVE_SUDO=""
+[[ "$PVE_SSH_USER" != "root" ]] && PVE_SUDO="sudo"
 # By-id symlink for the data disk; basename resolves to the raw block device.
 # matches variables.nix's haServerDrbdDisk.
 DATA_DISK_BYID="${DATA_DISK_BYID:-scsi-0QEMU_QEMU_HARDDISK_drive-${DATA_DISK_SLOT}}"
@@ -140,7 +142,7 @@ fi
 echo ""
 echo "Looking up VM IDs on ${PVE_HOST}..."
 
-QM_LIST=$(pve "qm list 2>/dev/null" || true)
+QM_LIST=$(pve "$PVE_SUDO qm list 2>/dev/null" || true)
 VMID1=$(echo "$QM_LIST" | awk -v name="$NODE1" '$0 ~ name {print $1}' | head -1)
 VMID2=$(echo "$QM_LIST" | awk -v name="$NODE2" '$0 ~ name {print $1}' | head -1)
 
@@ -182,12 +184,12 @@ echo "── Phase 1 — Proxmox disk resize (${DATA_DISK_SLOT} ${SIZE} on both
 
 echo "  ${DRY_PREFIX}qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}  ($NODE1 on ${PVE_HOST})"
 if ! $DRY_RUN; then
-  pve "qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}"
+  pve "$PVE_SUDO qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}"
 fi
 
 echo "  ${DRY_PREFIX}qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}  ($NODE2 on ${PVE_HOST})"
 if ! $DRY_RUN; then
-  pve "qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}"
+  pve "$PVE_SUDO qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}"
 fi
 
 echo "  Phase 1 done."
@@ -202,17 +204,13 @@ rescan_node() {
   local node_name=$1 run_fn=$2
 
   # Resolve block device name from the stable by-id symlink on the guest.
+  # Read-only lookup — safe to run even in dry-run so we show the real device.
   local blk_dev=""
-  if ! $DRY_RUN; then
-    blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
-  fi
+  blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
   if [[ -z "$blk_dev" ]]; then
-    # Fall back: scsi1 → index 1 → sdb, scsi2 → sdc, etc.
-    local slot_idx
-    slot_idx=$(echo "$DATA_DISK_SLOT" | grep -oE '[0-9]+$' || echo "1")
-    blk_dev=$(printf "sd%s" "$(echo "abcdefghij" | cut -c$((slot_idx + 1)))")
-    echo "  WARNING: could not resolve /dev/disk/by-id/${DATA_DISK_BYID} on $node_name;" \
-         "falling back to /dev/${blk_dev}" >&2
+    echo "  ERROR: /dev/disk/by-id/${DATA_DISK_BYID} not found on $node_name" >&2
+    echo "         Check DATA_DISK_BYID or DATA_DISK_SLOT configuration." >&2
+    exit 1
   fi
 
   echo "  ${DRY_PREFIX}Rescanning /dev/${blk_dev} on ${node_name}..."