fix(ha): resolve data disk device via by-id symlink even in dry-run

The by-id lookup is read-only so it's safe to run in dry-run mode.
Previously it was gated behind `if ! $DRY_RUN`, which always triggered
the sdb fallback warning in dry-run — making it look like the device
path wasn't reliable when the symlink actually exists on both servers.

Now the lookup always runs and the script errors out with a clear message
if the by-id symlink is genuinely missing, instead of silently falling
back to a guessed /dev/sd* name.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scripts/gc-hosts.sh

@@ -14,8 +14,8 @@
 # "nix-collect-garbage -d" if sudo needs a password — still collects
 # unreferenced store paths and old nixos-user profile generations, but leaves
 # old system generations in place.
-# pve1: runs "bash -l -c nix-collect-garbage -d" as the login user so
-# /etc/profile is sourced and the Nix daemon's PATH is set up automatically.
+# pve1: runs "nix-collect-garbage -d" as the login user (no system generations
+# on a non-NixOS host).
 #
 # Usage (from repo root):
 #   bash scripts/gc-hosts.sh [--dry-run]
@@ -30,12 +30,7 @@ source scripts/lib/nix-eval.sh 2>/dev/null || true
 : "${MAX_JOBS:=8}"
 : "${NIXOS_USER:=nixos}"
 : "${PVE1_SSH_USER:=${PROXMOX_SSH_USER:-wayne}}"
-
-# GC connections use BatchMode — no interactive prompts, just succeed or fail.
 SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10)
-# Discovery connections do NOT use BatchMode so that sudo can prompt if needed
-# (pct/qm list require root access on Proxmox).
-SSH_QUERY_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=10)
 
 DRY_RUN=0
 for arg in "$@"; do
@@ -49,7 +44,7 @@ done
 
 # ORDERED_HOSTS: names in display/execution order.
 # HOST_TARGET[name]: SSH target string (user@host).
-# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (login-shell gc).
+# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (user gc only).
 declare -a  ORDERED_HOSTS=()
 declare -A  HOST_TARGET=()
 declare -A  HOST_TYPE=()
@@ -83,10 +78,10 @@ if ! hostname_map="$(
   echo "  warning: flake eval failed — skipping dynamic host discovery" >&2
 fi
 
-# SSH_QUERY_OPTS (no BatchMode) so sudo can prompt if wayne's sudo needs a password.
-if ssh "${SSH_QUERY_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
+# Get names of all currently running guests from pve1.
+if ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
   running_guests="$(
-    ssh "${SSH_QUERY_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash <<'REMOTE'
+    ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash <<'REMOTE'
       { sudo pct list 2>/dev/null | awk 'NR>1 && $2=="running" { print $NF }';
         sudo qm  list 2>/dev/null | awk 'NR>1 && $3=="running" { print $2  }'; } | sort -u
 REMOTE
@@ -100,7 +95,7 @@ REMOTE
       | jq -r --arg g "$guest" '.[$g] // empty' 2>/dev/null || true)"
     [[ -z "$hostname" ]] && continue
 
-    # Exclude nix-cache and any hostname already in the list.
+    # Exclude nix-cache and any target whose hostname is already in our list.
     case "$hostname" in nix-cache) continue ;; esac
     if [[ -n "${_SEEN_HOSTNAMES[$hostname]+_}" ]]; then continue; fi
 
@@ -126,7 +121,7 @@ if [[ "$DRY_RUN" -eq 1 ]]; then
       echo "  ssh ${SSH_OPTS[*]} $target 'sudo -n nix-collect-garbage -d'"
       echo "  # fallback: ssh ... $target 'nix-collect-garbage -d'"
     else
-      echo "  ssh ${SSH_OPTS[*]} $target 'bash -l -c nix-collect-garbage -d'"
+      echo "  ssh ${SSH_OPTS[*]} $target '. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d'"
     fi
   done
   exit 0
@@ -152,10 +147,10 @@ gc_one() {
       echo "ok(user)"; return
     fi
   else
-    # Non-NixOS node: use a login shell so /etc/profile is sourced and the
-    # Nix daemon's bin dir is on PATH (set up by /etc/profile.d/nix-daemon.sh
-    # which the Nix installer adds to /etc/profile).
-    if ssh "${SSH_OPTS[@]}" "$target" "bash -l -c 'nix-collect-garbage -d'" \
+    # Non-NixOS node: BatchMode SSH doesn't source the Nix daemon profile, so
+    # nix-collect-garbage won't be on PATH unless we source it explicitly.
+    local nix_profile='. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null || true'
+    if ssh "${SSH_OPTS[@]}" "$target" "$nix_profile && nix-collect-garbage -d" \
         >>"$logfile" 2>>"$logfile"; then
       echo "ok"; return
     fi

scripts/ha/resize-data-disk.sh

@@ -204,17 +204,13 @@ rescan_node() {
   local node_name=$1 run_fn=$2
 
   # Resolve block device name from the stable by-id symlink on the guest.
+  # Read-only lookup — safe to run even in dry-run so we show the real device.
   local blk_dev=""
-  if ! $DRY_RUN; then
-    blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
-  fi
+  blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
   if [[ -z "$blk_dev" ]]; then
-    # Fall back: scsi1 → index 1 → sdb, scsi2 → sdc, etc.
-    local slot_idx
-    slot_idx=$(echo "$DATA_DISK_SLOT" | grep -oE '[0-9]+$' || echo "1")
-    blk_dev=$(printf "sd%s" "$(echo "abcdefghij" | cut -c$((slot_idx + 1)))")
-    echo "  WARNING: could not resolve /dev/disk/by-id/${DATA_DISK_BYID} on $node_name;" \
-         "falling back to /dev/${blk_dev}" >&2
+    echo "  ERROR: /dev/disk/by-id/${DATA_DISK_BYID} not found on $node_name" >&2
+    echo "         Check DATA_DISK_BYID or DATA_DISK_SLOT configuration." >&2
+    exit 1
   fi
 
   echo "  ${DRY_PREFIX}Rescanning /dev/${blk_dev} on ${node_name}..."