fix(ha): resolve data disk device via by-id symlink even in dry-run

The by-id lookup is read-only so it's safe to run in dry-run mode.
Previously it was gated behind `if ! $DRY_RUN`, which always triggered
the sdb fallback warning in dry-run — making it look like the device
path wasn't reliable when the symlink actually exists on both servers.

Now the lookup always runs and the script errors out with a clear message
if the by-id symlink is genuinely missing, instead of silently falling
back to a guessed /dev/sd* name.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scripts/gc-hosts.sh

@@ -14,8 +14,8 @@
 # "nix-collect-garbage -d" if sudo needs a password — still collects
 # unreferenced store paths and old nixos-user profile generations, but leaves
 # old system generations in place.
-# pve1: runs "bash -l -c nix-collect-garbage -d" as the login user so
-# /etc/profile is sourced and the Nix daemon's PATH is set up automatically.
+# pve1: runs "nix-collect-garbage -d" as the login user (no system generations
+# on a non-NixOS host).
 #
 # Usage (from repo root):
 #   bash scripts/gc-hosts.sh [--dry-run]
@@ -30,12 +30,7 @@ source scripts/lib/nix-eval.sh 2>/dev/null || true
 : "${MAX_JOBS:=8}"
 : "${NIXOS_USER:=nixos}"
 : "${PVE1_SSH_USER:=${PROXMOX_SSH_USER:-wayne}}"
-
-# GC connections use BatchMode — no interactive prompts, just succeed or fail.
 SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10)
-# Discovery connections do NOT use BatchMode so that sudo can prompt if needed
-# (pct/qm list require root access on Proxmox).
-SSH_QUERY_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=10)
 
 DRY_RUN=0
 for arg in "$@"; do
@@ -49,7 +44,7 @@ done
 
 # ORDERED_HOSTS: names in display/execution order.
 # HOST_TARGET[name]: SSH target string (user@host).
-# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (login-shell gc).
+# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (user gc only).
 declare -a  ORDERED_HOSTS=()
 declare -A  HOST_TARGET=()
 declare -A  HOST_TYPE=()
@@ -71,47 +66,42 @@ _add_host "nixos" "${NIXOS_USER}@nixos" "nixos"
 _add_host "pve1" "${PVE1_SSH_USER}@${PVE1_HOST}" "nix"
 
 # 3. Dynamically discover running NixOS guests on pve1
-#
-# create-proxmox-resource.sh names every guest after its NixOS hostname:
-#   pct create ... --hostname <nixos-hostname>  (LXC)
-#   qm  create ... --name    <nixos-hostname>   (VM)
-# So pct/qm list output already contains the NixOS hostname directly.
-# We validate against the flake to filter out non-NixOS guests on pve1
-# (e.g. FreeIPA, Proxmox Backup Server) that share the same Proxmox node.
 echo "Discovering running guests on ${PVE1_HOST}..."
 
-# Eval the flake once to get the set of hostnames that are actually NixOS.
-# Values are NixOS hostnames (e.g. "docker"); keys are flake targets ("lxc-docker").
-nixos_hostnames=""
-nixos_hostnames="$(
+# Evaluate the full flake hostname map in one shot.
+hostname_map="{}"
+if ! hostname_map="$(
   nix eval --json "${NIX_EVAL_FLAGS[@]}" .#nixosConfigurations \
-    --apply 'cfgs: builtins.attrValues (builtins.mapAttrs (_: cfg: cfg.config.networking.hostName) cfgs)' \
-    2>/dev/null | jq -r '.[]' | sort -u
-)" || { echo "  warning: flake eval failed — non-NixOS guests will not be filtered" >&2; }
+    --apply 'cfgs: builtins.mapAttrs (_: cfg: cfg.config.networking.hostName) cfgs' \
+    2>/dev/null
+)"; then
+  echo "  warning: flake eval failed — skipping dynamic host discovery" >&2
+fi
 
-# SSH_QUERY_OPTS (no BatchMode) so sudo can prompt if needed for pct/qm.
-if ssh "${SSH_QUERY_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
+# Get names of all currently running guests from pve1.
+if ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
   running_guests="$(
-    ssh "${SSH_QUERY_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash -s <<'DISCOVER'
-sudo pct list 2>/dev/null | awk 'NR>1 && $2=="running" { print $NF }'
-sudo qm  list 2>/dev/null | awk 'NR>1 && $3=="running" { print $2  }'
-DISCOVER
+    ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash <<'REMOTE'
+      { sudo pct list 2>/dev/null | awk 'NR>1 && $2=="running" { print $NF }';
+        sudo qm  list 2>/dev/null | awk 'NR>1 && $3=="running" { print $2  }'; } | sort -u
+REMOTE
   )" || running_guests=""
 
-  while IFS= read -r hostname; do
+  while IFS= read -r guest; do
+    [[ -z "$guest" ]] && continue
+
+    # Resolve flake target name → NixOS hostname.
+    hostname="$(printf '%s' "$hostname_map" \
+      | jq -r --arg g "$guest" '.[$g] // empty' 2>/dev/null || true)"
     [[ -z "$hostname" ]] && continue
-    # Exclude nix-cache.
-    case "$hostname" in *nix-cache*) continue ;; esac
-    # Skip if not a flake-managed NixOS host (filters non-NixOS pve1 guests).
-    if [[ -n "$nixos_hostnames" ]] && ! grep -qxF "$hostname" <<< "$nixos_hostnames"; then
-      continue
-    fi
-    # Skip if already in the list (e.g. a proxmox-gui guest whose hostname is nixos).
+
+    # Exclude nix-cache and any target whose hostname is already in our list.
+    case "$hostname" in nix-cache) continue ;; esac
     if [[ -n "${_SEEN_HOSTNAMES[$hostname]+_}" ]]; then continue; fi
 
-    echo "  + $hostname"
+    echo "  + $guest → $hostname"
     _add_host "$hostname" "${NIXOS_USER}@${hostname}" "nixos"
-  done <<< "$(echo "$running_guests" | sort -u)"
+  done <<< "$running_guests"
 else
   echo "  warning: ${PVE1_HOST} unreachable — skipping dynamic host discovery" >&2
 fi
@@ -131,7 +121,7 @@ if [[ "$DRY_RUN" -eq 1 ]]; then
       echo "  ssh ${SSH_OPTS[*]} $target 'sudo -n nix-collect-garbage -d'"
       echo "  # fallback: ssh ... $target 'nix-collect-garbage -d'"
     else
-      echo "  ssh ${SSH_OPTS[*]} $target 'bash -l -c nix-collect-garbage -d'"
+      echo "  ssh ${SSH_OPTS[*]} $target '. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d'"
     fi
   done
   exit 0
@@ -157,10 +147,10 @@ gc_one() {
       echo "ok(user)"; return
     fi
   else
-    # Non-NixOS node: use a login shell so /etc/profile is sourced and the
-    # Nix daemon's bin dir is on PATH (set up by /etc/profile.d/nix-daemon.sh
-    # which the Nix installer adds to /etc/profile).
-    if ssh "${SSH_OPTS[@]}" "$target" "bash -l -c 'nix-collect-garbage -d'" \
+    # Non-NixOS node: BatchMode SSH doesn't source the Nix daemon profile, so
+    # nix-collect-garbage won't be on PATH unless we source it explicitly.
+    local nix_profile='. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null || true'
+    if ssh "${SSH_OPTS[@]}" "$target" "$nix_profile && nix-collect-garbage -d" \
         >>"$logfile" 2>>"$logfile"; then
       echo "ok"; return
     fi

scripts/ha/resize-data-disk.sh

@@ -30,6 +30,8 @@ DATA_DISK_SLOT="${DATA_DISK_SLOT:-scsi1}"   # Proxmox disk name (scsi1 = data di
 HA_USER="${HA_USER:-nixos}"
 PVE_HOST="${PVE_HOST:-${PVE1_HOST}}"
 PVE_SSH_USER="${PVE_SSH_USER:-${PROXMOX_SSH_USER}}"
+PVE_SUDO=""
+[[ "$PVE_SSH_USER" != "root" ]] && PVE_SUDO="sudo"
 # By-id symlink for the data disk; basename resolves to the raw block device.
 # matches variables.nix's haServerDrbdDisk.
 DATA_DISK_BYID="${DATA_DISK_BYID:-scsi-0QEMU_QEMU_HARDDISK_drive-${DATA_DISK_SLOT}}"
@@ -140,7 +142,7 @@ fi
 echo ""
 echo "Looking up VM IDs on ${PVE_HOST}..."
 
-QM_LIST=$(pve "qm list 2>/dev/null" || true)
+QM_LIST=$(pve "$PVE_SUDO qm list 2>/dev/null" || true)
 VMID1=$(echo "$QM_LIST" | awk -v name="$NODE1" '$0 ~ name {print $1}' | head -1)
 VMID2=$(echo "$QM_LIST" | awk -v name="$NODE2" '$0 ~ name {print $1}' | head -1)
 
@@ -182,12 +184,12 @@ echo "── Phase 1 — Proxmox disk resize (${DATA_DISK_SLOT} ${SIZE} on both
 
 echo "  ${DRY_PREFIX}qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}  ($NODE1 on ${PVE_HOST})"
 if ! $DRY_RUN; then
-  pve "qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}"
+  pve "$PVE_SUDO qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}"
 fi
 
 echo "  ${DRY_PREFIX}qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}  ($NODE2 on ${PVE_HOST})"
 if ! $DRY_RUN; then
-  pve "qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}"
+  pve "$PVE_SUDO qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}"
 fi
 
 echo "  Phase 1 done."
@@ -202,17 +204,13 @@ rescan_node() {
   local node_name=$1 run_fn=$2
 
   # Resolve block device name from the stable by-id symlink on the guest.
+  # Read-only lookup — safe to run even in dry-run so we show the real device.
   local blk_dev=""
-  if ! $DRY_RUN; then
-    blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
-  fi
+  blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
   if [[ -z "$blk_dev" ]]; then
-    # Fall back: scsi1 → index 1 → sdb, scsi2 → sdc, etc.
-    local slot_idx
-    slot_idx=$(echo "$DATA_DISK_SLOT" | grep -oE '[0-9]+$' || echo "1")
-    blk_dev=$(printf "sd%s" "$(echo "abcdefghij" | cut -c$((slot_idx + 1)))")
-    echo "  WARNING: could not resolve /dev/disk/by-id/${DATA_DISK_BYID} on $node_name;" \
-         "falling back to /dev/${blk_dev}" >&2
+    echo "  ERROR: /dev/disk/by-id/${DATA_DISK_BYID} not found on $node_name" >&2
+    echo "         Check DATA_DISK_BYID or DATA_DISK_SLOT configuration." >&2
+    exit 1
   fi
 
   echo "  ${DRY_PREFIX}Rescanning /dev/${blk_dev} on ${node_name}..."

variables.nix

@@ -58,6 +58,7 @@
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZ9WKKAlP9Z7GQdgaZ1Xgw9C+vja2lqEZO5rJFpVqYN root@ha-server-1"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEGNKlaaMckd8nLWNGz4B2QokXjnnIvM+rEUv+R6h0sp root@ha-server-2"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK+XeMco7OxUpjrjZm54HogMs9QB5xlcKmElASRvrmlW root@nixos"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMseQwPpmaa6cgV5U8KhUsiVSYARG85zGa9rho0LJWks wayne@pve1"
   ];
 
   # Admin SSH public key, authorized on the primary user of every host and