fix(gc-hosts): filter discovered guests against flake to exclude non-NixOS hosts

pve1 runs non-NixOS guests (FreeIPA domain-controller, Proxmox Data Manager, etc.)
alongside NixOS ones. Validate each discovered hostname against the set of
hostnames defined in nixosConfigurations before adding it to the gc list.

Brings back the flake eval but uses it correctly: extract all hostname values
(not target-name keys) to build a filter set, then only include pve1 guests
whose name matches a flake-managed NixOS host.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scripts/gc-hosts.sh

@@ -1,159 +1,192 @@
 #!/usr/bin/env bash
-# gc-hosts.sh — Run nix-collect-garbage -d on all live NixOS hosts and pve1.
+# gc-hosts.sh — Run nix-collect-garbage -d on all live NixOS hosts.
 #
-# nix-cache is excluded: it is the shared binary cache for all other hosts, so
-# gc-ing it would evict cached store paths and force costly rebuilds elsewhere.
+# The host list is rebuilt on every run:
+#   1. This workstation (nixos) — always first
+#   2. pve1 — always second (non-NixOS Proxmox node with Nix installed)
+#   3. Every NixOS guest currently running on pve1 (discovered via pct/qm list)
 #
-# Runs SSH jobs in parallel (up to MAX_JOBS at a time) and prints a summary.
+# nix-cache is excluded: gc-ing the shared binary cache evicts store paths
+# that other hosts depend on for substitution.
+#
+# NixOS hosts: tries "sudo -n nix-collect-garbage -d" first (works when
+# wheelNeedsPassword = false, e.g. the HA cluster).  Falls back to user-level
+# "nix-collect-garbage -d" if sudo needs a password — still collects
+# unreferenced store paths and old nixos-user profile generations, but leaves
+# old system generations in place.
+# pve1: runs "bash -l -c nix-collect-garbage -d" as the login user so
+# /etc/profile is sourced and the Nix daemon's PATH is set up automatically.
 #
 # Usage (from repo root):
-#   bash scripts/gc-hosts.sh [--dry-run] [<hostname> ...]
-#
-# Options:
-#   --dry-run     Print the SSH commands without executing them.
-#   <hostname>    Limit to the given hostnames (bare names, no domain suffix).
-#                 Default: all hosts in the list below.
-#
-# Sudo notes:
-#   NixOS hosts: tries "sudo -n nix-collect-garbage -d" first (non-interactive,
-#   works when wheelNeedsPassword = false such as on the HA cluster).  Falls
-#   back to "nix-collect-garbage -d" as the nixos user if sudo requires a
-#   password — this still collects unreferenced store paths and removes old
-#   nixos-user profile generations, but will not remove old system generations.
-#   pve1: non-NixOS Proxmox node, runs nix-collect-garbage -d as wayne (no
-#   system generations to delete).
+#   bash scripts/gc-hosts.sh [--dry-run]
 
 set -euo pipefail
 cd "$(dirname "$0")/.."
 source scripts/env.sh 2>/dev/null || true
+source scripts/lib/nix-eval.sh 2>/dev/null || true
 
 # ── config ────────────────────────────────────────────────────────────────────
 
-: "${MAX_JOBS:=6}"
+: "${MAX_JOBS:=8}"
 : "${NIXOS_USER:=nixos}"
+: "${PVE1_SSH_USER:=${PROXMOX_SSH_USER:-wayne}}"
+
+# GC connections use BatchMode — no interactive prompts, just succeed or fail.
 SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10)
-
-# Map: hostname → ssh-user
-# Update this list when new hosts are added/removed.
-declare -A HOSTS=(
-  [docker]="$NIXOS_USER"
-  [ha-server-1]="$NIXOS_USER"
-  [ha-server-2]="$NIXOS_USER"
-  [nix-minimal]="$NIXOS_USER"
-  [nixos]="$NIXOS_USER"
-  [pxe-boot]="$NIXOS_USER"
-  [server]="$NIXOS_USER"
-  [tailscale-router]="$NIXOS_USER"
-  [tor-relay]="$NIXOS_USER"
-  [pve1]="${PROXMOX_SSH_USER:-wayne}"
-)
-
-# ── argument parsing ──────────────────────────────────────────────────────────
+# Discovery connections do NOT use BatchMode so that sudo can prompt if needed
+# (pct/qm list require root access on Proxmox).
+SSH_QUERY_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=10)
 
 DRY_RUN=0
-FILTER=()
-
 for arg in "$@"; do
   case "$arg" in
     --dry-run) DRY_RUN=1 ;;
-    --*)       echo "Unknown option: $arg" >&2; exit 1 ;;
-    *)         FILTER+=("$arg") ;;
+    *) echo "Unknown option: $arg" >&2; exit 1 ;;
   esac
 done
 
-# Resolve the set of (hostname, user) pairs to process.
-declare -A TARGET_USERS
-if [[ ${#FILTER[@]} -gt 0 ]]; then
-  for h in "${FILTER[@]}"; do
-    if [[ -z "${HOSTS[$h]+_}" ]]; then
-      echo "Unknown hostname: $h (not in the gc-hosts list)" >&2
-      exit 1
+# ── build the host list ───────────────────────────────────────────────────────
+
+# ORDERED_HOSTS: names in display/execution order.
+# HOST_TARGET[name]: SSH target string (user@host).
+# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (login-shell gc).
+declare -a  ORDERED_HOSTS=()
+declare -A  HOST_TARGET=()
+declare -A  HOST_TYPE=()
+declare -A  _SEEN_HOSTNAMES=()   # dedup tracker
+
+_add_host() {
+  local name="$1" target="$2" type="$3"
+  if [[ -n "${_SEEN_HOSTNAMES[$name]+_}" ]]; then return; fi
+  _SEEN_HOSTNAMES[$name]=1
+  ORDERED_HOSTS+=("$name")
+  HOST_TARGET[$name]="$target"
+  HOST_TYPE[$name]="$type"
+}
+
+# 1. Workstation (hard-wired first)
+_add_host "nixos" "${NIXOS_USER}@nixos" "nixos"
+
+# 2. pve1 (hard-wired second; non-NixOS, no system generations)
+_add_host "pve1" "${PVE1_SSH_USER}@${PVE1_HOST}" "nix"
+
+# 3. Dynamically discover running NixOS guests on pve1
+#
+# create-proxmox-resource.sh names every guest after its NixOS hostname:
+#   pct create ... --hostname <nixos-hostname>  (LXC)
+#   qm  create ... --name    <nixos-hostname>   (VM)
+# So pct/qm list output already contains the NixOS hostname directly.
+# We validate against the flake to filter out non-NixOS guests on pve1
+# (e.g. FreeIPA, Proxmox Backup Server) that share the same Proxmox node.
+echo "Discovering running guests on ${PVE1_HOST}..."
+
+# Eval the flake once to get the set of hostnames that are actually NixOS.
+# Values are NixOS hostnames (e.g. "docker"); keys are flake targets ("lxc-docker").
+nixos_hostnames=""
+nixos_hostnames="$(
+  nix eval --json "${NIX_EVAL_FLAGS[@]}" .#nixosConfigurations \
+    --apply 'cfgs: builtins.attrValues (builtins.mapAttrs (_: cfg: cfg.config.networking.hostName) cfgs)' \
+    2>/dev/null | jq -r '.[]' | sort -u
+)" || { echo "  warning: flake eval failed — non-NixOS guests will not be filtered" >&2; }
+
+# SSH_QUERY_OPTS (no BatchMode) so sudo can prompt if needed for pct/qm.
+if ssh "${SSH_QUERY_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
+  running_guests="$(
+    ssh "${SSH_QUERY_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash -s <<'DISCOVER'
+sudo pct list 2>/dev/null | awk 'NR>1 && $2=="running" { print $NF }'
+sudo qm  list 2>/dev/null | awk 'NR>1 && $3=="running" { print $2  }'
+DISCOVER
+  )" || running_guests=""
+
+  while IFS= read -r hostname; do
+    [[ -z "$hostname" ]] && continue
+    # Exclude nix-cache.
+    case "$hostname" in *nix-cache*) continue ;; esac
+    # Skip if not a flake-managed NixOS host (filters non-NixOS pve1 guests).
+    if [[ -n "$nixos_hostnames" ]] && ! grep -qxF "$hostname" <<< "$nixos_hostnames"; then
+      continue
     fi
-    TARGET_USERS[$h]="${HOSTS[$h]}"
-  done
+    # Skip if already in the list (e.g. a proxmox-gui guest whose hostname is nixos).
+    if [[ -n "${_SEEN_HOSTNAMES[$hostname]+_}" ]]; then continue; fi
+
+    echo "  + $hostname"
+    _add_host "$hostname" "${NIXOS_USER}@${hostname}" "nixos"
+  done <<< "$(echo "$running_guests" | sort -u)"
 else
-  for h in "${!HOSTS[@]}"; do
-    TARGET_USERS[$h]="${HOSTS[$h]}"
-  done
+  echo "  warning: ${PVE1_HOST} unreachable — skipping dynamic host discovery" >&2
 fi
 
-SORTED_HOSTS=($(printf '%s\n' "${!TARGET_USERS[@]}" | sort))
+echo ""
+echo "Hosts: ${ORDERED_HOSTS[*]}"
+echo ""
 
-# ── helpers ───────────────────────────────────────────────────────────────────
+# ── dry-run ───────────────────────────────────────────────────────────────────
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  echo "[dry-run] commands that would run:"
+  for host in "${ORDERED_HOSTS[@]}"; do
+    target="${HOST_TARGET[$host]}"
+    type="${HOST_TYPE[$host]}"
+    if [[ "$type" == "nixos" ]]; then
+      echo "  ssh ${SSH_OPTS[*]} $target 'sudo -n nix-collect-garbage -d'"
+      echo "  # fallback: ssh ... $target 'nix-collect-garbage -d'"
+    else
+      echo "  ssh ${SSH_OPTS[*]} $target 'bash -l -c nix-collect-garbage -d'"
+    fi
+  done
+  exit 0
+fi
+
+# ── gc worker ─────────────────────────────────────────────────────────────────
 
 gc_one() {
-  local host="$1" user="$2" logfile="$3"
-  local target="${user}@${host}"
+  local host="$1" target="${HOST_TARGET[$1]}" type="${HOST_TYPE[$1]}" logfile="$2"
 
   if ! ssh "${SSH_OPTS[@]}" "$target" "true" 2>>"$logfile"; then
-    echo "unreachable"
-    return
+    echo "unreachable"; return
   fi
 
-  # NixOS hosts: try passwordless sudo first.
-  if [[ "$user" == "$NIXOS_USER" ]]; then
+  if [[ "$type" == "nixos" ]]; then
     if ssh "${SSH_OPTS[@]}" "$target" "sudo -n nix-collect-garbage -d" \
         >>"$logfile" 2>>"$logfile"; then
-      echo "ok(sudo)"
-      return
+      echo "ok(sudo)"; return
     fi
-    # sudo required a password — fall back to user-level gc.
-    echo "[sudo needs password, falling back to user-level gc]" >>"$logfile"
+    echo "[sudo needs password — falling back to user-level gc]" >>"$logfile"
     if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \
         >>"$logfile" 2>>"$logfile"; then
-      echo "ok(user)"
-      return
+      echo "ok(user)"; return
     fi
   else
-    # Non-NixOS node (pve1): no system generations; just gc as the login user.
-    if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \
+    # Non-NixOS node: use a login shell so /etc/profile is sourced and the
+    # Nix daemon's bin dir is on PATH (set up by /etc/profile.d/nix-daemon.sh
+    # which the Nix installer adds to /etc/profile).
+    if ssh "${SSH_OPTS[@]}" "$target" "bash -l -c 'nix-collect-garbage -d'" \
         >>"$logfile" 2>>"$logfile"; then
-      echo "ok"
-      return
+      echo "ok"; return
     fi
   fi
 
   echo "failed:$?"
 }
 
-# ── dry-run ───────────────────────────────────────────────────────────────────
-
-if [[ "$DRY_RUN" -eq 1 ]]; then
-  echo "[dry-run] would run gc on: ${SORTED_HOSTS[*]}"
-  for host in "${SORTED_HOSTS[@]}"; do
-    user="${TARGET_USERS[$host]}"
-    if [[ "$user" == "$NIXOS_USER" ]]; then
-      echo "  ssh ${SSH_OPTS[*]} ${user}@${host} 'sudo -n nix-collect-garbage -d'"
-      echo "  # fallback: ssh ... 'nix-collect-garbage -d'"
-    else
-      echo "  ssh ${SSH_OPTS[*]} ${user}@${host} 'nix-collect-garbage -d'"
-    fi
-  done
-  exit 0
-fi
-
 # ── parallel execution ────────────────────────────────────────────────────────
 
+echo "Running gc on ${#ORDERED_HOSTS[@]} hosts (up to ${MAX_JOBS} parallel)..."
+echo ""
+
 TMPDIR_GC="$(mktemp -d)"
 trap 'rm -rf "$TMPDIR_GC"' EXIT
 
-declare -A LOGS
-
-echo "Running gc on ${#SORTED_HOSTS[@]} hosts (up to ${MAX_JOBS} parallel)..."
-echo ""
-
+declare -A LOGS=()
 job_count=0
-for host in "${SORTED_HOSTS[@]}"; do
-  user="${TARGET_USERS[$host]}"
+
+for host in "${ORDERED_HOSTS[@]}"; do
   logfile="${TMPDIR_GC}/${host}.log"
   resultfile="${TMPDIR_GC}/${host}.result"
   LOGS[$host]="$logfile"
   : > "$logfile"
 
-  (
-    result="$(gc_one "$host" "$user" "$logfile")"
-    echo "$result" > "$resultfile"
-  ) &
+  ( result="$(gc_one "$host" "$logfile")"; echo "$result" > "$resultfile" ) &
 
   (( job_count++ )) || true
   if [[ "$job_count" -ge "$MAX_JOBS" ]]; then
@@ -173,30 +206,24 @@ ok_hosts=()
 warn_hosts=()
 fail_hosts=()
 
-for host in "${SORTED_HOSTS[@]}"; do
-  resultfile="${TMPDIR_GC}/${host}.result"
-  result="$(cat "$resultfile" 2>/dev/null || echo "failed:missing")"
-
+for host in "${ORDERED_HOSTS[@]}"; do
+  result="$(cat "${TMPDIR_GC}/${host}.result" 2>/dev/null || echo "failed:missing")"
   case "$result" in
     ok|"ok(sudo)"|"ok(user)")
-      printf "  %-20s  %s\n" "$host" "$result"
-      ok_hosts+=("$host")
-      ;;
+      printf "  %-22s  %s\n" "$host" "$result"
+      ok_hosts+=("$host") ;;
     unreachable)
-      printf "  %-20s  UNREACHABLE\n" "$host"
-      warn_hosts+=("$host")
-      ;;
+      printf "  %-22s  UNREACHABLE\n" "$host"
+      warn_hosts+=("$host") ;;
     *)
-      printf "  %-20s  FAILED (%s)\n" "$host" "$result"
-      fail_hosts+=("$host")
-      ;;
+      printf "  %-22s  FAILED (%s)\n" "$host" "$result"
+      fail_hosts+=("$host") ;;
   esac
 done
 
 echo ""
 echo "  ${#ok_hosts[@]} succeeded, ${#warn_hosts[@]} unreachable, ${#fail_hosts[@]} failed"
 
-# Print logs for any non-ok host.
 for host in "${warn_hosts[@]+"${warn_hosts[@]}"}" "${fail_hosts[@]+"${fail_hosts[@]}"}"; do
   logfile="${LOGS[$host]}"
   if [[ -s "$logfile" ]]; then

variables.nix

@@ -58,7 +58,6 @@
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZ9WKKAlP9Z7GQdgaZ1Xgw9C+vja2lqEZO5rJFpVqYN root@ha-server-1"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEGNKlaaMckd8nLWNGz4B2QokXjnnIvM+rEUv+R6h0sp root@ha-server-2"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK+XeMco7OxUpjrjZm54HogMs9QB5xlcKmElASRvrmlW root@nixos"
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMseQwPpmaa6cgV5U8KhUsiVSYARG85zGa9rho0LJWks wayne@pve1"
   ];
 
   # Admin SSH public key, authorized on the primary user of every host and