Merge pull request 'feat(scripts): add gc-hosts.sh for parallel nix gc across all live hosts' (#101) from worktree-crispy-churning-kernighan into main

Reviewed-on: #101

scripts/gc-hosts.sh

@@ -1,192 +1,159 @@
 #!/usr/bin/env bash
-# gc-hosts.sh — Run nix-collect-garbage -d on all live NixOS hosts.
+# gc-hosts.sh — Run nix-collect-garbage -d on all live NixOS hosts and pve1.
 #
-# The host list is rebuilt on every run:
-#   1. This workstation (nixos) — always first
-#   2. pve1 — always second (non-NixOS Proxmox node with Nix installed)
-#   3. Every NixOS guest currently running on pve1 (discovered via pct/qm list)
+# nix-cache is excluded: it is the shared binary cache for all other hosts, so
+# gc-ing it would evict cached store paths and force costly rebuilds elsewhere.
 #
-# nix-cache is excluded: gc-ing the shared binary cache evicts store paths
-# that other hosts depend on for substitution.
-#
-# NixOS hosts: tries "sudo -n nix-collect-garbage -d" first (works when
-# wheelNeedsPassword = false, e.g. the HA cluster).  Falls back to user-level
-# "nix-collect-garbage -d" if sudo needs a password — still collects
-# unreferenced store paths and old nixos-user profile generations, but leaves
-# old system generations in place.
-# pve1: runs "bash -l -c nix-collect-garbage -d" as the login user so
-# /etc/profile is sourced and the Nix daemon's PATH is set up automatically.
+# Runs SSH jobs in parallel (up to MAX_JOBS at a time) and prints a summary.
 #
 # Usage (from repo root):
-#   bash scripts/gc-hosts.sh [--dry-run]
+#   bash scripts/gc-hosts.sh [--dry-run] [<hostname> ...]
+#
+# Options:
+#   --dry-run     Print the SSH commands without executing them.
+#   <hostname>    Limit to the given hostnames (bare names, no domain suffix).
+#                 Default: all hosts in the list below.
+#
+# Sudo notes:
+#   NixOS hosts: tries "sudo -n nix-collect-garbage -d" first (non-interactive,
+#   works when wheelNeedsPassword = false such as on the HA cluster).  Falls
+#   back to "nix-collect-garbage -d" as the nixos user if sudo requires a
+#   password — this still collects unreferenced store paths and removes old
+#   nixos-user profile generations, but will not remove old system generations.
+#   pve1: non-NixOS Proxmox node, runs nix-collect-garbage -d as wayne (no
+#   system generations to delete).
 
 set -euo pipefail
 cd "$(dirname "$0")/.."
 source scripts/env.sh 2>/dev/null || true
-source scripts/lib/nix-eval.sh 2>/dev/null || true
 
 # ── config ────────────────────────────────────────────────────────────────────
 
-: "${MAX_JOBS:=8}"
+: "${MAX_JOBS:=6}"
 : "${NIXOS_USER:=nixos}"
-: "${PVE1_SSH_USER:=${PROXMOX_SSH_USER:-wayne}}"
-
-# GC connections use BatchMode — no interactive prompts, just succeed or fail.
 SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10)
-# Discovery connections do NOT use BatchMode so that sudo can prompt if needed
-# (pct/qm list require root access on Proxmox).
-SSH_QUERY_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=10)
+
+# Map: hostname → ssh-user
+# Update this list when new hosts are added/removed.
+declare -A HOSTS=(
+  [docker]="$NIXOS_USER"
+  [ha-server-1]="$NIXOS_USER"
+  [ha-server-2]="$NIXOS_USER"
+  [nix-minimal]="$NIXOS_USER"
+  [nixos]="$NIXOS_USER"
+  [pxe-boot]="$NIXOS_USER"
+  [server]="$NIXOS_USER"
+  [tailscale-router]="$NIXOS_USER"
+  [tor-relay]="$NIXOS_USER"
+  [pve1]="${PROXMOX_SSH_USER:-wayne}"
+)
+
+# ── argument parsing ──────────────────────────────────────────────────────────
 
 DRY_RUN=0
+FILTER=()
+
 for arg in "$@"; do
   case "$arg" in
     --dry-run) DRY_RUN=1 ;;
-    *) echo "Unknown option: $arg" >&2; exit 1 ;;
+    --*)       echo "Unknown option: $arg" >&2; exit 1 ;;
+    *)         FILTER+=("$arg") ;;
   esac
 done
 
-# ── build the host list ───────────────────────────────────────────────────────
-
-# ORDERED_HOSTS: names in display/execution order.
-# HOST_TARGET[name]: SSH target string (user@host).
-# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (login-shell gc).
-declare -a  ORDERED_HOSTS=()
-declare -A  HOST_TARGET=()
-declare -A  HOST_TYPE=()
-declare -A  _SEEN_HOSTNAMES=()   # dedup tracker
-
-_add_host() {
-  local name="$1" target="$2" type="$3"
-  if [[ -n "${_SEEN_HOSTNAMES[$name]+_}" ]]; then return; fi
-  _SEEN_HOSTNAMES[$name]=1
-  ORDERED_HOSTS+=("$name")
-  HOST_TARGET[$name]="$target"
-  HOST_TYPE[$name]="$type"
-}
-
-# 1. Workstation (hard-wired first)
-_add_host "nixos" "${NIXOS_USER}@nixos" "nixos"
-
-# 2. pve1 (hard-wired second; non-NixOS, no system generations)
-_add_host "pve1" "${PVE1_SSH_USER}@${PVE1_HOST}" "nix"
-
-# 3. Dynamically discover running NixOS guests on pve1
-#
-# create-proxmox-resource.sh names every guest after its NixOS hostname:
-#   pct create ... --hostname <nixos-hostname>  (LXC)
-#   qm  create ... --name    <nixos-hostname>   (VM)
-# So pct/qm list output already contains the NixOS hostname directly.
-# We validate against the flake to filter out non-NixOS guests on pve1
-# (e.g. FreeIPA, Proxmox Backup Server) that share the same Proxmox node.
-echo "Discovering running guests on ${PVE1_HOST}..."
-
-# Eval the flake once to get the set of hostnames that are actually NixOS.
-# Values are NixOS hostnames (e.g. "docker"); keys are flake targets ("lxc-docker").
-nixos_hostnames=""
-nixos_hostnames="$(
-  nix eval --json "${NIX_EVAL_FLAGS[@]}" .#nixosConfigurations \
-    --apply 'cfgs: builtins.attrValues (builtins.mapAttrs (_: cfg: cfg.config.networking.hostName) cfgs)' \
-    2>/dev/null | jq -r '.[]' | sort -u
-)" || { echo "  warning: flake eval failed — non-NixOS guests will not be filtered" >&2; }
-
-# SSH_QUERY_OPTS (no BatchMode) so sudo can prompt if needed for pct/qm.
-if ssh "${SSH_QUERY_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
-  running_guests="$(
-    ssh "${SSH_QUERY_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash -s <<'DISCOVER'
-sudo pct list 2>/dev/null | awk 'NR>1 && $2=="running" { print $NF }'
-sudo qm  list 2>/dev/null | awk 'NR>1 && $3=="running" { print $2  }'
-DISCOVER
-  )" || running_guests=""
-
-  while IFS= read -r hostname; do
-    [[ -z "$hostname" ]] && continue
-    # Exclude nix-cache.
-    case "$hostname" in *nix-cache*) continue ;; esac
-    # Skip if not a flake-managed NixOS host (filters non-NixOS pve1 guests).
-    if [[ -n "$nixos_hostnames" ]] && ! grep -qxF "$hostname" <<< "$nixos_hostnames"; then
-      continue
-    fi
-    # Skip if already in the list (e.g. a proxmox-gui guest whose hostname is nixos).
-    if [[ -n "${_SEEN_HOSTNAMES[$hostname]+_}" ]]; then continue; fi
-
-    echo "  + $hostname"
-    _add_host "$hostname" "${NIXOS_USER}@${hostname}" "nixos"
-  done <<< "$(echo "$running_guests" | sort -u)"
-else
-  echo "  warning: ${PVE1_HOST} unreachable — skipping dynamic host discovery" >&2
-fi
-
-echo ""
-echo "Hosts: ${ORDERED_HOSTS[*]}"
-echo ""
-
-# ── dry-run ───────────────────────────────────────────────────────────────────
-
-if [[ "$DRY_RUN" -eq 1 ]]; then
-  echo "[dry-run] commands that would run:"
-  for host in "${ORDERED_HOSTS[@]}"; do
-    target="${HOST_TARGET[$host]}"
-    type="${HOST_TYPE[$host]}"
-    if [[ "$type" == "nixos" ]]; then
-      echo "  ssh ${SSH_OPTS[*]} $target 'sudo -n nix-collect-garbage -d'"
-      echo "  # fallback: ssh ... $target 'nix-collect-garbage -d'"
-    else
-      echo "  ssh ${SSH_OPTS[*]} $target 'bash -l -c nix-collect-garbage -d'"
+# Resolve the set of (hostname, user) pairs to process.
+declare -A TARGET_USERS
+if [[ ${#FILTER[@]} -gt 0 ]]; then
+  for h in "${FILTER[@]}"; do
+    if [[ -z "${HOSTS[$h]+_}" ]]; then
+      echo "Unknown hostname: $h (not in the gc-hosts list)" >&2
+      exit 1
     fi
+    TARGET_USERS[$h]="${HOSTS[$h]}"
+  done
+else
+  for h in "${!HOSTS[@]}"; do
+    TARGET_USERS[$h]="${HOSTS[$h]}"
   done
-  exit 0
 fi
 
-# ── gc worker ─────────────────────────────────────────────────────────────────
+SORTED_HOSTS=($(printf '%s\n' "${!TARGET_USERS[@]}" | sort))
+
+# ── helpers ───────────────────────────────────────────────────────────────────
 
 gc_one() {
-  local host="$1" target="${HOST_TARGET[$1]}" type="${HOST_TYPE[$1]}" logfile="$2"
+  local host="$1" user="$2" logfile="$3"
+  local target="${user}@${host}"
 
   if ! ssh "${SSH_OPTS[@]}" "$target" "true" 2>>"$logfile"; then
-    echo "unreachable"; return
+    echo "unreachable"
+    return
   fi
 
-  if [[ "$type" == "nixos" ]]; then
+  # NixOS hosts: try passwordless sudo first.
+  if [[ "$user" == "$NIXOS_USER" ]]; then
     if ssh "${SSH_OPTS[@]}" "$target" "sudo -n nix-collect-garbage -d" \
         >>"$logfile" 2>>"$logfile"; then
-      echo "ok(sudo)"; return
+      echo "ok(sudo)"
+      return
     fi
-    echo "[sudo needs password — falling back to user-level gc]" >>"$logfile"
+    # sudo required a password — fall back to user-level gc.
+    echo "[sudo needs password, falling back to user-level gc]" >>"$logfile"
     if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \
         >>"$logfile" 2>>"$logfile"; then
-      echo "ok(user)"; return
+      echo "ok(user)"
+      return
     fi
   else
-    # Non-NixOS node: use a login shell so /etc/profile is sourced and the
-    # Nix daemon's bin dir is on PATH (set up by /etc/profile.d/nix-daemon.sh
-    # which the Nix installer adds to /etc/profile).
-    if ssh "${SSH_OPTS[@]}" "$target" "bash -l -c 'nix-collect-garbage -d'" \
+    # Non-NixOS node (pve1): no system generations; just gc as the login user.
+    if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \
         >>"$logfile" 2>>"$logfile"; then
-      echo "ok"; return
+      echo "ok"
+      return
     fi
   fi
 
   echo "failed:$?"
 }
 
-# ── parallel execution ────────────────────────────────────────────────────────
+# ── dry-run ───────────────────────────────────────────────────────────────────
 
-echo "Running gc on ${#ORDERED_HOSTS[@]} hosts (up to ${MAX_JOBS} parallel)..."
-echo ""
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  echo "[dry-run] would run gc on: ${SORTED_HOSTS[*]}"
+  for host in "${SORTED_HOSTS[@]}"; do
+    user="${TARGET_USERS[$host]}"
+    if [[ "$user" == "$NIXOS_USER" ]]; then
+      echo "  ssh ${SSH_OPTS[*]} ${user}@${host} 'sudo -n nix-collect-garbage -d'"
+      echo "  # fallback: ssh ... 'nix-collect-garbage -d'"
+    else
+      echo "  ssh ${SSH_OPTS[*]} ${user}@${host} 'nix-collect-garbage -d'"
+    fi
+  done
+  exit 0
+fi
+
+# ── parallel execution ────────────────────────────────────────────────────────
 
 TMPDIR_GC="$(mktemp -d)"
 trap 'rm -rf "$TMPDIR_GC"' EXIT
 
-declare -A LOGS=()
-job_count=0
+declare -A LOGS
 
-for host in "${ORDERED_HOSTS[@]}"; do
+echo "Running gc on ${#SORTED_HOSTS[@]} hosts (up to ${MAX_JOBS} parallel)..."
+echo ""
+
+job_count=0
+for host in "${SORTED_HOSTS[@]}"; do
+  user="${TARGET_USERS[$host]}"
   logfile="${TMPDIR_GC}/${host}.log"
   resultfile="${TMPDIR_GC}/${host}.result"
   LOGS[$host]="$logfile"
   : > "$logfile"
 
-  ( result="$(gc_one "$host" "$logfile")"; echo "$result" > "$resultfile" ) &
+  (
+    result="$(gc_one "$host" "$user" "$logfile")"
+    echo "$result" > "$resultfile"
+  ) &
 
   (( job_count++ )) || true
   if [[ "$job_count" -ge "$MAX_JOBS" ]]; then
@@ -206,24 +173,30 @@ ok_hosts=()
 warn_hosts=()
 fail_hosts=()
 
-for host in "${ORDERED_HOSTS[@]}"; do
-  result="$(cat "${TMPDIR_GC}/${host}.result" 2>/dev/null || echo "failed:missing")"
+for host in "${SORTED_HOSTS[@]}"; do
+  resultfile="${TMPDIR_GC}/${host}.result"
+  result="$(cat "$resultfile" 2>/dev/null || echo "failed:missing")"
+
   case "$result" in
     ok|"ok(sudo)"|"ok(user)")
-      printf "  %-22s  %s\n" "$host" "$result"
-      ok_hosts+=("$host") ;;
+      printf "  %-20s  %s\n" "$host" "$result"
+      ok_hosts+=("$host")
+      ;;
     unreachable)
-      printf "  %-22s  UNREACHABLE\n" "$host"
-      warn_hosts+=("$host") ;;
+      printf "  %-20s  UNREACHABLE\n" "$host"
+      warn_hosts+=("$host")
+      ;;
     *)
-      printf "  %-22s  FAILED (%s)\n" "$host" "$result"
-      fail_hosts+=("$host") ;;
+      printf "  %-20s  FAILED (%s)\n" "$host" "$result"
+      fail_hosts+=("$host")
+      ;;
   esac
 done
 
 echo ""
 echo "  ${#ok_hosts[@]} succeeded, ${#warn_hosts[@]} unreachable, ${#fail_hosts[@]} failed"
 
+# Print logs for any non-ok host.
 for host in "${warn_hosts[@]+"${warn_hosts[@]}"}" "${fail_hosts[@]+"${fail_hosts[@]}"}"; do
   logfile="${LOGS[$host]}"
   if [[ -s "$logfile" ]]; then

variables.nix

@@ -58,6 +58,7 @@
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZ9WKKAlP9Z7GQdgaZ1Xgw9C+vja2lqEZO5rJFpVqYN root@ha-server-1"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEGNKlaaMckd8nLWNGz4B2QokXjnnIvM+rEUv+R6h0sp root@ha-server-2"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK+XeMco7OxUpjrjZm54HogMs9QB5xlcKmElASRvrmlW root@nixos"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMseQwPpmaa6cgV5U8KhUsiVSYARG85zGa9rho0LJWks wayne@pve1"
   ];
 
   # Admin SSH public key, authorized on the primary user of every host and