From 629c1457a9104963d32d6bc7d254546cb6f4b5c6 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Wed, 29 Jul 2026 13:44:26 +1000 Subject: [PATCH 1/2] refactor(gc-hosts): discover running pve1 guests dynamically each run Replace the static host list with dynamic discovery: workstation (nixos) and pve1 are hard-wired first and second; remaining hosts are discovered on every run by SSHing to pve1, listing running VMs/containers via pct/qm list, and resolving their NixOS hostnames from a single flake eval. Co-Authored-By: Claude Sonnet 4.6 --- scripts/gc-hosts.sh | 240 +++++++++++++++++++++++--------------------- 1 file changed, 127 insertions(+), 113 deletions(-) diff --git a/scripts/gc-hosts.sh b/scripts/gc-hosts.sh index 03f27f6..a9ba358 100755 --- a/scripts/gc-hosts.sh +++ b/scripts/gc-hosts.sh @@ -1,159 +1,179 @@ #!/usr/bin/env bash -# gc-hosts.sh — Run nix-collect-garbage -d on all live NixOS hosts and pve1. +# gc-hosts.sh — Run nix-collect-garbage -d on all live NixOS hosts. # -# nix-cache is excluded: it is the shared binary cache for all other hosts, so -# gc-ing it would evict cached store paths and force costly rebuilds elsewhere. +# The host list is rebuilt on every run: +# 1. This workstation (nixos) — always first +# 2. pve1 — always second (non-NixOS Proxmox node with Nix installed) +# 3. Every NixOS guest currently running on pve1 (discovered via pct/qm list) # -# Runs SSH jobs in parallel (up to MAX_JOBS at a time) and prints a summary. +# nix-cache is excluded: gc-ing the shared binary cache evicts store paths +# that other hosts depend on for substitution. +# +# NixOS hosts: tries "sudo -n nix-collect-garbage -d" first (works when +# wheelNeedsPassword = false, e.g. the HA cluster). Falls back to user-level +# "nix-collect-garbage -d" if sudo needs a password — still collects +# unreferenced store paths and old nixos-user profile generations, but leaves +# old system generations in place. +# pve1: runs "nix-collect-garbage -d" as the login user (no system generations +# on a non-NixOS host). # # Usage (from repo root): -# bash scripts/gc-hosts.sh [--dry-run] [ ...] -# -# Options: -# --dry-run Print the SSH commands without executing them. -# Limit to the given hostnames (bare names, no domain suffix). -# Default: all hosts in the list below. -# -# Sudo notes: -# NixOS hosts: tries "sudo -n nix-collect-garbage -d" first (non-interactive, -# works when wheelNeedsPassword = false such as on the HA cluster). Falls -# back to "nix-collect-garbage -d" as the nixos user if sudo requires a -# password — this still collects unreferenced store paths and removes old -# nixos-user profile generations, but will not remove old system generations. -# pve1: non-NixOS Proxmox node, runs nix-collect-garbage -d as wayne (no -# system generations to delete). +# bash scripts/gc-hosts.sh [--dry-run] set -euo pipefail cd "$(dirname "$0")/.." source scripts/env.sh 2>/dev/null || true +source scripts/lib/nix-eval.sh 2>/dev/null || true # ── config ──────────────────────────────────────────────────────────────────── -: "${MAX_JOBS:=6}" +: "${MAX_JOBS:=8}" : "${NIXOS_USER:=nixos}" +: "${PVE1_SSH_USER:=${PROXMOX_SSH_USER:-wayne}}" SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10) -# Map: hostname → ssh-user -# Update this list when new hosts are added/removed. -declare -A HOSTS=( - [docker]="$NIXOS_USER" - [ha-server-1]="$NIXOS_USER" - [ha-server-2]="$NIXOS_USER" - [nix-minimal]="$NIXOS_USER" - [nixos]="$NIXOS_USER" - [pxe-boot]="$NIXOS_USER" - [server]="$NIXOS_USER" - [tailscale-router]="$NIXOS_USER" - [tor-relay]="$NIXOS_USER" - [pve1]="${PROXMOX_SSH_USER:-wayne}" -) - -# ── argument parsing ────────────────────────────────────────────────────────── - DRY_RUN=0 -FILTER=() - for arg in "$@"; do case "$arg" in --dry-run) DRY_RUN=1 ;; - --*) echo "Unknown option: $arg" >&2; exit 1 ;; - *) FILTER+=("$arg") ;; + *) echo "Unknown option: $arg" >&2; exit 1 ;; esac done -# Resolve the set of (hostname, user) pairs to process. -declare -A TARGET_USERS -if [[ ${#FILTER[@]} -gt 0 ]]; then - for h in "${FILTER[@]}"; do - if [[ -z "${HOSTS[$h]+_}" ]]; then - echo "Unknown hostname: $h (not in the gc-hosts list)" >&2 - exit 1 - fi - TARGET_USERS[$h]="${HOSTS[$h]}" - done -else - for h in "${!HOSTS[@]}"; do - TARGET_USERS[$h]="${HOSTS[$h]}" - done +# ── build the host list ─────────────────────────────────────────────────────── + +# ORDERED_HOSTS: names in display/execution order. +# HOST_TARGET[name]: SSH target string (user@host). +# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (user gc only). +declare -a ORDERED_HOSTS=() +declare -A HOST_TARGET=() +declare -A HOST_TYPE=() +declare -A _SEEN_HOSTNAMES=() # dedup tracker + +_add_host() { + local name="$1" target="$2" type="$3" + if [[ -n "${_SEEN_HOSTNAMES[$name]+_}" ]]; then return; fi + _SEEN_HOSTNAMES[$name]=1 + ORDERED_HOSTS+=("$name") + HOST_TARGET[$name]="$target" + HOST_TYPE[$name]="$type" +} + +# 1. Workstation (hard-wired first) +_add_host "nixos" "${NIXOS_USER}@nixos" "nixos" + +# 2. pve1 (hard-wired second; non-NixOS, no system generations) +_add_host "pve1" "${PVE1_SSH_USER}@${PVE1_HOST}" "nix" + +# 3. Dynamically discover running NixOS guests on pve1 +echo "Discovering running guests on ${PVE1_HOST}..." + +# Evaluate the full flake hostname map in one shot. +hostname_map="{}" +if ! hostname_map="$( + nix eval --json "${NIX_EVAL_FLAGS[@]}" .#nixosConfigurations \ + --apply 'cfgs: builtins.mapAttrs (_: cfg: cfg.config.networking.hostName) cfgs' \ + 2>/dev/null +)"; then + echo " warning: flake eval failed — skipping dynamic host discovery" >&2 fi -SORTED_HOSTS=($(printf '%s\n' "${!TARGET_USERS[@]}" | sort)) +# Get names of all currently running guests from pve1. +if ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then + running_guests="$( + ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash <<'REMOTE' + { sudo pct list 2>/dev/null | awk 'NR>1 && $2=="running" { print $NF }'; + sudo qm list 2>/dev/null | awk 'NR>1 && $3=="running" { print $2 }'; } | sort -u +REMOTE + )" || running_guests="" -# ── helpers ─────────────────────────────────────────────────────────────────── + while IFS= read -r guest; do + [[ -z "$guest" ]] && continue + + # Resolve flake target name → NixOS hostname. + hostname="$(printf '%s' "$hostname_map" \ + | jq -r --arg g "$guest" '.[$g] // empty' 2>/dev/null || true)" + [[ -z "$hostname" ]] && continue + + # Exclude nix-cache and any target whose hostname is already in our list. + case "$hostname" in nix-cache) continue ;; esac + if [[ -n "${_SEEN_HOSTNAMES[$hostname]+_}" ]]; then continue; fi + + echo " + $guest → $hostname" + _add_host "$hostname" "${NIXOS_USER}@${hostname}" "nixos" + done <<< "$running_guests" +else + echo " warning: ${PVE1_HOST} unreachable — skipping dynamic host discovery" >&2 +fi + +echo "" +echo "Hosts: ${ORDERED_HOSTS[*]}" +echo "" + +# ── dry-run ─────────────────────────────────────────────────────────────────── + +if [[ "$DRY_RUN" -eq 1 ]]; then + echo "[dry-run] commands that would run:" + for host in "${ORDERED_HOSTS[@]}"; do + target="${HOST_TARGET[$host]}" + type="${HOST_TYPE[$host]}" + if [[ "$type" == "nixos" ]]; then + echo " ssh ${SSH_OPTS[*]} $target 'sudo -n nix-collect-garbage -d'" + echo " # fallback: ssh ... $target 'nix-collect-garbage -d'" + else + echo " ssh ${SSH_OPTS[*]} $target 'nix-collect-garbage -d'" + fi + done + exit 0 +fi + +# ── gc worker ───────────────────────────────────────────────────────────────── gc_one() { - local host="$1" user="$2" logfile="$3" - local target="${user}@${host}" + local host="$1" target="${HOST_TARGET[$1]}" type="${HOST_TYPE[$1]}" logfile="$2" if ! ssh "${SSH_OPTS[@]}" "$target" "true" 2>>"$logfile"; then - echo "unreachable" - return + echo "unreachable"; return fi - # NixOS hosts: try passwordless sudo first. - if [[ "$user" == "$NIXOS_USER" ]]; then + if [[ "$type" == "nixos" ]]; then if ssh "${SSH_OPTS[@]}" "$target" "sudo -n nix-collect-garbage -d" \ >>"$logfile" 2>>"$logfile"; then - echo "ok(sudo)" - return + echo "ok(sudo)"; return fi - # sudo required a password — fall back to user-level gc. - echo "[sudo needs password, falling back to user-level gc]" >>"$logfile" + echo "[sudo needs password — falling back to user-level gc]" >>"$logfile" if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \ >>"$logfile" 2>>"$logfile"; then - echo "ok(user)" - return + echo "ok(user)"; return fi else - # Non-NixOS node (pve1): no system generations; just gc as the login user. if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \ >>"$logfile" 2>>"$logfile"; then - echo "ok" - return + echo "ok"; return fi fi echo "failed:$?" } -# ── dry-run ─────────────────────────────────────────────────────────────────── - -if [[ "$DRY_RUN" -eq 1 ]]; then - echo "[dry-run] would run gc on: ${SORTED_HOSTS[*]}" - for host in "${SORTED_HOSTS[@]}"; do - user="${TARGET_USERS[$host]}" - if [[ "$user" == "$NIXOS_USER" ]]; then - echo " ssh ${SSH_OPTS[*]} ${user}@${host} 'sudo -n nix-collect-garbage -d'" - echo " # fallback: ssh ... 'nix-collect-garbage -d'" - else - echo " ssh ${SSH_OPTS[*]} ${user}@${host} 'nix-collect-garbage -d'" - fi - done - exit 0 -fi - # ── parallel execution ──────────────────────────────────────────────────────── +echo "Running gc on ${#ORDERED_HOSTS[@]} hosts (up to ${MAX_JOBS} parallel)..." +echo "" + TMPDIR_GC="$(mktemp -d)" trap 'rm -rf "$TMPDIR_GC"' EXIT -declare -A LOGS - -echo "Running gc on ${#SORTED_HOSTS[@]} hosts (up to ${MAX_JOBS} parallel)..." -echo "" - +declare -A LOGS=() job_count=0 -for host in "${SORTED_HOSTS[@]}"; do - user="${TARGET_USERS[$host]}" + +for host in "${ORDERED_HOSTS[@]}"; do logfile="${TMPDIR_GC}/${host}.log" resultfile="${TMPDIR_GC}/${host}.result" LOGS[$host]="$logfile" : > "$logfile" - ( - result="$(gc_one "$host" "$user" "$logfile")" - echo "$result" > "$resultfile" - ) & + ( result="$(gc_one "$host" "$logfile")"; echo "$result" > "$resultfile" ) & (( job_count++ )) || true if [[ "$job_count" -ge "$MAX_JOBS" ]]; then @@ -173,30 +193,24 @@ ok_hosts=() warn_hosts=() fail_hosts=() -for host in "${SORTED_HOSTS[@]}"; do - resultfile="${TMPDIR_GC}/${host}.result" - result="$(cat "$resultfile" 2>/dev/null || echo "failed:missing")" - +for host in "${ORDERED_HOSTS[@]}"; do + result="$(cat "${TMPDIR_GC}/${host}.result" 2>/dev/null || echo "failed:missing")" case "$result" in ok|"ok(sudo)"|"ok(user)") - printf " %-20s %s\n" "$host" "$result" - ok_hosts+=("$host") - ;; + printf " %-22s %s\n" "$host" "$result" + ok_hosts+=("$host") ;; unreachable) - printf " %-20s UNREACHABLE\n" "$host" - warn_hosts+=("$host") - ;; + printf " %-22s UNREACHABLE\n" "$host" + warn_hosts+=("$host") ;; *) - printf " %-20s FAILED (%s)\n" "$host" "$result" - fail_hosts+=("$host") - ;; + printf " %-22s FAILED (%s)\n" "$host" "$result" + fail_hosts+=("$host") ;; esac done echo "" echo " ${#ok_hosts[@]} succeeded, ${#warn_hosts[@]} unreachable, ${#fail_hosts[@]} failed" -# Print logs for any non-ok host. for host in "${warn_hosts[@]+"${warn_hosts[@]}"}" "${fail_hosts[@]+"${fail_hosts[@]}"}"; do logfile="${LOGS[$host]}" if [[ -s "$logfile" ]]; then -- 2.54.0 From 0ef8259225b982a504dacadb49f5254dd3c65709 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Wed, 29 Jul 2026 13:45:48 +1000 Subject: [PATCH 2/2] fix(gc-hosts): source nix-daemon profile on pve1 before running gc BatchMode SSH sessions don't source /etc/profile on non-NixOS hosts, so nix-collect-garbage isn't on PATH for the wayne user. Source the nix-daemon profile script explicitly, matching the pattern in scripts/lib/nix-bootstrap.sh. Co-Authored-By: Claude Sonnet 4.6 --- scripts/gc-hosts.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/gc-hosts.sh b/scripts/gc-hosts.sh index a9ba358..3a87cd0 100755 --- a/scripts/gc-hosts.sh +++ b/scripts/gc-hosts.sh @@ -121,7 +121,7 @@ if [[ "$DRY_RUN" -eq 1 ]]; then echo " ssh ${SSH_OPTS[*]} $target 'sudo -n nix-collect-garbage -d'" echo " # fallback: ssh ... $target 'nix-collect-garbage -d'" else - echo " ssh ${SSH_OPTS[*]} $target 'nix-collect-garbage -d'" + echo " ssh ${SSH_OPTS[*]} $target '. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d'" fi done exit 0 @@ -147,7 +147,10 @@ gc_one() { echo "ok(user)"; return fi else - if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \ + # Non-NixOS node: BatchMode SSH doesn't source the Nix daemon profile, so + # nix-collect-garbage won't be on PATH unless we source it explicitly. + local nix_profile='. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null || true' + if ssh "${SSH_OPTS[@]}" "$target" "$nix_profile && nix-collect-garbage -d" \ >>"$logfile" 2>>"$logfile"; then echo "ok"; return fi -- 2.54.0