Archived
refactor(gc-hosts): discover running pve1 guests dynamically each run
Replace the static host list with dynamic discovery: workstation (nixos) and pve1 are hard-wired first and second; remaining hosts are discovered on every run by SSHing to pve1, listing running VMs/containers via pct/qm list, and resolving their NixOS hostnames from a single flake eval. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
+125
-111
@@ -1,159 +1,179 @@
|
||||
#!/usr/bin/env bash
|
||||
# gc-hosts.sh — Run nix-collect-garbage -d on all live NixOS hosts and pve1.
|
||||
# gc-hosts.sh — Run nix-collect-garbage -d on all live NixOS hosts.
|
||||
#
|
||||
# nix-cache is excluded: it is the shared binary cache for all other hosts, so
|
||||
# gc-ing it would evict cached store paths and force costly rebuilds elsewhere.
|
||||
# The host list is rebuilt on every run:
|
||||
# 1. This workstation (nixos) — always first
|
||||
# 2. pve1 — always second (non-NixOS Proxmox node with Nix installed)
|
||||
# 3. Every NixOS guest currently running on pve1 (discovered via pct/qm list)
|
||||
#
|
||||
# Runs SSH jobs in parallel (up to MAX_JOBS at a time) and prints a summary.
|
||||
# nix-cache is excluded: gc-ing the shared binary cache evicts store paths
|
||||
# that other hosts depend on for substitution.
|
||||
#
|
||||
# NixOS hosts: tries "sudo -n nix-collect-garbage -d" first (works when
|
||||
# wheelNeedsPassword = false, e.g. the HA cluster). Falls back to user-level
|
||||
# "nix-collect-garbage -d" if sudo needs a password — still collects
|
||||
# unreferenced store paths and old nixos-user profile generations, but leaves
|
||||
# old system generations in place.
|
||||
# pve1: runs "nix-collect-garbage -d" as the login user (no system generations
|
||||
# on a non-NixOS host).
|
||||
#
|
||||
# Usage (from repo root):
|
||||
# bash scripts/gc-hosts.sh [--dry-run] [<hostname> ...]
|
||||
#
|
||||
# Options:
|
||||
# --dry-run Print the SSH commands without executing them.
|
||||
# <hostname> Limit to the given hostnames (bare names, no domain suffix).
|
||||
# Default: all hosts in the list below.
|
||||
#
|
||||
# Sudo notes:
|
||||
# NixOS hosts: tries "sudo -n nix-collect-garbage -d" first (non-interactive,
|
||||
# works when wheelNeedsPassword = false such as on the HA cluster). Falls
|
||||
# back to "nix-collect-garbage -d" as the nixos user if sudo requires a
|
||||
# password — this still collects unreferenced store paths and removes old
|
||||
# nixos-user profile generations, but will not remove old system generations.
|
||||
# pve1: non-NixOS Proxmox node, runs nix-collect-garbage -d as wayne (no
|
||||
# system generations to delete).
|
||||
# bash scripts/gc-hosts.sh [--dry-run]
|
||||
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
source scripts/env.sh 2>/dev/null || true
|
||||
source scripts/lib/nix-eval.sh 2>/dev/null || true
|
||||
|
||||
# ── config ────────────────────────────────────────────────────────────────────
|
||||
|
||||
: "${MAX_JOBS:=6}"
|
||||
: "${MAX_JOBS:=8}"
|
||||
: "${NIXOS_USER:=nixos}"
|
||||
: "${PVE1_SSH_USER:=${PROXMOX_SSH_USER:-wayne}}"
|
||||
SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10)
|
||||
|
||||
# Map: hostname → ssh-user
|
||||
# Update this list when new hosts are added/removed.
|
||||
declare -A HOSTS=(
|
||||
[docker]="$NIXOS_USER"
|
||||
[ha-server-1]="$NIXOS_USER"
|
||||
[ha-server-2]="$NIXOS_USER"
|
||||
[nix-minimal]="$NIXOS_USER"
|
||||
[nixos]="$NIXOS_USER"
|
||||
[pxe-boot]="$NIXOS_USER"
|
||||
[server]="$NIXOS_USER"
|
||||
[tailscale-router]="$NIXOS_USER"
|
||||
[tor-relay]="$NIXOS_USER"
|
||||
[pve1]="${PROXMOX_SSH_USER:-wayne}"
|
||||
)
|
||||
|
||||
# ── argument parsing ──────────────────────────────────────────────────────────
|
||||
|
||||
DRY_RUN=0
|
||||
FILTER=()
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--dry-run) DRY_RUN=1 ;;
|
||||
--*) echo "Unknown option: $arg" >&2; exit 1 ;;
|
||||
*) FILTER+=("$arg") ;;
|
||||
*) echo "Unknown option: $arg" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Resolve the set of (hostname, user) pairs to process.
|
||||
declare -A TARGET_USERS
|
||||
if [[ ${#FILTER[@]} -gt 0 ]]; then
|
||||
for h in "${FILTER[@]}"; do
|
||||
if [[ -z "${HOSTS[$h]+_}" ]]; then
|
||||
echo "Unknown hostname: $h (not in the gc-hosts list)" >&2
|
||||
exit 1
|
||||
# ── build the host list ───────────────────────────────────────────────────────
|
||||
|
||||
# ORDERED_HOSTS: names in display/execution order.
|
||||
# HOST_TARGET[name]: SSH target string (user@host).
|
||||
# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (user gc only).
|
||||
declare -a ORDERED_HOSTS=()
|
||||
declare -A HOST_TARGET=()
|
||||
declare -A HOST_TYPE=()
|
||||
declare -A _SEEN_HOSTNAMES=() # dedup tracker
|
||||
|
||||
_add_host() {
|
||||
local name="$1" target="$2" type="$3"
|
||||
if [[ -n "${_SEEN_HOSTNAMES[$name]+_}" ]]; then return; fi
|
||||
_SEEN_HOSTNAMES[$name]=1
|
||||
ORDERED_HOSTS+=("$name")
|
||||
HOST_TARGET[$name]="$target"
|
||||
HOST_TYPE[$name]="$type"
|
||||
}
|
||||
|
||||
# 1. Workstation (hard-wired first)
|
||||
_add_host "nixos" "${NIXOS_USER}@nixos" "nixos"
|
||||
|
||||
# 2. pve1 (hard-wired second; non-NixOS, no system generations)
|
||||
_add_host "pve1" "${PVE1_SSH_USER}@${PVE1_HOST}" "nix"
|
||||
|
||||
# 3. Dynamically discover running NixOS guests on pve1
|
||||
echo "Discovering running guests on ${PVE1_HOST}..."
|
||||
|
||||
# Evaluate the full flake hostname map in one shot.
|
||||
hostname_map="{}"
|
||||
if ! hostname_map="$(
|
||||
nix eval --json "${NIX_EVAL_FLAGS[@]}" .#nixosConfigurations \
|
||||
--apply 'cfgs: builtins.mapAttrs (_: cfg: cfg.config.networking.hostName) cfgs' \
|
||||
2>/dev/null
|
||||
)"; then
|
||||
echo " warning: flake eval failed — skipping dynamic host discovery" >&2
|
||||
fi
|
||||
TARGET_USERS[$h]="${HOSTS[$h]}"
|
||||
done
|
||||
|
||||
# Get names of all currently running guests from pve1.
|
||||
if ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
|
||||
running_guests="$(
|
||||
ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash <<'REMOTE'
|
||||
{ sudo pct list 2>/dev/null | awk 'NR>1 && $2=="running" { print $NF }';
|
||||
sudo qm list 2>/dev/null | awk 'NR>1 && $3=="running" { print $2 }'; } | sort -u
|
||||
REMOTE
|
||||
)" || running_guests=""
|
||||
|
||||
while IFS= read -r guest; do
|
||||
[[ -z "$guest" ]] && continue
|
||||
|
||||
# Resolve flake target name → NixOS hostname.
|
||||
hostname="$(printf '%s' "$hostname_map" \
|
||||
| jq -r --arg g "$guest" '.[$g] // empty' 2>/dev/null || true)"
|
||||
[[ -z "$hostname" ]] && continue
|
||||
|
||||
# Exclude nix-cache and any target whose hostname is already in our list.
|
||||
case "$hostname" in nix-cache) continue ;; esac
|
||||
if [[ -n "${_SEEN_HOSTNAMES[$hostname]+_}" ]]; then continue; fi
|
||||
|
||||
echo " + $guest → $hostname"
|
||||
_add_host "$hostname" "${NIXOS_USER}@${hostname}" "nixos"
|
||||
done <<< "$running_guests"
|
||||
else
|
||||
for h in "${!HOSTS[@]}"; do
|
||||
TARGET_USERS[$h]="${HOSTS[$h]}"
|
||||
done
|
||||
echo " warning: ${PVE1_HOST} unreachable — skipping dynamic host discovery" >&2
|
||||
fi
|
||||
|
||||
SORTED_HOSTS=($(printf '%s\n' "${!TARGET_USERS[@]}" | sort))
|
||||
echo ""
|
||||
echo "Hosts: ${ORDERED_HOSTS[*]}"
|
||||
echo ""
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
# ── dry-run ───────────────────────────────────────────────────────────────────
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "[dry-run] commands that would run:"
|
||||
for host in "${ORDERED_HOSTS[@]}"; do
|
||||
target="${HOST_TARGET[$host]}"
|
||||
type="${HOST_TYPE[$host]}"
|
||||
if [[ "$type" == "nixos" ]]; then
|
||||
echo " ssh ${SSH_OPTS[*]} $target 'sudo -n nix-collect-garbage -d'"
|
||||
echo " # fallback: ssh ... $target 'nix-collect-garbage -d'"
|
||||
else
|
||||
echo " ssh ${SSH_OPTS[*]} $target 'nix-collect-garbage -d'"
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── gc worker ─────────────────────────────────────────────────────────────────
|
||||
|
||||
gc_one() {
|
||||
local host="$1" user="$2" logfile="$3"
|
||||
local target="${user}@${host}"
|
||||
local host="$1" target="${HOST_TARGET[$1]}" type="${HOST_TYPE[$1]}" logfile="$2"
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "$target" "true" 2>>"$logfile"; then
|
||||
echo "unreachable"
|
||||
return
|
||||
echo "unreachable"; return
|
||||
fi
|
||||
|
||||
# NixOS hosts: try passwordless sudo first.
|
||||
if [[ "$user" == "$NIXOS_USER" ]]; then
|
||||
if [[ "$type" == "nixos" ]]; then
|
||||
if ssh "${SSH_OPTS[@]}" "$target" "sudo -n nix-collect-garbage -d" \
|
||||
>>"$logfile" 2>>"$logfile"; then
|
||||
echo "ok(sudo)"
|
||||
return
|
||||
echo "ok(sudo)"; return
|
||||
fi
|
||||
# sudo required a password — fall back to user-level gc.
|
||||
echo "[sudo needs password, falling back to user-level gc]" >>"$logfile"
|
||||
echo "[sudo needs password — falling back to user-level gc]" >>"$logfile"
|
||||
if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \
|
||||
>>"$logfile" 2>>"$logfile"; then
|
||||
echo "ok(user)"
|
||||
return
|
||||
echo "ok(user)"; return
|
||||
fi
|
||||
else
|
||||
# Non-NixOS node (pve1): no system generations; just gc as the login user.
|
||||
if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \
|
||||
>>"$logfile" 2>>"$logfile"; then
|
||||
echo "ok"
|
||||
return
|
||||
echo "ok"; return
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "failed:$?"
|
||||
}
|
||||
|
||||
# ── dry-run ───────────────────────────────────────────────────────────────────
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "[dry-run] would run gc on: ${SORTED_HOSTS[*]}"
|
||||
for host in "${SORTED_HOSTS[@]}"; do
|
||||
user="${TARGET_USERS[$host]}"
|
||||
if [[ "$user" == "$NIXOS_USER" ]]; then
|
||||
echo " ssh ${SSH_OPTS[*]} ${user}@${host} 'sudo -n nix-collect-garbage -d'"
|
||||
echo " # fallback: ssh ... 'nix-collect-garbage -d'"
|
||||
else
|
||||
echo " ssh ${SSH_OPTS[*]} ${user}@${host} 'nix-collect-garbage -d'"
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── parallel execution ────────────────────────────────────────────────────────
|
||||
|
||||
echo "Running gc on ${#ORDERED_HOSTS[@]} hosts (up to ${MAX_JOBS} parallel)..."
|
||||
echo ""
|
||||
|
||||
TMPDIR_GC="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMPDIR_GC"' EXIT
|
||||
|
||||
declare -A LOGS
|
||||
|
||||
echo "Running gc on ${#SORTED_HOSTS[@]} hosts (up to ${MAX_JOBS} parallel)..."
|
||||
echo ""
|
||||
|
||||
declare -A LOGS=()
|
||||
job_count=0
|
||||
for host in "${SORTED_HOSTS[@]}"; do
|
||||
user="${TARGET_USERS[$host]}"
|
||||
|
||||
for host in "${ORDERED_HOSTS[@]}"; do
|
||||
logfile="${TMPDIR_GC}/${host}.log"
|
||||
resultfile="${TMPDIR_GC}/${host}.result"
|
||||
LOGS[$host]="$logfile"
|
||||
: > "$logfile"
|
||||
|
||||
(
|
||||
result="$(gc_one "$host" "$user" "$logfile")"
|
||||
echo "$result" > "$resultfile"
|
||||
) &
|
||||
( result="$(gc_one "$host" "$logfile")"; echo "$result" > "$resultfile" ) &
|
||||
|
||||
(( job_count++ )) || true
|
||||
if [[ "$job_count" -ge "$MAX_JOBS" ]]; then
|
||||
@@ -173,30 +193,24 @@ ok_hosts=()
|
||||
warn_hosts=()
|
||||
fail_hosts=()
|
||||
|
||||
for host in "${SORTED_HOSTS[@]}"; do
|
||||
resultfile="${TMPDIR_GC}/${host}.result"
|
||||
result="$(cat "$resultfile" 2>/dev/null || echo "failed:missing")"
|
||||
|
||||
for host in "${ORDERED_HOSTS[@]}"; do
|
||||
result="$(cat "${TMPDIR_GC}/${host}.result" 2>/dev/null || echo "failed:missing")"
|
||||
case "$result" in
|
||||
ok|"ok(sudo)"|"ok(user)")
|
||||
printf " %-20s %s\n" "$host" "$result"
|
||||
ok_hosts+=("$host")
|
||||
;;
|
||||
printf " %-22s %s\n" "$host" "$result"
|
||||
ok_hosts+=("$host") ;;
|
||||
unreachable)
|
||||
printf " %-20s UNREACHABLE\n" "$host"
|
||||
warn_hosts+=("$host")
|
||||
;;
|
||||
printf " %-22s UNREACHABLE\n" "$host"
|
||||
warn_hosts+=("$host") ;;
|
||||
*)
|
||||
printf " %-20s FAILED (%s)\n" "$host" "$result"
|
||||
fail_hosts+=("$host")
|
||||
;;
|
||||
printf " %-22s FAILED (%s)\n" "$host" "$result"
|
||||
fail_hosts+=("$host") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " ${#ok_hosts[@]} succeeded, ${#warn_hosts[@]} unreachable, ${#fail_hosts[@]} failed"
|
||||
|
||||
# Print logs for any non-ok host.
|
||||
for host in "${warn_hosts[@]+"${warn_hosts[@]}"}" "${fail_hosts[@]+"${fail_hosts[@]}"}"; do
|
||||
logfile="${LOGS[$host]}"
|
||||
if [[ -s "$logfile" ]]; then
|
||||
|
||||
Reference in New Issue
Block a user