Run per-host/per-package nix eval and dry-run build concurrently

codex-maintenance.sh evaluated each affected host/package one at a time,
even though those calls are independent. Added scripts/lib/nix-parallel.sh
(run_nix_parallel) and wired it into the host-eval, package-eval, and
dry-run-build loops.

Concurrency defaults to core count capped by available memory (~1GB/job)
rather than plain nproc: empirically, nproc-many concurrent full-flake
evals OOM-killed each other on a 4GB/6-core box, while 3-4 ran clean and
were still ~2x faster than serial. Override via NIX_PARALLEL_JOBS.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

scripts/codex-maintenance.sh

@@ -16,6 +16,14 @@
 #
 # --dry-run: adds `nix build --dry-run --no-link` for whatever scope is
 # active (changed-files scope by default, full scope under --full-check).
+#
+# Per-host/per-package eval and dry-run build calls run concurrently (see
+# scripts/lib/nix-parallel.sh) since they're independent of each other.
+# Concurrency defaults to core count capped by available memory (~1GB/job)
+# rather than plain core count, since each concurrent `nix eval` evaluates a
+# whole NixOS system closure and can OOM a small/memory-constrained CI
+# runner otherwise; override via NIX_PARALLEL_JOBS if a runner has more (or
+# less) room than that estimate assumes.
 set -euo pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -23,6 +31,8 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${script_dir}/lib/nix-bootstrap.sh"
 # shellcheck source=lib/nix-eval.sh
 source "${script_dir}/lib/nix-eval.sh"
+# shellcheck source=lib/nix-parallel.sh
+source "${script_dir}/lib/nix-parallel.sh"
 
 repo_root="$(cd "${script_dir}/.." && pwd)"
 cd "$repo_root"
@@ -246,66 +256,64 @@ echo
 if [[ ${#hosts[@]} -eq 0 ]]; then
   echo "No hosts affected by changed files; skipping host eval."
 else
-  echo "Evaluating host toplevel derivations (${scope_desc})..."
+  echo "Evaluating host toplevel derivations (${scope_desc}, up to ${NIX_PARALLEL_JOBS} at a time)..."
-  for host in "${hosts[@]}"; do
-    echo "==> $host"
-    nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath"
-
   # lxc-* hosts deploy via a directly pct-restore-able tarball instead of
   # nixos-install (see docs/auto-installer.md); proxmox-* hosts can
   # alternatively be built as a standalone disk image (see
   # docs/proxmox-images.md). Both are otherwise-unvalidated buildable
   # surface, easy to silently break without this.
+  declare -a host_eval_jobs=()
+  for host in "${hosts[@]}"; do
+    host_eval_jobs+=("${host}${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.toplevel.drvPath")
     case "$host" in
       lxc-*)
-        echo "==> $host (tarball)"
+        host_eval_jobs+=("${host} (tarball)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.tarball.drvPath")
-        nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.tarball.drvPath"
         ;;
       proxmox-*)
-        echo "==> $host (diskoImagesScript)"
+        host_eval_jobs+=("${host} (diskoImagesScript)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.diskoImagesScript.drvPath")
-        nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.diskoImagesScript.drvPath"
         ;;
     esac
   done
+  run_nix_parallel host_eval_jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
 fi
 
 echo
 if ! $eval_packages; then
   echo "No packages affected by changed files; skipping package eval."
 else
-  echo "Evaluating buildable packages..."
+  echo "Evaluating buildable packages (up to ${NIX_PARALLEL_JOBS} at a time)..."
+  declare -a package_eval_jobs=()
   for pkg in "${all_packages[@]}"; do
-    echo "==> packages.x86_64-linux.${pkg}"
+    package_eval_jobs+=("packages.x86_64-linux.${pkg}${NIX_PARALLEL_SEP}.#packages.x86_64-linux.${pkg}")
-    nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#packages.x86_64-linux.${pkg}"
   done
+  run_nix_parallel package_eval_jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
 fi
 
 if $dry_run; then
   echo
-  echo "Running dry-run builds for the active scope. This will not create result symlinks."
+  echo "Running dry-run builds for the active scope (up to ${NIX_PARALLEL_JOBS} at a time). This will not create result symlinks."
+  declare -a host_build_jobs=()
   for host in "${hosts[@]:-}"; do
-    echo "==> Dry-run build: $host"
+    host_build_jobs+=("Dry-run build: ${host}${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.toplevel")
-    nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.toplevel"
-
     case "$host" in
       lxc-*)
-        echo "==> Dry-run build: $host (tarball)"
+        host_build_jobs+=("Dry-run build: ${host} (tarball)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.tarball")
-        nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.tarball"
         ;;
       proxmox-*)
-        echo "==> Dry-run build: $host (diskoImagesScript)"
+        host_build_jobs+=("Dry-run build: ${host} (diskoImagesScript)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.diskoImagesScript")
-        nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.diskoImagesScript"
         ;;
     esac
   done
+  run_nix_parallel host_build_jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
 
   if $eval_packages; then
     echo
     echo "Running dry-run builds for packages."
+    declare -a package_build_jobs=()
     for pkg in "${all_packages[@]}"; do
-      echo "==> Dry-run build: packages.x86_64-linux.${pkg}"
+      package_build_jobs+=("Dry-run build: packages.x86_64-linux.${pkg}${NIX_PARALLEL_SEP}.#packages.x86_64-linux.${pkg}")
-      nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#packages.x86_64-linux.${pkg}"
     done
+    run_nix_parallel package_build_jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
   fi
 fi
 

scripts/lib/nix-parallel.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# Shared parallel-nix-invocation helper for scripts/codex-maintenance.sh.
+# Source alongside nix-eval.sh:
+#   source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/nix-parallel.sh"
+#
+# The per-host/per-package `nix eval`/`nix build --dry-run` calls in
+# codex-maintenance.sh are independent of each other, so running them one at
+# a time leaves most cores idle for most of the sweep -- run_nix_parallel
+# fans a batch of them out across up to NIX_PARALLEL_JOBS processes instead.
+
+# NIX_PARALLEL_JOBS: how many `nix` invocations run_nix_parallel runs at
+# once. Defaults to core count capped by available memory (~1GB/job,
+# floor 1) rather than plain `nproc` -- each concurrent `nix eval` here
+# evaluates a whole NixOS system closure from scratch, and on a small/
+# memory-constrained CI runner, `nproc` concurrent evals can OOM-kill each
+# other (confirmed empirically: on a 4GB/6-core box, 5-6 concurrent evals
+# started getting killed while 3-4 ran clean and were still ~2x faster than
+# serial). Override via env if a given machine/CI runner has room to spare
+# or needs a tighter cap.
+default_nix_parallel_jobs() {
+  local cores mem_avail_kb mem_cap
+  cores="$(nproc 2>/dev/null || echo 4)"
+  mem_avail_kb="$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo 2>/dev/null)"
+  if [[ -z "$mem_avail_kb" ]]; then
+    echo "$cores"
+    return
+  fi
+  mem_cap=$((mem_avail_kb / 1024 / 1024))
+  ((mem_cap < 1)) && mem_cap=1
+  ((mem_cap < cores)) && echo "$mem_cap" || echo "$cores"
+}
+NIX_PARALLEL_JOBS="${NIX_PARALLEL_JOBS:-$(default_nix_parallel_jobs)}"
+
+# Separator between a job's label and its flake attr in the arrays
+# run_nix_parallel takes -- a control character so it can't collide with
+# anything a label or attr path would plausibly contain.
+NIX_PARALLEL_SEP=$'\x1f'
+
+# run_nix_parallel <jobs_array_name> <nix subcommand + flags...>
+#
+# jobs_array_name: name of an already-populated bash array whose entries are
+# "<label>${NIX_PARALLEL_SEP}<attr>" pairs, e.g.
+#   jobs=("proxmox-docker${NIX_PARALLEL_SEP}.#nixosConfigurations.proxmox-docker...drvPath")
+# Remaining args are passed to `nix` before the attr, e.g.:
+#   run_nix_parallel jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
+#   run_nix_parallel jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
+#
+# Prints "==> <label>" followed by that job's stdout+stderr for every job,
+# in submission order (not completion order) so a run stays readable and
+# diffable across invocations even though the work itself doesn't finish in
+# that order. Returns non-zero if any job failed, only after every job has
+# finished and been printed -- same "surface everything, then fail" contract
+# a `set -e` caller gets, just parallelized instead of stopping at the first
+# failure.
+run_nix_parallel() {
+  local -n jobs_ref="$1"
+  shift
+  local -a nix_args=("$@")
+
+  local n=${#jobs_ref[@]}
+  [[ $n -eq 0 ]] && return 0
+
+  local tmp_dir
+  tmp_dir="$(mktemp -d)"
+
+  local i=0 running=0
+  for job in "${jobs_ref[@]}"; do
+    local attr="${job#*"${NIX_PARALLEL_SEP}"}"
+    printf '%s\n' "${job%%"${NIX_PARALLEL_SEP}"*}" >"${tmp_dir}/${i}.label"
+    (
+      if nix "${nix_args[@]}" "$attr" >"${tmp_dir}/${i}.out" 2>&1; then
+        echo 0 >"${tmp_dir}/${i}.status"
+      else
+        echo 1 >"${tmp_dir}/${i}.status"
+      fi
+    ) &
+    i=$((i + 1))
+    running=$((running + 1))
+    if ((running >= NIX_PARALLEL_JOBS)); then
+      wait -n
+      running=$((running - 1))
+    fi
+  done
+  wait
+
+  local failed=0 j
+  for ((j = 0; j < n; j++)); do
+    echo "==> $(cat "${tmp_dir}/${j}.label")"
+    cat "${tmp_dir}/${j}.out"
+    [[ "$(cat "${tmp_dir}/${j}.status")" -ne 0 ]] && failed=1
+  done
+
+  rm -rf "$tmp_dir"
+  return $failed
+}