diff --git a/scripts/codex-maintenance.sh b/scripts/codex-maintenance.sh index ae1b630..26827ba 100755 --- a/scripts/codex-maintenance.sh +++ b/scripts/codex-maintenance.sh @@ -16,6 +16,14 @@ # # --dry-run: adds `nix build --dry-run --no-link` for whatever scope is # active (changed-files scope by default, full scope under --full-check). +# +# Per-host/per-package eval and dry-run build calls run concurrently (see +# scripts/lib/nix-parallel.sh) since they're independent of each other. +# Concurrency defaults to core count capped by available memory (~1GB/job) +# rather than plain core count, since each concurrent `nix eval` evaluates a +# whole NixOS system closure and can OOM a small/memory-constrained CI +# runner otherwise; override via NIX_PARALLEL_JOBS if a runner has more (or +# less) room than that estimate assumes. set -euo pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -23,6 +31,8 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${script_dir}/lib/nix-bootstrap.sh" # shellcheck source=lib/nix-eval.sh source "${script_dir}/lib/nix-eval.sh" +# shellcheck source=lib/nix-parallel.sh +source "${script_dir}/lib/nix-parallel.sh" repo_root="$(cd "${script_dir}/.." && pwd)" cd "$repo_root" @@ -246,66 +256,64 @@ echo if [[ ${#hosts[@]} -eq 0 ]]; then echo "No hosts affected by changed files; skipping host eval." else - echo "Evaluating host toplevel derivations (${scope_desc})..." + echo "Evaluating host toplevel derivations (${scope_desc}, up to ${NIX_PARALLEL_JOBS} at a time)..." + # lxc-* hosts deploy via a directly pct-restore-able tarball instead of + # nixos-install (see docs/auto-installer.md); proxmox-* hosts can + # alternatively be built as a standalone disk image (see + # docs/proxmox-images.md). Both are otherwise-unvalidated buildable + # surface, easy to silently break without this. + declare -a host_eval_jobs=() for host in "${hosts[@]}"; do - echo "==> $host" - nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath" - - # lxc-* hosts deploy via a directly pct-restore-able tarball instead of - # nixos-install (see docs/auto-installer.md); proxmox-* hosts can - # alternatively be built as a standalone disk image (see - # docs/proxmox-images.md). Both are otherwise-unvalidated buildable - # surface, easy to silently break without this. + host_eval_jobs+=("${host}${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.toplevel.drvPath") case "$host" in lxc-*) - echo "==> $host (tarball)" - nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.tarball.drvPath" + host_eval_jobs+=("${host} (tarball)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.tarball.drvPath") ;; proxmox-*) - echo "==> $host (diskoImagesScript)" - nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.diskoImagesScript.drvPath" + host_eval_jobs+=("${host} (diskoImagesScript)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.diskoImagesScript.drvPath") ;; esac done + run_nix_parallel host_eval_jobs eval --raw "${NIX_EVAL_FLAGS[@]}" fi echo if ! $eval_packages; then echo "No packages affected by changed files; skipping package eval." else - echo "Evaluating buildable packages..." + echo "Evaluating buildable packages (up to ${NIX_PARALLEL_JOBS} at a time)..." + declare -a package_eval_jobs=() for pkg in "${all_packages[@]}"; do - echo "==> packages.x86_64-linux.${pkg}" - nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#packages.x86_64-linux.${pkg}" + package_eval_jobs+=("packages.x86_64-linux.${pkg}${NIX_PARALLEL_SEP}.#packages.x86_64-linux.${pkg}") done + run_nix_parallel package_eval_jobs eval --raw "${NIX_EVAL_FLAGS[@]}" fi if $dry_run; then echo - echo "Running dry-run builds for the active scope. This will not create result symlinks." + echo "Running dry-run builds for the active scope (up to ${NIX_PARALLEL_JOBS} at a time). This will not create result symlinks." + declare -a host_build_jobs=() for host in "${hosts[@]:-}"; do - echo "==> Dry-run build: $host" - nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.toplevel" - + host_build_jobs+=("Dry-run build: ${host}${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.toplevel") case "$host" in lxc-*) - echo "==> Dry-run build: $host (tarball)" - nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.tarball" + host_build_jobs+=("Dry-run build: ${host} (tarball)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.tarball") ;; proxmox-*) - echo "==> Dry-run build: $host (diskoImagesScript)" - nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.diskoImagesScript" + host_build_jobs+=("Dry-run build: ${host} (diskoImagesScript)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.diskoImagesScript") ;; esac done + run_nix_parallel host_build_jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" if $eval_packages; then echo echo "Running dry-run builds for packages." + declare -a package_build_jobs=() for pkg in "${all_packages[@]}"; do - echo "==> Dry-run build: packages.x86_64-linux.${pkg}" - nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#packages.x86_64-linux.${pkg}" + package_build_jobs+=("Dry-run build: packages.x86_64-linux.${pkg}${NIX_PARALLEL_SEP}.#packages.x86_64-linux.${pkg}") done + run_nix_parallel package_build_jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" fi fi diff --git a/scripts/lib/nix-parallel.sh b/scripts/lib/nix-parallel.sh new file mode 100644 index 0000000..c6fe977 --- /dev/null +++ b/scripts/lib/nix-parallel.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Shared parallel-nix-invocation helper for scripts/codex-maintenance.sh. +# Source alongside nix-eval.sh: +# source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/nix-parallel.sh" +# +# The per-host/per-package `nix eval`/`nix build --dry-run` calls in +# codex-maintenance.sh are independent of each other, so running them one at +# a time leaves most cores idle for most of the sweep -- run_nix_parallel +# fans a batch of them out across up to NIX_PARALLEL_JOBS processes instead. + +# NIX_PARALLEL_JOBS: how many `nix` invocations run_nix_parallel runs at +# once. Defaults to core count capped by available memory (~1GB/job, +# floor 1) rather than plain `nproc` -- each concurrent `nix eval` here +# evaluates a whole NixOS system closure from scratch, and on a small/ +# memory-constrained CI runner, `nproc` concurrent evals can OOM-kill each +# other (confirmed empirically: on a 4GB/6-core box, 5-6 concurrent evals +# started getting killed while 3-4 ran clean and were still ~2x faster than +# serial). Override via env if a given machine/CI runner has room to spare +# or needs a tighter cap. +default_nix_parallel_jobs() { + local cores mem_avail_kb mem_cap + cores="$(nproc 2>/dev/null || echo 4)" + mem_avail_kb="$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo 2>/dev/null)" + if [[ -z "$mem_avail_kb" ]]; then + echo "$cores" + return + fi + mem_cap=$((mem_avail_kb / 1024 / 1024)) + ((mem_cap < 1)) && mem_cap=1 + ((mem_cap < cores)) && echo "$mem_cap" || echo "$cores" +} +NIX_PARALLEL_JOBS="${NIX_PARALLEL_JOBS:-$(default_nix_parallel_jobs)}" + +# Separator between a job's label and its flake attr in the arrays +# run_nix_parallel takes -- a control character so it can't collide with +# anything a label or attr path would plausibly contain. +NIX_PARALLEL_SEP=$'\x1f' + +# run_nix_parallel +# +# jobs_array_name: name of an already-populated bash array whose entries are +# "