Unmount nixos-install's leftover chroot bind mounts before zpool export

nixos-install bind-mounts /dev, /proc, /sys (and usually /run) into
/mnt to run the target's activation script in a chroot, and doesn't
unmount them again afterward. Left in place, those nested mounts made
ZFS refuse to unmount its own root dataset at /mnt: zpool export
failed with "cannot unmount '/mnt': pool or dataset busy", and because
of this script's set -e, that killed the script before it ever reached
reboot -- silently defeating the export-before-reboot fix from #40 on
every real run, which is why the ZFS-import stall kept recurring.

modules/platforms/baremetal.nix

@@ -25,4 +25,14 @@
     enable = true;
     enable32Bit = true;
   };
+
+  # The systemd-based initrd (default here since this host has a ZFS root --
+  # see modules/disko/baremetal.nix) locks the root account by default, so
+  # sulogin refuses to hand over a shell if something in the initrd (e.g.
+  # the ZFS pool import) fails and it drops to emergency mode -- confirmed
+  # live: it just loops re-entering the target instead of prompting. This
+  # only affects the pre-switch-root initrd shell, not the installed
+  # system's own login, and is worth the tradeoff on a box already reachable
+  # at the physical console.
+  boot.initrd.systemd.emergencyAccess = true;
 }

scripts/codex-maintenance.sh

@@ -16,6 +16,14 @@
 #
 # --dry-run: adds `nix build --dry-run --no-link` for whatever scope is
 # active (changed-files scope by default, full scope under --full-check).
+#
+# Per-host/per-package eval and dry-run build calls run concurrently (see
+# scripts/lib/nix-parallel.sh) since they're independent of each other.
+# Concurrency defaults to core count capped by available memory (~1GB/job)
+# rather than plain core count, since each concurrent `nix eval` evaluates a
+# whole NixOS system closure and can OOM a small/memory-constrained CI
+# runner otherwise; override via NIX_PARALLEL_JOBS if a runner has more (or
+# less) room than that estimate assumes.
 set -euo pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -23,6 +31,8 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${script_dir}/lib/nix-bootstrap.sh"
 # shellcheck source=lib/nix-eval.sh
 source "${script_dir}/lib/nix-eval.sh"
+# shellcheck source=lib/nix-parallel.sh
+source "${script_dir}/lib/nix-parallel.sh"
 
 repo_root="$(cd "${script_dir}/.." && pwd)"
 cd "$repo_root"
@@ -246,66 +256,64 @@ echo
 if [[ ${#hosts[@]} -eq 0 ]]; then
   echo "No hosts affected by changed files; skipping host eval."
 else
-  echo "Evaluating host toplevel derivations (${scope_desc})..."
+  echo "Evaluating host toplevel derivations (${scope_desc}, up to ${NIX_PARALLEL_JOBS} at a time)..."
-  for host in "${hosts[@]}"; do
-    echo "==> $host"
-    nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath"
-
   # lxc-* hosts deploy via a directly pct-restore-able tarball instead of
   # nixos-install (see docs/auto-installer.md); proxmox-* hosts can
   # alternatively be built as a standalone disk image (see
   # docs/proxmox-images.md). Both are otherwise-unvalidated buildable
   # surface, easy to silently break without this.
+  declare -a host_eval_jobs=()
+  for host in "${hosts[@]}"; do
+    host_eval_jobs+=("${host}${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.toplevel.drvPath")
     case "$host" in
       lxc-*)
-        echo "==> $host (tarball)"
+        host_eval_jobs+=("${host} (tarball)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.tarball.drvPath")
-        nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.tarball.drvPath"
         ;;
       proxmox-*)
-        echo "==> $host (diskoImagesScript)"
+        host_eval_jobs+=("${host} (diskoImagesScript)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.diskoImagesScript.drvPath")
-        nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.diskoImagesScript.drvPath"
         ;;
     esac
   done
+  run_nix_parallel host_eval_jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
 fi
 
 echo
 if ! $eval_packages; then
   echo "No packages affected by changed files; skipping package eval."
 else
-  echo "Evaluating buildable packages..."
+  echo "Evaluating buildable packages (up to ${NIX_PARALLEL_JOBS} at a time)..."
+  declare -a package_eval_jobs=()
   for pkg in "${all_packages[@]}"; do
-    echo "==> packages.x86_64-linux.${pkg}"
+    package_eval_jobs+=("packages.x86_64-linux.${pkg}${NIX_PARALLEL_SEP}.#packages.x86_64-linux.${pkg}")
-    nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#packages.x86_64-linux.${pkg}"
   done
+  run_nix_parallel package_eval_jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
 fi
 
 if $dry_run; then
   echo
-  echo "Running dry-run builds for the active scope. This will not create result symlinks."
+  echo "Running dry-run builds for the active scope (up to ${NIX_PARALLEL_JOBS} at a time). This will not create result symlinks."
+  declare -a host_build_jobs=()
   for host in "${hosts[@]:-}"; do
-    echo "==> Dry-run build: $host"
+    host_build_jobs+=("Dry-run build: ${host}${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.toplevel")
-    nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.toplevel"
-
     case "$host" in
       lxc-*)
-        echo "==> Dry-run build: $host (tarball)"
+        host_build_jobs+=("Dry-run build: ${host} (tarball)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.tarball")
-        nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.tarball"
         ;;
       proxmox-*)
-        echo "==> Dry-run build: $host (diskoImagesScript)"
+        host_build_jobs+=("Dry-run build: ${host} (diskoImagesScript)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.diskoImagesScript")
-        nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.diskoImagesScript"
         ;;
     esac
   done
+  run_nix_parallel host_build_jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
 
   if $eval_packages; then
     echo
     echo "Running dry-run builds for packages."
+    declare -a package_build_jobs=()
     for pkg in "${all_packages[@]}"; do
-      echo "==> Dry-run build: packages.x86_64-linux.${pkg}"
+      package_build_jobs+=("Dry-run build: packages.x86_64-linux.${pkg}${NIX_PARALLEL_SEP}.#packages.x86_64-linux.${pkg}")
-      nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#packages.x86_64-linux.${pkg}"
     done
+    run_nix_parallel package_build_jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
   fi
 fi
 

scripts/installer/auto-install.sh

@@ -1,17 +1,20 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -i bash -p jq disko nixos-install-tools
+#!nix-shell -i bash -p jq disko nixos-install-tools zfs
 # shellcheck shell=bash
 # The only genuinely external tools this script calls directly: `jq`
-# (parsing the `nix eval` host list) and `disko`/`nixos-install` (the
+# (parsing the `nix eval` host list), `disko`/`nixos-install` (the
-# install itself). Everything disko shells out to internally
+# install itself), and `zpool` (exporting a ZFS root pool before reboot,
-# (parted/sgdisk/mkfs.*/zfs/...) is self-contained -- disko's own
+# see the comment above that call below). Everything disko shells out to
-# generated scripts hardcode absolute Nix store paths for those, they
+# internally (parted/sgdisk/mkfs.*/zfs/...) is self-contained -- disko's
+# own generated scripts hardcode absolute Nix store paths for those, they
 # don't rely on this script's PATH at all (confirmed by inspecting a
 # generated system.build.formatScript). The built installer image
-# (modules/installer/common.nix) already has all three in
+# (modules/installer/common.nix, plus the upstream
-# environment.systemPackages, so this nix-shell wrapper is a fast no-op
+# installation-cd-minimal.nix it imports via iso.nix) already has all
-# there; it's what makes the script also work standalone (e.g. run
+# four in environment.systemPackages, so this nix-shell wrapper is a
-# directly from a checkout on a stock ISO), where they aren't.
+# fast no-op there; it's what makes the script also work standalone
+# (e.g. run directly from a checkout on a stock ISO), where they aren't
+# guaranteed.
 set -eux
 
 set -euo pipefail
@@ -162,5 +165,39 @@ nixos-install \
     # /etc/ssh/ssh_host_ed25519_key. Nothing NixOS-managed ever cleans this
     # up on its own since it was written imperatively, not declaratively.
     rm -rf /root/host-keys
+
+    # disko's --mode ...,mount left any ZFS root pool imported (that's what
+    # let nixos-install write into /mnt). If we reboot with it still
+    # imported, it isn't just "not exported" -- it's stamped with *this*
+    # live installer environment's hostid, which almost never matches the
+    # target's own networking.hostId (see hosts/*/host.nix; the installer
+    # itself sets none). modules/services/zfs/enable-service.nix and
+    # modules/common/configuration.nix both set boot.zfs.forceImportRoot =
+    # false deliberately (the safe option per that setting's own docs), so
+    # the freshly-installed system's first real boot sees a pool "in use by
+    # another system" and refuses to import it without -f -- which is what
+    # makes boot stall waiting on the ZFS import. Exporting here (a no-op
+    # if the chosen host has no ZFS root, e.g. proxmox-*/linode-*) clears
+    # that in-use state so the next import, from any hostid, succeeds.
+    #
+    # nixos-install bind-mounts /dev, /proc, /sys (and usually /run) into
+    # /mnt to run the target's activation script (switch-to-configuration
+    # boot) in a chroot, and doesn't unmount them again once it's done.
+    # Left in place, those nested mounts make ZFS refuse to unmount its own
+    # root dataset at /mnt -- confirmed live: zpool export failed with
+    # "cannot unmount '/mnt': pool or dataset busy", and because of this
+    # script's `set -e`, that killed the script before it ever reached
+    # reboot, silently defeating the whole point of exporting first.
+    for chroot_mount in dev proc sys run; do
+        if mountpoint -q "/mnt/${chroot_mount}"; then
+            umount -R "/mnt/${chroot_mount}"
+        fi
+    done
+
+    if [[ -n "$(zpool list -H -o name 2>/dev/null)" ]]; then
+        echo "Exporting ZFS pool(s) before reboot..."
+        zpool export -a
+    fi
+
     sleep 10
     reboot

scripts/lib/nix-parallel.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# Shared parallel-nix-invocation helper for scripts/codex-maintenance.sh.
+# Source alongside nix-eval.sh:
+#   source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/nix-parallel.sh"
+#
+# The per-host/per-package `nix eval`/`nix build --dry-run` calls in
+# codex-maintenance.sh are independent of each other, so running them one at
+# a time leaves most cores idle for most of the sweep -- run_nix_parallel
+# fans a batch of them out across up to NIX_PARALLEL_JOBS processes instead.
+
+# NIX_PARALLEL_JOBS: how many `nix` invocations run_nix_parallel runs at
+# once. Defaults to core count capped by available memory (~1GB/job,
+# floor 1) rather than plain `nproc` -- each concurrent `nix eval` here
+# evaluates a whole NixOS system closure from scratch, and on a small/
+# memory-constrained CI runner, `nproc` concurrent evals can OOM-kill each
+# other (confirmed empirically: on a 4GB/6-core box, 5-6 concurrent evals
+# started getting killed while 3-4 ran clean and were still ~2x faster than
+# serial). Override via env if a given machine/CI runner has room to spare
+# or needs a tighter cap.
+default_nix_parallel_jobs() {
+  local cores mem_avail_kb mem_cap
+  cores="$(nproc 2>/dev/null || echo 4)"
+  mem_avail_kb="$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo 2>/dev/null)"
+  if [[ -z "$mem_avail_kb" ]]; then
+    echo "$cores"
+    return
+  fi
+  mem_cap=$((mem_avail_kb / 1024 / 1024))
+  ((mem_cap < 1)) && mem_cap=1
+  ((mem_cap < cores)) && echo "$mem_cap" || echo "$cores"
+}
+NIX_PARALLEL_JOBS="${NIX_PARALLEL_JOBS:-$(default_nix_parallel_jobs)}"
+
+# Separator between a job's label and its flake attr in the arrays
+# run_nix_parallel takes -- a control character so it can't collide with
+# anything a label or attr path would plausibly contain.
+NIX_PARALLEL_SEP=$'\x1f'
+
+# run_nix_parallel <jobs_array_name> <nix subcommand + flags...>
+#
+# jobs_array_name: name of an already-populated bash array whose entries are
+# "<label>${NIX_PARALLEL_SEP}<attr>" pairs, e.g.
+#   jobs=("proxmox-docker${NIX_PARALLEL_SEP}.#nixosConfigurations.proxmox-docker...drvPath")
+# Remaining args are passed to `nix` before the attr, e.g.:
+#   run_nix_parallel jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
+#   run_nix_parallel jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
+#
+# Prints "==> <label>" followed by that job's stdout+stderr for every job,
+# in submission order (not completion order) so a run stays readable and
+# diffable across invocations even though the work itself doesn't finish in
+# that order. Returns non-zero if any job failed, only after every job has
+# finished and been printed -- same "surface everything, then fail" contract
+# a `set -e` caller gets, just parallelized instead of stopping at the first
+# failure.
+run_nix_parallel() {
+  local -n jobs_ref="$1"
+  shift
+  local -a nix_args=("$@")
+
+  local n=${#jobs_ref[@]}
+  [[ $n -eq 0 ]] && return 0
+
+  local tmp_dir
+  tmp_dir="$(mktemp -d)"
+
+  local i=0 running=0
+  for job in "${jobs_ref[@]}"; do
+    local attr="${job#*"${NIX_PARALLEL_SEP}"}"
+    printf '%s\n' "${job%%"${NIX_PARALLEL_SEP}"*}" >"${tmp_dir}/${i}.label"
+    (
+      if nix "${nix_args[@]}" "$attr" >"${tmp_dir}/${i}.out" 2>&1; then
+        echo 0 >"${tmp_dir}/${i}.status"
+      else
+        echo 1 >"${tmp_dir}/${i}.status"
+      fi
+    ) &
+    i=$((i + 1))
+    running=$((running + 1))
+    if ((running >= NIX_PARALLEL_JOBS)); then
+      wait -n
+      running=$((running - 1))
+    fi
+  done
+  wait
+
+  local failed=0 j
+  for ((j = 0; j < n; j++)); do
+    echo "==> $(cat "${tmp_dir}/${j}.label")"
+    cat "${tmp_dir}/${j}.out"
+    [[ "$(cat "${tmp_dir}/${j}.status")" -ne 0 ]] && failed=1
+  done
+
+  rm -rf "$tmp_dir"
+  return $failed
+}