Archived
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
79e8f9f2ce |
No files matched your search
@@ -195,6 +195,10 @@ default.
|
||||
`auto-install.sh` still supports the older manual path as a fallback: if a
|
||||
host's key isn't baked in (`/etc/host-keys`), it checks `/root/host-keys`
|
||||
next, where you can `scp` a key in after boot, same as before this migration.
|
||||
If neither has it and the script is running interactively (an actual
|
||||
operator at the other end of stdin, not an unattended run), it prompts for
|
||||
an arbitrary directory to check (a mounted USB stick, another filesystem,
|
||||
etc.) and copies the key pair into `/root/host-keys` from there if found.
|
||||
|
||||
## Storage
|
||||
|
||||
|
||||
@@ -16,14 +16,6 @@
|
||||
#
|
||||
# --dry-run: adds `nix build --dry-run --no-link` for whatever scope is
|
||||
# active (changed-files scope by default, full scope under --full-check).
|
||||
#
|
||||
# Per-host/per-package eval and dry-run build calls run concurrently (see
|
||||
# scripts/lib/nix-parallel.sh) since they're independent of each other.
|
||||
# Concurrency defaults to core count capped by available memory (~1GB/job)
|
||||
# rather than plain core count, since each concurrent `nix eval` evaluates a
|
||||
# whole NixOS system closure and can OOM a small/memory-constrained CI
|
||||
# runner otherwise; override via NIX_PARALLEL_JOBS if a runner has more (or
|
||||
# less) room than that estimate assumes.
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -31,8 +23,6 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "${script_dir}/lib/nix-bootstrap.sh"
|
||||
# shellcheck source=lib/nix-eval.sh
|
||||
source "${script_dir}/lib/nix-eval.sh"
|
||||
# shellcheck source=lib/nix-parallel.sh
|
||||
source "${script_dir}/lib/nix-parallel.sh"
|
||||
|
||||
repo_root="$(cd "${script_dir}/.." && pwd)"
|
||||
cd "$repo_root"
|
||||
@@ -256,64 +246,66 @@ echo
|
||||
if [[ ${#hosts[@]} -eq 0 ]]; then
|
||||
echo "No hosts affected by changed files; skipping host eval."
|
||||
else
|
||||
echo "Evaluating host toplevel derivations (${scope_desc}, up to ${NIX_PARALLEL_JOBS} at a time)..."
|
||||
# lxc-* hosts deploy via a directly pct-restore-able tarball instead of
|
||||
# nixos-install (see docs/auto-installer.md); proxmox-* hosts can
|
||||
# alternatively be built as a standalone disk image (see
|
||||
# docs/proxmox-images.md). Both are otherwise-unvalidated buildable
|
||||
# surface, easy to silently break without this.
|
||||
declare -a host_eval_jobs=()
|
||||
echo "Evaluating host toplevel derivations (${scope_desc})..."
|
||||
for host in "${hosts[@]}"; do
|
||||
host_eval_jobs+=("${host}${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.toplevel.drvPath")
|
||||
echo "==> $host"
|
||||
nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath"
|
||||
|
||||
# lxc-* hosts deploy via a directly pct-restore-able tarball instead of
|
||||
# nixos-install (see docs/auto-installer.md); proxmox-* hosts can
|
||||
# alternatively be built as a standalone disk image (see
|
||||
# docs/proxmox-images.md). Both are otherwise-unvalidated buildable
|
||||
# surface, easy to silently break without this.
|
||||
case "$host" in
|
||||
lxc-*)
|
||||
host_eval_jobs+=("${host} (tarball)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.tarball.drvPath")
|
||||
echo "==> $host (tarball)"
|
||||
nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.tarball.drvPath"
|
||||
;;
|
||||
proxmox-*)
|
||||
host_eval_jobs+=("${host} (diskoImagesScript)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.diskoImagesScript.drvPath")
|
||||
echo "==> $host (diskoImagesScript)"
|
||||
nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.diskoImagesScript.drvPath"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
run_nix_parallel host_eval_jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
|
||||
fi
|
||||
|
||||
echo
|
||||
if ! $eval_packages; then
|
||||
echo "No packages affected by changed files; skipping package eval."
|
||||
else
|
||||
echo "Evaluating buildable packages (up to ${NIX_PARALLEL_JOBS} at a time)..."
|
||||
declare -a package_eval_jobs=()
|
||||
echo "Evaluating buildable packages..."
|
||||
for pkg in "${all_packages[@]}"; do
|
||||
package_eval_jobs+=("packages.x86_64-linux.${pkg}${NIX_PARALLEL_SEP}.#packages.x86_64-linux.${pkg}")
|
||||
echo "==> packages.x86_64-linux.${pkg}"
|
||||
nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#packages.x86_64-linux.${pkg}"
|
||||
done
|
||||
run_nix_parallel package_eval_jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
|
||||
fi
|
||||
|
||||
if $dry_run; then
|
||||
echo
|
||||
echo "Running dry-run builds for the active scope (up to ${NIX_PARALLEL_JOBS} at a time). This will not create result symlinks."
|
||||
declare -a host_build_jobs=()
|
||||
echo "Running dry-run builds for the active scope. This will not create result symlinks."
|
||||
for host in "${hosts[@]:-}"; do
|
||||
host_build_jobs+=("Dry-run build: ${host}${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.toplevel")
|
||||
echo "==> Dry-run build: $host"
|
||||
nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.toplevel"
|
||||
|
||||
case "$host" in
|
||||
lxc-*)
|
||||
host_build_jobs+=("Dry-run build: ${host} (tarball)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.tarball")
|
||||
echo "==> Dry-run build: $host (tarball)"
|
||||
nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.tarball"
|
||||
;;
|
||||
proxmox-*)
|
||||
host_build_jobs+=("Dry-run build: ${host} (diskoImagesScript)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.diskoImagesScript")
|
||||
echo "==> Dry-run build: $host (diskoImagesScript)"
|
||||
nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.diskoImagesScript"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
run_nix_parallel host_build_jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
|
||||
|
||||
if $eval_packages; then
|
||||
echo
|
||||
echo "Running dry-run builds for packages."
|
||||
declare -a package_build_jobs=()
|
||||
for pkg in "${all_packages[@]}"; do
|
||||
package_build_jobs+=("Dry-run build: packages.x86_64-linux.${pkg}${NIX_PARALLEL_SEP}.#packages.x86_64-linux.${pkg}")
|
||||
echo "==> Dry-run build: packages.x86_64-linux.${pkg}"
|
||||
nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#packages.x86_64-linux.${pkg}"
|
||||
done
|
||||
run_nix_parallel package_build_jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@@ -114,15 +114,37 @@ elif [[ -f "/root/host-keys/${choice}_ssh_host_ed25519_key" ]]; then
|
||||
install -D -m 0600 "/root/host-keys/${choice}_ssh_host_ed25519_key" /mnt/etc/ssh/ssh_host_ed25519_key
|
||||
install -D -m 0644 "/root/host-keys/${choice}_ssh_host_ed25519_key.pub" /mnt/etc/ssh/ssh_host_ed25519_key.pub
|
||||
else
|
||||
echo "WARNING: no SSH host key found for ${choice} (checked /etc/host-keys and /root/host-keys)"
|
||||
echo "sops-nix secrets (including the login password) will NOT decrypt on first boot."
|
||||
echo "Run scripts/secrets/prepare-host-key.sh for host ${choice} on your admin workstation first,"
|
||||
echo "then either rebuild this image with NIXOS_HOST_KEYS_DIR set, or scp the result to"
|
||||
echo "/root/host-keys/ on this machine."
|
||||
read -rp "Continue without a pre-seeded key anyway? (y/N): " skip_key
|
||||
if [[ ! "$skip_key" =~ ^[Yy]$ ]]; then
|
||||
echo "Aborted."
|
||||
exit 1
|
||||
# Third place a key can come from: an arbitrary path the operator
|
||||
# points at interactively (e.g. a USB stick, a mount from another
|
||||
# machine) -- only offered when there's an actual human at the other
|
||||
# end of stdin to ask, never in a non-interactive run.
|
||||
key_copied=0
|
||||
if [[ -t 0 ]]; then
|
||||
echo "No SSH host key found for ${choice} (checked /etc/host-keys and /root/host-keys)."
|
||||
read -rp "Path to a directory containing ${choice}_ssh_host_ed25519_key(.pub) (blank to skip): " key_src_dir
|
||||
if [[ -n "$key_src_dir" && -f "${key_src_dir}/${choice}_ssh_host_ed25519_key" && -f "${key_src_dir}/${choice}_ssh_host_ed25519_key.pub" ]]; then
|
||||
cp "${key_src_dir}/${choice}_ssh_host_ed25519_key" "${key_src_dir}/${choice}_ssh_host_ed25519_key.pub" /root/host-keys/
|
||||
key_copied=1
|
||||
elif [[ -n "$key_src_dir" ]]; then
|
||||
echo "WARNING: ${choice}_ssh_host_ed25519_key(.pub) not found in ${key_src_dir}."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$key_copied" -eq 1 ]]; then
|
||||
echo "Copied SSH host key for ${choice} from ${key_src_dir}, installing to target..."
|
||||
install -D -m 0600 "/root/host-keys/${choice}_ssh_host_ed25519_key" /mnt/etc/ssh/ssh_host_ed25519_key
|
||||
install -D -m 0644 "/root/host-keys/${choice}_ssh_host_ed25519_key.pub" /mnt/etc/ssh/ssh_host_ed25519_key.pub
|
||||
else
|
||||
echo "WARNING: no SSH host key found for ${choice} (checked /etc/host-keys and /root/host-keys)"
|
||||
echo "sops-nix secrets (including the login password) will NOT decrypt on first boot."
|
||||
echo "Run scripts/secrets/prepare-host-key.sh for host ${choice} on your admin workstation first,"
|
||||
echo "then either rebuild this image with NIXOS_HOST_KEYS_DIR set, scp the result to"
|
||||
echo "/root/host-keys/ on this machine, or point at it when prompted above."
|
||||
read -rp "Continue without a pre-seeded key anyway? (y/N): " skip_key
|
||||
if [[ ! "$skip_key" =~ ^[Yy]$ ]]; then
|
||||
echo "Aborted."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared parallel-nix-invocation helper for scripts/codex-maintenance.sh.
|
||||
# Source alongside nix-eval.sh:
|
||||
# source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/nix-parallel.sh"
|
||||
#
|
||||
# The per-host/per-package `nix eval`/`nix build --dry-run` calls in
|
||||
# codex-maintenance.sh are independent of each other, so running them one at
|
||||
# a time leaves most cores idle for most of the sweep -- run_nix_parallel
|
||||
# fans a batch of them out across up to NIX_PARALLEL_JOBS processes instead.
|
||||
|
||||
# NIX_PARALLEL_JOBS: how many `nix` invocations run_nix_parallel runs at
|
||||
# once. Defaults to core count capped by available memory (~1GB/job,
|
||||
# floor 1) rather than plain `nproc` -- each concurrent `nix eval` here
|
||||
# evaluates a whole NixOS system closure from scratch, and on a small/
|
||||
# memory-constrained CI runner, `nproc` concurrent evals can OOM-kill each
|
||||
# other (confirmed empirically: on a 4GB/6-core box, 5-6 concurrent evals
|
||||
# started getting killed while 3-4 ran clean and were still ~2x faster than
|
||||
# serial). Override via env if a given machine/CI runner has room to spare
|
||||
# or needs a tighter cap.
|
||||
default_nix_parallel_jobs() {
|
||||
local cores mem_avail_kb mem_cap
|
||||
cores="$(nproc 2>/dev/null || echo 4)"
|
||||
mem_avail_kb="$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo 2>/dev/null)"
|
||||
if [[ -z "$mem_avail_kb" ]]; then
|
||||
echo "$cores"
|
||||
return
|
||||
fi
|
||||
mem_cap=$((mem_avail_kb / 1024 / 1024))
|
||||
((mem_cap < 1)) && mem_cap=1
|
||||
((mem_cap < cores)) && echo "$mem_cap" || echo "$cores"
|
||||
}
|
||||
NIX_PARALLEL_JOBS="${NIX_PARALLEL_JOBS:-$(default_nix_parallel_jobs)}"
|
||||
|
||||
# Separator between a job's label and its flake attr in the arrays
|
||||
# run_nix_parallel takes -- a control character so it can't collide with
|
||||
# anything a label or attr path would plausibly contain.
|
||||
NIX_PARALLEL_SEP=$'\x1f'
|
||||
|
||||
# run_nix_parallel <jobs_array_name> <nix subcommand + flags...>
|
||||
#
|
||||
# jobs_array_name: name of an already-populated bash array whose entries are
|
||||
# "<label>${NIX_PARALLEL_SEP}<attr>" pairs, e.g.
|
||||
# jobs=("proxmox-docker${NIX_PARALLEL_SEP}.#nixosConfigurations.proxmox-docker...drvPath")
|
||||
# Remaining args are passed to `nix` before the attr, e.g.:
|
||||
# run_nix_parallel jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
|
||||
# run_nix_parallel jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
|
||||
#
|
||||
# Prints "==> <label>" followed by that job's stdout+stderr for every job,
|
||||
# in submission order (not completion order) so a run stays readable and
|
||||
# diffable across invocations even though the work itself doesn't finish in
|
||||
# that order. Returns non-zero if any job failed, only after every job has
|
||||
# finished and been printed -- same "surface everything, then fail" contract
|
||||
# a `set -e` caller gets, just parallelized instead of stopping at the first
|
||||
# failure.
|
||||
run_nix_parallel() {
|
||||
local -n jobs_ref="$1"
|
||||
shift
|
||||
local -a nix_args=("$@")
|
||||
|
||||
local n=${#jobs_ref[@]}
|
||||
[[ $n -eq 0 ]] && return 0
|
||||
|
||||
local tmp_dir
|
||||
tmp_dir="$(mktemp -d)"
|
||||
|
||||
local i=0 running=0
|
||||
for job in "${jobs_ref[@]}"; do
|
||||
local attr="${job#*"${NIX_PARALLEL_SEP}"}"
|
||||
printf '%s\n' "${job%%"${NIX_PARALLEL_SEP}"*}" >"${tmp_dir}/${i}.label"
|
||||
(
|
||||
if nix "${nix_args[@]}" "$attr" >"${tmp_dir}/${i}.out" 2>&1; then
|
||||
echo 0 >"${tmp_dir}/${i}.status"
|
||||
else
|
||||
echo 1 >"${tmp_dir}/${i}.status"
|
||||
fi
|
||||
) &
|
||||
i=$((i + 1))
|
||||
running=$((running + 1))
|
||||
if ((running >= NIX_PARALLEL_JOBS)); then
|
||||
wait -n
|
||||
running=$((running - 1))
|
||||
fi
|
||||
done
|
||||
wait
|
||||
|
||||
local failed=0 j
|
||||
for ((j = 0; j < n; j++)); do
|
||||
echo "==> $(cat "${tmp_dir}/${j}.label")"
|
||||
cat "${tmp_dir}/${j}.out"
|
||||
[[ "$(cat "${tmp_dir}/${j}.status")" -ne 0 ]] && failed=1
|
||||
done
|
||||
|
||||
rm -rf "$tmp_dir"
|
||||
return $failed
|
||||
}
|
||||
Reference in new issue
Block a user