Archived
Compare commits
34
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5497a5b0ae | ||
|
|
e10e493ddd | ||
|
|
ae9acecbf3 | ||
|
|
a3be05538b | ||
|
|
289163c712 | ||
|
|
2123e4ad69 | ||
|
|
177950dd3d | ||
|
|
cbf1239be4 | ||
|
|
8a282ee32e | ||
|
|
25079a7f0a | ||
|
|
4952e5224d | ||
|
|
98409f4502 | ||
|
|
7779f3e137 | ||
|
|
1462829aa6 | ||
|
|
48ce2c4097 | ||
|
|
bcae177d8e | ||
|
|
d35aca3138 | ||
|
|
147cb3803a | ||
|
|
98445565d6 | ||
|
|
eef4b05254 | ||
|
|
619324589a | ||
|
|
400af07154 | ||
|
|
9bb626327f | ||
|
|
1f8bf8c852 | ||
|
|
5d7a6327b7 | ||
|
|
0b9f124713 | ||
|
|
9479d56e11 | ||
|
|
c53c1940d6 | ||
|
|
5fe575d362 | ||
|
|
33b1d5ec79 | ||
|
|
a91634c460 | ||
|
|
42919ea15c | ||
|
|
0f78e96b81 | ||
|
|
f237a6a3d2 |
@@ -19,11 +19,13 @@
|
||||
nextcloud-client
|
||||
# vscode
|
||||
chromium
|
||||
claude-code
|
||||
];
|
||||
|
||||
# Optional: set environment vars
|
||||
sessionVariables = {
|
||||
EDITOR = "vim";
|
||||
SOPS_AGE_KEY_FILE = "/home/nixos/Nextcloud/Filing Cabinet/keys/nixos-sops-age-key-txt";
|
||||
};
|
||||
|
||||
file = {
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
];
|
||||
|
||||
boot.loader.grub.useOSProber = true;
|
||||
|
||||
programs.direnv.enable = true;
|
||||
services = {
|
||||
xserver = {
|
||||
enable = true;
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
btop
|
||||
git
|
||||
gcr
|
||||
jq
|
||||
];
|
||||
|
||||
# Secrets shared by every host, decrypted at activation via each host's
|
||||
@@ -76,6 +77,7 @@
|
||||
openssh.authorizedKeys.keys = [
|
||||
vars.adminSshKey
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface"
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
|
||||
];
|
||||
};
|
||||
|
||||
|
||||
@@ -25,4 +25,14 @@
|
||||
enable = true;
|
||||
enable32Bit = true;
|
||||
};
|
||||
|
||||
# The systemd-based initrd (default here since this host has a ZFS root --
|
||||
# see modules/disko/baremetal.nix) locks the root account by default, so
|
||||
# sulogin refuses to hand over a shell if something in the initrd (e.g.
|
||||
# the ZFS pool import) fails and it drops to emergency mode -- confirmed
|
||||
# live: it just loops re-entering the target instead of prompting. This
|
||||
# only affects the pre-switch-root initrd shell, not the installed
|
||||
# system's own login, and is worth the tradeoff on a box already reachable
|
||||
# at the physical console.
|
||||
boot.initrd.systemd.emergencyAccess = true;
|
||||
}
|
||||
|
||||
@@ -16,6 +16,14 @@
|
||||
#
|
||||
# --dry-run: adds `nix build --dry-run --no-link` for whatever scope is
|
||||
# active (changed-files scope by default, full scope under --full-check).
|
||||
#
|
||||
# Per-host/per-package eval and dry-run build calls run concurrently (see
|
||||
# scripts/lib/nix-parallel.sh) since they're independent of each other.
|
||||
# Concurrency defaults to core count capped by available memory (~1GB/job)
|
||||
# rather than plain core count, since each concurrent `nix eval` evaluates a
|
||||
# whole NixOS system closure and can OOM a small/memory-constrained CI
|
||||
# runner otherwise; override via NIX_PARALLEL_JOBS if a runner has more (or
|
||||
# less) room than that estimate assumes.
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -23,6 +31,8 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "${script_dir}/lib/nix-bootstrap.sh"
|
||||
# shellcheck source=lib/nix-eval.sh
|
||||
source "${script_dir}/lib/nix-eval.sh"
|
||||
# shellcheck source=lib/nix-parallel.sh
|
||||
source "${script_dir}/lib/nix-parallel.sh"
|
||||
|
||||
repo_root="$(cd "${script_dir}/.." && pwd)"
|
||||
cd "$repo_root"
|
||||
@@ -246,66 +256,64 @@ echo
|
||||
if [[ ${#hosts[@]} -eq 0 ]]; then
|
||||
echo "No hosts affected by changed files; skipping host eval."
|
||||
else
|
||||
echo "Evaluating host toplevel derivations (${scope_desc})..."
|
||||
echo "Evaluating host toplevel derivations (${scope_desc}, up to ${NIX_PARALLEL_JOBS} at a time)..."
|
||||
# lxc-* hosts deploy via a directly pct-restore-able tarball instead of
|
||||
# nixos-install (see docs/auto-installer.md); proxmox-* hosts can
|
||||
# alternatively be built as a standalone disk image (see
|
||||
# docs/proxmox-images.md). Both are otherwise-unvalidated buildable
|
||||
# surface, easy to silently break without this.
|
||||
declare -a host_eval_jobs=()
|
||||
for host in "${hosts[@]}"; do
|
||||
echo "==> $host"
|
||||
nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath"
|
||||
|
||||
# lxc-* hosts deploy via a directly pct-restore-able tarball instead of
|
||||
# nixos-install (see docs/auto-installer.md); proxmox-* hosts can
|
||||
# alternatively be built as a standalone disk image (see
|
||||
# docs/proxmox-images.md). Both are otherwise-unvalidated buildable
|
||||
# surface, easy to silently break without this.
|
||||
host_eval_jobs+=("${host}${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.toplevel.drvPath")
|
||||
case "$host" in
|
||||
lxc-*)
|
||||
echo "==> $host (tarball)"
|
||||
nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.tarball.drvPath"
|
||||
host_eval_jobs+=("${host} (tarball)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.tarball.drvPath")
|
||||
;;
|
||||
proxmox-*)
|
||||
echo "==> $host (diskoImagesScript)"
|
||||
nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.diskoImagesScript.drvPath"
|
||||
host_eval_jobs+=("${host} (diskoImagesScript)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.diskoImagesScript.drvPath")
|
||||
;;
|
||||
esac
|
||||
done
|
||||
run_nix_parallel host_eval_jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
|
||||
fi
|
||||
|
||||
echo
|
||||
if ! $eval_packages; then
|
||||
echo "No packages affected by changed files; skipping package eval."
|
||||
else
|
||||
echo "Evaluating buildable packages..."
|
||||
echo "Evaluating buildable packages (up to ${NIX_PARALLEL_JOBS} at a time)..."
|
||||
declare -a package_eval_jobs=()
|
||||
for pkg in "${all_packages[@]}"; do
|
||||
echo "==> packages.x86_64-linux.${pkg}"
|
||||
nix eval --raw "${NIX_EVAL_FLAGS[@]}" ".#packages.x86_64-linux.${pkg}"
|
||||
package_eval_jobs+=("packages.x86_64-linux.${pkg}${NIX_PARALLEL_SEP}.#packages.x86_64-linux.${pkg}")
|
||||
done
|
||||
run_nix_parallel package_eval_jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
|
||||
fi
|
||||
|
||||
if $dry_run; then
|
||||
echo
|
||||
echo "Running dry-run builds for the active scope. This will not create result symlinks."
|
||||
echo "Running dry-run builds for the active scope (up to ${NIX_PARALLEL_JOBS} at a time). This will not create result symlinks."
|
||||
declare -a host_build_jobs=()
|
||||
for host in "${hosts[@]:-}"; do
|
||||
echo "==> Dry-run build: $host"
|
||||
nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.toplevel"
|
||||
|
||||
host_build_jobs+=("Dry-run build: ${host}${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.toplevel")
|
||||
case "$host" in
|
||||
lxc-*)
|
||||
echo "==> Dry-run build: $host (tarball)"
|
||||
nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.tarball"
|
||||
host_build_jobs+=("Dry-run build: ${host} (tarball)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.tarball")
|
||||
;;
|
||||
proxmox-*)
|
||||
echo "==> Dry-run build: $host (diskoImagesScript)"
|
||||
nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#nixosConfigurations.${host}.config.system.build.diskoImagesScript"
|
||||
host_build_jobs+=("Dry-run build: ${host} (diskoImagesScript)${NIX_PARALLEL_SEP}.#nixosConfigurations.${host}.config.system.build.diskoImagesScript")
|
||||
;;
|
||||
esac
|
||||
done
|
||||
run_nix_parallel host_build_jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
|
||||
|
||||
if $eval_packages; then
|
||||
echo
|
||||
echo "Running dry-run builds for packages."
|
||||
declare -a package_build_jobs=()
|
||||
for pkg in "${all_packages[@]}"; do
|
||||
echo "==> Dry-run build: packages.x86_64-linux.${pkg}"
|
||||
nix build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}" ".#packages.x86_64-linux.${pkg}"
|
||||
package_build_jobs+=("Dry-run build: packages.x86_64-linux.${pkg}${NIX_PARALLEL_SEP}.#packages.x86_64-linux.${pkg}")
|
||||
done
|
||||
run_nix_parallel package_build_jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@@ -68,6 +68,7 @@ cat > "$HOME/.config/nix/nix.conf" <<'EOF'
|
||||
experimental-features = nix-command flakes
|
||||
accept-flake-config = false
|
||||
warn-dirty = false
|
||||
build-users-group =
|
||||
EOF
|
||||
|
||||
echo "Nix version:"
|
||||
|
||||
+2
-2
@@ -22,7 +22,7 @@
|
||||
: "${PVE1_HOST:=pve1.sweet.home}"
|
||||
: "${PVE_TEST_HOST:=pve-test.sweet.home}"
|
||||
: "${PROXMOX_HOST:=$PVE1_HOST}"
|
||||
: "${PROXMOX_SSH_USER:=root}"
|
||||
: "${PROXMOX_SSH_USER:=wayne}"
|
||||
|
||||
# Where this flake repo lives on the Proxmox node itself.
|
||||
# scripts/proxmox/create-proxmox-resource.sh builds images directly on the node
|
||||
@@ -30,7 +30,7 @@
|
||||
# (from this checkout's own `origin` remote) the first time it doesn't
|
||||
# find it, installing build tooling via scripts/codex-setup.sh, then
|
||||
# `git pull`s it before every subsequent build.
|
||||
: "${PROXMOX_REMOTE_REPO_DIR:=/root/nixos}"
|
||||
: "${PROXMOX_REMOTE_REPO_DIR:=/home/${PROXMOX_SSH_USER}/nixos}"
|
||||
|
||||
# Storage pool names -- Proxmox's own stock-install defaults, but this
|
||||
# varies a lot by setup (ZFS pool name, custom LVM-thin volume, etc.).
|
||||
|
||||
@@ -1,17 +1,20 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i bash -p jq disko nixos-install-tools
|
||||
#!nix-shell -i bash -p jq disko nixos-install-tools zfs
|
||||
# shellcheck shell=bash
|
||||
# The only genuinely external tools this script calls directly: `jq`
|
||||
# (parsing the `nix eval` host list) and `disko`/`nixos-install` (the
|
||||
# install itself). Everything disko shells out to internally
|
||||
# (parted/sgdisk/mkfs.*/zfs/...) is self-contained -- disko's own
|
||||
# generated scripts hardcode absolute Nix store paths for those, they
|
||||
# (parsing the `nix eval` host list), `disko`/`nixos-install` (the
|
||||
# install itself), and `zpool` (exporting a ZFS root pool before reboot,
|
||||
# see the comment above that call below). Everything disko shells out to
|
||||
# internally (parted/sgdisk/mkfs.*/zfs/...) is self-contained -- disko's
|
||||
# own generated scripts hardcode absolute Nix store paths for those, they
|
||||
# don't rely on this script's PATH at all (confirmed by inspecting a
|
||||
# generated system.build.formatScript). The built installer image
|
||||
# (modules/installer/common.nix) already has all three in
|
||||
# environment.systemPackages, so this nix-shell wrapper is a fast no-op
|
||||
# there; it's what makes the script also work standalone (e.g. run
|
||||
# directly from a checkout on a stock ISO), where they aren't.
|
||||
# (modules/installer/common.nix, plus the upstream
|
||||
# installation-cd-minimal.nix it imports via iso.nix) already has all
|
||||
# four in environment.systemPackages, so this nix-shell wrapper is a
|
||||
# fast no-op there; it's what makes the script also work standalone
|
||||
# (e.g. run directly from a checkout on a stock ISO), where they aren't
|
||||
# guaranteed.
|
||||
set -eux
|
||||
|
||||
set -euo pipefail
|
||||
@@ -162,5 +165,42 @@ nixos-install \
|
||||
# /etc/ssh/ssh_host_ed25519_key. Nothing NixOS-managed ever cleans this
|
||||
# up on its own since it was written imperatively, not declaratively.
|
||||
rm -rf /root/host-keys
|
||||
|
||||
# disko's --mode ...,mount left any ZFS root pool imported (that's what
|
||||
# let nixos-install write into /mnt). If we reboot with it still
|
||||
# imported, it isn't just "not exported" -- it's stamped with *this*
|
||||
# live installer environment's hostid, which almost never matches the
|
||||
# target's own networking.hostId (see hosts/*/host.nix; the installer
|
||||
# itself sets none). modules/services/zfs/enable-service.nix and
|
||||
# modules/common/configuration.nix both set boot.zfs.forceImportRoot =
|
||||
# false deliberately (the safe option per that setting's own docs), so
|
||||
# the freshly-installed system's first real boot sees a pool "in use by
|
||||
# another system" and refuses to import it without -f -- which is what
|
||||
# makes boot stall waiting on the ZFS import. Exporting here (a no-op
|
||||
# if the chosen host has no ZFS root, e.g. proxmox-*/linode-*) clears
|
||||
# that in-use state so the next import, from any hostid, succeeds.
|
||||
#
|
||||
# Anything still mounted under /mnt -- nixos-install's own leftover
|
||||
# chroot bind mounts for running the target's activation script
|
||||
# (/mnt/dev, /mnt/proc, /mnt/sys, /mnt/run), and disko's own /mnt/boot
|
||||
# ESP mount (modules/disko/baremetal.nix) -- blocks ZFS from unmounting
|
||||
# its root dataset at /mnt, the same way any nested mount blocks
|
||||
# unmounting its parent. Confirmed live: zpool export failed with
|
||||
# "cannot unmount '/mnt': pool or dataset busy" even after handling the
|
||||
# chroot mounts alone, because /mnt/boot was still mounted too. Because
|
||||
# of this script's `set -e`, that killed the script before it ever
|
||||
# reached reboot, silently defeating the whole point of exporting first.
|
||||
# Unmounting everything under /mnt up front (recursively, so nested
|
||||
# mounts like /mnt/dev/pts come along for free) sidesteps needing to
|
||||
# enumerate every mount disko/nixos-install might leave behind.
|
||||
if mountpoint -q /mnt; then
|
||||
umount -R /mnt
|
||||
fi
|
||||
|
||||
if [[ -n "$(zpool list -H -o name 2>/dev/null)" ]]; then
|
||||
echo "Exporting ZFS pool(s) before reboot..."
|
||||
zpool export -a
|
||||
fi
|
||||
|
||||
sleep 10
|
||||
reboot
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared parallel-nix-invocation helper for scripts/codex-maintenance.sh.
|
||||
# Source alongside nix-eval.sh:
|
||||
# source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/nix-parallel.sh"
|
||||
#
|
||||
# The per-host/per-package `nix eval`/`nix build --dry-run` calls in
|
||||
# codex-maintenance.sh are independent of each other, so running them one at
|
||||
# a time leaves most cores idle for most of the sweep -- run_nix_parallel
|
||||
# fans a batch of them out across up to NIX_PARALLEL_JOBS processes instead.
|
||||
|
||||
# NIX_PARALLEL_JOBS: how many `nix` invocations run_nix_parallel runs at
|
||||
# once. Defaults to core count capped by available memory (~1GB/job,
|
||||
# floor 1) rather than plain `nproc` -- each concurrent `nix eval` here
|
||||
# evaluates a whole NixOS system closure from scratch, and on a small/
|
||||
# memory-constrained CI runner, `nproc` concurrent evals can OOM-kill each
|
||||
# other (confirmed empirically: on a 4GB/6-core box, 5-6 concurrent evals
|
||||
# started getting killed while 3-4 ran clean and were still ~2x faster than
|
||||
# serial). Override via env if a given machine/CI runner has room to spare
|
||||
# or needs a tighter cap.
|
||||
default_nix_parallel_jobs() {
|
||||
local cores mem_avail_kb mem_cap
|
||||
cores="$(nproc 2>/dev/null || echo 4)"
|
||||
mem_avail_kb="$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo 2>/dev/null)"
|
||||
if [[ -z "$mem_avail_kb" ]]; then
|
||||
echo "$cores"
|
||||
return
|
||||
fi
|
||||
mem_cap=$((mem_avail_kb / 1024 / 1024))
|
||||
((mem_cap < 1)) && mem_cap=1
|
||||
((mem_cap < cores)) && echo "$mem_cap" || echo "$cores"
|
||||
}
|
||||
NIX_PARALLEL_JOBS="${NIX_PARALLEL_JOBS:-$(default_nix_parallel_jobs)}"
|
||||
|
||||
# Separator between a job's label and its flake attr in the arrays
|
||||
# run_nix_parallel takes -- a control character so it can't collide with
|
||||
# anything a label or attr path would plausibly contain.
|
||||
NIX_PARALLEL_SEP=$'\x1f'
|
||||
|
||||
# run_nix_parallel <jobs_array_name> <nix subcommand + flags...>
|
||||
#
|
||||
# jobs_array_name: name of an already-populated bash array whose entries are
|
||||
# "<label>${NIX_PARALLEL_SEP}<attr>" pairs, e.g.
|
||||
# jobs=("proxmox-docker${NIX_PARALLEL_SEP}.#nixosConfigurations.proxmox-docker...drvPath")
|
||||
# Remaining args are passed to `nix` before the attr, e.g.:
|
||||
# run_nix_parallel jobs eval --raw "${NIX_EVAL_FLAGS[@]}"
|
||||
# run_nix_parallel jobs build --dry-run --no-link "${NIX_EVAL_FLAGS[@]}"
|
||||
#
|
||||
# Prints "==> <label>" followed by that job's stdout+stderr for every job,
|
||||
# in submission order (not completion order) so a run stays readable and
|
||||
# diffable across invocations even though the work itself doesn't finish in
|
||||
# that order. Returns non-zero if any job failed, only after every job has
|
||||
# finished and been printed -- same "surface everything, then fail" contract
|
||||
# a `set -e` caller gets, just parallelized instead of stopping at the first
|
||||
# failure.
|
||||
run_nix_parallel() {
|
||||
local -n jobs_ref="$1"
|
||||
shift
|
||||
local -a nix_args=("$@")
|
||||
|
||||
local n=${#jobs_ref[@]}
|
||||
[[ $n -eq 0 ]] && return 0
|
||||
|
||||
local tmp_dir
|
||||
tmp_dir="$(mktemp -d)"
|
||||
|
||||
local i=0 running=0
|
||||
for job in "${jobs_ref[@]}"; do
|
||||
local attr="${job#*"${NIX_PARALLEL_SEP}"}"
|
||||
printf '%s\n' "${job%%"${NIX_PARALLEL_SEP}"*}" >"${tmp_dir}/${i}.label"
|
||||
(
|
||||
if nix "${nix_args[@]}" "$attr" >"${tmp_dir}/${i}.out" 2>&1; then
|
||||
echo 0 >"${tmp_dir}/${i}.status"
|
||||
else
|
||||
echo 1 >"${tmp_dir}/${i}.status"
|
||||
fi
|
||||
) &
|
||||
i=$((i + 1))
|
||||
running=$((running + 1))
|
||||
if ((running >= NIX_PARALLEL_JOBS)); then
|
||||
wait -n
|
||||
running=$((running - 1))
|
||||
fi
|
||||
done
|
||||
wait
|
||||
|
||||
local failed=0 j
|
||||
for ((j = 0; j < n; j++)); do
|
||||
echo "==> $(cat "${tmp_dir}/${j}.label")"
|
||||
cat "${tmp_dir}/${j}.out"
|
||||
[[ "$(cat "${tmp_dir}/${j}.status")" -ne 0 ]] && failed=1
|
||||
done
|
||||
|
||||
rm -rf "$tmp_dir"
|
||||
return $failed
|
||||
}
|
||||
@@ -6,27 +6,31 @@
|
||||
#
|
||||
# This is the non-NixOS equivalent of modules/nix-cache/client.nix +
|
||||
# modules/nix-cache/remote-builder-client.nix -- those two only apply to
|
||||
# hosts built from this flake. A plain Debian box with Nix installed
|
||||
# (single- or multi-user install, nix-daemon running) has no NixOS module
|
||||
# system to pick that config up, so this edits /etc/nix/nix.conf by hand
|
||||
# instead. Run this ON the target Debian machine, as root.
|
||||
# hosts built from this flake. A plain Debian box with Nix installed has no
|
||||
# NixOS module system to pick that config up, so this edits nix.conf by hand.
|
||||
#
|
||||
# Two modes depending on who runs it:
|
||||
#
|
||||
# root (multi-user / daemon install):
|
||||
# Writes /etc/nix/nix.conf, /etc/ssh/ssh_known_hosts, restarts nix-daemon.
|
||||
# Requires /etc/nix/nix.conf to already exist (i.e. nix-daemon is set up).
|
||||
# Run as: sudo ./configure-nix-cache-client.sh [options]
|
||||
#
|
||||
# non-root (single-user install):
|
||||
# Writes ~/.config/nix/nix.conf, ~/.ssh/known_hosts. No daemon to restart.
|
||||
# Run as: ./configure-nix-cache-client.sh [options]
|
||||
#
|
||||
# The values below mirror variables.nix / modules/nix-cache/client.nix in
|
||||
# this repo -- update both if nix-cache is ever rebuilt with a new host
|
||||
# key or the cache signing key is rotated (see docs/nix-cache.md).
|
||||
#
|
||||
# REMOTE_BUILDER_KEY defaults to this machine's own default root SSH
|
||||
# identity (matches modules/nix-cache/remote-builder-client.nix's
|
||||
# convention for real NixOS clients: authenticate as nixremote with the
|
||||
# host's own default key, added individually to
|
||||
# vars.remoteBuilderAuthorizedKeys, rather than a separately-named or
|
||||
# shared keypair) -- generate one with
|
||||
# `ssh-keygen -t ed25519 -N '' -f /root/.ssh/id_ed25519` if this machine
|
||||
# doesn't have one yet, then add its .pub to vars.remoteBuilderAuthorizedKeys
|
||||
# and rebuild nix-cache.
|
||||
# REMOTE_BUILDER_KEY defaults to the running user's default SSH identity
|
||||
# (root: /root/.ssh/id_ed25519, other user: ~/.ssh/id_ed25519). That key
|
||||
# must be listed in vars.remoteBuilderAuthorizedKeys in this repo and
|
||||
# nix-cache rebuilt before remote building works.
|
||||
#
|
||||
# Usage:
|
||||
# sudo ./configure-nix-cache-client.sh [--dry-run] [--no-remote-builder] [--no-restart]
|
||||
# ./configure-nix-cache-client.sh [--dry-run] [--no-remote-builder] [--no-restart]
|
||||
#
|
||||
# Env overrides (defaults match variables.nix):
|
||||
# NIX_CACHE_HOST, NIX_CACHE_HOST_KEY, REMOTE_BUILDER_USER, REMOTE_BUILDER_KEY
|
||||
@@ -36,17 +40,28 @@ set -euo pipefail
|
||||
: "${NIX_CACHE_HOST:=nix-cache}"
|
||||
: "${NIX_CACHE_HOST_KEY:=ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPeWgMsdaiz4axT/deFc1+0B5bN+GX/NOeW9bbQ0c/IT lxc-nix-cache}"
|
||||
: "${REMOTE_BUILDER_USER:=nixremote}"
|
||||
: "${REMOTE_BUILDER_KEY:=/root/.ssh/id_ed25519}"
|
||||
|
||||
CACHE_PUB_KEY="cache.local-1:usoWYanY3Kpq2+kDIS2nhWoLZiRxanmdysdzqCFBHW4="
|
||||
FALLBACK_URL="https://cache.nixos.org/"
|
||||
FALLBACK_PUB_KEY="cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
|
||||
|
||||
NIX_CONF="/etc/nix/nix.conf"
|
||||
KNOWN_HOSTS="/etc/ssh/ssh_known_hosts"
|
||||
MARKER_BEGIN="# BEGIN nix-cache client config (configure-nix-cache-client.sh)"
|
||||
MARKER_END="# END nix-cache client config"
|
||||
|
||||
# Mode: root uses system-wide paths and restarts the daemon; non-root uses
|
||||
# user-level paths and has no daemon to restart.
|
||||
if [[ "$EUID" -eq 0 ]]; then
|
||||
install_mode="multi"
|
||||
NIX_CONF="/etc/nix/nix.conf"
|
||||
KNOWN_HOSTS="/etc/ssh/ssh_known_hosts"
|
||||
: "${REMOTE_BUILDER_KEY:=/root/.ssh/id_ed25519}"
|
||||
else
|
||||
install_mode="single"
|
||||
NIX_CONF="${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf"
|
||||
KNOWN_HOSTS="$HOME/.ssh/known_hosts"
|
||||
: "${REMOTE_BUILDER_KEY:=$HOME/.ssh/id_ed25519}"
|
||||
fi
|
||||
|
||||
dry_run=0
|
||||
with_remote_builder=1
|
||||
restart_daemon=1
|
||||
@@ -57,7 +72,7 @@ for arg in "$@"; do
|
||||
--no-remote-builder) with_remote_builder=0 ;;
|
||||
--no-restart) restart_daemon=0 ;;
|
||||
-h|--help)
|
||||
sed -n '2,20p' "$0"
|
||||
sed -n '2,37p' "$0"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
@@ -67,21 +82,22 @@ for arg in "$@"; do
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$dry_run" -eq 0 && "$EUID" -ne 0 ]]; then
|
||||
echo "ERROR: must run as root (writes $NIX_CONF and, unless --no-remote-builder, $KNOWN_HOSTS)." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v nix >/dev/null 2>&1; then
|
||||
echo "ERROR: no 'nix' binary on PATH -- install the Nix package manager first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$NIX_CONF" ]]; then
|
||||
if [[ "$install_mode" == "multi" && ! -f "$NIX_CONF" ]]; then
|
||||
echo "ERROR: $NIX_CONF not found -- expected an existing multi-user Nix install." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Single-user: create the config file if it doesn't exist yet.
|
||||
if [[ "$install_mode" == "single" && "$dry_run" -eq 0 ]]; then
|
||||
mkdir -p "$(dirname "$NIX_CONF")"
|
||||
[[ -f "$NIX_CONF" ]] || touch "$NIX_CONF"
|
||||
fi
|
||||
|
||||
builder_line=""
|
||||
if [[ "$with_remote_builder" -eq 1 ]]; then
|
||||
if [[ -f "$REMOTE_BUILDER_KEY" ]]; then
|
||||
@@ -117,7 +133,7 @@ fi
|
||||
block="${block}
|
||||
$MARKER_END"
|
||||
|
||||
echo "== nix.conf block to install =="
|
||||
echo "== nix.conf block to install ($NIX_CONF) =="
|
||||
echo "$block"
|
||||
echo "================================"
|
||||
|
||||
@@ -159,7 +175,8 @@ if [[ "$with_remote_builder" -eq 1 ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$dry_run" -eq 0 && "$restart_daemon" -eq 1 ]]; then
|
||||
# Only restart the daemon for multi-user installs -- single-user has no daemon.
|
||||
if [[ "$dry_run" -eq 0 && "$restart_daemon" -eq 1 && "$install_mode" == "multi" ]]; then
|
||||
if command -v systemctl >/dev/null 2>&1 && systemctl is-active --quiet nix-daemon 2>/dev/null; then
|
||||
systemctl restart nix-daemon
|
||||
echo "Restarted nix-daemon to pick up the new config."
|
||||
|
||||
@@ -193,6 +193,16 @@ done
|
||||
|
||||
ssh_target="${PROXMOX_SSH_USER}@${node}"
|
||||
|
||||
# Proxmox tools (pvesh, qm, pct) require root access to the cluster IPC
|
||||
# socket. When SSH-ing as a non-root user with sudo, prefix every remote
|
||||
# Proxmox command with sudo.
|
||||
sudo_prefix=""
|
||||
sudo_display=""
|
||||
if [[ "$PROXMOX_SSH_USER" != "root" ]]; then
|
||||
sudo_prefix="sudo"
|
||||
sudo_display="sudo "
|
||||
fi
|
||||
|
||||
remote() {
|
||||
if [[ "$dry_run" -eq 1 ]]; then
|
||||
echo "[dry-run] ssh ${ssh_target} -- $*"
|
||||
@@ -214,10 +224,10 @@ cmd_modify() {
|
||||
|
||||
echo "Looking up VMID ${vmid} on ${node}..."
|
||||
local kind current_cores current_memory disk_key
|
||||
if ssh "$ssh_target" "qm status ${vmid}" >/dev/null 2>&1; then
|
||||
if ssh "$ssh_target" "${sudo_prefix} qm status ${vmid}" >/dev/null 2>&1; then
|
||||
kind="vm"
|
||||
disk_key="scsi0"
|
||||
elif ssh "$ssh_target" "pct status ${vmid}" >/dev/null 2>&1; then
|
||||
elif ssh "$ssh_target" "${sudo_prefix} pct status ${vmid}" >/dev/null 2>&1; then
|
||||
kind="lxc"
|
||||
disk_key="rootfs"
|
||||
else
|
||||
@@ -225,8 +235,8 @@ cmd_modify() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local config_cmd="qm config ${vmid}"
|
||||
[[ "$kind" == "lxc" ]] && config_cmd="pct config ${vmid}"
|
||||
local config_cmd="${sudo_prefix} qm config ${vmid}"
|
||||
[[ "$kind" == "lxc" ]] && config_cmd="${sudo_prefix} pct config ${vmid}"
|
||||
local current_config
|
||||
current_config="$(ssh "$ssh_target" "$config_cmd")"
|
||||
current_cores="$(echo "$current_config" | grep -oP '^cores:\s*\K\S+' || echo '?')"
|
||||
@@ -250,9 +260,9 @@ cmd_modify() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local set_cmd="qm set"
|
||||
local resize_cmd="qm resize"
|
||||
[[ "$kind" == "lxc" ]] && set_cmd="pct set" && resize_cmd="pct resize"
|
||||
local set_cmd="${sudo_prefix} qm set"
|
||||
local resize_cmd="${sudo_prefix} qm resize"
|
||||
[[ "$kind" == "lxc" ]] && set_cmd="${sudo_prefix} pct set" && resize_cmd="${sudo_prefix} pct resize"
|
||||
|
||||
if [[ -n "$cores" || -n "$memory" ]]; then
|
||||
local args=""
|
||||
@@ -359,14 +369,15 @@ else
|
||||
echo
|
||||
echo "==> Checking ${node} for an existing VM/CT identified as '${host}'..."
|
||||
ssh_check_status=0
|
||||
existing="$(ssh "$ssh_target" bash -s -- "$host" <<'REMOTE_SCRIPT'
|
||||
existing="$(ssh "$ssh_target" bash -s -- "$host" "$sudo_prefix" <<'REMOTE_SCRIPT'
|
||||
target="$1"
|
||||
for id in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
|
||||
n="$(qm config "$id" 2>/dev/null | grep -oP '^name:\s*\K\S+' || true)"
|
||||
sudo_pfx="$2"
|
||||
for id in $($sudo_pfx qm list 2>/dev/null | awk 'NR>1{print $1}'); do
|
||||
n="$($sudo_pfx qm config "$id" 2>/dev/null | grep -oP '^name:\s*\K\S+' || true)"
|
||||
[[ "$n" == "$target" ]] && echo "vm ${id} ${n}"
|
||||
done
|
||||
for id in $(pct list 2>/dev/null | awk 'NR>1{print $1}'); do
|
||||
n="$(pct config "$id" 2>/dev/null | grep -oP '^hostname:\s*\K\S+' || true)"
|
||||
for id in $($sudo_pfx pct list 2>/dev/null | awk 'NR>1{print $1}'); do
|
||||
n="$($sudo_pfx pct config "$id" 2>/dev/null | grep -oP '^hostname:\s*\K\S+' || true)"
|
||||
[[ "$n" == "$target" ]] && echo "lxc ${id} ${n}"
|
||||
done
|
||||
exit 0
|
||||
@@ -453,12 +464,12 @@ REMOTE_SCRIPT
|
||||
if [[ "$kind" == "vm" ]]; then
|
||||
# qm destroy has no --force to stop-then-destroy in one call (pct's
|
||||
# does) -- stop explicitly first if it's running.
|
||||
if ssh "$ssh_target" "qm status ${id}" 2>/dev/null | grep -q running; then
|
||||
ssh "$ssh_target" "qm stop ${id}"
|
||||
if ssh "$ssh_target" "${sudo_prefix} qm status ${id}" 2>/dev/null | grep -q running; then
|
||||
ssh "$ssh_target" "${sudo_prefix} qm stop ${id}"
|
||||
fi
|
||||
ssh "$ssh_target" "qm destroy ${id} --purge 1"
|
||||
ssh "$ssh_target" "${sudo_prefix} qm destroy ${id} --purge 1"
|
||||
else
|
||||
ssh "$ssh_target" "pct destroy ${id} --force 1 --purge 1"
|
||||
ssh "$ssh_target" "${sudo_prefix} pct destroy ${id} --force 1 --purge 1"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
@@ -487,7 +498,7 @@ if [[ -z "$vmid" ]]; then
|
||||
vmid="<next-free-vmid>"
|
||||
echo "[dry-run] would ask ${node} for the next free VMID (pvesh get /cluster/nextid)"
|
||||
else
|
||||
vmid="$(ssh "$ssh_target" "pvesh get /cluster/nextid" | tr -d '[:space:]')"
|
||||
vmid="$(ssh "$ssh_target" "${sudo_prefix} pvesh get /cluster/nextid" | tr -d '[:space:]')"
|
||||
echo "Auto-assigned VMID: ${vmid}"
|
||||
fi
|
||||
else
|
||||
@@ -501,8 +512,8 @@ if [[ "$dry_run" -eq 0 ]]; then
|
||||
# both. Any success here means something is already using this ID --
|
||||
# refuse to go anywhere near it. (Reconfiguring an existing resource is
|
||||
# --modify's job, not this one's.)
|
||||
if ssh "$ssh_target" "qm status ${vmid}" >/dev/null 2>&1 \
|
||||
|| ssh "$ssh_target" "pct status ${vmid}" >/dev/null 2>&1; then
|
||||
if ssh "$ssh_target" "${sudo_prefix} qm status ${vmid}" >/dev/null 2>&1 \
|
||||
|| ssh "$ssh_target" "${sudo_prefix} pct status ${vmid}" >/dev/null 2>&1; then
|
||||
echo "ERROR: VMID ${vmid} already exists on ${node}. Refusing to touch an" >&2
|
||||
echo "existing resource here -- use --modify to reconfigure it, pick a" >&2
|
||||
echo "different --vmid, or omit it to auto-assign." >&2
|
||||
@@ -672,11 +683,11 @@ if [[ "$image_already_remote" -eq 0 && -z "$local_image" ]]; then
|
||||
# hands the result to the remote shell to re-split, which would
|
||||
# otherwise scatter NIX_EXTRA_OPTS (itself several space-separated,
|
||||
# %q-quoted tokens) across the wrong positional parameters below.
|
||||
printf -v remote_cmd 'bash -s -- %q %q %q %q %q' \
|
||||
"$remote_repo_dir" "$flake_target" "$remote_dir" "$remote_filename" "$NIX_EXTRA_OPTS"
|
||||
printf -v remote_cmd 'bash -s -- %q %q %q %q %q %q' \
|
||||
"$remote_repo_dir" "$flake_target" "$remote_dir" "$remote_filename" "$NIX_EXTRA_OPTS" "$sudo_prefix"
|
||||
ssh "$ssh_target" "$remote_cmd" <<'REMOTE_SCRIPT'
|
||||
set -euo pipefail
|
||||
repo_dir="$1"; target="$2"; dest_dir="$3"; dest_name="$4"; nix_extra_opts_str="$5"
|
||||
repo_dir="$1"; target="$2"; dest_dir="$3"; dest_name="$4"; nix_extra_opts_str="$5"; sudo_pfx="$6"
|
||||
declare -a NIX_OPTS=()
|
||||
[[ -n "$nix_extra_opts_str" ]] && eval "NIX_OPTS=(${nix_extra_opts_str})"
|
||||
cd "$repo_dir"
|
||||
@@ -694,23 +705,14 @@ if [[ -z "$built" ]]; then
|
||||
echo "ERROR: no tarball found under result-${target}/tarball after build." >&2
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p "$dest_dir"
|
||||
cp "$built" "${dest_dir}/${dest_name}"
|
||||
$sudo_pfx mkdir -p "$dest_dir"
|
||||
$sudo_pfx cp "$built" "${dest_dir}/${dest_name}"
|
||||
echo "Built and staged: ${dest_dir}/${dest_name}"
|
||||
REMOTE_SCRIPT
|
||||
local_image="$remote_path"
|
||||
echo "Built on ${node}: ${remote_path}"
|
||||
fi
|
||||
else
|
||||
# PROXMOX_SSH_USER defaults to root (env.sh), which needs no sudo and
|
||||
# can't assume it's even installed on a minimal node -- only shell out
|
||||
# through sudo when actually running as a non-root SSH user.
|
||||
sudo_prefix="sudo"
|
||||
sudo_display="sudo "
|
||||
if [[ "$PROXMOX_SSH_USER" == "root" ]]; then
|
||||
sudo_prefix=""
|
||||
sudo_display=""
|
||||
fi
|
||||
if [[ "$dry_run" -eq 1 ]]; then
|
||||
echo "[dry-run] would build on ${node}: nix build --no-use-registries --no-accept-flake-config${nix_opts_display} \\"
|
||||
echo "[dry-run] .#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript"
|
||||
@@ -728,7 +730,7 @@ REMOTE_SCRIPT
|
||||
"$remote_repo_dir" "$flake_target" "$remote_dir" "$remote_filename" "$NIX_EXTRA_OPTS" "$sudo_prefix"
|
||||
ssh "$ssh_target" "$remote_cmd" <<'REMOTE_SCRIPT'
|
||||
set -euo pipefail
|
||||
repo_dir="$1"; target="$2"; dest_dir="$3"; dest_name="$4"; nix_extra_opts_str="$5"; sudo_prefix="$6"
|
||||
repo_dir="$1"; target="$2"; dest_dir="$3"; dest_name="$4"; nix_extra_opts_str="$5"; sudo_pfx="$6"
|
||||
declare -a NIX_OPTS=()
|
||||
[[ -n "$nix_extra_opts_str" ]] && eval "NIX_OPTS=(${nix_extra_opts_str})"
|
||||
cd "$repo_dir"
|
||||
@@ -737,7 +739,7 @@ ensure_nix_profile
|
||||
nix build --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \
|
||||
".#nixosConfigurations.${target}.config.system.build.diskoImagesScript" \
|
||||
--out-link "result-${target}"
|
||||
$sudo_prefix "./result-${target}" \
|
||||
$sudo_pfx "./result-${target}" \
|
||||
--pre-format-files "host-keys/${target}_ssh_host_ed25519_key" /etc/ssh/ssh_host_ed25519_key \
|
||||
--pre-format-files "host-keys/${target}_ssh_host_ed25519_key.pub" /etc/ssh/ssh_host_ed25519_key.pub \
|
||||
--build-memory 2048
|
||||
@@ -746,8 +748,8 @@ if [[ -z "$built" ]]; then
|
||||
echo "ERROR: no .raw image found in ${repo_dir} after build." >&2
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p "$dest_dir"
|
||||
mv "$built" "${dest_dir}/${dest_name}"
|
||||
$sudo_pfx mkdir -p "$dest_dir"
|
||||
$sudo_pfx mv "$built" "${dest_dir}/${dest_name}"
|
||||
echo "Built and staged: ${dest_dir}/${dest_name}"
|
||||
REMOTE_SCRIPT
|
||||
local_image="$remote_path"
|
||||
@@ -812,9 +814,9 @@ if [[ "$type" == "lxc" ]]; then
|
||||
# hands the whole string to `ssh` as a single command for the *remote*
|
||||
# shell to parse -- unquoted, that `;` would be read as a remote
|
||||
# command separator and silently truncate this into two commands.
|
||||
create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --unprivileged ${unprivileged_flag} --features '${PROXMOX_DEFAULT_LXC_FEATURES}' --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --swap ${local_swap} --net0 name=eth0,bridge=${bridge},ip=dhcp"
|
||||
create_cmd="${sudo_prefix} pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --unprivileged ${unprivileged_flag} --features '${PROXMOX_DEFAULT_LXC_FEATURES}' --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --swap ${local_swap} --net0 name=eth0,bridge=${bridge},ip=dhcp"
|
||||
remote "$create_cmd"
|
||||
remote "pct start ${vmid}"
|
||||
remote "${sudo_prefix} pct start ${vmid}"
|
||||
else
|
||||
echo "==> Creating VM ${vmid} (${name})..."
|
||||
# pre-enrolled-keys=0 disables OVMF's Secure Boot key pre-enrollment --
|
||||
@@ -825,29 +827,29 @@ else
|
||||
# without this flag Proxmox never creates the channel it listens on, so
|
||||
# `qm guest exec`/`qm agent` and the UI's IP-address display silently
|
||||
# never work for any VM this script creates.
|
||||
remote "qm create ${vmid} --name ${name} --memory ${memory} --cores ${cores} \
|
||||
remote "${sudo_prefix} qm create ${vmid} --name ${name} --memory ${memory} --cores ${cores} \
|
||||
--net0 virtio,bridge=${bridge} --bios ovmf --machine q35 --scsihw virtio-scsi-pci \
|
||||
--efidisk0 ${storage}:1,efitype=4m,pre-enrolled-keys=0 --agent enabled=1"
|
||||
|
||||
if [[ "$dry_run" -eq 1 ]]; then
|
||||
echo "[dry-run] ssh ${ssh_target} -- qm importdisk ${vmid} ${remote_path} ${storage}"
|
||||
echo "[dry-run] ssh ${ssh_target} -- ${sudo_display}qm importdisk ${vmid} ${remote_path} ${storage}"
|
||||
echo "[dry-run] (would parse the resulting disk identifier from that output)"
|
||||
echo "[dry-run] ssh ${ssh_target} -- qm set ${vmid} --scsi0 ${storage}:<parsed-disk-id>"
|
||||
echo "[dry-run] ssh ${ssh_target} -- ${sudo_display}qm set ${vmid} --scsi0 ${storage}:<parsed-disk-id>"
|
||||
else
|
||||
importdisk_output="$(ssh "$ssh_target" "qm importdisk ${vmid} ${remote_path} ${storage}")"
|
||||
importdisk_output="$(ssh "$ssh_target" "${sudo_prefix} qm importdisk ${vmid} ${remote_path} ${storage}")"
|
||||
echo "$importdisk_output"
|
||||
disk_id="$(echo "$importdisk_output" | grep -oP "(?<=Successfully imported disk as ')[^']+" | sed 's/^unused[0-9]*://')"
|
||||
if [[ -z "$disk_id" ]]; then
|
||||
echo "ERROR: couldn't parse the imported disk identifier from qm importdisk's output above." >&2
|
||||
echo "The VM shell (${vmid}) and imported disk both exist -- finish attaching it by hand:" >&2
|
||||
echo " ssh ${ssh_target} -- qm set ${vmid} --scsi0 ${storage}:<disk-id-from-output-above>" >&2
|
||||
echo " ssh ${ssh_target} -- qm set ${vmid} --boot order=scsi0" >&2
|
||||
echo " ssh ${ssh_target} -- ${sudo_display}qm set ${vmid} --scsi0 ${storage}:<disk-id-from-output-above>" >&2
|
||||
echo " ssh ${ssh_target} -- ${sudo_display}qm set ${vmid} --boot order=scsi0" >&2
|
||||
exit 1
|
||||
fi
|
||||
remote "qm set ${vmid} --scsi0 ${disk_id}"
|
||||
remote "${sudo_prefix} qm set ${vmid} --scsi0 ${disk_id}"
|
||||
fi
|
||||
remote "qm set ${vmid} --boot order=scsi0"
|
||||
remote "qm start ${vmid}"
|
||||
remote "${sudo_prefix} qm set ${vmid} --boot order=scsi0"
|
||||
remote "${sudo_prefix} qm start ${vmid}"
|
||||
fi
|
||||
|
||||
echo
|
||||
|
||||
Reference in New Issue
Block a user