From 23b910a0119909734f3fd306415047f02caed3c6 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 20 Jul 2026 15:07:04 +0000 Subject: [PATCH 1/2] Add script to configure nix-cache as substituter/remote builder on Debian clients Non-NixOS machines with just the Nix package manager installed have no module system to pick up modules/nix-cache/client.nix, so this edits /etc/nix/nix.conf directly (extra-substituters/extra-trusted-public-keys, plus the SSH remote-builder config once the nixremote key is installed), falling back to cache.nixos.org when nix-cache is unreachable. Co-Authored-By: Claude Sonnet 5 --- CLAUDE.md | 14 +++ scripts/configure-nix-cache-client.sh | 172 ++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100755 scripts/configure-nix-cache-client.sh diff --git a/CLAUDE.md b/CLAUDE.md index 8921f0c..43018cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -101,6 +101,20 @@ Beyond `codex-setup.sh`/`codex-maintenance.sh` above, `scripts/` also has: - `scripts/env.sh` — shared config (`PROXMOX_HOST`, storage pool, bridge, default cores/memory) sourced by `create-proxmox-resource.sh`. Add new cross-script config here instead of duplicating it per-script. +- `scripts/configure-nix-cache-client.sh [--dry-run] [--no-remote-builder] + [--no-restart]` — the non-NixOS equivalent of + `modules/nix-cache/client.nix`/`remote-builder-client.nix`, for a plain + Debian machine with the Nix package manager (not NixOS) already + installed: run as root *on that machine* to add nix-cache as a + substituter in `/etc/nix/nix.conf` (`https://cache.nixos.org/` kept as + fallback) via `extra-substituters`/`extra-trusted-public-keys` so it + layers on top of whatever's already there instead of clobbering it, and, + if `/root/.ssh/nixremote` is already present (see docs/nix-cache.md + "Remote builder SSH keys"), configures it as a distributed-build + machine too and trusts nix-cache's SSH host key in + `/etc/ssh/ssh_known_hosts`. Idempotent (re-running replaces its own + marked block rather than duplicating it); restarts `nix-daemon` by + default so the change takes effect immediately. - `scripts/bump-nixpkgs-release.sh` — bumps `flake.nix`'s `nixpkgs.url`/ `home-manager.url` in place. Exists because flake input URLs can't reference `variables.nix` (confirmed empirically — `nix flake metadata` diff --git a/scripts/configure-nix-cache-client.sh b/scripts/configure-nix-cache-client.sh new file mode 100755 index 0000000..6c6c66f --- /dev/null +++ b/scripts/configure-nix-cache-client.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# Points a non-NixOS Debian machine's Nix install at nix-cache: adds it as +# a substituter (with cache.nixos.org kept as fallback) and, once the +# remote-builder private key is installed, as a distributed-build machine +# too. +# +# This is the non-NixOS equivalent of modules/nix-cache/client.nix + +# modules/nix-cache/remote-builder-client.nix -- those two only apply to +# hosts built from this flake. A plain Debian box with Nix installed +# (single- or multi-user install, nix-daemon running) has no NixOS module +# system to pick that config up, so this edits /etc/nix/nix.conf by hand +# instead. Run this ON the target Debian machine, as root. +# +# The values below mirror variables.nix / modules/nix-cache/client.nix in +# this repo -- update both if nix-cache is ever rebuilt with a new host +# key or the cache signing key is rotated (see docs/nix-cache.md). +# +# Usage: +# sudo ./configure-nix-cache-client.sh [--dry-run] [--no-remote-builder] [--no-restart] +# +# Env overrides (defaults match variables.nix): +# NIX_CACHE_HOST, NIX_CACHE_HOST_KEY, REMOTE_BUILDER_USER, REMOTE_BUILDER_KEY + +set -euo pipefail + +: "${NIX_CACHE_HOST:=nix-cache}" +: "${NIX_CACHE_HOST_KEY:=ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHrMKZlIGUd3pH9G3AqbsruqUGjxIXMAZw52u9MwiBCn lxc-nix-cache}" +: "${REMOTE_BUILDER_USER:=nixremote}" +: "${REMOTE_BUILDER_KEY:=/root/.ssh/nixremote}" + +CACHE_PUB_KEY="cache.local-1:usoWYanY3Kpq2+kDIS2nhWoLZiRxanmdysdzqCFBHW4=" +FALLBACK_URL="https://cache.nixos.org/" +FALLBACK_PUB_KEY="cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + +NIX_CONF="/etc/nix/nix.conf" +KNOWN_HOSTS="/etc/ssh/ssh_known_hosts" +MARKER_BEGIN="# BEGIN nix-cache client config (configure-nix-cache-client.sh)" +MARKER_END="# END nix-cache client config" + +dry_run=0 +with_remote_builder=1 +restart_daemon=1 + +for arg in "$@"; do + case "$arg" in + --dry-run) dry_run=1 ;; + --no-remote-builder) with_remote_builder=0 ;; + --no-restart) restart_daemon=0 ;; + -h|--help) + sed -n '2,20p' "$0" + exit 0 + ;; + *) + echo "ERROR: unknown argument: $arg" >&2 + exit 1 + ;; + esac +done + +if [[ "$dry_run" -eq 0 && "$EUID" -ne 0 ]]; then + echo "ERROR: must run as root (writes $NIX_CONF and, unless --no-remote-builder, $KNOWN_HOSTS)." >&2 + exit 1 +fi + +if ! command -v nix >/dev/null 2>&1; then + echo "ERROR: no 'nix' binary on PATH -- install the Nix package manager first." >&2 + exit 1 +fi + +if [[ ! -f "$NIX_CONF" ]]; then + echo "ERROR: $NIX_CONF not found -- expected an existing multi-user Nix install." >&2 + exit 1 +fi + +builder_line="" +if [[ "$with_remote_builder" -eq 1 ]]; then + if [[ -f "$REMOTE_BUILDER_KEY" ]]; then + case "$(uname -m)" in + x86_64) nix_system="x86_64-linux" ;; + aarch64) nix_system="aarch64-linux" ;; + *) + echo "WARNING: unrecognized architecture '$(uname -m)' -- skipping remote builder, keeping substituter config." >&2 + with_remote_builder=0 + ;; + esac + if [[ "$with_remote_builder" -eq 1 ]]; then + builder_line="builders = ssh://${REMOTE_BUILDER_USER}@${NIX_CACHE_HOST} ${nix_system} ${REMOTE_BUILDER_KEY} 4 2 big-parallel,kvm,nixos-test,benchmark" + fi + else + echo "WARNING: $REMOTE_BUILDER_KEY not found -- skipping remote builder config (substituter still configured)." >&2 + echo " See docs/nix-cache.md 'Remote builder SSH keys' for how to install it, then re-run this script." >&2 + with_remote_builder=0 + fi +fi + +block="$(cat < "$tmp_conf" + else + cp "$NIX_CONF" "$tmp_conf" + printf '\n%s\n' "$block" >> "$tmp_conf" + fi + + cp "$NIX_CONF" "${NIX_CONF}.bak.$(date +%Y%m%d%H%M%S)" + install -m 0644 "$tmp_conf" "$NIX_CONF" + echo "Updated $NIX_CONF (backup saved alongside it)." +fi + +if [[ "$with_remote_builder" -eq 1 ]]; then + known_hosts_line="${NIX_CACHE_HOST} ${NIX_CACHE_HOST_KEY}" + if [[ "$dry_run" -eq 1 ]]; then + echo "(--dry-run: would ensure this line is present in $KNOWN_HOSTS)" + echo " $known_hosts_line" + else + mkdir -p "$(dirname "$KNOWN_HOSTS")" + touch "$KNOWN_HOSTS" + if ! grep -qF "$known_hosts_line" "$KNOWN_HOSTS" 2>/dev/null; then + echo "$known_hosts_line" >> "$KNOWN_HOSTS" + echo "Added nix-cache's SSH host key to $KNOWN_HOSTS." + fi + fi +fi + +if [[ "$dry_run" -eq 0 && "$restart_daemon" -eq 1 ]]; then + if command -v systemctl >/dev/null 2>&1 && systemctl is-active --quiet nix-daemon 2>/dev/null; then + systemctl restart nix-daemon + echo "Restarted nix-daemon to pick up the new config." + else + echo "nix-daemon not managed by systemd (or not running) -- restart it manually to pick up the new config." + fi +fi + +cat < Date: Mon, 20 Jul 2026 15:09:57 +0000 Subject: [PATCH 2/2] Wire configure-nix-cache-client.sh into create-proxmox-resource.sh's tooling bootstrap Run it once, right after a node's first-time Nix bootstrap (not on every invocation, and not inside codex-setup.sh/codex-maintenance.sh themselves), so a freshly-bootstrapped Proxmox node substitutes from and can offload builds to nix-cache on every subsequent run. Non-fatal on failure -- the build still proceeds, just without nix-cache. Co-Authored-By: Claude Sonnet 5 --- CLAUDE.md | 7 ++++++- scripts/create-proxmox-resource.sh | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 43018cf..0c613bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,7 +97,12 @@ Beyond `codex-setup.sh`/`codex-maintenance.sh` above, `scripts/` also has: Refuses to create a target whose host identity already exists live on the node (checked directly via `qm`/`pct`, not any file in this repo) unless `--allow-duplicate-host` is passed. `--dry-run` throughout both - modes. + modes. The first time it has to bootstrap build tooling on a node (i.e. + `nix` wasn't already on its `PATH`), it also runs + `scripts/configure-nix-cache-client.sh` there (non-fatally — a failure + just falls back to building from source / `cache.nixos.org`) so the + node substitutes from and can offload builds to nix-cache on every + subsequent run, not just this one. - `scripts/env.sh` — shared config (`PROXMOX_HOST`, storage pool, bridge, default cores/memory) sourced by `create-proxmox-resource.sh`. Add new cross-script config here instead of duplicating it per-script. diff --git a/scripts/create-proxmox-resource.sh b/scripts/create-proxmox-resource.sh index 6d303df..e4fb715 100755 --- a/scripts/create-proxmox-resource.sh +++ b/scripts/create-proxmox-resource.sh @@ -524,7 +524,7 @@ ensure_remote_repo() { echo echo "==> Ensuring ${remote_repo_dir} exists and is current on ${node}..." if [[ "$dry_run" -eq 1 ]]; then - echo "[dry-run] would ensure ${remote_repo_dir} exists on ${node} (clone if missing, git pull if present), and would verify/bootstrap build tooling there (scripts/codex-setup.sh) if \`nix\` isn't already on PATH" + echo "[dry-run] would ensure ${remote_repo_dir} exists on ${node} (clone if missing, git pull if present), and would verify/bootstrap build tooling there (scripts/codex-setup.sh) if \`nix\` isn't already on PATH -- and if that bootstrap actually ran, would also configure ${node} as a nix-cache client (scripts/configure-nix-cache-client.sh)" return fi @@ -571,6 +571,21 @@ ensure_remote_repo() { else echo "==> Bootstrapping build tooling on ${node} (scripts/codex-setup.sh)..." ssh "$ssh_target" "cd '${remote_repo_dir}' && bash scripts/codex-setup.sh" + + # Only on this first-time bootstrap, not every run -- a node that + # already has tooling either already went through this once, or had + # it configured some other way, and re-running is harmless but + # pointless. Non-fatal: this only makes the node's own builds faster + # (substitute from nix-cache instead of building from source) and + # offloadable to it as a remote builder -- worth trying, not worth + # aborting the image build over if nix-cache happens to be down right + # now. Needs ensure_nix_profile first, same as the tooling_check_cmd + # above -- ssh's non-interactive command execution won't have picked + # up a freshly single-user-installed `nix` otherwise. + echo "==> Configuring ${node} as a nix-cache substituter/remote-builder client..." + if ! ssh "$ssh_target" "cd '${remote_repo_dir}' && . scripts/lib/nix-bootstrap.sh && ensure_nix_profile && bash scripts/configure-nix-cache-client.sh"; then + echo "WARNING: configure-nix-cache-client.sh failed on ${node} -- continuing without it (${node} will build from source / against cache.nixos.org only)." >&2 + fi fi } -- 2.54.0