From a3e85b50793d47af2a03de6bcefec829f8aae41b Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 20 Jul 2026 11:05:26 +1000 Subject: [PATCH] Fix nix-cache retry storms and lxc creation in create-proxmox-resource.sh Two independent problems found while actually running the script: 1. nix build/nix-shell retry each unreachable substituter/builder up to 5x with backoff, per store path -- with nix-cache down this compounds into minutes of noise. scripts/env.sh gains nix_extra_opts(), which probes http://nix-cache and nixremote@nix-cache:22 once via plain curl/TCP (bypassing Nix's own retry logic entirely -- confirmed nix store ping still retries 5x even with a short connect-timeout) and exports the decision so create-proxmox-resource.sh and the sync-host-keys.sh subprocess it shells out to both reuse it instead of probing independently. 2. The actual failure: "archive contains no configuration file". pct restore expects a vzdump backup archive with embedded config; config.system.build.tarball is a plain CT template tarball -- wrong Proxmox mechanism entirely. Fixed to pct create against it as a vztmpl template instead, uploaded to /var/lib/vz/template/cache/ rather than /var/lib/vz/dump/. This same wrong claim had propagated into docs/auto-installer.md, README.md, and CLAUDE.md from when the script was first written -- corrected everywhere. Also: checks for an already-uploaded image on the node (fixed .tar.xz/.raw naming) before building, skipping build+upload entirely if found (--force-rebuild to always rebuild). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01La55Nsss8jZ7ZuzUV9mfot --- CLAUDE.md | 22 +++-- README.md | 6 +- docs/auto-installer.md | 24 ++++- scripts/create-proxmox-resource.sh | 142 +++++++++++++++++++---------- scripts/env.sh | 54 ++++++++++- scripts/sync-host-keys.sh | 16 ++-- 6 files changed, 193 insertions(+), 71 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 02be9f1..73ccc15 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,12 +86,16 @@ Beyond `codex-setup.sh`/`codex-maintenance.sh` above, `scripts/` also has: has. - `scripts/create-proxmox-resource.sh` — builds a `lxc-*`/`proxmox-*` target's tarball/disk image and creates it on a real Proxmox node - (`pct restore` / `qm create`+`importdisk`), or reconfigures an existing - resource's cores/memory/disk size (`--modify`, always requires typing the - VMID back to confirm). Refuses to create a target whose host identity - already has a real deployment elsewhere (`variables.nix`'s - `deployedTargets`) unless `--allow-duplicate-host` is passed. - `--dry-run` throughout both modes. + (`pct create` against the tarball as a CT template / `qm create`+ + `importdisk`), or reconfigures an existing resource's cores/memory/disk + size (`--modify`, always requires typing the VMID back to confirm). + Checks for an already-uploaded image on the node before building + (`--force-rebuild` to skip that and always rebuild), and probes + nix-cache's substituter/remote-builder reachability once up front rather + than letting every `nix build` call retry against it individually. + Refuses to create a target whose host identity already has a real + deployment elsewhere (`variables.nix`'s `deployedTargets`) unless + `--allow-duplicate-host` is passed. `--dry-run` throughout both modes. - `scripts/env.sh` — shared config (`PROXMOX_HOST`, storage pool, bridge, default cores/memory) sourced by `create-proxmox-resource.sh`. Add new cross-script config here instead of duplicating it per-script. @@ -153,8 +157,10 @@ removing a host. from `flake.nix`. `lxc.nix` has no hardware-configuration counterpart since containers share the host kernel; instead it imports nixpkgs' own `virtualisation/proxmox-lxc.nix`, which gives every `lxc-*` host a - `config.system.build.tarball` output (`pct restore`-ready, no install step - — see `docs/auto-installer.md`). + `config.system.build.tarball` output — a plain rootfs tarball, used as a + `pct create ... vztmpl` CT template (**not** `pct restore`, which expects + `vzdump` backup-archive metadata this doesn't have), no install step — + see `docs/auto-installer.md`. - `modules/build-types/*.nix` — what a system is for: minimal/server/docker/gui/pxe-boot/nix-cache. - `modules/common/configuration.nix` — base NixOS config imported by every diff --git a/README.md b/README.md index 4da0d60..643798a 100644 --- a/README.md +++ b/README.md @@ -103,15 +103,15 @@ Three different paths depending on target, none of them involving a manual section covers how this stays non-destructive for `linode-*`, whose disks Linode itself provisions ahead of time). - `lxc-*` targets: not installed at all — build a ready-to-run container - tarball and `pct restore` it directly. `docs/auto-installer.md` covers why - (and the installer's menu excludes them for the same reason). + tarball and `pct create` it as a CT template directly. `docs/auto-installer.md` + covers why (and the installer's menu excludes them for the same reason). - `proxmox-*` targets: can alternatively be built as a standalone `.raw` disk image and attached to a new VM with no install step — see `docs/proxmox-images.md`. `scripts/create-proxmox-resource.sh --type lxc|vm --host ` automates either of the last two end to end (build, host-key registration, upload, -`pct restore`/`qm create`), with `--dry-run` and a guard against duplicating +`pct create`/`qm create`), with `--dry-run` and a guard against duplicating an already-deployed host's identity. See its `--help`. ## Security Notes diff --git a/docs/auto-installer.md b/docs/auto-installer.md index 5c1050f..cccc46b 100644 --- a/docs/auto-installer.md +++ b/docs/auto-installer.md @@ -34,10 +34,26 @@ container image, no install step at all: nix build .#nixosConfigurations.lxc-minimal.config.system.build.tarball ``` -Then, on the Proxmox host, `pct restore` (or the GUI's "Create CT" → upload -template flow) that tarball directly as a new container. First boot runs -`boot.postBootCommands` (registers the Nix store DB and system profile) — -there's no separate activation step to run yourself. +This is a plain rootfs tarball, not a `vzdump` backup archive — restoring it +with `pct restore` fails ("archive contains no configuration file"), since +that command expects backup-archive metadata this tarball doesn't have. Use +it as a CT *template* instead: drop it under Proxmox's template storage +(conventionally `/var/lib/vz/template/cache/` for the `local` storage, or +the GUI's "Create CT" → upload-as-template flow) and create a container +from it, supplying all config on the command line since a template has none +of its own: + +```sh +pct create local:vztmpl/.tar.xz \ + --rootfs local-lvm:8 --hostname --cores 2 --memory 2048 \ + --net0 name=eth0,bridge=vmbr0,ip=dhcp +pct start +``` + +First boot runs `boot.postBootCommands` (registers the Nix store DB and +system profile) — there's no separate activation step to run yourself. +`scripts/create-proxmox-resource.sh --type lxc --host ` automates all +of this (build, host-key handling, upload, `pct create`) — see its `--help`. Host keys still need pre-seeding the same way as any other host (see "Host keys" below) — the sops-nix activation-vs-first-boot race is identical diff --git a/scripts/create-proxmox-resource.sh b/scripts/create-proxmox-resource.sh index f21aeaf..8a201dc 100755 --- a/scripts/create-proxmox-resource.sh +++ b/scripts/create-proxmox-resource.sh @@ -34,7 +34,7 @@ Usage: $0 --type lxc|vm --host [options] (create) $0 --modify --vmid [options] (reconfigure) Create mode (default): - --type lxc|vm lxc = container, built as a pct-restorable tarball. + --type lxc|vm lxc = container, built as a CT template tarball. vm = VM, built as a Disko .raw disk image (UEFI/OVMF). --host Which host identity to deploy -- matches config.networking.hostName (server, docker, @@ -45,10 +45,13 @@ Create mode (default): --vmid Numeric VMID (default: next free, via \`pvesh get /cluster/nextid\` on the node). Refuses to run if this ID already exists. - --disk-size lxc only, at create time: overrides the - restored rootfs's absolute size. + --disk-size lxc only: rootfs size for \`pct create\` + (default: \$PROXMOX_DEFAULT_LXC_DISK_GB, ${PROXMOX_DEFAULT_LXC_DISK_GB}). --image Use this local image/tarball instead of - building one from the flake. + checking the node / building one from the flake. + --force-rebuild Skip the "does the node already have this + image" check -- always build fresh and + overwrite what's there. --allow-duplicate-host Required if --host already has a real deployment elsewhere (variables.nix's deployedTargets) -- otherwise refused, since @@ -101,6 +104,7 @@ bridge="$PROXMOX_BRIDGE" node="$PROXMOX_HOST" do_list=0 allow_duplicate_host=0 +force_rebuild=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -119,6 +123,7 @@ while [[ $# -gt 0 ]]; do --node) node="$2"; shift 2 ;; --list) do_list=1; shift ;; --allow-duplicate-host) allow_duplicate_host=1; shift ;; + --force-rebuild) force_rebuild=1; shift ;; --modify) modify=1; shift ;; --dry-run) dry_run=1; shift ;; -h | --help) usage; exit 0 ;; @@ -288,6 +293,13 @@ fi echo "Target: ${flake_target} (host=${host}, type=${type}) -> Proxmox resource '${name}'" +# Decide on nix-cache once, here -- this is the earliest point that needs +# it (sync-host-keys.sh below needs nix-shell packages regardless of +# whether an image ends up getting built later), and the decision is +# exported so that subprocess -- and this script's own later build step, +# if it gets there -- both reuse it instead of probing again. +nix_extra_opts + # --- make sure this target has a registered host key -------------------- echo echo "==> Ensuring host key exists and is registered..." @@ -325,61 +337,93 @@ if [[ "$dry_run" -eq 0 ]]; then fi fi -# --- build (or reuse) the image ------------------------------------------ +# --- resolve the remote path -- fixed naming (not the nix store's own +# derivation-hash-based filename), so a later run can check for it by name. +# lxc uploads as a CT *template* (Proxmox's "vztmpl" content type, under +# iso_storage) -- config.system.build.tarball is a plain rootfs tarball, +# not a vzdump backup archive, so it's created with `pct create ... vztmpl`, +# not restored with `pct restore` (that expects backup-archive metadata +# this tarball doesn't have, and fails with "archive contains no +# configuration file"). +remote_dir="/var/lib/vz/import" +remote_filename="${flake_target}.raw" +if [[ "$type" == "lxc" ]]; then + remote_dir="/var/lib/vz/template/cache" + remote_filename="${flake_target}.tar.xz" +fi +remote_path="${remote_dir}/${remote_filename}" + +# --- build (or reuse an image already on the node) ------------------------ echo local_image="" -remote_dir="" +image_already_remote=0 + if [[ -n "$image" ]]; then [[ -f "$image" ]] || { echo "ERROR: --image '${image}' not found." >&2; exit 1; } local_image="$image" echo "Using provided image: ${local_image}" -elif [[ "$type" == "lxc" ]]; then - remote_dir="/var/lib/vz/dump" - if [[ "$dry_run" -eq 1 ]]; then - echo "[dry-run] would build: NIXOS_HOST_KEYS_DIR=${repo_root}/host-keys nix build --impure \\" - echo "[dry-run] .#nixosConfigurations.${flake_target}.config.system.build.tarball" - local_image="" - else - echo "==> Building LXC tarball for ${flake_target}..." - NIXOS_HOST_KEYS_DIR="${repo_root}/host-keys" nix build --impure \ - --no-use-registries --no-accept-flake-config \ - ".#nixosConfigurations.${flake_target}.config.system.build.tarball" \ - --out-link "${repo_root}/result-${flake_target}" - local_image="$(find "${repo_root}/result-${flake_target}/tarball" -maxdepth 1 -type f | head -1)" - echo "Built: ${local_image}" - fi +elif [[ "$force_rebuild" -eq 1 ]]; then + echo "--force-rebuild: skipping the existing-image check on ${node}." else - remote_dir="/var/lib/vz/import" + echo "==> Checking whether ${node} already has ${remote_path}..." if [[ "$dry_run" -eq 1 ]]; then - echo "[dry-run] would build: nix build .#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript" - echo "[dry-run] would run: sudo ./result-${flake_target} \\" - echo "[dry-run] --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key \\" - echo "[dry-run] --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key.pub /etc/ssh/ssh_host_ed25519_key.pub \\" - echo "[dry-run] --build-memory 2048" - local_image=".raw" + echo "[dry-run] would check: ssh ${ssh_target} -- test -f ${remote_path}" + elif ssh "$ssh_target" "test -f '${remote_path}'" 2>/dev/null; then + echo "Found it -- reusing, skipping build and upload (use --force-rebuild to override)." + image_already_remote=1 else - echo "==> Building Disko image script for ${flake_target}..." - nix build --no-use-registries --no-accept-flake-config \ - ".#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript" \ - --out-link "${repo_root}/result-${flake_target}" - echo "==> Running it (builds the .raw image in a temporary QEMU VM, needs sudo)..." - ( cd "$repo_root" && sudo "./result-${flake_target}" \ - --pre-format-files "host-keys/${flake_target}_ssh_host_ed25519_key" /etc/ssh/ssh_host_ed25519_key \ - --pre-format-files "host-keys/${flake_target}_ssh_host_ed25519_key.pub" /etc/ssh/ssh_host_ed25519_key.pub \ - --build-memory 2048 ) - local_image="$(find "$repo_root" -maxdepth 1 -name "*.raw" -newer "${repo_root}/result-${flake_target}" | head -1)" - if [[ -z "$local_image" ]]; then - echo "ERROR: expected a .raw image after the build but didn't find one in ${repo_root}." >&2 - exit 1 - fi - echo "Built: ${local_image}" + echo "Not found -- will build." fi fi -# --- upload --------------------------------------------------------------- +if [[ "$image_already_remote" -eq 0 && -z "$local_image" ]]; then + if [[ "$type" == "lxc" ]]; then + if [[ "$dry_run" -eq 1 ]]; then + echo "[dry-run] would build: NIXOS_HOST_KEYS_DIR=${repo_root}/host-keys nix build --impure \\" + echo "[dry-run] .#nixosConfigurations.${flake_target}.config.system.build.tarball" + local_image="" + else + echo "==> Building LXC tarball for ${flake_target}..." + NIXOS_HOST_KEYS_DIR="${repo_root}/host-keys" nix build --impure \ + --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \ + ".#nixosConfigurations.${flake_target}.config.system.build.tarball" \ + --out-link "${repo_root}/result-${flake_target}" + local_image="$(find "${repo_root}/result-${flake_target}/tarball" -maxdepth 1 -type f | head -1)" + echo "Built: ${local_image}" + fi + else + if [[ "$dry_run" -eq 1 ]]; then + echo "[dry-run] would build: nix build .#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript" + echo "[dry-run] would run: sudo ./result-${flake_target} \\" + echo "[dry-run] --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key \\" + echo "[dry-run] --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key.pub /etc/ssh/ssh_host_ed25519_key.pub \\" + echo "[dry-run] --build-memory 2048" + local_image=".raw" + else + echo "==> Building Disko image script for ${flake_target}..." + nix build --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \ + ".#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript" \ + --out-link "${repo_root}/result-${flake_target}" + echo "==> Running it (builds the .raw image in a temporary QEMU VM, needs sudo)..." + ( cd "$repo_root" && sudo "./result-${flake_target}" \ + --pre-format-files "host-keys/${flake_target}_ssh_host_ed25519_key" /etc/ssh/ssh_host_ed25519_key \ + --pre-format-files "host-keys/${flake_target}_ssh_host_ed25519_key.pub" /etc/ssh/ssh_host_ed25519_key.pub \ + --build-memory 2048 ) + local_image="$(find "$repo_root" -maxdepth 1 -name "*.raw" -newer "${repo_root}/result-${flake_target}" | head -1)" + if [[ -z "$local_image" ]]; then + echo "ERROR: expected a .raw image after the build but didn't find one in ${repo_root}." >&2 + exit 1 + fi + echo "Built: ${local_image}" + fi + fi +fi + +# --- upload (skip entirely if reusing an image already on the node) ------ echo -remote_path="${remote_dir}/$(basename "${local_image}")" -if [[ "$dry_run" -eq 1 ]]; then +if [[ "$image_already_remote" -eq 1 ]]; then + : # nothing to upload +elif [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would upload: scp ${local_image} ${ssh_target}:${remote_path}" else echo "==> Uploading to ${node}:${remote_path}..." @@ -391,9 +435,9 @@ fi echo if [[ "$type" == "lxc" ]]; then echo "==> Creating LXC container ${vmid} (${name})..." - restore_cmd="pct restore ${vmid} ${remote_path} --storage ${storage} --hostname ${name} --cores ${cores} --memory ${memory} --net0 name=eth0,bridge=${bridge},ip=dhcp" - [[ -n "$disk_size" ]] && restore_cmd="${restore_cmd} --rootfs ${storage}:${disk_size}" - remote "$restore_cmd" + local_disk_size="${disk_size:-$PROXMOX_DEFAULT_LXC_DISK_GB}" + create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --net0 name=eth0,bridge=${bridge},ip=dhcp" + remote "$create_cmd" remote "pct start ${vmid}" else echo "==> Creating VM ${vmid} (${name})..." diff --git a/scripts/env.sh b/scripts/env.sh index 1109541..b777d20 100755 --- a/scripts/env.sh +++ b/scripts/env.sh @@ -26,5 +26,57 @@ : "${PROXMOX_DEFAULT_CORES:=2}" : "${PROXMOX_DEFAULT_MEMORY_MB:=2048}" +# `pct create` (unlike `pct restore`) requires an explicit rootfs size -- +# no backup metadata to infer it from. Matches Proxmox's own GUI default. +: "${PROXMOX_DEFAULT_LXC_DISK_GB:=8}" + export PROXMOX_HOST PROXMOX_SSH_USER PROXMOX_STORAGE PROXMOX_ISO_STORAGE \ - PROXMOX_BRIDGE PROXMOX_DEFAULT_CORES PROXMOX_DEFAULT_MEMORY_MB + PROXMOX_BRIDGE PROXMOX_DEFAULT_CORES PROXMOX_DEFAULT_MEMORY_MB PROXMOX_DEFAULT_LXC_DISK_GB + +# Matches variables.nix's nixCacheHost -- update both if it ever changes. +: "${NIX_CACHE_HOST:=nix-cache}" +export NIX_CACHE_HOST + +# nix_extra_opts: call as a plain statement (NOT inside $(...)/<(...) -- +# that forks a subshell, and the whole point is exporting a decision back +# into *this* shell) to populate the global NIX_OPTS array with whatever +# extra `nix`/`nix-shell` CLI options are needed to avoid nix-cache when +# it's unreachable: +# nix_extra_opts +# nix build "${NIX_OPTS[@]}" ... +# +# Without this, every single `nix eval`/`nix build` call retries each +# store path against a dead substituter up to 5 times with backoff +# (confirmed: ~15s+ per lookup even with a short connect-timeout, because +# nix's own retry count isn't controllable that way), and separately +# tries it as a remote builder too -- both fail independently, so both +# are checked. +# +# Checked with a single fast `curl`/TCP probe (bypassing nix's retry logic +# entirely) the first time this is called in a given process, and the +# result is exported as NIX_EXTRA_OPTS so a script that shells out to +# another script in this repo (e.g. create-proxmox-resource.sh calling +# sync-host-keys.sh) reuses the same decision instead of probing twice. +declare -a NIX_OPTS=() + +nix_extra_opts() { + if [[ -n "${NIX_EXTRA_OPTS_DECIDED:-}" ]]; then + if [[ -n "${NIX_EXTRA_OPTS:-}" ]]; then + eval "NIX_OPTS=(${NIX_EXTRA_OPTS})" + else + NIX_OPTS=() + fi + return + fi + export NIX_EXTRA_OPTS_DECIDED=1 + NIX_OPTS=() + if ! curl --silent --fail --max-time 3 "http://${NIX_CACHE_HOST}/nix-cache-info" >/dev/null 2>&1; then + echo "nix-cache (http://${NIX_CACHE_HOST}) is unreachable -- skipping it (substituter + remote builder) for the rest of this run." >&2 + NIX_OPTS=(--option substituters "https://cache.nixos.org/" --builders "") + elif ! timeout 3 bash -c "cat < /dev/tcp/${NIX_CACHE_HOST}/22" >/dev/null 2>&1; then + echo "nix-cache's SSH remote builder (nixremote@${NIX_CACHE_HOST}:22) is unreachable -- disabling remote builds for the rest of this run." >&2 + NIX_OPTS=(--builders "") + fi + printf -v NIX_EXTRA_OPTS '%q ' "${NIX_OPTS[@]}" + export NIX_EXTRA_OPTS +} diff --git a/scripts/sync-host-keys.sh b/scripts/sync-host-keys.sh index cbe3bc6..2893fbd 100755 --- a/scripts/sync-host-keys.sh +++ b/scripts/sync-host-keys.sh @@ -29,6 +29,9 @@ sops_yaml="${repo_root}/.sops.yaml" keydir="${repo_root}/host-keys" editor="${repo_root}/scripts/lib/sync-host-keys-edit-sops.py" +# shellcheck source=env.sh +source "${repo_root}/scripts/env.sh" + mkdir -p "$keydir" usage() { @@ -88,9 +91,9 @@ ensure_admin_decrypt_key() { echo "No sops age decryption key found (checked \$SOPS_AGE_KEY, \$SOPS_AGE_KEY_FILE, ${key_file})." echo "Generating a new one at ${key_file}..." mkdir -p "$(dirname "$key_file")" - nix-shell -p age --run "age-keygen -o '${key_file}'" 2>&1 | grep -v "^Public key:" || true + nix-shell "${NIX_OPTS[@]}" -p age --run "age-keygen -o '${key_file}'" 2>&1 | grep -v "^Public key:" || true local new_pub - new_pub="$(nix-shell -p age --run "age-keygen -y '${key_file}'")" + new_pub="$(nix-shell "${NIX_OPTS[@]}" -p age --run "age-keygen -y '${key_file}'")" cat < ${host}: generating host key" - nix-shell -p openssh --run "ssh-keygen -t ed25519 -N '' -C '${host}' -f '${keyfile}'" >/dev/null + nix-shell "${NIX_OPTS[@]}" -p openssh --run "ssh-keygen -t ed25519 -N '' -C '${host}' -f '${keyfile}'" >/dev/null fi else echo "==> ${host}: host key already present" @@ -167,7 +170,7 @@ queue_host_sync() { if [[ "$dry_run" -eq 1 ]]; then age_pub="dry-run-placeholder-not-a-real-key" else - age_pub="$(nix-shell -p ssh-to-age --run "ssh-to-age -i '${keyfile}.pub'")" + age_pub="$(nix-shell "${NIX_OPTS[@]}" -p ssh-to-age --run "ssh-to-age -i '${keyfile}.pub'")" fi add_keys_json="$(jq --arg host "$host" --arg key "$age_pub" \ '. + [{host: $host, age_key: $key}]' <<<"$add_keys_json")" @@ -203,7 +206,7 @@ apply_edit_plan() { fi local result - result="$(echo "$plan" | nix-shell -p python3 --run "python3 '${editor}' '${target}'")" + result="$(echo "$plan" | nix-shell "${NIX_OPTS[@]}" -p python3 --run "python3 '${editor}' '${target}'")" [[ -n "$tmpfile" ]] && rm -f "$tmpfile" local added removed changed @@ -234,7 +237,7 @@ apply_edit_plan() { while IFS= read -r basename; do [[ -z "$basename" ]] && continue echo "==> secrets/${basename}" - nix-shell -p sops --run "sops updatekeys --yes '${repo_root}/secrets/${basename}'" + nix-shell "${NIX_OPTS[@]}" -p sops --run "sops updatekeys --yes '${repo_root}/secrets/${basename}'" done <<<"$changed" fi fi @@ -408,6 +411,7 @@ main() { echo fi + nix_extra_opts ensure_admin_decrypt_key case "${1:-}" in