#!/usr/bin/env bash # Creates new Proxmox VMs/LXC containers from this flake, and reconfigures # existing ones -- the manual workflows in docs/proxmox-images.md (VM) and # docs/auto-installer.md's "LXC hosts" section (container), automated. # # Images are built directly on the Proxmox node (PROXMOX_REMOTE_REPO_DIR / # --remote-repo-dir in scripts/env.sh), not on whatever machine runs this # script -- there's no multi-gigabyte image to transfer afterward. The first # time a node doesn't have that repo path yet, it's bootstrapped: cloned from # this checkout's own `origin` remote, then scripts/codex-setup.sh installs # the build tooling (Nix, etc.). Every run after that just `git pull`s it and # copies over the locally-managed host-keys/ (gitignored, so a git pull # alone wouldn't carry it) before building. # # Usage: # scripts/proxmox/create-proxmox-resource.sh --type lxc|vm --host [options] # scripts/proxmox/create-proxmox-resource.sh --type lxc|vm --list # scripts/proxmox/create-proxmox-resource.sh --modify --vmid [--cores N] [--memory MB] [--grow-disk GB] # # SAFETY: # - The default (create) mode only ever creates a NEW resource -- it # refuses to run if the target VMID already exists on the node, or if # a VM/CT identified as --host already exists under any other VMID # (checked live against the node; --allow-duplicate-host overrides). # - --allow-duplicate-host distinguishes an exact match (same --type # *and* --host, e.g. re-running --type lxc --host docker while an # lxc-docker container already exists -- almost always a redeploy of # the same target to pick up a rebuilt image) from a cross-type match # (a different platform sharing the same host identity, e.g. a # proxmox-docker VM coexisting with lxc-docker). Only the exact match # is destroyed and replaced, after typing the hostname back to # confirm (outside --dry-run) -- a cross-type match is always left # untouched, matching-or-not. # - --modify only ever touches a resource you name explicitly via # --vmid, shows exactly what will change first, and (outside # --dry-run) always requires typing that VMID back to confirm before # anything is sent to the node. There is no bulk/implicit modify. # - Outside of --allow-duplicate-host's exact-match replace above, # neither mode can start/stop/delete a resource. # # See --help for the full option list. set -euo pipefail repo_root="$(cd "$(dirname "$0")/../.." && pwd)" # shellcheck source=../env.sh source "${repo_root}/scripts/env.sh" # shellcheck source=../lib/nix-eval.sh source "${repo_root}/scripts/lib/nix-eval.sh" # shellcheck source=../lib/confirm.sh source "${repo_root}/scripts/lib/confirm.sh" sync_keys="${repo_root}/scripts/secrets/sync-host-keys.sh" usage() { cat < [options] (create) $0 --type lxc|vm --list (list --host values) $0 --modify --vmid [options] (reconfigure) Create mode (default): --type lxc|vm lxc = container, built as a CT template tarball. vm = VM, built as a Disko .raw disk image (UEFI/OVMF). --host Which host identity to deploy -- matches config.networking.hostName (server, docker, nix-cache, nixos, pxe-boot, nix-minimal). Use --list to see what's available for --type. --name Proxmox display name/hostname (default: --host's value, e.g. nix-cache -- for lxc this becomes the guest's real networking.hostName too, since proxmoxLXC.manageHostName pulls it from Proxmox's own container config, so it must match host.nix regardless of build type) --vmid Numeric VMID (default: next free, via \`pvesh get /cluster/nextid\` on the node). Refuses to run if this ID already exists. --disk-size lxc only: rootfs size for \`pct create\` (default: \$PROXMOX_DEFAULT_LXC_DISK_GB, ${PROXMOX_DEFAULT_LXC_DISK_GB}). --image Use this local image/tarball (uploaded to the node via scp) instead of checking the node / building one there from the flake. --force-rebuild Skip the "does the node already have this image" check -- always build fresh and overwrite what's there. --remote-repo-dir Where this flake repo lives (or gets cloned) on the node, and is built from (default: \$PROXMOX_REMOTE_REPO_DIR, ${PROXMOX_REMOTE_REPO_DIR}). --allow-duplicate-host Required if a VM/CT identified as --host already exists on the node (checked live via qm/pct, not any file in this repo) -- otherwise refused, since it'd share that host's hostName/hostId. An existing resource of this *same* --type (e.g. re-running --type lxc --host docker over an existing lxc-docker) is destroyed and replaced, after confirming -- a different --type sharing the same --host (e.g. a proxmox-docker VM) is always left untouched. Modify mode (reconfigure an EXISTING resource -- requires --modify): --modify Switch to modify mode. --vmid Required: which existing resource to change. Type/VM-vs-CT is auto-detected on the node. --grow-disk Grow the primary disk by this many GB (qm/pct resize; Proxmox only supports growing, never shrinking, an existing disk). At least one of --cores / --memory / --grow-disk is required. Always prints the current -> new values and requires typing the VMID back to confirm, even outside --dry-run. Shared: --cores create: default \$PROXMOX_DEFAULT_CORES (${PROXMOX_DEFAULT_CORES}). modify: omit to leave unchanged. --memory create: default \$PROXMOX_DEFAULT_MEMORY_MB (${PROXMOX_DEFAULT_MEMORY_MB}). modify: omit to leave unchanged. --swap lxc only, create time: \`--memory\` doesn't touch swap -- it silently stays at Proxmox's own 512M default otherwise. (default: matches whatever --memory resolves to) --storage (default: \$PROXMOX_STORAGE, ${PROXMOX_STORAGE}) --iso-storage (default: \$PROXMOX_ISO_STORAGE, ${PROXMOX_ISO_STORAGE}) --bridge (default: \$PROXMOX_BRIDGE, ${PROXMOX_BRIDGE}) --node Proxmox node to SSH into (default: \$PROXMOX_HOST, ${PROXMOX_HOST}) --dry-run Print the full plan; touch nothing local or remote, no prompts. -h, --help Config for --storage/--bridge/--node/etc. lives in scripts/env.sh -- edit that instead of passing the same flag every time. EOF } dry_run=0 modify=0 type="" host="" name="" vmid="" cores="" memory="" swap="" disk_size="" grow_disk="" image="" storage="$PROXMOX_STORAGE" iso_storage="$PROXMOX_ISO_STORAGE" bridge="$PROXMOX_BRIDGE" node="$PROXMOX_HOST" remote_repo_dir="$PROXMOX_REMOTE_REPO_DIR" do_list=0 allow_duplicate_host=0 force_rebuild=0 while [[ $# -gt 0 ]]; do case "$1" in --type) type="$2"; shift 2 ;; --host) host="$2"; shift 2 ;; --name) name="$2"; shift 2 ;; --vmid) vmid="$2"; shift 2 ;; --cores) cores="$2"; shift 2 ;; --memory) memory="$2"; shift 2 ;; --swap) swap="$2"; shift 2 ;; --disk-size) disk_size="$2"; shift 2 ;; --grow-disk) grow_disk="$2"; shift 2 ;; --image) image="$2"; shift 2 ;; --storage) storage="$2"; shift 2 ;; --iso-storage) iso_storage="$2"; shift 2 ;; --bridge) bridge="$2"; shift 2 ;; --node) node="$2"; shift 2 ;; --remote-repo-dir) remote_repo_dir="$2"; shift 2 ;; --list) do_list=1; shift ;; --allow-duplicate-host) allow_duplicate_host=1; shift ;; --force-rebuild) force_rebuild=1; shift ;; --modify) modify=1; shift ;; --dry-run) dry_run=1; shift ;; -h | --help) usage; exit 0 ;; *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; esac done ssh_target="${PROXMOX_SSH_USER}@${node}" remote() { if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] ssh ${ssh_target} -- $*" else ssh "$ssh_target" "$@" fi } # ============================================================ modify mode cmd_modify() { if [[ -z "$vmid" ]]; then echo "ERROR: --modify requires --vmid." >&2 exit 1 fi if [[ -z "$cores" && -z "$memory" && -z "$grow_disk" ]]; then echo "ERROR: --modify needs at least one of --cores / --memory / --grow-disk." >&2 exit 1 fi echo "Looking up VMID ${vmid} on ${node}..." local kind current_cores current_memory disk_key if ssh "$ssh_target" "qm status ${vmid}" >/dev/null 2>&1; then kind="vm" disk_key="scsi0" elif ssh "$ssh_target" "pct status ${vmid}" >/dev/null 2>&1; then kind="lxc" disk_key="rootfs" else echo "ERROR: VMID ${vmid} doesn't exist on ${node} -- nothing to modify." >&2 exit 1 fi local config_cmd="qm config ${vmid}" [[ "$kind" == "lxc" ]] && config_cmd="pct config ${vmid}" local current_config current_config="$(ssh "$ssh_target" "$config_cmd")" current_cores="$(echo "$current_config" | grep -oP '^cores:\s*\K\S+' || echo '?')" current_memory="$(echo "$current_config" | grep -oP '^memory:\s*\K\S+' || echo '?')" echo echo "VMID ${vmid} is a ${kind} on ${node}. Planned changes:" [[ -n "$cores" ]] && echo " cores: ${current_cores} -> ${cores}" [[ -n "$memory" ]] && echo " memory: ${current_memory} MB -> ${memory} MB" [[ -n "$grow_disk" ]] && echo " ${disk_key}: grow by +${grow_disk}G (Proxmox can only grow, not shrink, an existing disk)" if [[ "$dry_run" -eq 1 ]]; then echo echo "[dry-run] Nothing was changed." return fi echo if ! confirm_typed "$vmid" "Type the VMID (${vmid}) to confirm these changes: "; then echo "Cancelled -- input didn't match ${vmid}." exit 1 fi local set_cmd="qm set" local resize_cmd="qm resize" [[ "$kind" == "lxc" ]] && set_cmd="pct set" && resize_cmd="pct resize" if [[ -n "$cores" || -n "$memory" ]]; then local args="" [[ -n "$cores" ]] && args="${args} --cores ${cores}" [[ -n "$memory" ]] && args="${args} --memory ${memory}" remote "${set_cmd} ${vmid}${args}" fi if [[ -n "$grow_disk" ]]; then remote "${resize_cmd} ${vmid} ${disk_key} +${grow_disk}G" fi echo echo "Done. VMID ${vmid} updated." } if [[ "$modify" -eq 1 ]]; then cmd_modify exit 0 fi # ============================================================= create mode if [[ "$type" != "lxc" && "$type" != "vm" ]]; then echo "ERROR: --type must be 'lxc' or 'vm'." >&2 usage >&2 exit 1 fi platform_prefix="lxc" [[ "$type" == "vm" ]] && platform_prefix="proxmox" [[ -z "$cores" ]] && cores="$PROXMOX_DEFAULT_CORES" [[ -z "$memory" ]] && memory="$PROXMOX_DEFAULT_MEMORY_MB" # --- discover / resolve the flake target from --host -------------------- # Emits "\t" pairs for every ${platform_prefix}-* flake # target -- the one source both --list and the --host lookup below read # from, so they can never see a different set of targets from each other. targets_for_platform() { local target for target in $(list_flake_targets "$repo_root" 2>/dev/null | grep -- "^${platform_prefix}-"); do printf '%s\t%s\n' "$target" "$(flake_target_hostname "$repo_root" "$target")" done } list_hosts() { local target hostname while IFS=$'\t' read -r target hostname; do printf ' %-12s -> %s\n' "$hostname" "$target" done < <(targets_for_platform) } if [[ "$do_list" -eq 1 ]]; then echo "Available --host values for --type ${type}:" list_hosts exit 0 fi if [[ -z "$host" ]]; then echo "ERROR: --host is required (or use --list to see options)." >&2 exit 1 fi flake_target="" while IFS=$'\t' read -r target hostname; do if [[ "$hostname" == "$host" ]]; then flake_target="$target" break fi done < <(targets_for_platform) if [[ -z "$flake_target" ]]; then echo "ERROR: no ${platform_prefix}-* target has hostName '${host}'." >&2 echo "Available:" >&2 list_hosts >&2 exit 1 fi # The container/VM's real identity is --host (e.g. "nix-cache"), validated # above against config.networking.hostName -- not the flake target name # (e.g. "lxc-nix-cache"), which is build-type-specific and only exists to # pick which platform variant to build. Defaulting --name to the flake # target would make lxc's --hostname (which proxmoxLXC.manageHostName # feeds straight into the guest's real hostname) disagree with host.nix. [[ -z "$name" ]] && name="$host" # --- refuse to duplicate a host that's already live on the node --------- # Queries the node itself (qm/pct's own name/hostname config), not any # static list in this repo -- a file can't track whether a resource still # actually exists, and this used to be checked against variables.nix's # deployedTargets, which drifted stale (it kept naming a VM as "the real # deployment" well after that VM had been destroyed, blocking its own # redeploy) until that list was dropped in favour of this live check. This # only catches guests identified with the default --name (== --host, what # this script itself always uses unless --name is overridden) -- a guest # manually renamed on the node afterwards wouldn't match, but nothing here # creates guests that way. if [[ "$dry_run" -eq 1 ]]; then echo echo "[dry-run] would check ${node} for an existing VM/CT identified as '${host}'" if [[ "$allow_duplicate_host" -eq 1 ]]; then echo "[dry-run] --allow-duplicate-host: an existing ${type} named '${host}' would be" \ "destroyed and replaced; a different-type match would be left untouched" fi else echo echo "==> Checking ${node} for an existing VM/CT identified as '${host}'..." ssh_check_status=0 existing="$(ssh "$ssh_target" bash -s -- "$host" <<'REMOTE_SCRIPT' target="$1" for id in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do n="$(qm config "$id" 2>/dev/null | grep -oP '^name:\s*\K\S+' || true)" [[ "$n" == "$target" ]] && echo "vm ${id} ${n}" done for id in $(pct list 2>/dev/null | awk 'NR>1{print $1}'); do n="$(pct config "$id" 2>/dev/null | grep -oP '^hostname:\s*\K\S+' || true)" [[ "$n" == "$target" ]] && echo "lxc ${id} ${n}" done exit 0 REMOTE_SCRIPT )" || ssh_check_status=$? if [[ "$ssh_check_status" -ne 0 ]]; then echo "ERROR: couldn't reach ${node} (ssh exited ${ssh_check_status}) to check for an" >&2 echo "existing '${host}' resource -- refusing to guess. Fix connectivity and retry," >&2 echo "or pass --allow-duplicate-host if you're sure none exists (this skips the" >&2 echo "check entirely)." >&2 exit 1 fi # Split into "exact" (same resource kind as --type -- i.e. literally this # same host+platform combo already exists, almost always a redeploy of # the same target to test a rebuilt image) vs "cross-type" (a different # platform sharing this host identity, e.g. a stopped proxmox-docker VM # coexisting with an lxc-docker container -- a deliberate, valid setup # this script has never managed and still won't). Read via a herestring # (not a pipe) so the appends below survive outside the loop. this_kind="$type" exact_matches="" cross_matches="" if [[ -n "$existing" ]]; then while read -r kind id n; do [[ -z "$kind" ]] && continue if [[ "$kind" == "$this_kind" ]]; then exact_matches+="${kind} ${id} ${n}"$'\n' else cross_matches+="${kind} ${id} ${n}"$'\n' fi done <<<"$existing" fi if [[ -n "$exact_matches" && "$allow_duplicate_host" -ne 1 ]]; then echo "ERROR: '${host}' already exists on ${node} as this same resource type:" >&2 echo "$exact_matches" | while read -r kind id n; do [[ -z "$kind" ]] && continue echo " - ${kind} VMID ${id} (${n})" >&2 done echo "Refusing to create a second ${this_kind} sharing this identity. Pass" >&2 echo "--allow-duplicate-host to destroy it and create a fresh one in its place" >&2 echo "(after confirming), or use --modify to reconfigure the existing one instead." >&2 exit 1 fi if [[ -n "$cross_matches" && "$allow_duplicate_host" -ne 1 ]]; then echo "ERROR: '${host}' already exists on ${node} as a different resource type:" >&2 echo "$cross_matches" | while read -r kind id n; do [[ -z "$kind" ]] && continue echo " - ${kind} VMID ${id} (${n})" >&2 done echo "Refusing to create a second resource sharing this identity. Pass" >&2 echo "--allow-duplicate-host to create one anyway (it gets its own distinct" >&2 echo "sops key and VMID -- the existing resource above is left untouched)," >&2 echo "or use --modify to reconfigure the existing one instead." >&2 exit 1 fi if [[ -n "$cross_matches" ]]; then echo "--allow-duplicate-host: '${host}' also exists on ${node} as a different resource" \ "type -- leaving it untouched:" echo "$cross_matches" | while read -r kind id n; do [[ -z "$kind" ]] && continue echo " - ${kind} VMID ${id} (${n})" done fi if [[ -n "$exact_matches" ]]; then echo "--allow-duplicate-host: '${host}' already exists on ${node} as this same resource" \ "type -- it will be destroyed and replaced:" echo "$exact_matches" | while read -r kind id n; do [[ -z "$kind" ]] && continue echo " - ${kind} VMID ${id} (${n})" done echo if ! confirm_typed "$host" "Type the hostname (${host}) to confirm destroying the above and replacing it: "; then echo "Cancelled -- input didn't match ${host}." >&2 exit 1 fi echo "$exact_matches" | while read -r kind id n; do [[ -z "$kind" ]] && continue echo "==> Destroying ${kind} VMID ${id} (${n})..." if [[ "$kind" == "vm" ]]; then # qm destroy has no --force to stop-then-destroy in one call (pct's # does) -- stop explicitly first if it's running. if ssh "$ssh_target" "qm status ${id}" 2>/dev/null | grep -q running; then ssh "$ssh_target" "qm stop ${id}" fi ssh "$ssh_target" "qm destroy ${id} --purge 1" else ssh "$ssh_target" "pct destroy ${id} --force 1 --purge 1" fi done fi fi echo "Target: ${flake_target} (host=${host}, type=${type}) -> Proxmox resource '${name}'" # Decide on nix-cache once, here -- this is the earliest point that needs # it (sync-host-keys.sh below needs nix-shell packages regardless of # whether an image ends up getting built later), and the decision is # exported so that subprocess -- and this script's own later build step, # if it gets there -- both reuse it instead of probing again. nix_extra_opts # --- make sure this target has a registered host key -------------------- echo echo "==> Ensuring host key exists and is registered..." sync_args=("$flake_target") [[ "$dry_run" -eq 1 ]] && sync_args+=(--dry-run) bash "$sync_keys" "${sync_args[@]}" # --- VMID: pick one, and refuse to touch anything that already exists --- echo if [[ -z "$vmid" ]]; then if [[ "$dry_run" -eq 1 ]]; then vmid="" echo "[dry-run] would ask ${node} for the next free VMID (pvesh get /cluster/nextid)" else vmid="$(ssh "$ssh_target" "pvesh get /cluster/nextid" | tr -d '[:space:]')" echo "Auto-assigned VMID: ${vmid}" fi else echo "Requested VMID: ${vmid}" fi if [[ "$dry_run" -eq 0 ]]; then # qm/pct status exits non-zero (and prints "does not exist") for a free # ID on that resource type -- but a VMID could exist as the OTHER # resource type (e.g. requested a CT id that's actually a VM), so check # both. Any success here means something is already using this ID -- # refuse to go anywhere near it. (Reconfiguring an existing resource is # --modify's job, not this one's.) if ssh "$ssh_target" "qm status ${vmid}" >/dev/null 2>&1 \ || ssh "$ssh_target" "pct status ${vmid}" >/dev/null 2>&1; then echo "ERROR: VMID ${vmid} already exists on ${node}. Refusing to touch an" >&2 echo "existing resource here -- use --modify to reconfigure it, pick a" >&2 echo "different --vmid, or omit it to auto-assign." >&2 exit 1 fi fi # --- resolve the remote path -- fixed naming (not the nix store's own # derivation-hash-based filename), so a later run can check for it by name. # lxc uploads as a CT *template* (Proxmox's "vztmpl" content type, under # iso_storage) -- config.system.build.tarball is a plain rootfs tarball, # not a vzdump backup archive, so it's created with `pct create ... vztmpl`, # not restored with `pct restore` (that expects backup-archive metadata # this tarball doesn't have, and fails with "archive contains no # configuration file"). remote_dir="/var/lib/vz/import" remote_filename="${flake_target}.raw" if [[ "$type" == "lxc" ]]; then remote_dir="/var/lib/vz/template/cache" remote_filename="${flake_target}.tar.xz" fi remote_path="${remote_dir}/${remote_filename}" # --- ensure the flake repo (+ tooling) exists on the node, and is current -- # Bootstraps once (git clone from this checkout's own `origin`, then # scripts/codex-setup.sh installs Nix + friends) if ${remote_repo_dir} # doesn't exist yet on the node; otherwise just `git pull`s it, so the image # built there reflects what's actually committed and pushed. Only called # right before an actual remote build below -- reusing an image already on # the node, or an explicit --image, never touch the node's checkout at all. ensure_remote_repo() { echo echo "==> Ensuring ${remote_repo_dir} exists and is current on ${node}..." if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would ensure ${remote_repo_dir} exists on ${node} (clone if missing, git pull if present), and would verify/bootstrap build tooling there (scripts/codex-setup.sh) if \`nix\` isn't already on PATH -- and if that bootstrap actually ran, would also configure ${node} as a nix-cache client (scripts/proxmox/configure-nix-cache-client.sh)" return fi if ssh "$ssh_target" "test -d '${remote_repo_dir}/.git'"; then echo "Repo present -- pulling latest..." ssh "$ssh_target" "cd '${remote_repo_dir}' && git pull --ff-only" else local origin_url origin_url="$(git -C "$repo_root" remote get-url origin 2>/dev/null || true)" if [[ -z "$origin_url" ]]; then echo "ERROR: ${remote_repo_dir} doesn't exist on ${node}, and this checkout has no" >&2 echo "'origin' remote to clone from. Set one (git remote add origin ) or create" >&2 echo "${remote_repo_dir} on ${node} yourself (e.g. git clone), then re-run." >&2 exit 1 fi echo "Not present -- cloning from ${origin_url}..." ssh "$ssh_target" "git clone '${origin_url}' '${remote_repo_dir}'" fi # Trivial check, run every time (not just right after a fresh clone) -- # confirmed live: a first bootstrap can clone the repo successfully and # still leave the node without a working `nix` (e.g. the node had no # `sudo`, which the Nix installer's root path depends on -- see the fix # in scripts/codex-setup.sh), and a later run with the repo already # present would otherwise never retry it. Sources # scripts/lib/nix-bootstrap.sh's ensure_nix_profile first -- a # single-user Nix install typically only gets sourced into login shells, # and ssh's non-interactive command execution is neither, so a # freshly-installed `nix` still wouldn't be on PATH here without it. # # Just `nix` today -- the only thing the remote build commands below # actually invoke -- but a list (not a single hardcoded check) so a # future remote step needing another tool can add itself here instead of # growing a parallel check. local remote_required_cmds=(nix) local tooling_check_cmd="cd '${remote_repo_dir}' && . scripts/lib/nix-bootstrap.sh && ensure_nix_profile" local cmd for cmd in "${remote_required_cmds[@]}"; do tooling_check_cmd="${tooling_check_cmd} && command -v ${cmd}" done if ssh "$ssh_target" "$tooling_check_cmd" >/dev/null 2>&1; then echo "Build tooling already present on ${node}." else echo "==> Bootstrapping build tooling on ${node} (scripts/codex-setup.sh)..." ssh "$ssh_target" "cd '${remote_repo_dir}' && bash scripts/codex-setup.sh" # Only on this first-time bootstrap, not every run -- a node that # already has tooling either already went through this once, or had # it configured some other way, and re-running is harmless but # pointless. Non-fatal: this only makes the node's own builds faster # (substitute from nix-cache instead of building from source) and # offloadable to it as a remote builder -- worth trying, not worth # aborting the image build over if nix-cache happens to be down right # now. Needs ensure_nix_profile first, same as the tooling_check_cmd # above -- ssh's non-interactive command execution won't have picked # up a freshly single-user-installed `nix` otherwise. echo "==> Configuring ${node} as a nix-cache substituter/remote-builder client..." if ! ssh "$ssh_target" "cd '${remote_repo_dir}' && . scripts/lib/nix-bootstrap.sh && ensure_nix_profile && bash scripts/proxmox/configure-nix-cache-client.sh"; then echo "WARNING: configure-nix-cache-client.sh failed on ${node} -- continuing without it (${node} will build from source / against cache.nixos.org only)." >&2 fi fi } # --- sync locally-managed host-keys/ to the node --------------------------- # Gitignored (see .gitignore), so `git pull` above never carries it -- both # build paths need it present as NIXOS_HOST_KEYS_DIR / --pre-format-files # input on the node itself now that the build runs there. scp (not rsync, # not already a dependency anywhere else in this repo) mirrors how this # script already transfers the --image case below. sync_remote_host_keys() { echo echo "==> Syncing host-keys/ to ${node}..." if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would copy ${repo_root}/host-keys/ to ${ssh_target}:${remote_repo_dir}/host-keys/" return fi ssh "$ssh_target" "mkdir -p '${remote_repo_dir}/host-keys'" scp -pr "${repo_root}/host-keys/." "${ssh_target}:${remote_repo_dir}/host-keys/" } # --- build (or reuse an image already on the node) ------------------------ echo local_image="" image_already_remote=0 if [[ -n "$image" ]]; then [[ -f "$image" ]] || { echo "ERROR: --image '${image}' not found." >&2; exit 1; } local_image="$image" echo "Using provided image: ${local_image}" elif [[ "$force_rebuild" -eq 1 ]]; then echo "--force-rebuild: skipping the existing-image check on ${node}." else echo "==> Checking whether ${node} already has ${remote_path}..." if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would check: ssh ${ssh_target} -- test -f ${remote_path}" elif ssh "$ssh_target" "test -f '${remote_path}'" 2>/dev/null; then echo "Found it -- reusing, skipping build (use --force-rebuild to override)." image_already_remote=1 else echo "Not found -- will build." fi fi if [[ "$image_already_remote" -eq 0 && -z "$local_image" ]]; then ensure_remote_repo sync_remote_host_keys # Relayed into the remote build below exactly as decided by the local # nix_extra_opts call earlier in this script -- that decision (whether # nix-cache is reachable) is made once, locally, same as it always has # been; only *where* the resulting "${NIX_OPTS[@]}" gets used as a `nix # build` flag moves to the node. NIX_EXTRA_OPTS is already a %q-quoted # string built for exactly this eval-based reconstruction (see env.sh). nix_opts_display="" if [[ ${#NIX_OPTS[@]} -gt 0 ]]; then printf -v nix_opts_display '%q ' "${NIX_OPTS[@]}" nix_opts_display=" ${nix_opts_display% }" fi if [[ "$type" == "lxc" ]]; then if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would build on ${node}: NIXOS_HOST_KEYS_DIR=\$(pwd)/host-keys nix build --impure \\" echo "[dry-run] --no-use-registries --no-accept-flake-config${nix_opts_display} \\" echo "[dry-run] .#nixosConfigurations.${flake_target}.config.system.build.tarball" echo "[dry-run] would stage the result at ${remote_path}" local_image="" else echo "==> Building LXC tarball for ${flake_target} on ${node}..." # Built as a single already-%q-quoted command string, not separate ssh # argv elements -- ssh joins remote command args with plain spaces and # hands the result to the remote shell to re-split, which would # otherwise scatter NIX_EXTRA_OPTS (itself several space-separated, # %q-quoted tokens) across the wrong positional parameters below. printf -v remote_cmd 'bash -s -- %q %q %q %q %q' \ "$remote_repo_dir" "$flake_target" "$remote_dir" "$remote_filename" "$NIX_EXTRA_OPTS" ssh "$ssh_target" "$remote_cmd" <<'REMOTE_SCRIPT' set -euo pipefail repo_dir="$1"; target="$2"; dest_dir="$3"; dest_name="$4"; nix_extra_opts_str="$5" declare -a NIX_OPTS=() [[ -n "$nix_extra_opts_str" ]] && eval "NIX_OPTS=(${nix_extra_opts_str})" cd "$repo_dir" # A single-user Nix install only gets sourced into login shells; this ssh # session is neither, so `nix` wouldn't otherwise be on PATH here even # right after a successful install. . scripts/lib/nix-bootstrap.sh ensure_nix_profile NIXOS_HOST_KEYS_DIR="$(pwd)/host-keys" nix build --impure \ --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \ ".#nixosConfigurations.${target}.config.system.build.tarball" \ --out-link "result-${target}" built="$(find "result-${target}/tarball" -maxdepth 1 -type f | head -1)" if [[ -z "$built" ]]; then echo "ERROR: no tarball found under result-${target}/tarball after build." >&2 exit 1 fi mkdir -p "$dest_dir" cp "$built" "${dest_dir}/${dest_name}" echo "Built and staged: ${dest_dir}/${dest_name}" REMOTE_SCRIPT local_image="$remote_path" echo "Built on ${node}: ${remote_path}" fi else # PROXMOX_SSH_USER defaults to root (env.sh), which needs no sudo and # can't assume it's even installed on a minimal node -- only shell out # through sudo when actually running as a non-root SSH user. sudo_prefix="sudo" sudo_display="sudo " if [[ "$PROXMOX_SSH_USER" == "root" ]]; then sudo_prefix="" sudo_display="" fi if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would build on ${node}: nix build --no-use-registries --no-accept-flake-config${nix_opts_display} \\" echo "[dry-run] .#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript" echo "[dry-run] would run: ${sudo_display}./result-${flake_target} \\" echo "[dry-run] --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key \\" echo "[dry-run] --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key.pub /etc/ssh/ssh_host_ed25519_key.pub \\" echo "[dry-run] --build-memory 2048" echo "[dry-run] would stage the result at ${remote_path}" local_image=".raw" else echo "==> Building Disko image for ${flake_target} on ${node}..." # See the LXC branch above for why this is one %q-quoted command # string rather than separate ssh argv elements. printf -v remote_cmd 'bash -s -- %q %q %q %q %q %q' \ "$remote_repo_dir" "$flake_target" "$remote_dir" "$remote_filename" "$NIX_EXTRA_OPTS" "$sudo_prefix" ssh "$ssh_target" "$remote_cmd" <<'REMOTE_SCRIPT' set -euo pipefail repo_dir="$1"; target="$2"; dest_dir="$3"; dest_name="$4"; nix_extra_opts_str="$5"; sudo_prefix="$6" declare -a NIX_OPTS=() [[ -n "$nix_extra_opts_str" ]] && eval "NIX_OPTS=(${nix_extra_opts_str})" cd "$repo_dir" . scripts/lib/nix-bootstrap.sh ensure_nix_profile nix build --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \ ".#nixosConfigurations.${target}.config.system.build.diskoImagesScript" \ --out-link "result-${target}" $sudo_prefix "./result-${target}" \ --pre-format-files "host-keys/${target}_ssh_host_ed25519_key" /etc/ssh/ssh_host_ed25519_key \ --pre-format-files "host-keys/${target}_ssh_host_ed25519_key.pub" /etc/ssh/ssh_host_ed25519_key.pub \ --build-memory 2048 built="$(find . -maxdepth 1 -name '*.raw' -newer "result-${target}" | head -1)" if [[ -z "$built" ]]; then echo "ERROR: no .raw image found in ${repo_dir} after build." >&2 exit 1 fi mkdir -p "$dest_dir" mv "$built" "${dest_dir}/${dest_name}" echo "Built and staged: ${dest_dir}/${dest_name}" REMOTE_SCRIPT local_image="$remote_path" echo "Built on ${node}: ${remote_path}" fi fi fi # --- upload -- only for an explicit --image; a build above stages its # result directly at ${remote_path} on the node already, and reusing an # image already on the node needs nothing transferred either. ------------ echo if [[ -n "$image" ]]; then if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would upload: scp ${local_image} ${ssh_target}:${remote_path}" else echo "==> Uploading to ${node}:${remote_path}..." ssh "$ssh_target" "mkdir -p ${remote_dir}" scp "$local_image" "${ssh_target}:${remote_path}" fi fi # --- create ----------------------------------------------------------------- echo if [[ "$type" == "lxc" ]]; then echo "==> Creating LXC container ${vmid} (${name})..." local_disk_size="${disk_size:-$PROXMOX_DEFAULT_LXC_DISK_GB}" # --memory doesn't touch swap -- it silently stays at Proxmox's own # 512M default otherwise (confirmed live: --memory 2048 left swap at # 512). Default to matching whatever --memory resolved to above. local_swap="${swap:-$memory}" # --unprivileged: read back from modules/platforms/lxc.nix's own # proxmoxLXC.privileged (via flake_target_lxc_privileged) rather than # hardcoded, since that's no longer the same for every lxc-* target -- # lxc-docker sets it true so the container's NFS mounts work at all (the # kernel's NFS client can't mount from inside any unprivileged # container's user namespace, no matter what AppArmor allows -- see that # option's own comment). The NixOS config inside the image bakes in # cgroup/capability/mount expectations matching whichever value it was # built with, so this must stay in sync with it -- `pct create`'s own # CLI default for this flag is privileged (unlike the web UI, which # defaults its checkbox the other way), so leaving it unset would create # a privileged container running a NixOS config that assumes # unprivileged for every target except lxc-docker, a real mismatch. privileged_eval="$(flake_target_lxc_privileged "$repo_root" "$flake_target")" unprivileged_flag=1 [[ "$privileged_eval" == "true" ]] && unprivileged_flag=0 # # --features nesting=1,keyctl=1: required for a modern (v247+) systemd # guest to actually boot unprivileged -- confirmed live: without this, # AppArmor denies the nested user namespaces and credential mounts # systemd routinely uses (even plain getty units), and every getty # crash-loops on a denied mount every ~3s (visible as garbage on the # console) while core services like nsncd fail the same way. # # ...,mount=nfs;nfs4: without it AppArmor blanket-denies the `nfs`/ # `rpc_pipefs` mount syscalls any NFS client share needs -- confirmed # live on lxc-docker: `mount: /var/lib/nfs/rpc_pipefs: permission # denied`. The value's `;` (Proxmox's own multi-fstype separator for # this one feature, per PVE::LXC's use of PVE::ParseUtils::split_list) # must stay single-quoted here: create_cmd is sent to `remote()`, which # hands the whole string to `ssh` as a single command for the *remote* # shell to parse -- unquoted, that `;` would be read as a remote # command separator and silently truncate this into two commands. create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --unprivileged ${unprivileged_flag} --features '${PROXMOX_DEFAULT_LXC_FEATURES}' --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --swap ${local_swap} --net0 name=eth0,bridge=${bridge},ip=dhcp" remote "$create_cmd" remote "pct start ${vmid}" else echo "==> Creating VM ${vmid} (${name})..." # pre-enrolled-keys=0 disables OVMF's Secure Boot key pre-enrollment -- # required, or systemd-boot (unsigned) can't be trusted by the firmware. # --agent 1: wires up the virtio-serial channel QEMU exposes to the guest. # modules/common/configuration.nix sets services.qemuGuest.enable = true # on every host, so the guest-side qemu-ga daemon is already running -- # without this flag Proxmox never creates the channel it listens on, so # `qm guest exec`/`qm agent` and the UI's IP-address display silently # never work for any VM this script creates. remote "qm create ${vmid} --name ${name} --memory ${memory} --cores ${cores} \ --net0 virtio,bridge=${bridge} --bios ovmf --machine q35 --scsihw virtio-scsi-pci \ --efidisk0 ${storage}:1,efitype=4m,pre-enrolled-keys=0 --agent enabled=1" if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] ssh ${ssh_target} -- qm importdisk ${vmid} ${remote_path} ${storage}" echo "[dry-run] (would parse the resulting disk identifier from that output)" echo "[dry-run] ssh ${ssh_target} -- qm set ${vmid} --scsi0 ${storage}:" else importdisk_output="$(ssh "$ssh_target" "qm importdisk ${vmid} ${remote_path} ${storage}")" echo "$importdisk_output" disk_id="$(echo "$importdisk_output" | grep -oP "(?<=Successfully imported disk as ')[^']+" | sed 's/^unused[0-9]*://')" if [[ -z "$disk_id" ]]; then echo "ERROR: couldn't parse the imported disk identifier from qm importdisk's output above." >&2 echo "The VM shell (${vmid}) and imported disk both exist -- finish attaching it by hand:" >&2 echo " ssh ${ssh_target} -- qm set ${vmid} --scsi0 ${storage}:" >&2 echo " ssh ${ssh_target} -- qm set ${vmid} --boot order=scsi0" >&2 exit 1 fi remote "qm set ${vmid} --scsi0 ${disk_id}" fi remote "qm set ${vmid} --boot order=scsi0" remote "qm start ${vmid}" fi echo if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] Nothing was built, uploaded, or created." else echo "Done. ${name} (VMID ${vmid}) should be booting on ${node}." fi