#!/usr/bin/env bash # Creates new Proxmox VMs/LXC containers from this flake, and reconfigures # existing ones -- the manual workflows in docs/proxmox-images.md (VM) and # docs/auto-installer.md's "LXC hosts" section (container), automated. # # Usage: # scripts/create-proxmox-resource.sh --type lxc|vm --host [options] # scripts/create-proxmox-resource.sh --type lxc|vm --list # scripts/create-proxmox-resource.sh --modify --vmid [--cores N] [--memory MB] [--grow-disk GB] # # SAFETY: # - The default (create) mode only ever creates a NEW resource -- it # refuses to run if the target VMID already exists on the node. # - --modify only ever touches a resource you name explicitly via # --vmid, shows exactly what will change first, and (outside # --dry-run) always requires typing that VMID back to confirm before # anything is sent to the node. There is no bulk/implicit modify. # - Neither mode can start/stop/delete a resource. Not implemented on # purpose -- ask before adding it. # # See --help for the full option list. set -euo pipefail repo_root="$(cd "$(dirname "$0")/.." && pwd)" # shellcheck source=env.sh source "${repo_root}/scripts/env.sh" sync_keys="${repo_root}/scripts/sync-host-keys.sh" usage() { cat < [options] (create) $0 --type lxc|vm --list (list --host values) $0 --modify --vmid [options] (reconfigure) Create mode (default): --type lxc|vm lxc = container, built as a CT template tarball. vm = VM, built as a Disko .raw disk image (UEFI/OVMF). --host Which host identity to deploy -- matches config.networking.hostName (server, docker, nix-cache, nixos, pxe-boot, nix-minimal). Use --list to see what's available for --type. --name Proxmox display name/hostname (default: --host's value, e.g. nix-cache -- for lxc this becomes the guest's real networking.hostName too, since proxmoxLXC.manageHostName pulls it from Proxmox's own container config, so it must match host.nix regardless of build type) --vmid Numeric VMID (default: next free, via \`pvesh get /cluster/nextid\` on the node). Refuses to run if this ID already exists. --disk-size lxc only: rootfs size for \`pct create\` (default: \$PROXMOX_DEFAULT_LXC_DISK_GB, ${PROXMOX_DEFAULT_LXC_DISK_GB}). --image Use this local image/tarball instead of checking the node / building one from the flake. --force-rebuild Skip the "does the node already have this image" check -- always build fresh and overwrite what's there. --allow-duplicate-host Required if --host already has a real deployment elsewhere (variables.nix's deployedTargets) -- otherwise refused, since it'd share that host's hostName/hostId. Modify mode (reconfigure an EXISTING resource -- requires --modify): --modify Switch to modify mode. --vmid Required: which existing resource to change. Type/VM-vs-CT is auto-detected on the node. --grow-disk Grow the primary disk by this many GB (qm/pct resize; Proxmox only supports growing, never shrinking, an existing disk). At least one of --cores / --memory / --grow-disk is required. Always prints the current -> new values and requires typing the VMID back to confirm, even outside --dry-run. Shared: --cores create: default \$PROXMOX_DEFAULT_CORES (${PROXMOX_DEFAULT_CORES}). modify: omit to leave unchanged. --memory create: default \$PROXMOX_DEFAULT_MEMORY_MB (${PROXMOX_DEFAULT_MEMORY_MB}). modify: omit to leave unchanged. --swap lxc only, create time: \`--memory\` doesn't touch swap -- it silently stays at Proxmox's own 512M default otherwise. (default: matches whatever --memory resolves to) --storage (default: \$PROXMOX_STORAGE, ${PROXMOX_STORAGE}) --iso-storage (default: \$PROXMOX_ISO_STORAGE, ${PROXMOX_ISO_STORAGE}) --bridge (default: \$PROXMOX_BRIDGE, ${PROXMOX_BRIDGE}) --node Proxmox node to SSH into (default: \$PROXMOX_HOST, ${PROXMOX_HOST}) --dry-run Print the full plan; touch nothing local or remote, no prompts. -h, --help Config for --storage/--bridge/--node/etc. lives in scripts/env.sh -- edit that instead of passing the same flag every time. EOF } dry_run=0 modify=0 type="" host="" name="" vmid="" cores="" memory="" swap="" disk_size="" grow_disk="" image="" storage="$PROXMOX_STORAGE" iso_storage="$PROXMOX_ISO_STORAGE" bridge="$PROXMOX_BRIDGE" node="$PROXMOX_HOST" do_list=0 allow_duplicate_host=0 force_rebuild=0 while [[ $# -gt 0 ]]; do case "$1" in --type) type="$2"; shift 2 ;; --host) host="$2"; shift 2 ;; --name) name="$2"; shift 2 ;; --vmid) vmid="$2"; shift 2 ;; --cores) cores="$2"; shift 2 ;; --memory) memory="$2"; shift 2 ;; --swap) swap="$2"; shift 2 ;; --disk-size) disk_size="$2"; shift 2 ;; --grow-disk) grow_disk="$2"; shift 2 ;; --image) image="$2"; shift 2 ;; --storage) storage="$2"; shift 2 ;; --iso-storage) iso_storage="$2"; shift 2 ;; --bridge) bridge="$2"; shift 2 ;; --node) node="$2"; shift 2 ;; --list) do_list=1; shift ;; --allow-duplicate-host) allow_duplicate_host=1; shift ;; --force-rebuild) force_rebuild=1; shift ;; --modify) modify=1; shift ;; --dry-run) dry_run=1; shift ;; -h | --help) usage; exit 0 ;; *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; esac done ssh_target="${PROXMOX_SSH_USER}@${node}" remote() { if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] ssh ${ssh_target} -- $*" else ssh "$ssh_target" "$@" fi } # ============================================================ modify mode cmd_modify() { if [[ -z "$vmid" ]]; then echo "ERROR: --modify requires --vmid." >&2 exit 1 fi if [[ -z "$cores" && -z "$memory" && -z "$grow_disk" ]]; then echo "ERROR: --modify needs at least one of --cores / --memory / --grow-disk." >&2 exit 1 fi echo "Looking up VMID ${vmid} on ${node}..." local kind current_cores current_memory disk_key if ssh "$ssh_target" "qm status ${vmid}" >/dev/null 2>&1; then kind="vm" disk_key="scsi0" elif ssh "$ssh_target" "pct status ${vmid}" >/dev/null 2>&1; then kind="lxc" disk_key="rootfs" else echo "ERROR: VMID ${vmid} doesn't exist on ${node} -- nothing to modify." >&2 exit 1 fi local config_cmd="qm config ${vmid}" [[ "$kind" == "lxc" ]] && config_cmd="pct config ${vmid}" local current_config current_config="$(ssh "$ssh_target" "$config_cmd")" current_cores="$(echo "$current_config" | grep -oP '^cores:\s*\K\S+' || echo '?')" current_memory="$(echo "$current_config" | grep -oP '^memory:\s*\K\S+' || echo '?')" echo echo "VMID ${vmid} is a ${kind} on ${node}. Planned changes:" [[ -n "$cores" ]] && echo " cores: ${current_cores} -> ${cores}" [[ -n "$memory" ]] && echo " memory: ${current_memory} MB -> ${memory} MB" [[ -n "$grow_disk" ]] && echo " ${disk_key}: grow by +${grow_disk}G (Proxmox can only grow, not shrink, an existing disk)" if [[ "$dry_run" -eq 1 ]]; then echo echo "[dry-run] Nothing was changed." return fi echo read -rp "Type the VMID (${vmid}) to confirm these changes: " confirm if [[ "$confirm" != "$vmid" ]]; then echo "Cancelled -- input didn't match ${vmid}." exit 1 fi local set_cmd="qm set" local resize_cmd="qm resize" [[ "$kind" == "lxc" ]] && set_cmd="pct set" && resize_cmd="pct resize" if [[ -n "$cores" || -n "$memory" ]]; then local args="" [[ -n "$cores" ]] && args="${args} --cores ${cores}" [[ -n "$memory" ]] && args="${args} --memory ${memory}" remote "${set_cmd} ${vmid}${args}" fi if [[ -n "$grow_disk" ]]; then remote "${resize_cmd} ${vmid} ${disk_key} +${grow_disk}G" fi echo echo "Done. VMID ${vmid} updated." } if [[ "$modify" -eq 1 ]]; then cmd_modify exit 0 fi # ============================================================= create mode if [[ "$type" != "lxc" && "$type" != "vm" ]]; then echo "ERROR: --type must be 'lxc' or 'vm'." >&2 usage >&2 exit 1 fi platform_prefix="lxc" [[ "$type" == "vm" ]] && platform_prefix="proxmox" [[ -z "$cores" ]] && cores="$PROXMOX_DEFAULT_CORES" [[ -z "$memory" ]] && memory="$PROXMOX_DEFAULT_MEMORY_MB" # --- discover / resolve the flake target from --host -------------------- list_hosts() { local target hostname for target in $(nix eval --json --no-use-registries --no-accept-flake-config \ "${repo_root}#nixosConfigurations" --apply builtins.attrNames 2>/dev/null \ | jq -r --arg p "${platform_prefix}-" '.[] | select(startswith($p))'); do hostname="$(nix eval --raw --no-use-registries --no-accept-flake-config \ "${repo_root}#nixosConfigurations.${target}.config.networking.hostName" 2>/dev/null)" printf ' %-12s -> %s\n' "$hostname" "$target" done } if [[ "$do_list" -eq 1 ]]; then echo "Available --host values for --type ${type}:" list_hosts exit 0 fi if [[ -z "$host" ]]; then echo "ERROR: --host is required (or use --list to see options)." >&2 exit 1 fi flake_target="" for target in $(nix eval --json --no-use-registries --no-accept-flake-config \ "${repo_root}#nixosConfigurations" --apply builtins.attrNames \ | jq -r --arg p "${platform_prefix}-" '.[] | select(startswith($p))'); do hn="$(nix eval --raw --no-use-registries --no-accept-flake-config \ "${repo_root}#nixosConfigurations.${target}.config.networking.hostName")" if [[ "$hn" == "$host" ]]; then flake_target="$target" break fi done if [[ -z "$flake_target" ]]; then echo "ERROR: no ${platform_prefix}-* target has hostName '${host}'." >&2 echo "Available:" >&2 list_hosts >&2 exit 1 fi # The container/VM's real identity is --host (e.g. "nix-cache"), validated # above against config.networking.hostName -- not the flake target name # (e.g. "lxc-nix-cache"), which is build-type-specific and only exists to # pick which platform variant to build. Defaulting --name to the flake # target would make lxc's --hostname (which proxmoxLXC.manageHostName # feeds straight into the guest's real hostname) disagree with host.nix. [[ -z "$name" ]] && name="$host" # --- refuse to duplicate a host that's already really deployed ---------- # Checked by hostName, not exact flake target: proxmox-server being # deployed also blocks --type lxc --host server, since both would carry # the same hosts/server/host.nix identity (hostName, hostId). if [[ "$allow_duplicate_host" -eq 0 ]]; then deployed_targets_json="$(nix eval --json --no-use-registries --no-accept-flake-config \ --file "${repo_root}/variables.nix" deployedTargets)" for dt in $(echo "$deployed_targets_json" | jq -r '.[]'); do dt_hostname="$(nix eval --raw --no-use-registries --no-accept-flake-config \ "${repo_root}#nixosConfigurations.${dt}.config.networking.hostName" 2>/dev/null || true)" if [[ "$dt_hostname" == "$host" ]]; then echo "ERROR: '${host}' already has a real deployment (${dt}, per variables.nix's" >&2 echo "deployedTargets). Creating ${flake_target} would share its hostName/hostId --" >&2 echo "refusing by default. Pass --allow-duplicate-host if you really mean to spin" >&2 echo "up a separate test instance of this host (it'll still get its own distinct" >&2 echo "sops key and VMID, never touching ${dt})." >&2 exit 1 fi done fi echo "Target: ${flake_target} (host=${host}, type=${type}) -> Proxmox resource '${name}'" # Decide on nix-cache once, here -- this is the earliest point that needs # it (sync-host-keys.sh below needs nix-shell packages regardless of # whether an image ends up getting built later), and the decision is # exported so that subprocess -- and this script's own later build step, # if it gets there -- both reuse it instead of probing again. nix_extra_opts # --- make sure this target has a registered host key -------------------- echo echo "==> Ensuring host key exists and is registered..." sync_args=("$flake_target") [[ "$dry_run" -eq 1 ]] && sync_args+=(--dry-run) bash "$sync_keys" "${sync_args[@]}" # --- VMID: pick one, and refuse to touch anything that already exists --- echo if [[ -z "$vmid" ]]; then if [[ "$dry_run" -eq 1 ]]; then vmid="" echo "[dry-run] would ask ${node} for the next free VMID (pvesh get /cluster/nextid)" else vmid="$(ssh "$ssh_target" "pvesh get /cluster/nextid" | tr -d '[:space:]')" echo "Auto-assigned VMID: ${vmid}" fi else echo "Requested VMID: ${vmid}" fi if [[ "$dry_run" -eq 0 ]]; then # qm/pct status exits non-zero (and prints "does not exist") for a free # ID on that resource type -- but a VMID could exist as the OTHER # resource type (e.g. requested a CT id that's actually a VM), so check # both. Any success here means something is already using this ID -- # refuse to go anywhere near it. (Reconfiguring an existing resource is # --modify's job, not this one's.) if ssh "$ssh_target" "qm status ${vmid}" >/dev/null 2>&1 \ || ssh "$ssh_target" "pct status ${vmid}" >/dev/null 2>&1; then echo "ERROR: VMID ${vmid} already exists on ${node}. Refusing to touch an" >&2 echo "existing resource here -- use --modify to reconfigure it, pick a" >&2 echo "different --vmid, or omit it to auto-assign." >&2 exit 1 fi fi # --- resolve the remote path -- fixed naming (not the nix store's own # derivation-hash-based filename), so a later run can check for it by name. # lxc uploads as a CT *template* (Proxmox's "vztmpl" content type, under # iso_storage) -- config.system.build.tarball is a plain rootfs tarball, # not a vzdump backup archive, so it's created with `pct create ... vztmpl`, # not restored with `pct restore` (that expects backup-archive metadata # this tarball doesn't have, and fails with "archive contains no # configuration file"). remote_dir="/var/lib/vz/import" remote_filename="${flake_target}.raw" if [[ "$type" == "lxc" ]]; then remote_dir="/var/lib/vz/template/cache" remote_filename="${flake_target}.tar.xz" fi remote_path="${remote_dir}/${remote_filename}" # --- build (or reuse an image already on the node) ------------------------ echo local_image="" image_already_remote=0 if [[ -n "$image" ]]; then [[ -f "$image" ]] || { echo "ERROR: --image '${image}' not found." >&2; exit 1; } local_image="$image" echo "Using provided image: ${local_image}" elif [[ "$force_rebuild" -eq 1 ]]; then echo "--force-rebuild: skipping the existing-image check on ${node}." else echo "==> Checking whether ${node} already has ${remote_path}..." if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would check: ssh ${ssh_target} -- test -f ${remote_path}" elif ssh "$ssh_target" "test -f '${remote_path}'" 2>/dev/null; then echo "Found it -- reusing, skipping build and upload (use --force-rebuild to override)." image_already_remote=1 else echo "Not found -- will build." fi fi if [[ "$image_already_remote" -eq 0 && -z "$local_image" ]]; then if [[ "$type" == "lxc" ]]; then if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would build: NIXOS_HOST_KEYS_DIR=${repo_root}/host-keys nix build --impure \\" echo "[dry-run] .#nixosConfigurations.${flake_target}.config.system.build.tarball" local_image="" else echo "==> Building LXC tarball for ${flake_target}..." NIXOS_HOST_KEYS_DIR="${repo_root}/host-keys" nix build --impure \ --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \ ".#nixosConfigurations.${flake_target}.config.system.build.tarball" \ --out-link "${repo_root}/result-${flake_target}" local_image="$(find "${repo_root}/result-${flake_target}/tarball" -maxdepth 1 -type f | head -1)" echo "Built: ${local_image}" fi else if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would build: nix build .#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript" echo "[dry-run] would run: sudo ./result-${flake_target} \\" echo "[dry-run] --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key \\" echo "[dry-run] --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key.pub /etc/ssh/ssh_host_ed25519_key.pub \\" echo "[dry-run] --build-memory 2048" local_image=".raw" else echo "==> Building Disko image script for ${flake_target}..." nix build --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \ ".#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript" \ --out-link "${repo_root}/result-${flake_target}" echo "==> Running it (builds the .raw image in a temporary QEMU VM, needs sudo)..." ( cd "$repo_root" && sudo "./result-${flake_target}" \ --pre-format-files "host-keys/${flake_target}_ssh_host_ed25519_key" /etc/ssh/ssh_host_ed25519_key \ --pre-format-files "host-keys/${flake_target}_ssh_host_ed25519_key.pub" /etc/ssh/ssh_host_ed25519_key.pub \ --build-memory 2048 ) local_image="$(find "$repo_root" -maxdepth 1 -name "*.raw" -newer "${repo_root}/result-${flake_target}" | head -1)" if [[ -z "$local_image" ]]; then echo "ERROR: expected a .raw image after the build but didn't find one in ${repo_root}." >&2 exit 1 fi echo "Built: ${local_image}" fi fi fi # --- upload (skip entirely if reusing an image already on the node) ------ echo if [[ "$image_already_remote" -eq 1 ]]; then : # nothing to upload elif [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] would upload: scp ${local_image} ${ssh_target}:${remote_path}" else echo "==> Uploading to ${node}:${remote_path}..." ssh "$ssh_target" "mkdir -p ${remote_dir}" scp "$local_image" "${ssh_target}:${remote_path}" fi # --- create ----------------------------------------------------------------- echo if [[ "$type" == "lxc" ]]; then echo "==> Creating LXC container ${vmid} (${name})..." local_disk_size="${disk_size:-$PROXMOX_DEFAULT_LXC_DISK_GB}" # --memory doesn't touch swap -- it silently stays at Proxmox's own # 512M default otherwise (confirmed live: --memory 2048 left swap at # 512). Default to matching whatever --memory resolved to above. local_swap="${swap:-$memory}" # --unprivileged 1: modules/platforms/lxc.nix sets proxmoxLXC.privileged # = false, so the NixOS config inside the image assumes it's running as # an unprivileged container (cgroup/capability/mount expectations baked # in at boot). `pct create`'s own CLI default for this flag is # privileged (unlike the web UI, which defaults its checkbox the other # way) -- leaving it unset creates a privileged container running a # NixOS config that assumes unprivileged, a real mismatch. # # --features nesting=1,keyctl=1: required for a modern (v247+) systemd # guest to actually boot unprivileged -- confirmed live: without this, # AppArmor denies the nested user namespaces and credential mounts # systemd routinely uses (even plain getty units), and every getty # crash-loops on a denied mount every ~3s (visible as garbage on the # console) while core services like nsncd fail the same way. create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --unprivileged 1 --features ${PROXMOX_DEFAULT_LXC_FEATURES} --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --swap ${local_swap} --net0 name=eth0,bridge=${bridge},ip=dhcp" remote "$create_cmd" remote "pct start ${vmid}" else echo "==> Creating VM ${vmid} (${name})..." # pre-enrolled-keys=0 disables OVMF's Secure Boot key pre-enrollment -- # required, or systemd-boot (unsigned) can't be trusted by the firmware. remote "qm create ${vmid} --name ${name} --memory ${memory} --cores ${cores} \ --net0 virtio,bridge=${bridge} --bios ovmf --machine q35 --scsihw virtio-scsi-pci \ --efidisk0 ${storage}:1,efitype=4m,pre-enrolled-keys=0" if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] ssh ${ssh_target} -- qm importdisk ${vmid} ${remote_path} ${storage}" echo "[dry-run] (would parse the resulting disk identifier from that output)" echo "[dry-run] ssh ${ssh_target} -- qm set ${vmid} --scsi0 ${storage}:" else importdisk_output="$(ssh "$ssh_target" "qm importdisk ${vmid} ${remote_path} ${storage}")" echo "$importdisk_output" disk_id="$(echo "$importdisk_output" | grep -oP "(?<=Successfully imported disk as ')[^']+" | sed 's/^unused[0-9]*://')" if [[ -z "$disk_id" ]]; then echo "ERROR: couldn't parse the imported disk identifier from qm importdisk's output above." >&2 echo "The VM shell (${vmid}) and imported disk both exist -- finish attaching it by hand:" >&2 echo " ssh ${ssh_target} -- qm set ${vmid} --scsi0 ${storage}:" >&2 echo " ssh ${ssh_target} -- qm set ${vmid} --boot order=scsi0" >&2 exit 1 fi remote "qm set ${vmid} --scsi0 ${disk_id}" fi remote "qm set ${vmid} --boot order=scsi0" remote "qm start ${vmid}" fi echo if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] Nothing was built, uploaded, or created." else echo "Done. ${name} (VMID ${vmid}) should be booting on ${node}." fi