nixos/scripts/create-proxmox-resource.sh

#!/usr/bin/env bash
# Creates new Proxmox VMs/LXC containers from this flake, and reconfigures
# existing ones -- the manual workflows in docs/proxmox-images.md (VM) and
# docs/auto-installer.md's "LXC hosts" section (container), automated.
#
# Usage:
#   scripts/create-proxmox-resource.sh --type lxc|vm --host <name> [options]
#   scripts/create-proxmox-resource.sh --type lxc|vm --list
#   scripts/create-proxmox-resource.sh --modify --vmid <n> [--cores N] [--memory MB] [--grow-disk GB]
#
# SAFETY:
#   - The default (create) mode only ever creates a NEW resource -- it
#     refuses to run if the target VMID already exists on the node.
#   - --modify only ever touches a resource you name explicitly via
#     --vmid, shows exactly what will change first, and (outside
#     --dry-run) always requires typing that VMID back to confirm before
#     anything is sent to the node. There is no bulk/implicit modify.
#   - Neither mode can start/stop/delete a resource. Not implemented on
#     purpose -- ask before adding it.
#
# See --help for the full option list.
set -euo pipefail

repo_root="$(cd "$(dirname "$0")/.." && pwd)"
# shellcheck source=env.sh
source "${repo_root}/scripts/env.sh"

sync_keys="${repo_root}/scripts/sync-host-keys.sh"

usage() {
  cat <<EOF
Usage: $0 --type lxc|vm --host <name> [options]      (create)
       $0 --type lxc|vm --list                        (list --host values)
       $0 --modify --vmid <n> [options]                (reconfigure)

Create mode (default):
  --type lxc|vm        lxc = container, built as a CT template tarball.
                        vm  = VM, built as a Disko .raw disk image (UEFI/OVMF).
  --host <name>         Which host identity to deploy -- matches
                        config.networking.hostName (server, docker,
                        nix-cache, nixos, pxe-boot, nix-minimal). Use
                        --list to see what's available for --type.
  --name <name>          Proxmox display name/hostname (default: the flake
                          target name, e.g. lxc-server)
  --vmid <n>              Numeric VMID (default: next free, via
                          \`pvesh get /cluster/nextid\` on the node).
                          Refuses to run if this ID already exists.
  --disk-size <GB>           lxc only: rootfs size for \`pct create\`
                              (default: \$PROXMOX_DEFAULT_LXC_DISK_GB, ${PROXMOX_DEFAULT_LXC_DISK_GB}).
  --image <path>              Use this local image/tarball instead of
                              checking the node / building one from the flake.
  --force-rebuild                Skip the "does the node already have this
                                  image" check -- always build fresh and
                                  overwrite what's there.
  --allow-duplicate-host        Required if --host already has a real
                                deployment elsewhere (variables.nix's
                                deployedTargets) -- otherwise refused, since
                                it'd share that host's hostName/hostId.

Modify mode (reconfigure an EXISTING resource -- requires --modify):
  --modify                Switch to modify mode.
  --vmid <n>                Required: which existing resource to change.
                            Type/VM-vs-CT is auto-detected on the node.
  --grow-disk <GB>            Grow the primary disk by this many GB
                              (qm/pct resize; Proxmox only supports
                              growing, never shrinking, an existing disk).
  At least one of --cores / --memory / --grow-disk is required. Always
  prints the current -> new values and requires typing the VMID back to
  confirm, even outside --dry-run.

Shared:
  --cores <n>              create: default \$PROXMOX_DEFAULT_CORES (${PROXMOX_DEFAULT_CORES}).
                            modify: omit to leave unchanged.
  --memory <MB>              create: default \$PROXMOX_DEFAULT_MEMORY_MB (${PROXMOX_DEFAULT_MEMORY_MB}).
                              modify: omit to leave unchanged.
  --storage <pool>              (default: \$PROXMOX_STORAGE, ${PROXMOX_STORAGE})
  --iso-storage <pool>            (default: \$PROXMOX_ISO_STORAGE, ${PROXMOX_ISO_STORAGE})
  --bridge <bridge>                (default: \$PROXMOX_BRIDGE, ${PROXMOX_BRIDGE})
  --node <host>                     Proxmox node to SSH into (default:
                                    \$PROXMOX_HOST, ${PROXMOX_HOST})
  --dry-run                           Print the full plan; touch nothing
                                      local or remote, no prompts.
  -h, --help

Config for --storage/--bridge/--node/etc. lives in scripts/env.sh -- edit
that instead of passing the same flag every time.
EOF
}

dry_run=0
modify=0
type=""
host=""
name=""
vmid=""
cores=""
memory=""
disk_size=""
grow_disk=""
image=""
storage="$PROXMOX_STORAGE"
iso_storage="$PROXMOX_ISO_STORAGE"
bridge="$PROXMOX_BRIDGE"
node="$PROXMOX_HOST"
do_list=0
allow_duplicate_host=0
force_rebuild=0

while [[ $# -gt 0 ]]; do
  case "$1" in
  --type) type="$2"; shift 2 ;;
  --host) host="$2"; shift 2 ;;
  --name) name="$2"; shift 2 ;;
  --vmid) vmid="$2"; shift 2 ;;
  --cores) cores="$2"; shift 2 ;;
  --memory) memory="$2"; shift 2 ;;
  --disk-size) disk_size="$2"; shift 2 ;;
  --grow-disk) grow_disk="$2"; shift 2 ;;
  --image) image="$2"; shift 2 ;;
  --storage) storage="$2"; shift 2 ;;
  --iso-storage) iso_storage="$2"; shift 2 ;;
  --bridge) bridge="$2"; shift 2 ;;
  --node) node="$2"; shift 2 ;;
  --list) do_list=1; shift ;;
  --allow-duplicate-host) allow_duplicate_host=1; shift ;;
  --force-rebuild) force_rebuild=1; shift ;;
  --modify) modify=1; shift ;;
  --dry-run) dry_run=1; shift ;;
  -h | --help) usage; exit 0 ;;
  *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;;
  esac
done

ssh_target="${PROXMOX_SSH_USER}@${node}"

remote() {
  if [[ "$dry_run" -eq 1 ]]; then
    echo "[dry-run] ssh ${ssh_target} -- $*"
  else
    ssh "$ssh_target" "$@"
  fi
}

# ============================================================ modify mode
cmd_modify() {
  if [[ -z "$vmid" ]]; then
    echo "ERROR: --modify requires --vmid." >&2
    exit 1
  fi
  if [[ -z "$cores" && -z "$memory" && -z "$grow_disk" ]]; then
    echo "ERROR: --modify needs at least one of --cores / --memory / --grow-disk." >&2
    exit 1
  fi

  echo "Looking up VMID ${vmid} on ${node}..."
  local kind current_cores current_memory disk_key
  if ssh "$ssh_target" "qm status ${vmid}" >/dev/null 2>&1; then
    kind="vm"
    disk_key="scsi0"
  elif ssh "$ssh_target" "pct status ${vmid}" >/dev/null 2>&1; then
    kind="lxc"
    disk_key="rootfs"
  else
    echo "ERROR: VMID ${vmid} doesn't exist on ${node} -- nothing to modify." >&2
    exit 1
  fi

  local config_cmd="qm config ${vmid}"
  [[ "$kind" == "lxc" ]] && config_cmd="pct config ${vmid}"
  local current_config
  current_config="$(ssh "$ssh_target" "$config_cmd")"
  current_cores="$(echo "$current_config" | grep -oP '^cores:\s*\K\S+' || echo '?')"
  current_memory="$(echo "$current_config" | grep -oP '^memory:\s*\K\S+' || echo '?')"

  echo
  echo "VMID ${vmid} is a ${kind} on ${node}. Planned changes:"
  [[ -n "$cores" ]] && echo "  cores:  ${current_cores} -> ${cores}"
  [[ -n "$memory" ]] && echo "  memory: ${current_memory} MB -> ${memory} MB"
  [[ -n "$grow_disk" ]] && echo "  ${disk_key}: grow by +${grow_disk}G (Proxmox can only grow, not shrink, an existing disk)"

  if [[ "$dry_run" -eq 1 ]]; then
    echo
    echo "[dry-run] Nothing was changed."
    return
  fi

  echo
  read -rp "Type the VMID (${vmid}) to confirm these changes: " confirm
  if [[ "$confirm" != "$vmid" ]]; then
    echo "Cancelled -- input didn't match ${vmid}."
    exit 1
  fi

  local set_cmd="qm set"
  local resize_cmd="qm resize"
  [[ "$kind" == "lxc" ]] && set_cmd="pct set" && resize_cmd="pct resize"

  if [[ -n "$cores" || -n "$memory" ]]; then
    local args=""
    [[ -n "$cores" ]] && args="${args} --cores ${cores}"
    [[ -n "$memory" ]] && args="${args} --memory ${memory}"
    remote "${set_cmd} ${vmid}${args}"
  fi
  if [[ -n "$grow_disk" ]]; then
    remote "${resize_cmd} ${vmid} ${disk_key} +${grow_disk}G"
  fi

  echo
  echo "Done. VMID ${vmid} updated."
}

if [[ "$modify" -eq 1 ]]; then
  cmd_modify
  exit 0
fi

# ============================================================= create mode
if [[ "$type" != "lxc" && "$type" != "vm" ]]; then
  echo "ERROR: --type must be 'lxc' or 'vm'." >&2
  usage >&2
  exit 1
fi

platform_prefix="lxc"
[[ "$type" == "vm" ]] && platform_prefix="proxmox"
[[ -z "$cores" ]] && cores="$PROXMOX_DEFAULT_CORES"
[[ -z "$memory" ]] && memory="$PROXMOX_DEFAULT_MEMORY_MB"

# --- discover / resolve the flake target from --host --------------------
list_hosts() {
  local target hostname
  for target in $(nix eval --json --no-use-registries --no-accept-flake-config \
    "${repo_root}#nixosConfigurations" --apply builtins.attrNames 2>/dev/null \
    | jq -r --arg p "${platform_prefix}-" '.[] | select(startswith($p))'); do
    hostname="$(nix eval --raw --no-use-registries --no-accept-flake-config \
      "${repo_root}#nixosConfigurations.${target}.config.networking.hostName" 2>/dev/null)"
    printf '  %-12s -> %s\n' "$hostname" "$target"
  done
}

if [[ "$do_list" -eq 1 ]]; then
  echo "Available --host values for --type ${type}:"
  list_hosts
  exit 0
fi

if [[ -z "$host" ]]; then
  echo "ERROR: --host is required (or use --list to see options)." >&2
  exit 1
fi

flake_target=""
for target in $(nix eval --json --no-use-registries --no-accept-flake-config \
  "${repo_root}#nixosConfigurations" --apply builtins.attrNames \
  | jq -r --arg p "${platform_prefix}-" '.[] | select(startswith($p))'); do
  hn="$(nix eval --raw --no-use-registries --no-accept-flake-config \
    "${repo_root}#nixosConfigurations.${target}.config.networking.hostName")"
  if [[ "$hn" == "$host" ]]; then
    flake_target="$target"
    break
  fi
done

if [[ -z "$flake_target" ]]; then
  echo "ERROR: no ${platform_prefix}-* target has hostName '${host}'." >&2
  echo "Available:" >&2
  list_hosts >&2
  exit 1
fi

[[ -z "$name" ]] && name="$flake_target"

# --- refuse to duplicate a host that's already really deployed ----------
# Checked by hostName, not exact flake target: proxmox-server being
# deployed also blocks --type lxc --host server, since both would carry
# the same hosts/server/host.nix identity (hostName, hostId).
if [[ "$allow_duplicate_host" -eq 0 ]]; then
  deployed_targets_json="$(nix eval --json --no-use-registries --no-accept-flake-config \
    --file "${repo_root}/variables.nix" deployedTargets)"
  for dt in $(echo "$deployed_targets_json" | jq -r '.[]'); do
    dt_hostname="$(nix eval --raw --no-use-registries --no-accept-flake-config \
      "${repo_root}#nixosConfigurations.${dt}.config.networking.hostName" 2>/dev/null || true)"
    if [[ "$dt_hostname" == "$host" ]]; then
      echo "ERROR: '${host}' already has a real deployment (${dt}, per variables.nix's" >&2
      echo "deployedTargets). Creating ${flake_target} would share its hostName/hostId --" >&2
      echo "refusing by default. Pass --allow-duplicate-host if you really mean to spin" >&2
      echo "up a separate test instance of this host (it'll still get its own distinct" >&2
      echo "sops key and VMID, never touching ${dt})." >&2
      exit 1
    fi
  done
fi

echo "Target: ${flake_target} (host=${host}, type=${type}) -> Proxmox resource '${name}'"

# Decide on nix-cache once, here -- this is the earliest point that needs
# it (sync-host-keys.sh below needs nix-shell packages regardless of
# whether an image ends up getting built later), and the decision is
# exported so that subprocess -- and this script's own later build step,
# if it gets there -- both reuse it instead of probing again.
nix_extra_opts

# --- make sure this target has a registered host key --------------------
echo
echo "==> Ensuring host key exists and is registered..."
sync_args=("$flake_target")
[[ "$dry_run" -eq 1 ]] && sync_args+=(--dry-run)
bash "$sync_keys" "${sync_args[@]}"

# --- VMID: pick one, and refuse to touch anything that already exists ---
echo
if [[ -z "$vmid" ]]; then
  if [[ "$dry_run" -eq 1 ]]; then
    vmid="<next-free-vmid>"
    echo "[dry-run] would ask ${node} for the next free VMID (pvesh get /cluster/nextid)"
  else
    vmid="$(ssh "$ssh_target" "pvesh get /cluster/nextid" | tr -d '[:space:]')"
    echo "Auto-assigned VMID: ${vmid}"
  fi
else
  echo "Requested VMID: ${vmid}"
fi

if [[ "$dry_run" -eq 0 ]]; then
  # qm/pct status exits non-zero (and prints "does not exist") for a free
  # ID on that resource type -- but a VMID could exist as the OTHER
  # resource type (e.g. requested a CT id that's actually a VM), so check
  # both. Any success here means something is already using this ID --
  # refuse to go anywhere near it. (Reconfiguring an existing resource is
  # --modify's job, not this one's.)
  if ssh "$ssh_target" "qm status ${vmid}" >/dev/null 2>&1 \
    || ssh "$ssh_target" "pct status ${vmid}" >/dev/null 2>&1; then
    echo "ERROR: VMID ${vmid} already exists on ${node}. Refusing to touch an" >&2
    echo "existing resource here -- use --modify to reconfigure it, pick a" >&2
    echo "different --vmid, or omit it to auto-assign." >&2
    exit 1
  fi
fi

# --- resolve the remote path -- fixed naming (not the nix store's own
# derivation-hash-based filename), so a later run can check for it by name.
# lxc uploads as a CT *template* (Proxmox's "vztmpl" content type, under
# iso_storage) -- config.system.build.tarball is a plain rootfs tarball,
# not a vzdump backup archive, so it's created with `pct create ... vztmpl`,
# not restored with `pct restore` (that expects backup-archive metadata
# this tarball doesn't have, and fails with "archive contains no
# configuration file").
remote_dir="/var/lib/vz/import"
remote_filename="${flake_target}.raw"
if [[ "$type" == "lxc" ]]; then
  remote_dir="/var/lib/vz/template/cache"
  remote_filename="${flake_target}.tar.xz"
fi
remote_path="${remote_dir}/${remote_filename}"

# --- build (or reuse an image already on the node) ------------------------
echo
local_image=""
image_already_remote=0

if [[ -n "$image" ]]; then
  [[ -f "$image" ]] || { echo "ERROR: --image '${image}' not found." >&2; exit 1; }
  local_image="$image"
  echo "Using provided image: ${local_image}"
elif [[ "$force_rebuild" -eq 1 ]]; then
  echo "--force-rebuild: skipping the existing-image check on ${node}."
else
  echo "==> Checking whether ${node} already has ${remote_path}..."
  if [[ "$dry_run" -eq 1 ]]; then
    echo "[dry-run] would check: ssh ${ssh_target} -- test -f ${remote_path}"
  elif ssh "$ssh_target" "test -f '${remote_path}'" 2>/dev/null; then
    echo "Found it -- reusing, skipping build and upload (use --force-rebuild to override)."
    image_already_remote=1
  else
    echo "Not found -- will build."
  fi
fi

if [[ "$image_already_remote" -eq 0 && -z "$local_image" ]]; then
  if [[ "$type" == "lxc" ]]; then
    if [[ "$dry_run" -eq 1 ]]; then
      echo "[dry-run] would build: NIXOS_HOST_KEYS_DIR=${repo_root}/host-keys nix build --impure \\"
      echo "[dry-run]   .#nixosConfigurations.${flake_target}.config.system.build.tarball"
      local_image="<built-tarball>"
    else
      echo "==> Building LXC tarball for ${flake_target}..."
      NIXOS_HOST_KEYS_DIR="${repo_root}/host-keys" nix build --impure \
        --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \
        ".#nixosConfigurations.${flake_target}.config.system.build.tarball" \
        --out-link "${repo_root}/result-${flake_target}"
      local_image="$(find "${repo_root}/result-${flake_target}/tarball" -maxdepth 1 -type f | head -1)"
      echo "Built: ${local_image}"
    fi
  else
    if [[ "$dry_run" -eq 1 ]]; then
      echo "[dry-run] would build: nix build .#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript"
      echo "[dry-run] would run:   sudo ./result-${flake_target} \\"
      echo "[dry-run]                --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key \\"
      echo "[dry-run]                --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key.pub /etc/ssh/ssh_host_ed25519_key.pub \\"
      echo "[dry-run]                --build-memory 2048"
      local_image="<built-image>.raw"
    else
      echo "==> Building Disko image script for ${flake_target}..."
      nix build --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \
        ".#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript" \
        --out-link "${repo_root}/result-${flake_target}"
      echo "==> Running it (builds the .raw image in a temporary QEMU VM, needs sudo)..."
      ( cd "$repo_root" && sudo "./result-${flake_target}" \
          --pre-format-files "host-keys/${flake_target}_ssh_host_ed25519_key" /etc/ssh/ssh_host_ed25519_key \
          --pre-format-files "host-keys/${flake_target}_ssh_host_ed25519_key.pub" /etc/ssh/ssh_host_ed25519_key.pub \
          --build-memory 2048 )
      local_image="$(find "$repo_root" -maxdepth 1 -name "*.raw" -newer "${repo_root}/result-${flake_target}" | head -1)"
      if [[ -z "$local_image" ]]; then
        echo "ERROR: expected a .raw image after the build but didn't find one in ${repo_root}." >&2
        exit 1
      fi
      echo "Built: ${local_image}"
    fi
  fi
fi

# --- upload (skip entirely if reusing an image already on the node) ------
echo
if [[ "$image_already_remote" -eq 1 ]]; then
  : # nothing to upload
elif [[ "$dry_run" -eq 1 ]]; then
  echo "[dry-run] would upload: scp ${local_image} ${ssh_target}:${remote_path}"
else
  echo "==> Uploading to ${node}:${remote_path}..."
  ssh "$ssh_target" "mkdir -p ${remote_dir}"
  scp "$local_image" "${ssh_target}:${remote_path}"
fi

# --- create -----------------------------------------------------------------
echo
if [[ "$type" == "lxc" ]]; then
  echo "==> Creating LXC container ${vmid} (${name})..."
  local_disk_size="${disk_size:-$PROXMOX_DEFAULT_LXC_DISK_GB}"
  create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --net0 name=eth0,bridge=${bridge},ip=dhcp"
  remote "$create_cmd"
  remote "pct start ${vmid}"
else
  echo "==> Creating VM ${vmid} (${name})..."
  # pre-enrolled-keys=0 disables OVMF's Secure Boot key pre-enrollment --
  # required, or systemd-boot (unsigned) can't be trusted by the firmware.
  remote "qm create ${vmid} --name ${name} --memory ${memory} --cores ${cores} \
--net0 virtio,bridge=${bridge} --bios ovmf --machine q35 --scsihw virtio-scsi-pci \
--efidisk0 ${storage}:1,efitype=4m,pre-enrolled-keys=0"

  if [[ "$dry_run" -eq 1 ]]; then
    echo "[dry-run] ssh ${ssh_target} -- qm importdisk ${vmid} ${remote_path} ${storage}"
    echo "[dry-run] (would parse the resulting disk identifier from that output)"
    echo "[dry-run] ssh ${ssh_target} -- qm set ${vmid} --scsi0 ${storage}:<parsed-disk-id>"
  else
    importdisk_output="$(ssh "$ssh_target" "qm importdisk ${vmid} ${remote_path} ${storage}")"
    echo "$importdisk_output"
    disk_id="$(echo "$importdisk_output" | grep -oP "(?<=Successfully imported disk as ')[^']+" | sed 's/^unused[0-9]*://')"
    if [[ -z "$disk_id" ]]; then
      echo "ERROR: couldn't parse the imported disk identifier from qm importdisk's output above." >&2
      echo "The VM shell (${vmid}) and imported disk both exist -- finish attaching it by hand:" >&2
      echo "  ssh ${ssh_target} -- qm set ${vmid} --scsi0 ${storage}:<disk-id-from-output-above>" >&2
      echo "  ssh ${ssh_target} -- qm set ${vmid} --boot order=scsi0" >&2
      exit 1
    fi
    remote "qm set ${vmid} --scsi0 ${disk_id}"
  fi
  remote "qm set ${vmid} --boot order=scsi0"
  remote "qm start ${vmid}"
fi

echo
if [[ "$dry_run" -eq 1 ]]; then
  echo "[dry-run] Nothing was built, uploaded, or created."
else
  echo "Done. ${name} (VMID ${vmid}) should be booting on ${node}."
fi