Archived
beatzaplenty
619324589a
Check NixOS configurations / eval-hosts (pull_request) Successful in 10m28s
The previous fix (#42) only unmounted /mnt/{dev,proc,sys,run}, but disko also mounts the ESP at /mnt/boot (modules/disko/baremetal.nix) -- another nested mount blocking ZFS from unmounting its own root dataset at /mnt the same way. Confirmed live: zpool export still failed with "cannot unmount '/mnt': pool or dataset busy" after the chroot-only fix. Replace the manual dev/proc/sys/run list with a single recursive `umount -R /mnt`, which clears everything nested under /mnt -- current and future mountpoints alike -- rather than needing to keep enumerating whatever nixos-install/disko happen to leave mounted.
207 lines
10 KiB
Bash
Executable File
207 lines
10 KiB
Bash
Executable File
#!/usr/bin/env nix-shell
|
|
#!nix-shell -i bash -p jq disko nixos-install-tools zfs
|
|
# shellcheck shell=bash
|
|
# The only genuinely external tools this script calls directly: `jq`
|
|
# (parsing the `nix eval` host list), `disko`/`nixos-install` (the
|
|
# install itself), and `zpool` (exporting a ZFS root pool before reboot,
|
|
# see the comment above that call below). Everything disko shells out to
|
|
# internally (parted/sgdisk/mkfs.*/zfs/...) is self-contained -- disko's
|
|
# own generated scripts hardcode absolute Nix store paths for those, they
|
|
# don't rely on this script's PATH at all (confirmed by inspecting a
|
|
# generated system.build.formatScript). The built installer image
|
|
# (modules/installer/common.nix, plus the upstream
|
|
# installation-cd-minimal.nix it imports via iso.nix) already has all
|
|
# four in environment.systemPackages, so this nix-shell wrapper is a
|
|
# fast no-op there; it's what makes the script also work standalone
|
|
# (e.g. run directly from a checkout on a stock ISO), where they aren't
|
|
# guaranteed.
|
|
set -eux
|
|
|
|
set -euo pipefail
|
|
|
|
# shellcheck source=../env.sh
|
|
source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/env.sh"
|
|
|
|
export FLAKE_BASE_URL="git+https://${LAN_DOMAIN}/beatzaplenty/nixos.git"
|
|
|
|
echo "Fetching available NixOS hosts from flake..."
|
|
# Two categories deliberately excluded from the menu:
|
|
# lxc-* — these build a config.system.build.tarball meant for
|
|
# `pct restore` on Proxmox directly, not an install.
|
|
# Running nixos-install against one here would
|
|
# bind-mount / onto /mnt and then refuse to touch the
|
|
# filesystem it's currently running on — see
|
|
# docs/auto-installer.md.
|
|
# installer — this *is* the installer image's own flake target,
|
|
# not a deployable host; "installing" it means
|
|
# nixos-install-ing a copy of the installer into
|
|
# itself.
|
|
mapfile -t options < <(
|
|
nix eval --json --no-use-registries --no-accept-flake-config --extra-experimental-features "flakes nix-command" \
|
|
"${FLAKE_BASE_URL}#nixosConfigurations" \
|
|
--apply builtins.attrNames \
|
|
| jq -r '.[]
|
|
| select(startswith("lxc-") | not)
|
|
| select(. != "installer")'
|
|
)
|
|
|
|
if [[ ${#options[@]} -eq 0 ]]; then
|
|
echo "ERROR: No NixOS hosts found in ${FLAKE_BASE_URL}#nixosConfigurations" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Note: lxc-* targets aren't installed this way — build them with"
|
|
echo " nix build .#nixosConfigurations.<name>.config.system.build.tarball"
|
|
echo "and 'pct restore' the result on Proxmox directly. See docs/auto-installer.md."
|
|
|
|
echo "Choose the flake profile to install:"
|
|
select choice in "${options[@]}"; do
|
|
if [[ -n "$choice" ]]; then
|
|
echo "You selected: $choice"
|
|
break
|
|
else
|
|
echo "Invalid selection. Try again."
|
|
fi
|
|
done
|
|
|
|
echo "Starting install with flake: ${FLAKE_BASE_URL}#${choice}"
|
|
|
|
# Optional: confirm before proceeding
|
|
read -rp "Proceed with installation? (y/N): " confirm
|
|
if [[ ! "$confirm" =~ ^[Yy]$ ]]; then
|
|
echo "Aborted."
|
|
exit 1
|
|
fi
|
|
|
|
# A nix-cache host is *the* substituter/remote-builder for every other
|
|
# host once installed (its own config explicitly excludes itself from
|
|
# using either — see buildType != "nix-cache" in the nixos flake.nix).
|
|
# Installing one shouldn't depend on a nix-cache substituter either,
|
|
# for the same reason — plus in practice "nix-cache" only resolves over
|
|
# Tailscale, which a fresh installer environment was never connected to
|
|
# anyway, so it's dead weight even for non-nix-cache installs until
|
|
# that's sorted out. Override it away here specifically for nix-cache
|
|
# targets to keep install-time behaviour consistent with run-time.
|
|
nix_extra_opts=()
|
|
if [[ "${choice}" == *-nix-cache ]]; then
|
|
echo "Installing a nix-cache host — skipping the nix-cache substituter."
|
|
nix_extra_opts+=(--option substituters "https://cache.nixos.org/")
|
|
fi
|
|
|
|
# Every host reachable through this menu has a Disko config (lxc-*
|
|
# is filtered out above, and is the only category that doesn't —
|
|
# see docs/auto-installer.md), so this can run unconditionally: no
|
|
# need to probe the flake first and branch on whether Disko applies.
|
|
disko --mode destroy,format,mount \
|
|
--flake "${FLAKE_BASE_URL}#${choice}" "${nix_extra_opts[@]}" --yes-wipe-all-disks
|
|
|
|
# sops-nix derives this host's decryption key from its own SSH host key
|
|
# at *activation* time, which runs before systemd would otherwise
|
|
# generate one on first boot. Without pre-seeding it here, secrets
|
|
# (including the login password) fail to decrypt on first boot.
|
|
# Generate the key with scripts/secrets/prepare-host-key.sh first.
|
|
#
|
|
# Two places a key can come from, checked in order:
|
|
# /etc/host-keys — baked into this image at build time (see
|
|
# modules/installer/host-keys.nix; only present
|
|
# if built with NIXOS_HOST_KEYS_DIR set)
|
|
# /root/host-keys — scp'd in manually after boot (older fallback,
|
|
# still supported for images built without keys)
|
|
mkdir -p /root/host-keys
|
|
if [[ -f "/etc/host-keys/${choice}_ssh_host_ed25519_key" ]]; then
|
|
echo "Found baked-in SSH host key for ${choice}, installing to target..."
|
|
install -D -m 0600 "/etc/host-keys/${choice}_ssh_host_ed25519_key" /mnt/etc/ssh/ssh_host_ed25519_key
|
|
install -D -m 0644 "/etc/host-keys/${choice}_ssh_host_ed25519_key.pub" /mnt/etc/ssh/ssh_host_ed25519_key.pub
|
|
elif [[ -f "/root/host-keys/${choice}_ssh_host_ed25519_key" ]]; then
|
|
echo "Found pre-seeded SSH host key for ${choice}, installing to target..."
|
|
install -D -m 0600 "/root/host-keys/${choice}_ssh_host_ed25519_key" /mnt/etc/ssh/ssh_host_ed25519_key
|
|
install -D -m 0644 "/root/host-keys/${choice}_ssh_host_ed25519_key.pub" /mnt/etc/ssh/ssh_host_ed25519_key.pub
|
|
else
|
|
# Third place a key can come from: an arbitrary path the operator
|
|
# points at interactively (e.g. a USB stick, a mount from another
|
|
# machine) -- only offered when there's an actual human at the other
|
|
# end of stdin to ask, never in a non-interactive run.
|
|
key_copied=0
|
|
if [[ -t 0 ]]; then
|
|
echo "No SSH host key found for ${choice} (checked /etc/host-keys and /root/host-keys)."
|
|
read -rp "Path to a directory containing ${choice}_ssh_host_ed25519_key(.pub) (blank to skip): " key_src_dir
|
|
if [[ -n "$key_src_dir" && -f "${key_src_dir}/${choice}_ssh_host_ed25519_key" && -f "${key_src_dir}/${choice}_ssh_host_ed25519_key.pub" ]]; then
|
|
cp "${key_src_dir}/${choice}_ssh_host_ed25519_key" "${key_src_dir}/${choice}_ssh_host_ed25519_key.pub" /root/host-keys/
|
|
key_copied=1
|
|
elif [[ -n "$key_src_dir" ]]; then
|
|
echo "WARNING: ${choice}_ssh_host_ed25519_key(.pub) not found in ${key_src_dir}."
|
|
fi
|
|
fi
|
|
|
|
if [[ "$key_copied" -eq 1 ]]; then
|
|
echo "Copied SSH host key for ${choice} from ${key_src_dir}, installing to target..."
|
|
install -D -m 0600 "/root/host-keys/${choice}_ssh_host_ed25519_key" /mnt/etc/ssh/ssh_host_ed25519_key
|
|
install -D -m 0644 "/root/host-keys/${choice}_ssh_host_ed25519_key.pub" /mnt/etc/ssh/ssh_host_ed25519_key.pub
|
|
else
|
|
echo "WARNING: no SSH host key found for ${choice} (checked /etc/host-keys and /root/host-keys)"
|
|
echo "sops-nix secrets (including the login password) will NOT decrypt on first boot."
|
|
echo "Run scripts/secrets/prepare-host-key.sh for host ${choice} on your admin workstation first,"
|
|
echo "then either rebuild this image with NIXOS_HOST_KEYS_DIR set, scp the result to"
|
|
echo "/root/host-keys/ on this machine, or point at it when prompted above."
|
|
read -rp "Continue without a pre-seeded key anyway? (y/N): " skip_key
|
|
if [[ ! "$skip_key" =~ ^[Yy]$ ]]; then
|
|
echo "Aborted."
|
|
exit 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
mkdir -p /mnt/install-tmp
|
|
export TMPDIR=/mnt/install-tmp
|
|
|
|
nixos-install \
|
|
--flake "${FLAKE_BASE_URL}#${choice}" \
|
|
"${nix_extra_opts[@]}" \
|
|
--no-root-password
|
|
|
|
|
|
rm -rf /mnt/install-tmp
|
|
# Redundant copy of the host's private key — the real one is now at
|
|
# /etc/ssh/ssh_host_ed25519_key. Nothing NixOS-managed ever cleans this
|
|
# up on its own since it was written imperatively, not declaratively.
|
|
rm -rf /root/host-keys
|
|
|
|
# disko's --mode ...,mount left any ZFS root pool imported (that's what
|
|
# let nixos-install write into /mnt). If we reboot with it still
|
|
# imported, it isn't just "not exported" -- it's stamped with *this*
|
|
# live installer environment's hostid, which almost never matches the
|
|
# target's own networking.hostId (see hosts/*/host.nix; the installer
|
|
# itself sets none). modules/services/zfs/enable-service.nix and
|
|
# modules/common/configuration.nix both set boot.zfs.forceImportRoot =
|
|
# false deliberately (the safe option per that setting's own docs), so
|
|
# the freshly-installed system's first real boot sees a pool "in use by
|
|
# another system" and refuses to import it without -f -- which is what
|
|
# makes boot stall waiting on the ZFS import. Exporting here (a no-op
|
|
# if the chosen host has no ZFS root, e.g. proxmox-*/linode-*) clears
|
|
# that in-use state so the next import, from any hostid, succeeds.
|
|
#
|
|
# Anything still mounted under /mnt -- nixos-install's own leftover
|
|
# chroot bind mounts for running the target's activation script
|
|
# (/mnt/dev, /mnt/proc, /mnt/sys, /mnt/run), and disko's own /mnt/boot
|
|
# ESP mount (modules/disko/baremetal.nix) -- blocks ZFS from unmounting
|
|
# its root dataset at /mnt, the same way any nested mount blocks
|
|
# unmounting its parent. Confirmed live: zpool export failed with
|
|
# "cannot unmount '/mnt': pool or dataset busy" even after handling the
|
|
# chroot mounts alone, because /mnt/boot was still mounted too. Because
|
|
# of this script's `set -e`, that killed the script before it ever
|
|
# reached reboot, silently defeating the whole point of exporting first.
|
|
# Unmounting everything under /mnt up front (recursively, so nested
|
|
# mounts like /mnt/dev/pts come along for free) sidesteps needing to
|
|
# enumerate every mount disko/nixos-install might leave behind.
|
|
if mountpoint -q /mnt; then
|
|
umount -R /mnt
|
|
fi
|
|
|
|
if [[ -n "$(zpool list -H -o name 2>/dev/null)" ]]; then
|
|
echo "Exporting ZFS pool(s) before reboot..."
|
|
zpool export -a
|
|
fi
|
|
|
|
sleep 10
|
|
reboot
|