Archived
Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
997918e2f7 | ||
|
|
2860f750b4 | ||
|
|
781b1d324e | ||
|
|
3014a45936 | ||
|
|
cda2132d6a | ||
|
|
6c1cc821a0 | ||
|
|
4be064572d | ||
|
|
89d506180d | ||
|
|
6e1e992652 | ||
|
|
5467c2e140 | ||
|
|
123cd2b3d7 | ||
|
|
006dd8097a | ||
|
|
18cd6e884e | ||
|
|
3102d66337 | ||
|
|
dfa5452af5 | ||
|
|
852ba2240f | ||
|
|
096dff4fa0 | ||
|
|
b5f749daa9 | ||
|
|
adaf53d647 |
@@ -77,6 +77,14 @@ creation_rules:
|
|||||||
- *admin
|
- *admin
|
||||||
- *lxc-tor-relay
|
- *lxc-tor-relay
|
||||||
|
|
||||||
|
- path_regex: secrets/tailscale-router\.yaml$
|
||||||
|
key_groups:
|
||||||
|
- age:
|
||||||
|
- *admin
|
||||||
|
- *linode-tailscale-router
|
||||||
|
- *lxc-tailscale-router
|
||||||
|
- *proxmox-tailscale-router
|
||||||
|
|
||||||
# gui-host-specific secrets (currently: wifi-password, see
|
# gui-host-specific secrets (currently: wifi-password, see
|
||||||
# modules/networking/wifi.nix). Only *lxc-gui has a registered key today
|
# modules/networking/wifi.nix). Only *lxc-gui has a registered key today
|
||||||
# -- proxmox-gui/linode-gui/baremetal-gui haven't been provisioned via
|
# -- proxmox-gui/linode-gui/baremetal-gui haven't been provisioned via
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ nix eval --json .#nixosConfigurations --apply builtins.attrNames | jq -r '.[]'
|
|||||||
| `modules/nix-cache/` | Binary cache and remote builder client/server modules |
|
| `modules/nix-cache/` | Binary cache and remote builder client/server modules |
|
||||||
| `modules/installer/` | Auto-installer environment (ISO, also served as PXE netboot) — see `docs/auto-installer.md` |
|
| `modules/installer/` | Auto-installer environment (ISO, also served as PXE netboot) — see `docs/auto-installer.md` |
|
||||||
| `host-keys/` | Gitignored; only used by the auto-installer environment for pre-seeding SSH host keys before first boot — see `docs/auto-installer.md`. All deployed hosts use clan vars (`vars/per-machine/<target>/openssh/`) instead |
|
| `host-keys/` | Gitignored; only used by the auto-installer environment for pre-seeding SSH host keys before first boot — see `docs/auto-installer.md`. All deployed hosts use clan vars (`vars/per-machine/<target>/openssh/`) instead |
|
||||||
| `vars/per-machine/` | Clan vars: committed, sops-encrypted SSH host keys for deployed LXC hosts; read by `create-proxmox-resource.sh` at deploy time |
|
| `vars/per-machine/` | Clan vars: committed, sops-encrypted SSH host keys for all deployed hosts; read by `create-proxmox-resource.sh` at deploy time |
|
||||||
| `docs/` | Operational notes for cache, builders, lock updates, boot services, the auto-installer, and Proxmox image builds |
|
| `docs/` | Operational notes for cache, builders, lock updates, boot services, the auto-installer, and Proxmox image builds |
|
||||||
| `scripts/` | Codex setup, validation, host-key, release-bump, and Proxmox resource helpers |
|
| `scripts/` | Codex setup, validation, host-key, release-bump, and Proxmox resource helpers |
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,19 @@
|
|||||||
_:
|
_:
|
||||||
|
|
||||||
{
|
{
|
||||||
|
imports = [
|
||||||
|
(import ../../modules/beszel/host-token.nix {
|
||||||
|
name = "tailscale-router";
|
||||||
|
sopsFile = ../../secrets/tailscale-router.yaml;
|
||||||
|
})
|
||||||
|
];
|
||||||
|
|
||||||
networking.hostName = "tailscale-router";
|
networking.hostName = "tailscale-router";
|
||||||
|
|
||||||
|
services.beszel.agent.environment = {
|
||||||
|
KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
|
||||||
|
};
|
||||||
|
|
||||||
# No networking.hostId: only ZFS-touching hosts (server, docker) need one
|
# No networking.hostId: only ZFS-touching hosts (server, docker) need one
|
||||||
# for pool-import safety, and this host does neither.
|
# for pool-import safety, and this host does neither.
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
{ ... }:
|
{ vars, ... }:
|
||||||
|
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../tailscale/subnet-router.nix
|
../tailscale/subnet-router.nix
|
||||||
|
../beszel/enable-agent.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
# "server", not "both": this build type advertises LAN subnet routes but
|
# "server", not "both": this build type advertises LAN subnet routes but
|
||||||
@@ -12,8 +13,33 @@
|
|||||||
# own setting) so the intent is clear at the build-type level.
|
# own setting) so the intent is clear at the build-type level.
|
||||||
services.tailscale.useRoutingFeatures = "server";
|
services.tailscale.useRoutingFeatures = "server";
|
||||||
|
|
||||||
# Forwarded subnet-router traffic arrives on tailscale0 already
|
# Advertise the LAN subnet so Tailscale peers can route back to LAN machines.
|
||||||
# tailscale-authenticated -- the firewall's normal per-port allow-list
|
# Must also be approved in the Tailscale admin console (Machines → Edit route settings).
|
||||||
# would otherwise drop it. Standard NixOS/Tailscale subnet-router guidance.
|
services.tailscale.extraUpFlags = [ "--advertise-routes=${vars.lanCidr}" ];
|
||||||
networking.firewall.trustedInterfaces = [ "tailscale0" ];
|
|
||||||
|
networking = {
|
||||||
|
# SNAT traffic from LAN machines going out through Tailscale so the remote
|
||||||
|
# peer sees it sourced from this router's Tailscale IP (100.x.x.x) rather
|
||||||
|
# than a raw LAN IP. Without this, Tailscale drops the forwarded packets
|
||||||
|
# because the source is not a recognised Tailscale address.
|
||||||
|
#
|
||||||
|
# networking.nat.externalInterface alone does not insert a MASQUERADE rule
|
||||||
|
# (it only does so when internalInterfaces is also set). We use
|
||||||
|
# extraCommands to add the rule into the nixos-nat-post chain that
|
||||||
|
# networking.nat.enable creates, and extraStopCommands to clean it up.
|
||||||
|
nat.enable = true;
|
||||||
|
|
||||||
|
firewall = {
|
||||||
|
# Forwarded subnet-router traffic arrives on tailscale0 already
|
||||||
|
# tailscale-authenticated -- the firewall's normal per-port allow-list
|
||||||
|
# would otherwise drop it. Standard NixOS/Tailscale subnet-router guidance.
|
||||||
|
trustedInterfaces = [ "tailscale0" ];
|
||||||
|
extraCommands = ''
|
||||||
|
iptables -t nat -A nixos-nat-post -s ${vars.lanCidr} -o tailscale0 -j MASQUERADE
|
||||||
|
'';
|
||||||
|
extraStopCommands = ''
|
||||||
|
iptables -t nat -D nixos-nat-post -s ${vars.lanCidr} -o tailscale0 -j MASQUERADE 2>/dev/null || true
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -61,24 +61,32 @@
|
|||||||
!include ${config.sops.templates."nix-github-token.conf".path}
|
!include ${config.sops.templates."nix-github-token.conf".path}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
#Set root password
|
users = {
|
||||||
users.users.root = {
|
# With mutableUsers = false, update-users-groups.pl enforces hashedPasswordFile
|
||||||
hashedPasswordFile = config.sops.secrets."root-hashedPassword".path;
|
# on every activation regardless of whether the account already exists in
|
||||||
};
|
# /etc/shadow. The default (true) only applies hashedPasswordFile to newly-
|
||||||
|
# created accounts — which means a freshly-built proxmox disk image (where
|
||||||
|
# activation runs without a usable sops key, so both accounts land in shadow
|
||||||
|
# with ‘!’) will never have its passwords fixed by subsequent boots.
|
||||||
|
mutableUsers = false;
|
||||||
|
|
||||||
# Define a user account. Don't forget to set a password with ‘passwd’.
|
users.root = {
|
||||||
users.users.${vars.primaryUser} = {
|
hashedPasswordFile = config.sops.secrets."root-hashedPassword".path;
|
||||||
isNormalUser = true;
|
};
|
||||||
extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user.
|
|
||||||
packages = with pkgs; [
|
users.${vars.primaryUser} = {
|
||||||
tree
|
isNormalUser = true;
|
||||||
];
|
extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user.
|
||||||
hashedPasswordFile = config.sops.secrets."nixos-hashedPassword".path;
|
packages = with pkgs; [
|
||||||
openssh.authorizedKeys.keys = [
|
tree
|
||||||
vars.adminSshKey
|
];
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface"
|
hashedPasswordFile = config.sops.secrets."nixos-hashedPassword".path;
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
|
openssh.authorizedKeys.keys = [
|
||||||
];
|
vars.adminSshKey
|
||||||
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface"
|
||||||
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
|
||||||
|
];
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,81 @@
|
|||||||
{ ... }:
|
{ lib, flakeTarget, ... }:
|
||||||
|
|
||||||
|
let
|
||||||
|
# Bakes this exact flake target's pre-generated SSH host key straight
|
||||||
|
# into /etc/ssh/ -- mirrors lxc.nix's builtins.getEnv pattern (impure
|
||||||
|
# and empty under normal `nix build`/`nix eval`, so this is a no-op
|
||||||
|
# unless explicitly opted into with NIXOS_HOST_KEYS_DIR=... --impure).
|
||||||
|
#
|
||||||
|
# Unlike --pre-format-files (which places files on the QEMU builder VM's
|
||||||
|
# rootfs, not the target disk), embedding via environment.etc here means
|
||||||
|
# nixos-install's own activation step installs the key onto the target
|
||||||
|
# disk. sshd-keygen then finds it already present and skips generation,
|
||||||
|
# so the disk image boots with the clan-registered key and sops can
|
||||||
|
# decrypt on first boot.
|
||||||
|
#
|
||||||
|
# Without this, nixos-install's sshd-keygen activation generates a fresh
|
||||||
|
# key (unregistered in .sops.yaml), sops decryption fails permanently,
|
||||||
|
# and password hashes are never applied -- confirmed live: passwords
|
||||||
|
# stayed '!' even with mutableUsers = false because hashedPasswordFile
|
||||||
|
# pointed to a path that sops never wrote.
|
||||||
|
hostKeysDirStr = builtins.getEnv "NIXOS_HOST_KEYS_DIR";
|
||||||
|
hasHostKeysDir = hostKeysDirStr != "" && builtins.pathExists hostKeysDirStr;
|
||||||
|
hostKeysDir = /. + hostKeysDirStr;
|
||||||
|
|
||||||
|
privKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key";
|
||||||
|
pubKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key.pub";
|
||||||
|
hasKeyForThisTarget =
|
||||||
|
hasHostKeysDir
|
||||||
|
&& builtins.pathExists privKeyFile
|
||||||
|
&& builtins.pathExists pubKeyFile;
|
||||||
|
in
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../hardware-configuration/vm/proxmox.nix
|
../hardware-configuration/vm/proxmox.nix
|
||||||
../boot/efi.nix
|
../boot/efi.nix
|
||||||
../disko/proxmox.nix
|
../disko/proxmox.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
|
environment.etc = lib.mkIf hasKeyForThisTarget {
|
||||||
|
"ssh/ssh_host_ed25519_key" = {
|
||||||
|
source = privKeyFile;
|
||||||
|
mode = "0600";
|
||||||
|
};
|
||||||
|
"ssh/ssh_host_ed25519_key.pub" = {
|
||||||
|
source = pubKeyFile;
|
||||||
|
mode = "0644";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# NixOS's etc activation removes any /etc file that was in the previous
|
||||||
|
# generation's environment.etc but is absent from the current one. Since
|
||||||
|
# the SSH key is only in environment.etc during the --impure build (when
|
||||||
|
# NIXOS_HOST_KEYS_DIR is set), normal rebuilds would remove it as
|
||||||
|
# "obsolete". These scripts mirror lxc.nix's approach: save the live key
|
||||||
|
# before etc runs, restore it after. Without the explicit deps, the
|
||||||
|
# topological sort places preserveSshHostKey after etc (confirmed live on
|
||||||
|
# lxc-tor-relay: position 7 vs etc's position 5), so the key is gone
|
||||||
|
# before it can be saved.
|
||||||
|
system.activationScripts = {
|
||||||
|
preserveSshHostKey = ''
|
||||||
|
if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
|
||||||
|
cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
|
||||||
|
cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
|
||||||
|
fi
|
||||||
|
'';
|
||||||
|
|
||||||
|
restoreSshHostKey = {
|
||||||
|
deps = [ "etc" ];
|
||||||
|
text = ''
|
||||||
|
if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
|
||||||
|
install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
|
||||||
|
install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
|
||||||
|
fi
|
||||||
|
rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
etc = { deps = [ "preserveSshHostKey" ]; };
|
||||||
|
setupSecrets = { deps = [ "restoreSshHostKey" ]; };
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
_:
|
{ pkgs, ... }:
|
||||||
|
|
||||||
{
|
{
|
||||||
imports = [ ./enable-service.nix ];
|
imports = [ ./enable-service.nix ];
|
||||||
@@ -12,4 +12,24 @@ _:
|
|||||||
# instead of relaying through DERP.
|
# instead of relaying through DERP.
|
||||||
openFirewall = true;
|
openFirewall = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# Tailscale recommends these ethtool flags on the uplink interface to get
|
||||||
|
# full UDP GRO throughput on subnet routers (https://tailscale.com/s/ethtool-config-udp-gro).
|
||||||
|
# The interface is derived from the default route so it works regardless of
|
||||||
|
# what the NIC is named on a given host.
|
||||||
|
systemd.services.tailscale-udp-gro = {
|
||||||
|
description = "Enable UDP GRO forwarding on uplink for Tailscale subnet router";
|
||||||
|
after = [ "network-online.target" ];
|
||||||
|
wants = [ "network-online.target" ];
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
path = [ pkgs.ethtool pkgs.iproute2 ];
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "oneshot";
|
||||||
|
RemainAfterExit = true;
|
||||||
|
ExecStart = pkgs.writeShellScript "tailscale-udp-gro" ''
|
||||||
|
NETDEV=$(ip -o route get 8.8.8.8 | cut -f 5 -d " ")
|
||||||
|
ethtool -K "$NETDEV" rx-udp-gro-forwarding on rx-gro-list off
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -302,6 +302,12 @@ platform_prefix="lxc"
|
|||||||
[[ -z "$cores" ]] && cores="$PROXMOX_DEFAULT_CORES"
|
[[ -z "$cores" ]] && cores="$PROXMOX_DEFAULT_CORES"
|
||||||
[[ -z "$memory" ]] && memory="$PROXMOX_DEFAULT_MEMORY_MB"
|
[[ -z "$memory" ]] && memory="$PROXMOX_DEFAULT_MEMORY_MB"
|
||||||
|
|
||||||
|
if [[ "$type" == "vm" && -n "$disk_size" ]]; then
|
||||||
|
echo "WARNING: --disk-size is LXC-only for create mode and is ignored for VMs." >&2
|
||||||
|
echo " VM disk size comes from proxmoxImageSize in variables.nix (currently ${disk_size}G was requested)." >&2
|
||||||
|
echo " To expand after creation, use: --modify --vmid <n> --grow-disk <GB>" >&2
|
||||||
|
fi
|
||||||
|
|
||||||
# --- discover / resolve the flake target from --host --------------------
|
# --- discover / resolve the flake target from --host --------------------
|
||||||
# Emits "<target>\t<hostName>" pairs for every ${platform_prefix}-* flake
|
# Emits "<target>\t<hostName>" pairs for every ${platform_prefix}-* flake
|
||||||
# target -- the one source both --list and the --host lookup below read
|
# target -- the one source both --list and the --host lookup below read
|
||||||
@@ -354,6 +360,14 @@ fi
|
|||||||
# feeds straight into the guest's real hostname) disagree with host.nix.
|
# feeds straight into the guest's real hostname) disagree with host.nix.
|
||||||
[[ -z "$name" ]] && name="$host"
|
[[ -z "$name" ]] && name="$host"
|
||||||
|
|
||||||
|
# For VM builds: the diskoImagesScript (run via QEMU on the node) writes the
|
||||||
|
# raw disk image as <hostname>.raw into the CWD it was called from (the remote
|
||||||
|
# repo dir), not to /var/lib/vz/import/ or anywhere else. Import directly from
|
||||||
|
# there -- no intermediate mv that can fail crossing filesystem boundaries or
|
||||||
|
# leave a stale file on error.
|
||||||
|
vm_built_raw=""
|
||||||
|
[[ "$type" == "vm" ]] && vm_built_raw="${remote_repo_dir}/${host}.raw"
|
||||||
|
|
||||||
# --- refuse to duplicate a host that's already live on the node ---------
|
# --- refuse to duplicate a host that's already live on the node ---------
|
||||||
# Queries the node itself (qm/pct's own name/hostname config), not any
|
# Queries the node itself (qm/pct's own name/hostname config), not any
|
||||||
# static list in this repo -- a file can't track whether a resource still
|
# static list in this repo -- a file can't track whether a resource still
|
||||||
@@ -688,10 +702,14 @@ if [[ -n "$image" ]]; then
|
|||||||
elif [[ "$force_rebuild" -eq 1 ]]; then
|
elif [[ "$force_rebuild" -eq 1 ]]; then
|
||||||
echo "--force-rebuild: skipping the existing-image check on ${node}."
|
echo "--force-rebuild: skipping the existing-image check on ${node}."
|
||||||
else
|
else
|
||||||
echo "==> Checking whether ${node} already has ${remote_path}..."
|
# VMs: check for the raw image in the remote repo dir (where disko writes it).
|
||||||
|
# LXC: check for the tarball in iso_storage (where the LXC build stages it).
|
||||||
|
_check_path="$remote_path"
|
||||||
|
[[ "$type" == "vm" ]] && _check_path="$vm_built_raw"
|
||||||
|
echo "==> Checking whether ${node} already has ${_check_path}..."
|
||||||
if [[ "$dry_run" -eq 1 ]]; then
|
if [[ "$dry_run" -eq 1 ]]; then
|
||||||
echo "[dry-run] would check: ssh ${ssh_target} -- test -f ${remote_path}"
|
echo "[dry-run] would check: ssh ${ssh_target} -- test -f ${_check_path}"
|
||||||
elif ssh "$ssh_target" "test -f '${remote_path}'" 2>/dev/null; then
|
elif ssh "$ssh_target" "test -f '${_check_path}'" 2>/dev/null; then
|
||||||
echo "Found it -- reusing, skipping build (use --force-rebuild to override)."
|
echo "Found it -- reusing, skipping build (use --force-rebuild to override)."
|
||||||
image_already_remote=1
|
image_already_remote=1
|
||||||
else
|
else
|
||||||
@@ -766,23 +784,29 @@ REMOTE_SCRIPT
|
|||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
if [[ "$dry_run" -eq 1 ]]; then
|
if [[ "$dry_run" -eq 1 ]]; then
|
||||||
echo "[dry-run] would build on ${node}: nix build --no-use-registries --no-accept-flake-config${nix_opts_display} \\"
|
echo "[dry-run] would build on ${node}: NIXOS_HOST_KEYS_DIR=\$(pwd)/host-keys nix build --impure --no-use-registries --no-accept-flake-config${nix_opts_display} \\"
|
||||||
echo "[dry-run] .#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript"
|
echo "[dry-run] .#nixosConfigurations.${flake_target}.config.system.build.diskoImagesScript"
|
||||||
echo "[dry-run] would run: ${sudo_display}./result-${flake_target} \\"
|
echo "[dry-run] would run: ${sudo_display}./result-${flake_target} --build-memory 2048"
|
||||||
echo "[dry-run] --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key \\"
|
echo "[dry-run] image will be at ${vm_built_raw} (imported from there; no mv to /var/lib/vz/import/)"
|
||||||
echo "[dry-run] --pre-format-files host-keys/${flake_target}_ssh_host_ed25519_key.pub /etc/ssh/ssh_host_ed25519_key.pub \\"
|
|
||||||
echo "[dry-run] --build-memory 2048"
|
|
||||||
echo "[dry-run] would stage the result at ${remote_path}"
|
|
||||||
local_image="<built-image>.raw"
|
local_image="<built-image>.raw"
|
||||||
else
|
else
|
||||||
echo "==> Building Disko image for ${flake_target} on ${node}..."
|
echo "==> Building Disko image for ${flake_target} on ${node}..."
|
||||||
# See the LXC branch above for why this is one %q-quoted command
|
# See the LXC branch above for why this is one %q-quoted command
|
||||||
# string rather than separate ssh argv elements.
|
# string rather than separate ssh argv elements.
|
||||||
printf -v remote_cmd 'bash -s -- %q %q %q %q %q %q' \
|
# $7 = image_name (hostname, the diskoImagesScript's own output filename).
|
||||||
"$remote_repo_dir" "$flake_target" "$remote_dir" "$remote_filename" "$NIX_EXTRA_OPTS" "$sudo_prefix"
|
#
|
||||||
|
# NIXOS_HOST_KEYS_DIR + --impure: modules/platforms/proxmox.nix reads
|
||||||
|
# this env var at eval time (like lxc.nix) to embed the clan SSH host
|
||||||
|
# key in environment.etc. nixos-install's own activation then places the
|
||||||
|
# key on the target disk, so sshd-keygen finds it already present and
|
||||||
|
# skips generation. --pre-format-files put the key on the QEMU builder
|
||||||
|
# VM's rootfs (not the target disk), so sshd-keygen regenerated a fresh
|
||||||
|
# key -- one not registered in .sops.yaml -- and sops could never decrypt.
|
||||||
|
printf -v remote_cmd 'bash -s -- %q %q %q %q %q %q %q' \
|
||||||
|
"$remote_repo_dir" "$flake_target" "$remote_dir" "$remote_filename" "$NIX_EXTRA_OPTS" "$sudo_prefix" "$host"
|
||||||
ssh "$ssh_target" "$remote_cmd" <<'REMOTE_SCRIPT'
|
ssh "$ssh_target" "$remote_cmd" <<'REMOTE_SCRIPT'
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
repo_dir="$1"; target="$2"; dest_dir="$3"; dest_name="$4"; nix_extra_opts_str="$5"; sudo_pfx="$6"
|
repo_dir="$1"; target="$2"; dest_dir="$3"; dest_name="$4"; nix_extra_opts_str="$5"; sudo_pfx="$6"; image_name="$7"
|
||||||
declare -a NIX_OPTS=()
|
declare -a NIX_OPTS=()
|
||||||
[[ -n "$nix_extra_opts_str" ]] && eval "NIX_OPTS=(${nix_extra_opts_str})"
|
[[ -n "$nix_extra_opts_str" ]] && eval "NIX_OPTS=(${nix_extra_opts_str})"
|
||||||
cd "$repo_dir"
|
cd "$repo_dir"
|
||||||
@@ -794,24 +818,26 @@ if [[ ! -f "host-keys/${target}_ssh_host_ed25519_key" ]]; then
|
|||||||
echo "and ensure it was synced here before starting the build." >&2
|
echo "and ensure it was synced here before starting the build." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
nix build --no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \
|
# Build diskoImagesScript with NIXOS_HOST_KEYS_DIR so proxmox.nix embeds the
|
||||||
|
# clan SSH key in environment.etc (same as lxc.nix). This causes nixos-install
|
||||||
|
# to place the key on the target disk, so sshd-keygen finds it and skips
|
||||||
|
# generation -- the disk image boots with the registered key, sops decrypts.
|
||||||
|
NIXOS_HOST_KEYS_DIR="$(pwd)/host-keys" nix build --impure \
|
||||||
|
--no-use-registries --no-accept-flake-config "${NIX_OPTS[@]}" \
|
||||||
".#nixosConfigurations.${target}.config.system.build.diskoImagesScript" \
|
".#nixosConfigurations.${target}.config.system.build.diskoImagesScript" \
|
||||||
--out-link "result-${target}"
|
--out-link "result-${target}"
|
||||||
$sudo_pfx "./result-${target}" \
|
# Remove any stale .raw from a previous failed build so the post-build check
|
||||||
--pre-format-files "$(pwd)/host-keys/${target}_ssh_host_ed25519_key" /etc/ssh/ssh_host_ed25519_key \
|
# below is unambiguous (diskoImagesScript writes to CWD as ${image_name}.raw).
|
||||||
--pre-format-files "$(pwd)/host-keys/${target}_ssh_host_ed25519_key.pub" /etc/ssh/ssh_host_ed25519_key.pub \
|
$sudo_pfx rm -f "${image_name}.raw" 2>/dev/null || true
|
||||||
--build-memory 2048
|
$sudo_pfx "./result-${target}" --build-memory 2048
|
||||||
built="$(find . -maxdepth 1 -name '*.raw' -newer "result-${target}" | head -1)"
|
if [[ ! -f "${image_name}.raw" ]]; then
|
||||||
if [[ -z "$built" ]]; then
|
echo "ERROR: ${image_name}.raw not found in ${repo_dir} after build -- disko/QEMU may have failed." >&2
|
||||||
echo "ERROR: no .raw image found in ${repo_dir} after build." >&2
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
$sudo_pfx mkdir -p "$dest_dir"
|
echo "Built image: ${repo_dir}/${image_name}.raw"
|
||||||
$sudo_pfx mv "$built" "${dest_dir}/${dest_name}"
|
|
||||||
echo "Built and staged: ${dest_dir}/${dest_name}"
|
|
||||||
REMOTE_SCRIPT
|
REMOTE_SCRIPT
|
||||||
local_image="$remote_path"
|
local_image="$vm_built_raw"
|
||||||
echo "Built on ${node}: ${remote_path}"
|
echo "Built on ${node}: ${vm_built_raw}"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
@@ -889,14 +915,35 @@ else
|
|||||||
--net0 virtio,bridge=${bridge} --bios ovmf --machine q35 --scsihw virtio-scsi-pci \
|
--net0 virtio,bridge=${bridge} --bios ovmf --machine q35 --scsihw virtio-scsi-pci \
|
||||||
--efidisk0 ${storage}:1,efitype=4m,pre-enrolled-keys=0 --agent enabled=1"
|
--efidisk0 ${storage}:1,efitype=4m,pre-enrolled-keys=0 --agent enabled=1"
|
||||||
|
|
||||||
|
# VMs built on the node: import from the repo dir (where disko/QEMU wrote it).
|
||||||
|
# VMs from --image: import from remote_path (where scp uploaded it).
|
||||||
|
_import_path="${remote_path}"
|
||||||
|
[[ -z "$image" ]] && _import_path="${vm_built_raw}"
|
||||||
if [[ "$dry_run" -eq 1 ]]; then
|
if [[ "$dry_run" -eq 1 ]]; then
|
||||||
echo "[dry-run] ssh ${ssh_target} -- ${sudo_display}qm importdisk ${vmid} ${remote_path} ${storage}"
|
echo "[dry-run] ssh ${ssh_target} -- ${sudo_display}qm importdisk ${vmid} ${_import_path} ${storage}"
|
||||||
echo "[dry-run] (would parse the resulting disk identifier from that output)"
|
echo "[dry-run] (would parse the resulting disk identifier from that output)"
|
||||||
echo "[dry-run] ssh ${ssh_target} -- ${sudo_display}qm set ${vmid} --scsi0 ${storage}:<parsed-disk-id>"
|
echo "[dry-run] ssh ${ssh_target} -- ${sudo_display}qm set ${vmid} --scsi0 ${storage}:<parsed-disk-id>"
|
||||||
else
|
else
|
||||||
importdisk_output="$(ssh "$ssh_target" "${sudo_prefix} qm importdisk ${vmid} ${remote_path} ${storage}")"
|
if ! importdisk_output="$(ssh "$ssh_target" "${sudo_prefix} qm importdisk ${vmid} ${_import_path} ${storage}" 2>&1)"; then
|
||||||
|
echo "ERROR: qm importdisk failed:" >&2
|
||||||
|
echo "${importdisk_output}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "$importdisk_output"
|
echo "$importdisk_output"
|
||||||
disk_id="$(echo "$importdisk_output" | grep -oP "(?<=Successfully imported disk as ')[^']+" | sed 's/^unused[0-9]*://')"
|
# PVE output format: "unusedN: successfully imported disk '<storage>:<vol>'"
|
||||||
|
# (lowercase "successfully", no "as"; the primary regex targets this form; the
|
||||||
|
# || true inside the substitution prevents set -e from aborting when grep finds
|
||||||
|
# no match -- without it the script would silently exit before reaching the
|
||||||
|
# fallback whenever the PVE format doesn't match).
|
||||||
|
disk_id="$(echo "$importdisk_output" | grep -oP "successfully imported disk '\\K[^']+" || true)"
|
||||||
|
if [[ -z "$disk_id" ]]; then
|
||||||
|
# Fallback for other PVE output variants: read qm config directly.
|
||||||
|
unused_line="$(ssh "$ssh_target" "${sudo_prefix} qm config ${vmid}" | grep '^unused[0-9]*:' | head -1 || true)"
|
||||||
|
if [[ -n "$unused_line" ]]; then
|
||||||
|
disk_id="${unused_line#*: }"
|
||||||
|
echo "Note: disk ID resolved from qm config: ${disk_id}"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
if [[ -z "$disk_id" ]]; then
|
if [[ -z "$disk_id" ]]; then
|
||||||
echo "ERROR: couldn't parse the imported disk identifier from qm importdisk's output above." >&2
|
echo "ERROR: couldn't parse the imported disk identifier from qm importdisk's output above." >&2
|
||||||
echo "The VM shell (${vmid}) and imported disk both exist -- finish attaching it by hand:" >&2
|
echo "The VM shell (${vmid}) and imported disk both exist -- finish attaching it by hand:" >&2
|
||||||
@@ -905,6 +952,12 @@ else
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
remote "${sudo_prefix} qm set ${vmid} --scsi0 ${disk_id}"
|
remote "${sudo_prefix} qm set ${vmid} --scsi0 ${disk_id}"
|
||||||
|
# The disk data is now in ZFS; remove the source raw file (only for images
|
||||||
|
# we built on the node -- --image uploads are the operator's to manage).
|
||||||
|
if [[ -z "$image" ]]; then
|
||||||
|
ssh "$ssh_target" "${sudo_prefix} rm -f '${_import_path}'" 2>/dev/null || \
|
||||||
|
echo "Warning: couldn't remove ${_import_path} from ${node} -- you can delete it manually" >&2
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
remote "${sudo_prefix} qm set ${vmid} --boot order=scsi0"
|
remote "${sudo_prefix} qm set ${vmid} --boot order=scsi0"
|
||||||
remote "${sudo_prefix} qm start ${vmid}"
|
remote "${sudo_prefix} qm start ${vmid}"
|
||||||
|
|||||||
@@ -11,17 +11,16 @@
|
|||||||
# sync-host-keys.sh --regenerate-all-keys Remove and freshly regenerate
|
# sync-host-keys.sh --regenerate-all-keys Remove and freshly regenerate
|
||||||
# every locally-managed key.
|
# every locally-managed key.
|
||||||
#
|
#
|
||||||
# "Generate/register" is idempotent and additive only: an existing
|
# "Generate/register" is idempotent and additive only: an existing clan
|
||||||
# host-keys/ file is never touched, and .sops.yaml only ever gains an
|
# var is never overwritten, and .sops.yaml only ever gains an anchor/alias
|
||||||
# anchor/alias it doesn't already have -- safe to re-run any time, e.g.
|
# it doesn't already have -- safe to re-run any time, e.g. right after
|
||||||
# right after adding a new host to flake.nix.
|
# adding a new host to flake.nix.
|
||||||
#
|
#
|
||||||
# --remove and --regenerate-all-keys only ever operate on anchors that have
|
# --remove and --regenerate-all-keys only ever operate on anchors that
|
||||||
# a corresponding host-keys/<name>_ssh_host_ed25519_key file. Anchors
|
# have a corresponding clan var (vars/per-machine/<name>/openssh/) or
|
||||||
# without one (&admin, and any anchor for an already-deployed host whose
|
# host-keys/ file. Anchors without either (&admin) are never listed,
|
||||||
# real /etc/ssh key was registered by hand, e.g. &docker/&server/&nix-cache
|
# removed, or regenerated -- this tooling only ever touches keys it itself
|
||||||
# today) are never listed, removed, or regenerated -- this tooling only
|
# manages.
|
||||||
# ever touches keys it itself manages.
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
|
repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
@@ -56,13 +55,14 @@ Usage: $0 --all [--dry-run]
|
|||||||
<flake-target> Same, for just one target (e.g. lxc-server).
|
<flake-target> Same, for just one target (e.g. lxc-server).
|
||||||
Reports if it already has one.
|
Reports if it already has one.
|
||||||
--remove Interactively pick one locally-managed key to
|
--remove Interactively pick one locally-managed key to
|
||||||
remove from .sops.yaml and host-keys/.
|
remove from .sops.yaml and vars/per-machine/
|
||||||
|
(or host-keys/ for legacy keys).
|
||||||
--regenerate-all-keys Remove every locally-managed key and generate
|
--regenerate-all-keys Remove every locally-managed key and generate
|
||||||
fresh replacements for every current flake
|
fresh clan-var replacements for every current
|
||||||
target. Destructive -- requires typed
|
flake target. Destructive -- requires typed
|
||||||
confirmation.
|
confirmation.
|
||||||
--dry-run Combine with any of the above: print what would
|
--dry-run Combine with any of the above: print what would
|
||||||
change (host-keys/ files, .sops.yaml anchors and
|
change (clan vars, .sops.yaml anchors and
|
||||||
key_groups, which secrets/*.yaml would be
|
key_groups, which secrets/*.yaml would be
|
||||||
re-encrypted) without touching anything. No keys
|
re-encrypted) without touching anything. No keys
|
||||||
generated, no files written, no sops calls,
|
generated, no files written, no sops calls,
|
||||||
@@ -315,7 +315,7 @@ cmd_remove() {
|
|||||||
local hosts
|
local hosts
|
||||||
mapfile -t hosts < <(locally_managed_hosts)
|
mapfile -t hosts < <(locally_managed_hosts)
|
||||||
if [[ "${#hosts[@]}" -eq 0 ]]; then
|
if [[ "${#hosts[@]}" -eq 0 ]]; then
|
||||||
echo "No locally-managed keys in host-keys/ -- nothing to remove."
|
echo "No locally-managed keys found (checked host-keys/ and vars/per-machine/) -- nothing to remove."
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -372,7 +372,7 @@ cmd_regenerate_all() {
|
|||||||
local hosts
|
local hosts
|
||||||
mapfile -t hosts < <(locally_managed_hosts)
|
mapfile -t hosts < <(locally_managed_hosts)
|
||||||
if [[ "${#hosts[@]}" -eq 0 ]]; then
|
if [[ "${#hosts[@]}" -eq 0 ]]; then
|
||||||
echo "No locally-managed keys in host-keys/ -- nothing to regenerate."
|
echo "No locally-managed keys found (checked host-keys/ and vars/per-machine/) -- nothing to regenerate."
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -166,7 +166,7 @@
|
|||||||
# build (modules/disko/proxmox.nix, config.system.build.diskoImagesScript
|
# build (modules/disko/proxmox.nix, config.system.build.diskoImagesScript
|
||||||
# — see docs/proxmox-images.md). Root fills whatever's left after the ESP
|
# — see docs/proxmox-images.md). Root fills whatever's left after the ESP
|
||||||
# and swap partitions within this total.
|
# and swap partitions within this total.
|
||||||
proxmoxImageSize = "20G";
|
proxmoxImageSize = "50G";
|
||||||
|
|
||||||
# nix-cache's Nix store garbage collection retention
|
# nix-cache's Nix store garbage collection retention
|
||||||
# (modules/nix-cache/server.nix).
|
# (modules/nix-cache/server.nix).
|
||||||
|
|||||||
Reference in New Issue
Block a user