Archived
beatzaplentyandClaude Sonnet 5
d973da487c
The real root cause behind the original nix-cache 502, traced all the way through: modules/installer/host-keys.nix (which NIXOS_HOST_KEYS_DIR=... --impure actually wires up) is only ever imported by the installer's own modules/installer/common.nix -- modules/platforms/lxc.nix, which every real lxc-* host build actually uses, never imported anything like it. docs/auto-installer.md previously claimed NIXOS_HOST_KEYS_DIR bakes a key into lxc-* tarballs "the same way it does for the ISO/PXE installer images" -- that was never actually true; I wrote it without verifying the mechanism existed for lxc.nix specifically. In practice this meant every lxc-* container booted with a freshly self-generated SSH host key that could never match whatever .sops.yaml actually trusts for that target, so *every* secret -- not just cache-priv-key -- silently failed to decrypt. No error surfaces in the boot log for this: the activation step that installs secrets only runs on a genuinely fresh first activation and silently no-ops once /run/current-system already exists, so by the time anyone looks the window has closed. Found by manually invoking sops-install-secrets directly: "Error getting data key: 0 successful groups required, got 0". Fixed by giving modules/platforms/lxc.nix the same key-baking mechanism the installer has, but keyed to its own exact flake target and placing the key directly at /etc/ssh/ssh_host_ed25519_key (no copy step to stage for, unlike the installer's /etc/host-keys/ staging area -- an lxc-* tarball has no install step). The target name comes in via specialArgs.flakeTarget (new, set by flake.nix's mkTarget) rather than being read back from config.environment.etc."flake-target" -- reading that back from within a module that also contributes to environment.etc is circular (confirmed: "infinite recursion encountered"). Verified live end-to-end against the real test container (lxc-nix-cache, VMID 100 on pve.sweet.home): destroyed it, rebuilt the tarball fresh with the fix, recreated it, and confirmed /run/secrets/ now has all three secrets this host needs (beszel-token, cache-priv-key, nix-github-token), nix-serve is active (running), and curl http://localhost/nix-cache-info succeeds both directly and through nginx. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01La55Nsss8jZ7ZuzUV9mfot
89 lines
4.0 KiB
Nix
89 lines
4.0 KiB
Nix
{ lib, modulesPath, flakeTarget, ... }:
|
|
|
|
let
|
|
# Bakes this exact flake target's pre-generated SSH host key straight
|
|
# into /etc/ssh/ -- mirrors modules/installer/host-keys.nix's
|
|
# builtins.getEnv pattern (impure and empty under normal `nix
|
|
# build`/`nix eval`, so this is a no-op unless explicitly opted into
|
|
# with NIXOS_HOST_KEYS_DIR=... --impure), but places the key directly
|
|
# rather than staging it under /etc/host-keys/ for a later manual copy
|
|
# -- this is the whole system for a `lxc-*` host, built straight to a
|
|
# pct-restorable tarball with no install step, so there's no later copy
|
|
# step to stage for.
|
|
#
|
|
# Without this, config.system.build.tarball's built-in system just
|
|
# generates a fresh host key at first boot like any other host would --
|
|
# but sops-nix derives its decryption key from *this* file, and
|
|
# .sops.yaml only trusts whatever key scripts/sync-host-keys.sh already
|
|
# registered for this exact target name. A freshly-generated key can
|
|
# never match that, so every secret (including this host's own login)
|
|
# permanently fails to decrypt. Confirmed live: sops-install-secrets
|
|
# errored with "Error getting data key: 0 successful groups required,
|
|
# got 0" -- the container's actual host key's age fingerprint didn't
|
|
# match the one registered in .sops.yaml at all.
|
|
hostKeysDirStr = builtins.getEnv "NIXOS_HOST_KEYS_DIR";
|
|
hasHostKeysDir = hostKeysDirStr != "" && builtins.pathExists hostKeysDirStr;
|
|
hostKeysDir = /. + hostKeysDirStr;
|
|
|
|
# flakeTarget ("${platform}-${buildType}") comes in via specialArgs from
|
|
# flake.nix's mkTarget -- exactly the name scripts/sync-host-keys.sh
|
|
# registers keys under. Deliberately not read back from
|
|
# config.environment.etc."flake-target" (which is set to the same value)
|
|
# -- this module also *contributes* to environment.etc below, and a
|
|
# module reading the merged value of an option it's still defining is a
|
|
# circular dependency (confirmed: "infinite recursion encountered").
|
|
privKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key";
|
|
pubKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key.pub";
|
|
hasKeyForThisTarget =
|
|
hasHostKeysDir
|
|
&& builtins.pathExists privKeyFile
|
|
&& builtins.pathExists pubKeyFile;
|
|
in
|
|
{
|
|
# LXC containers share the host kernel — Proxmox starts them by exec'ing
|
|
# /sbin/init directly, no bootloader/initrd involved — and Proxmox has its
|
|
# own container hostname/network provisioning outside Nix. nixpkgs' own
|
|
# virtualisation/proxmox-lxc.nix module already handles all of this
|
|
# correctly (boot.isContainer, loader.initScript, systemd-networkd) and,
|
|
# critically, provides config.system.build.tarball — a directly
|
|
# `pct restore`-able container image, no nixos-install/bind-mount needed
|
|
# (nixos-install refuses to touch the filesystem it's currently running
|
|
# on, which is exactly what bind-mounting / onto /mnt for an installer
|
|
# LXC container does).
|
|
imports = [
|
|
(modulesPath + "/virtualisation/proxmox-lxc.nix")
|
|
];
|
|
|
|
proxmoxLXC = {
|
|
# host.nix declares each host's real hostname (networking.hostName);
|
|
# keep that instead of letting Proxmox's ambient container config win.
|
|
manageHostName = true;
|
|
# Unprivileged matches how these containers are actually created.
|
|
privileged = false;
|
|
};
|
|
|
|
boot.loader = {
|
|
grub.enable = false;
|
|
systemd-boot.enable = false;
|
|
};
|
|
|
|
# NetworkManager depends on a running udevd to enumerate/classify devices,
|
|
# which boot.isContainer disables (see nixpkgs' container-config.nix) —
|
|
# that's what broke DHCP-hostname registration in Pi-hole. The imported
|
|
# proxmox-lxc.nix module already switches networking to systemd-networkd
|
|
# for the same reason; it just doesn't disable NetworkManager itself,
|
|
# which modules/common/configuration.nix enables for every host.
|
|
networking.networkmanager.enable = lib.mkForce false;
|
|
|
|
environment.etc = lib.mkIf hasKeyForThisTarget {
|
|
"ssh/ssh_host_ed25519_key" = {
|
|
source = privKeyFile;
|
|
mode = "0600";
|
|
};
|
|
"ssh/ssh_host_ed25519_key.pub" = {
|
|
source = pubKeyFile;
|
|
mode = "0644";
|
|
};
|
|
};
|
|
}
|