From d973da487c99bdf99b684099223b24e2a24bf1a9 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 20 Jul 2026 12:29:01 +1000 Subject: [PATCH] Fix lxc-* hosts having no host-key pre-seeding mechanism at all The real root cause behind the original nix-cache 502, traced all the way through: modules/installer/host-keys.nix (which NIXOS_HOST_KEYS_DIR=... --impure actually wires up) is only ever imported by the installer's own modules/installer/common.nix -- modules/platforms/lxc.nix, which every real lxc-* host build actually uses, never imported anything like it. docs/auto-installer.md previously claimed NIXOS_HOST_KEYS_DIR bakes a key into lxc-* tarballs "the same way it does for the ISO/PXE installer images" -- that was never actually true; I wrote it without verifying the mechanism existed for lxc.nix specifically. In practice this meant every lxc-* container booted with a freshly self-generated SSH host key that could never match whatever .sops.yaml actually trusts for that target, so *every* secret -- not just cache-priv-key -- silently failed to decrypt. No error surfaces in the boot log for this: the activation step that installs secrets only runs on a genuinely fresh first activation and silently no-ops once /run/current-system already exists, so by the time anyone looks the window has closed. Found by manually invoking sops-install-secrets directly: "Error getting data key: 0 successful groups required, got 0". Fixed by giving modules/platforms/lxc.nix the same key-baking mechanism the installer has, but keyed to its own exact flake target and placing the key directly at /etc/ssh/ssh_host_ed25519_key (no copy step to stage for, unlike the installer's /etc/host-keys/ staging area -- an lxc-* tarball has no install step). The target name comes in via specialArgs.flakeTarget (new, set by flake.nix's mkTarget) rather than being read back from config.environment.etc."flake-target" -- reading that back from within a module that also contributes to environment.etc is circular (confirmed: "infinite recursion encountered"). Verified live end-to-end against the real test container (lxc-nix-cache, VMID 100 on pve.sweet.home): destroyed it, rebuilt the tarball fresh with the fix, recreated it, and confirmed /run/secrets/ now has all three secrets this host needs (beszel-token, cache-priv-key, nix-github-token), nix-serve is active (running), and curl http://localhost/nix-cache-info succeeds both directly and through nginx. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01La55Nsss8jZ7ZuzUV9mfot --- docs/auto-installer.md | 32 ++++++++++++++++++++---- flake.nix | 13 ++++++++-- modules/platforms/lxc.nix | 52 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 8 deletions(-) diff --git a/docs/auto-installer.md b/docs/auto-installer.md index 19a4cb9..1ced5b7 100644 --- a/docs/auto-installer.md +++ b/docs/auto-installer.md @@ -77,11 +77,33 @@ system profile) — there's no separate activation step to run yourself. of this (build, host-key handling, upload, `pct create` with the flags above) — see its `--help`. -Host keys still need pre-seeding the same way as any other host (see "Host -keys" below) — the sops-nix activation-vs-first-boot race is identical -regardless of how the image reaches the machine. `NIXOS_HOST_KEYS_DIR=... -nix build ... --impure` bakes the matching key into the tarball the same way -it does for the ISO/PXE installer images. +Host keys still need pre-seeding the same way as any other host — the +sops-nix activation-vs-first-boot race is identical regardless of how the +image reaches the machine. Unlike the ISO/PXE installer (where +`modules/installer/host-keys.nix` bakes *every* `host-keys/` entry into +`/etc/host-keys/` for `auto-install.sh` to pick from and copy at install +time — see "Host keys" below), an `lxc-*` tarball has no install step to +copy anything during, so `modules/platforms/lxc.nix` bakes this *one* +target's key straight into `/etc/ssh/ssh_host_ed25519_key(.pub)` directly, +keyed by its own exact flake target name (`config.environment.etc` can't +be read back from within a module still contributing to it, so this comes +in via `specialArgs.flakeTarget`, set by `flake.nix`'s `mkTarget`): + +```sh +NIXOS_HOST_KEYS_DIR="$(pwd)/host-keys" \ + nix build .#nixosConfigurations.lxc-nix-cache.config.system.build.tarball --impure +``` + +Confirmed the hard way: without this, the tarball's own built-in system +just generates a fresh host key at first boot like any host would, which +can never match whatever `.sops.yaml` actually trusts for that target — +`sops-install-secrets` fails with `Error getting data key: 0 successful +groups required, got 0`, and *every* secret (including this host's own +login) permanently fails to decrypt, silently — no error in the boot log +at all, since the activation step that would install secrets only runs on +a from-scratch first activation and skips silently once `/run/current-system` +already exists. `scripts/create-proxmox-resource.sh` always builds with +`NIXOS_HOST_KEYS_DIR` set for this reason. ## Layout diff --git a/flake.nix b/flake.nix index 9a0c8af..f4beec8 100644 --- a/flake.nix +++ b/flake.nix @@ -33,6 +33,9 @@ # nix-cache itself consumes the nix-cache substituter and remote # builder. mkTarget = { platform, buildType, hostPath, homeFile ? ./modules/common/home.nix }: + let + flakeTarget = "${platform}-${buildType}"; + in nixpkgs.lib.nixosSystem { inherit system; modules = [ @@ -42,7 +45,7 @@ ./modules/platforms/${platform}.nix ./modules/build-types/${buildType}.nix hostPath - { environment.etc."flake-target".text = "${platform}-${buildType}"; } + { environment.etc."flake-target".text = flakeTarget; } home-manager.nixosModules.home-manager { home-manager = { @@ -56,7 +59,13 @@ ./modules/nix-cache/client.nix ./modules/nix-cache/remote-builder-client.nix ]; - specialArgs = { inherit inputs vars netbootSystem; }; + # flakeTarget is passed via specialArgs (not read back from + # config.environment.etc."flake-target" above) specifically so + # modules/platforms/lxc.nix can use it to select its own host key + # file without a same-option circular dependency (a module + # contributing to environment.etc can't read the merged + # environment.etc it's itself contributing to). + specialArgs = { inherit inputs vars netbootSystem flakeTarget; }; }; # Generated platform x build-type matrix. pxe-boot has no linode diff --git a/modules/platforms/lxc.nix b/modules/platforms/lxc.nix index 31d6fe7..dca5389 100644 --- a/modules/platforms/lxc.nix +++ b/modules/platforms/lxc.nix @@ -1,5 +1,44 @@ -{ lib, modulesPath, ... }: +{ lib, modulesPath, flakeTarget, ... }: +let + # Bakes this exact flake target's pre-generated SSH host key straight + # into /etc/ssh/ -- mirrors modules/installer/host-keys.nix's + # builtins.getEnv pattern (impure and empty under normal `nix + # build`/`nix eval`, so this is a no-op unless explicitly opted into + # with NIXOS_HOST_KEYS_DIR=... --impure), but places the key directly + # rather than staging it under /etc/host-keys/ for a later manual copy + # -- this is the whole system for a `lxc-*` host, built straight to a + # pct-restorable tarball with no install step, so there's no later copy + # step to stage for. + # + # Without this, config.system.build.tarball's built-in system just + # generates a fresh host key at first boot like any other host would -- + # but sops-nix derives its decryption key from *this* file, and + # .sops.yaml only trusts whatever key scripts/sync-host-keys.sh already + # registered for this exact target name. A freshly-generated key can + # never match that, so every secret (including this host's own login) + # permanently fails to decrypt. Confirmed live: sops-install-secrets + # errored with "Error getting data key: 0 successful groups required, + # got 0" -- the container's actual host key's age fingerprint didn't + # match the one registered in .sops.yaml at all. + hostKeysDirStr = builtins.getEnv "NIXOS_HOST_KEYS_DIR"; + hasHostKeysDir = hostKeysDirStr != "" && builtins.pathExists hostKeysDirStr; + hostKeysDir = /. + hostKeysDirStr; + + # flakeTarget ("${platform}-${buildType}") comes in via specialArgs from + # flake.nix's mkTarget -- exactly the name scripts/sync-host-keys.sh + # registers keys under. Deliberately not read back from + # config.environment.etc."flake-target" (which is set to the same value) + # -- this module also *contributes* to environment.etc below, and a + # module reading the merged value of an option it's still defining is a + # circular dependency (confirmed: "infinite recursion encountered"). + privKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key"; + pubKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key.pub"; + hasKeyForThisTarget = + hasHostKeysDir + && builtins.pathExists privKeyFile + && builtins.pathExists pubKeyFile; +in { # LXC containers share the host kernel — Proxmox starts them by exec'ing # /sbin/init directly, no bootloader/initrd involved — and Proxmox has its @@ -35,4 +74,15 @@ # for the same reason; it just doesn't disable NetworkManager itself, # which modules/common/configuration.nix enables for every host. networking.networkmanager.enable = lib.mkForce false; + + environment.etc = lib.mkIf hasKeyForThisTarget { + "ssh/ssh_host_ed25519_key" = { + source = privKeyFile; + mode = "0600"; + }; + "ssh/ssh_host_ed25519_key.pub" = { + source = pubKeyFile; + mode = "0644"; + }; + }; }