Archived
The real root cause behind the original nix-cache 502, traced all the way through: modules/installer/host-keys.nix (which NIXOS_HOST_KEYS_DIR=... --impure actually wires up) is only ever imported by the installer's own modules/installer/common.nix -- modules/platforms/lxc.nix, which every real lxc-* host build actually uses, never imported anything like it. docs/auto-installer.md previously claimed NIXOS_HOST_KEYS_DIR bakes a key into lxc-* tarballs "the same way it does for the ISO/PXE installer images" -- that was never actually true; I wrote it without verifying the mechanism existed for lxc.nix specifically. In practice this meant every lxc-* container booted with a freshly self-generated SSH host key that could never match whatever .sops.yaml actually trusts for that target, so *every* secret -- not just cache-priv-key -- silently failed to decrypt. No error surfaces in the boot log for this: the activation step that installs secrets only runs on a genuinely fresh first activation and silently no-ops once /run/current-system already exists, so by the time anyone looks the window has closed. Found by manually invoking sops-install-secrets directly: "Error getting data key: 0 successful groups required, got 0". Fixed by giving modules/platforms/lxc.nix the same key-baking mechanism the installer has, but keyed to its own exact flake target and placing the key directly at /etc/ssh/ssh_host_ed25519_key (no copy step to stage for, unlike the installer's /etc/host-keys/ staging area -- an lxc-* tarball has no install step). The target name comes in via specialArgs.flakeTarget (new, set by flake.nix's mkTarget) rather than being read back from config.environment.etc."flake-target" -- reading that back from within a module that also contributes to environment.etc is circular (confirmed: "infinite recursion encountered"). Verified live end-to-end against the real test container (lxc-nix-cache, VMID 100 on pve.sweet.home): destroyed it, rebuilt the tarball fresh with the fix, recreated it, and confirmed /run/secrets/ now has all three secrets this host needs (beszel-token, cache-priv-key, nix-github-token), nix-serve is active (running), and curl http://localhost/nix-cache-info succeeds both directly and through nginx. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01La55Nsss8jZ7ZuzUV9mfot
153 lines
7.1 KiB
Nix
153 lines
7.1 KiB
Nix
{
|
|
description = "LAN NixOS configs";
|
|
|
|
inputs = {
|
|
nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05";
|
|
nixos-conf-editor.url = "github:snowfallorg/nixos-conf-editor";
|
|
home-manager = {
|
|
url = "github:nix-community/home-manager/release-26.05";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
};
|
|
disko = {
|
|
url = "github:nix-community/disko";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
};
|
|
sops-nix = {
|
|
url = "github:Mic92/sops-nix";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
};
|
|
};
|
|
|
|
outputs = { self, nixpkgs, nixos-conf-editor, home-manager, sops-nix, ... } @ inputs:
|
|
|
|
let
|
|
system = "x86_64-linux";
|
|
inherit (nixpkgs) lib;
|
|
pkgs = nixpkgs.legacyPackages.${system};
|
|
vars = import ./variables.nix;
|
|
|
|
# Generates a nixosConfiguration from a platform (what it runs on) and
|
|
# a build type (what it's for), plus the per-identity host.nix that
|
|
# carries the bits that must stay fixed regardless of platform
|
|
# (hostName, hostId, per-machine secrets). Every build type except
|
|
# nix-cache itself consumes the nix-cache substituter and remote
|
|
# builder.
|
|
mkTarget = { platform, buildType, hostPath, homeFile ? ./modules/common/home.nix }:
|
|
let
|
|
flakeTarget = "${platform}-${buildType}";
|
|
in
|
|
nixpkgs.lib.nixosSystem {
|
|
inherit system;
|
|
modules = [
|
|
inputs.disko.nixosModules.disko
|
|
sops-nix.nixosModules.sops
|
|
./modules/common/configuration.nix
|
|
./modules/platforms/${platform}.nix
|
|
./modules/build-types/${buildType}.nix
|
|
hostPath
|
|
{ environment.etc."flake-target".text = flakeTarget; }
|
|
home-manager.nixosModules.home-manager
|
|
{
|
|
home-manager = {
|
|
useGlobalPkgs = true;
|
|
useUserPackages = true;
|
|
extraSpecialArgs = { inherit vars; };
|
|
users.nixos = import homeFile;
|
|
};
|
|
}
|
|
] ++ lib.optionals (buildType != "nix-cache") [
|
|
./modules/nix-cache/client.nix
|
|
./modules/nix-cache/remote-builder-client.nix
|
|
];
|
|
# flakeTarget is passed via specialArgs (not read back from
|
|
# config.environment.etc."flake-target" above) specifically so
|
|
# modules/platforms/lxc.nix can use it to select its own host key
|
|
# file without a same-option circular dependency (a module
|
|
# contributing to environment.etc can't read the merged
|
|
# environment.etc it's itself contributing to).
|
|
specialArgs = { inherit inputs vars netbootSystem flakeTarget; };
|
|
};
|
|
|
|
# Generated platform x build-type matrix. pxe-boot has no linode
|
|
# variant (PXE/DHCP/TFTP need LAN L2 adjacency, which a Linode VPS
|
|
# doesn't have).
|
|
generatedTargets = {
|
|
linode-minimal = mkTarget { platform = "linode"; buildType = "minimal"; hostPath = ./hosts/nix-minimal/host.nix; };
|
|
proxmox-minimal = mkTarget { platform = "proxmox"; buildType = "minimal"; hostPath = ./hosts/nix-minimal/host.nix; };
|
|
lxc-minimal = mkTarget { platform = "lxc"; buildType = "minimal"; hostPath = ./hosts/nix-minimal/host.nix; };
|
|
|
|
linode-nix-cache = mkTarget { platform = "linode"; buildType = "nix-cache"; hostPath = ./hosts/nix-cache/host.nix; };
|
|
proxmox-nix-cache = mkTarget { platform = "proxmox"; buildType = "nix-cache"; hostPath = ./hosts/nix-cache/host.nix; };
|
|
lxc-nix-cache = mkTarget { platform = "lxc"; buildType = "nix-cache"; hostPath = ./hosts/nix-cache/host.nix; };
|
|
|
|
linode-server = mkTarget { platform = "linode"; buildType = "server"; hostPath = ./hosts/server/host.nix; };
|
|
proxmox-server = mkTarget { platform = "proxmox"; buildType = "server"; hostPath = ./hosts/server/host.nix; };
|
|
lxc-server = mkTarget { platform = "lxc"; buildType = "server"; hostPath = ./hosts/server/host.nix; };
|
|
|
|
linode-docker = mkTarget { platform = "linode"; buildType = "docker"; hostPath = ./hosts/docker/host.nix; };
|
|
proxmox-docker = mkTarget { platform = "proxmox"; buildType = "docker"; hostPath = ./hosts/docker/host.nix; };
|
|
lxc-docker = mkTarget { platform = "lxc"; buildType = "docker"; hostPath = ./hosts/docker/host.nix; };
|
|
|
|
linode-gui = mkTarget { platform = "linode"; buildType = "gui"; hostPath = ./hosts/nixos/host.nix; homeFile = ./hosts/nixos/home.nix; };
|
|
proxmox-gui = mkTarget { platform = "proxmox"; buildType = "gui"; hostPath = ./hosts/nixos/host.nix; homeFile = ./hosts/nixos/home.nix; };
|
|
lxc-gui = mkTarget { platform = "lxc"; buildType = "gui"; hostPath = ./hosts/nixos/host.nix; homeFile = ./hosts/nixos/home.nix; };
|
|
|
|
proxmox-pxe-boot = mkTarget { platform = "proxmox"; buildType = "pxe-boot"; hostPath = ./hosts/pxe-boot/host.nix; };
|
|
lxc-pxe-boot = mkTarget { platform = "lxc"; buildType = "pxe-boot"; hostPath = ./hosts/pxe-boot/host.nix; };
|
|
};
|
|
|
|
# Auto-install environments (migrated from the former nix-auto-installer
|
|
# flake): a self-contained NixOS installer that boots, discovers this
|
|
# flake's own nixosConfigurations over the network, and runs
|
|
# nixos-install against whichever one the operator picks. These are
|
|
# deliberately not part of the platform x build-type matrix above —
|
|
# they're throwaway boot media, not persistent hosts, so they skip
|
|
# disko/sops-nix/home-manager and just need `vars`.
|
|
installerTargets = {
|
|
installer = nixpkgs.lib.nixosSystem {
|
|
inherit system;
|
|
modules = [ ./modules/installer/iso.nix ];
|
|
specialArgs = { inherit vars; };
|
|
};
|
|
};
|
|
|
|
# Same installer environment, built as netboot (kernel + initrd +
|
|
# iPXE script) instead of an ISO — this is what packages.pxe bundles.
|
|
netbootSystem = nixpkgs.lib.nixosSystem {
|
|
inherit system;
|
|
modules = [
|
|
./modules/installer/iso.nix
|
|
({ modulesPath, ... }: {
|
|
imports = [
|
|
(modulesPath + "/installer/netboot/netboot-minimal.nix")
|
|
];
|
|
})
|
|
];
|
|
specialArgs = { inherit vars; };
|
|
};
|
|
|
|
in
|
|
{
|
|
|
|
nixosConfigurations = generatedTargets // installerTargets;
|
|
|
|
# Buildable auto-installer artifacts (`nix build .#<name>`). No `lxc`
|
|
# variant (installer-boots-as-an-LXC-container) or `all` bundle
|
|
# anymore — lxc-* and proxmox-* hosts deploy via their own tarball/
|
|
# disk-image outputs instead (see docs/auto-installer.md and
|
|
# docs/proxmox-images.md), which left the installer's own LXC form
|
|
# with no real use case: it's excluded from the install menu (same
|
|
# bind-mount problem as any LXC nixos-install target) and nothing
|
|
# else needed booting the installer itself as a container.
|
|
packages.${system} = {
|
|
iso = installerTargets.installer.config.system.build.isoImage;
|
|
|
|
pxe = pkgs.linkFarm "pxe" [
|
|
{ name = "netboot.ipxe"; path = netbootSystem.config.system.build.netbootIpxeScript; }
|
|
{ name = "initrd"; path = netbootSystem.config.system.build.netbootRamdisk; }
|
|
{ name = "kernel"; path = netbootSystem.config.system.build.kernel; }
|
|
];
|
|
};
|
|
};
|
|
}
|