Archived
virtualisation/proxmox-lxc.nix registers the Nix store DB via a systemd service, never an activation script -- so neededForUsers sops secrets (password hashes) and the user-creation step that consumes them never ran on a real first boot, leaving /etc/shadow stuck with build-time placeholder entries. boot.postBootCommands looked like the right hook (stage-2-init.sh does invoke it) but switch-to-configuration behaves unreliably that early, before systemd itself is up. Fixed with a genuine oneshot systemd service, gated by ConditionPathExists so it only ever runs once. Confirmed live via a from-scratch destroy+rebuild+redeploy of the lxc-nix-cache test container: real password hashes applied automatically, systemctl is-system-running -> running, zero failed units. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01La55Nsss8jZ7ZuzUV9mfot
149 lines
7.3 KiB
Nix
149 lines
7.3 KiB
Nix
{ lib, modulesPath, flakeTarget, ... }:
|
|
|
|
let
|
|
# Bakes this exact flake target's pre-generated SSH host key straight
|
|
# into /etc/ssh/ -- mirrors modules/installer/host-keys.nix's
|
|
# builtins.getEnv pattern (impure and empty under normal `nix
|
|
# build`/`nix eval`, so this is a no-op unless explicitly opted into
|
|
# with NIXOS_HOST_KEYS_DIR=... --impure), but places the key directly
|
|
# rather than staging it under /etc/host-keys/ for a later manual copy
|
|
# -- this is the whole system for a `lxc-*` host, built straight to a
|
|
# pct-restorable tarball with no install step, so there's no later copy
|
|
# step to stage for.
|
|
#
|
|
# Without this, config.system.build.tarball's built-in system just
|
|
# generates a fresh host key at first boot like any other host would --
|
|
# but sops-nix derives its decryption key from *this* file, and
|
|
# .sops.yaml only trusts whatever key scripts/sync-host-keys.sh already
|
|
# registered for this exact target name. A freshly-generated key can
|
|
# never match that, so every secret (including this host's own login)
|
|
# permanently fails to decrypt. Confirmed live: sops-install-secrets
|
|
# errored with "Error getting data key: 0 successful groups required,
|
|
# got 0" -- the container's actual host key's age fingerprint didn't
|
|
# match the one registered in .sops.yaml at all.
|
|
hostKeysDirStr = builtins.getEnv "NIXOS_HOST_KEYS_DIR";
|
|
hasHostKeysDir = hostKeysDirStr != "" && builtins.pathExists hostKeysDirStr;
|
|
hostKeysDir = /. + hostKeysDirStr;
|
|
|
|
# flakeTarget ("${platform}-${buildType}") comes in via specialArgs from
|
|
# flake.nix's mkTarget -- exactly the name scripts/sync-host-keys.sh
|
|
# registers keys under. Deliberately not read back from
|
|
# config.environment.etc."flake-target" (which is set to the same value)
|
|
# -- this module also *contributes* to environment.etc below, and a
|
|
# module reading the merged value of an option it's still defining is a
|
|
# circular dependency (confirmed: "infinite recursion encountered").
|
|
privKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key";
|
|
pubKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key.pub";
|
|
hasKeyForThisTarget =
|
|
hasHostKeysDir
|
|
&& builtins.pathExists privKeyFile
|
|
&& builtins.pathExists pubKeyFile;
|
|
in
|
|
{
|
|
# LXC containers share the host kernel — Proxmox starts them by exec'ing
|
|
# /sbin/init directly, no bootloader/initrd involved — and Proxmox has its
|
|
# own container hostname/network provisioning outside Nix. nixpkgs' own
|
|
# virtualisation/proxmox-lxc.nix module already handles all of this
|
|
# correctly (boot.isContainer, loader.initScript, systemd-networkd) and,
|
|
# critically, provides config.system.build.tarball — a directly
|
|
# `pct restore`-able container image, no nixos-install/bind-mount needed
|
|
# (nixos-install refuses to touch the filesystem it's currently running
|
|
# on, which is exactly what bind-mounting / onto /mnt for an installer
|
|
# LXC container does).
|
|
imports = [
|
|
(modulesPath + "/virtualisation/proxmox-lxc.nix")
|
|
];
|
|
|
|
proxmoxLXC = {
|
|
# host.nix declares each host's real hostname (networking.hostName);
|
|
# keep that instead of letting Proxmox's ambient container config win.
|
|
manageHostName = true;
|
|
# Unprivileged matches how these containers are actually created.
|
|
privileged = false;
|
|
};
|
|
|
|
boot.loader = {
|
|
grub.enable = false;
|
|
systemd-boot.enable = false;
|
|
};
|
|
|
|
# NetworkManager depends on a running udevd to enumerate/classify devices,
|
|
# which boot.isContainer disables (see nixpkgs' container-config.nix) —
|
|
# that's what broke DHCP-hostname registration in Pi-hole. The imported
|
|
# proxmox-lxc.nix module already switches networking to systemd-networkd
|
|
# for the same reason; it just doesn't disable NetworkManager itself,
|
|
# which modules/common/configuration.nix enables for every host.
|
|
networking.networkmanager.enable = lib.mkForce false;
|
|
|
|
environment.etc = lib.mkIf hasKeyForThisTarget {
|
|
"ssh/ssh_host_ed25519_key" = {
|
|
source = privKeyFile;
|
|
mode = "0600";
|
|
};
|
|
"ssh/ssh_host_ed25519_key.pub" = {
|
|
source = pubKeyFile;
|
|
mode = "0644";
|
|
};
|
|
};
|
|
|
|
# virtualisation/proxmox-lxc.nix (imported above) registers the Nix
|
|
# store DB via a systemd service (register-nix-paths) -- it never runs
|
|
# an activation script at all. Confirmed live this means neither
|
|
# sops-nix's "for users" secrets (password hashes -- installed by the
|
|
# activation script itself, not a systemd service, since they need to
|
|
# exist *before* user creation) nor the user-creation step that
|
|
# consumes them ever run on a real lxc-* boot. Regular secrets
|
|
# (nix-serve's key, beszel's token, etc.) work anyway because sops-nix
|
|
# provides its own systemd service for those.
|
|
#
|
|
# A systemd service, not boot.postBootCommands: tried that first (it's
|
|
# a genuine, generally-invoked hook -- nixos/modules/system/boot/stage-2-init.sh,
|
|
# which becomes this container's actual /sbin/init, unconditionally
|
|
# runs it) but switch-to-configuration behaves differently that early in
|
|
# boot (raw stage-2-init.sh, before systemd itself has even started) --
|
|
# confirmed live it silently failed to rewrite /etc/shadow from there
|
|
# even in "test" mode, despite the exact same command working reliably
|
|
# every time when run post-boot (i.e. as a normal systemd service, which
|
|
# is what this is). Not fully root-caused why the early context
|
|
# specifically breaks it; a real systemd service sidesteps needing to.
|
|
#
|
|
# /etc/shadow already has PLACEHOLDER entries for every declared user
|
|
# baked in at build time (part of constructing the system closure).
|
|
# update-users-groups.pl deliberately never overwrites an *existing*
|
|
# shadow entry -- a correct safety property in general (don't clobber a
|
|
# real user's real password on a config rebuild) -- but on a genuine
|
|
# first boot that only means the real hashedPasswordFile-derived hash
|
|
# never gets the chance to be applied either, since the placeholder is
|
|
# already "seen". Safe to clear here specifically: there is no real
|
|
# password yet to protect on a first boot.
|
|
#
|
|
# "test" mode, not "boot": confirmed live "boot" mode aborts partway
|
|
# through (before rewriting /etc/shadow) on a warning that "/boot" is on
|
|
# a different filesystem -- a real check for a host with a bootloader to
|
|
# update, meaningless for a container that has none
|
|
# (boot.loader.{grub,systemd-boot}.enable are both false above), but it
|
|
# still aborts the script. "test" runs every activation step without
|
|
# touching boot-loader state at all.
|
|
#
|
|
# ConditionPathExists (systemd-native, not a bash-level check) means
|
|
# this only ever runs once, on the genuine first boot -- systemd itself
|
|
# skips even starting it on every later boot once the marker exists.
|
|
# switch-to-configuration is otherwise the operator's call per this
|
|
# repo's own safety rules, not something to run on every boot.
|
|
systemd.services.nixos-lxc-first-boot-activate = {
|
|
description = "Complete first-boot NixOS activation (users, secrets) for this LXC container";
|
|
wantedBy = [ "multi-user.target" ];
|
|
unitConfig.ConditionPathExists = "!/var/lib/nixos-lxc-first-boot-activated";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
};
|
|
script = ''
|
|
rm -f /etc/shadow
|
|
/run/current-system/bin/switch-to-configuration test
|
|
mkdir -p /var/lib
|
|
touch /var/lib/nixos-lxc-first-boot-activated
|
|
'';
|
|
};
|
|
}
|