This repository has been archived on 2026-07-30. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
nixos/modules/platforms/lxc.nix
T
beatzaplentyandClaude Sonnet 5 b68bb3a997 Fix lxc-* hosts never completing first-boot user/secrets activation
virtualisation/proxmox-lxc.nix registers the Nix store DB via a systemd
service, never an activation script -- so neededForUsers sops secrets
(password hashes) and the user-creation step that consumes them never ran
on a real first boot, leaving /etc/shadow stuck with build-time placeholder
entries. boot.postBootCommands looked like the right hook (stage-2-init.sh
does invoke it) but switch-to-configuration behaves unreliably that early,
before systemd itself is up. Fixed with a genuine oneshot systemd service,
gated by ConditionPathExists so it only ever runs once.

Confirmed live via a from-scratch destroy+rebuild+redeploy of the
lxc-nix-cache test container: real password hashes applied automatically,
systemctl is-system-running -> running, zero failed units.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01La55Nsss8jZ7ZuzUV9mfot
2026-07-20 13:38:21 +10:00

149 lines
7.3 KiB
Nix

{ lib, modulesPath, flakeTarget, ... }:
let
# Bakes this exact flake target's pre-generated SSH host key straight
# into /etc/ssh/ -- mirrors modules/installer/host-keys.nix's
# builtins.getEnv pattern (impure and empty under normal `nix
# build`/`nix eval`, so this is a no-op unless explicitly opted into
# with NIXOS_HOST_KEYS_DIR=... --impure), but places the key directly
# rather than staging it under /etc/host-keys/ for a later manual copy
# -- this is the whole system for a `lxc-*` host, built straight to a
# pct-restorable tarball with no install step, so there's no later copy
# step to stage for.
#
# Without this, config.system.build.tarball's built-in system just
# generates a fresh host key at first boot like any other host would --
# but sops-nix derives its decryption key from *this* file, and
# .sops.yaml only trusts whatever key scripts/sync-host-keys.sh already
# registered for this exact target name. A freshly-generated key can
# never match that, so every secret (including this host's own login)
# permanently fails to decrypt. Confirmed live: sops-install-secrets
# errored with "Error getting data key: 0 successful groups required,
# got 0" -- the container's actual host key's age fingerprint didn't
# match the one registered in .sops.yaml at all.
hostKeysDirStr = builtins.getEnv "NIXOS_HOST_KEYS_DIR";
hasHostKeysDir = hostKeysDirStr != "" && builtins.pathExists hostKeysDirStr;
hostKeysDir = /. + hostKeysDirStr;
# flakeTarget ("${platform}-${buildType}") comes in via specialArgs from
# flake.nix's mkTarget -- exactly the name scripts/sync-host-keys.sh
# registers keys under. Deliberately not read back from
# config.environment.etc."flake-target" (which is set to the same value)
# -- this module also *contributes* to environment.etc below, and a
# module reading the merged value of an option it's still defining is a
# circular dependency (confirmed: "infinite recursion encountered").
privKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key";
pubKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key.pub";
hasKeyForThisTarget =
hasHostKeysDir
&& builtins.pathExists privKeyFile
&& builtins.pathExists pubKeyFile;
in
{
# LXC containers share the host kernel — Proxmox starts them by exec'ing
# /sbin/init directly, no bootloader/initrd involved — and Proxmox has its
# own container hostname/network provisioning outside Nix. nixpkgs' own
# virtualisation/proxmox-lxc.nix module already handles all of this
# correctly (boot.isContainer, loader.initScript, systemd-networkd) and,
# critically, provides config.system.build.tarball — a directly
# `pct restore`-able container image, no nixos-install/bind-mount needed
# (nixos-install refuses to touch the filesystem it's currently running
# on, which is exactly what bind-mounting / onto /mnt for an installer
# LXC container does).
imports = [
(modulesPath + "/virtualisation/proxmox-lxc.nix")
];
proxmoxLXC = {
# host.nix declares each host's real hostname (networking.hostName);
# keep that instead of letting Proxmox's ambient container config win.
manageHostName = true;
# Unprivileged matches how these containers are actually created.
privileged = false;
};
boot.loader = {
grub.enable = false;
systemd-boot.enable = false;
};
# NetworkManager depends on a running udevd to enumerate/classify devices,
# which boot.isContainer disables (see nixpkgs' container-config.nix) —
# that's what broke DHCP-hostname registration in Pi-hole. The imported
# proxmox-lxc.nix module already switches networking to systemd-networkd
# for the same reason; it just doesn't disable NetworkManager itself,
# which modules/common/configuration.nix enables for every host.
networking.networkmanager.enable = lib.mkForce false;
environment.etc = lib.mkIf hasKeyForThisTarget {
"ssh/ssh_host_ed25519_key" = {
source = privKeyFile;
mode = "0600";
};
"ssh/ssh_host_ed25519_key.pub" = {
source = pubKeyFile;
mode = "0644";
};
};
# virtualisation/proxmox-lxc.nix (imported above) registers the Nix
# store DB via a systemd service (register-nix-paths) -- it never runs
# an activation script at all. Confirmed live this means neither
# sops-nix's "for users" secrets (password hashes -- installed by the
# activation script itself, not a systemd service, since they need to
# exist *before* user creation) nor the user-creation step that
# consumes them ever run on a real lxc-* boot. Regular secrets
# (nix-serve's key, beszel's token, etc.) work anyway because sops-nix
# provides its own systemd service for those.
#
# A systemd service, not boot.postBootCommands: tried that first (it's
# a genuine, generally-invoked hook -- nixos/modules/system/boot/stage-2-init.sh,
# which becomes this container's actual /sbin/init, unconditionally
# runs it) but switch-to-configuration behaves differently that early in
# boot (raw stage-2-init.sh, before systemd itself has even started) --
# confirmed live it silently failed to rewrite /etc/shadow from there
# even in "test" mode, despite the exact same command working reliably
# every time when run post-boot (i.e. as a normal systemd service, which
# is what this is). Not fully root-caused why the early context
# specifically breaks it; a real systemd service sidesteps needing to.
#
# /etc/shadow already has PLACEHOLDER entries for every declared user
# baked in at build time (part of constructing the system closure).
# update-users-groups.pl deliberately never overwrites an *existing*
# shadow entry -- a correct safety property in general (don't clobber a
# real user's real password on a config rebuild) -- but on a genuine
# first boot that only means the real hashedPasswordFile-derived hash
# never gets the chance to be applied either, since the placeholder is
# already "seen". Safe to clear here specifically: there is no real
# password yet to protect on a first boot.
#
# "test" mode, not "boot": confirmed live "boot" mode aborts partway
# through (before rewriting /etc/shadow) on a warning that "/boot" is on
# a different filesystem -- a real check for a host with a bootloader to
# update, meaningless for a container that has none
# (boot.loader.{grub,systemd-boot}.enable are both false above), but it
# still aborts the script. "test" runs every activation step without
# touching boot-loader state at all.
#
# ConditionPathExists (systemd-native, not a bash-level check) means
# this only ever runs once, on the genuine first boot -- systemd itself
# skips even starting it on every later boot once the marker exists.
# switch-to-configuration is otherwise the operator's call per this
# repo's own safety rules, not something to run on every boot.
systemd.services.nixos-lxc-first-boot-activate = {
description = "Complete first-boot NixOS activation (users, secrets) for this LXC container";
wantedBy = [ "multi-user.target" ];
unitConfig.ConditionPathExists = "!/var/lib/nixos-lxc-first-boot-activated";
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
script = ''
rm -f /etc/shadow
/run/current-system/bin/switch-to-configuration test
mkdir -p /var/lib
touch /var/lib/nixos-lxc-first-boot-activated
'';
};
}