From 055577ee9134426a436ee4f3ba851c0d5f9ae58a Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Sat, 25 Jul 2026 19:07:30 +1000 Subject: [PATCH] fix(lxc): fix activation ordering and add boot-time sops reinstall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs prevented nixos-rebuild switch from working on lxc-* hosts after first boot, both confirmed live on a deployed lxc-tor-relay container: 1. Ordering bug: preserveSshHostKey had no explicit deps, so the topological sort placed it at position 7 — after etc at position 5. By the time it tried to save the SSH key, etc had already removed it as "obsolete" (absent from the current generation's environment.etc when built without NIXOS_HOST_KEYS_DIR). Consolidate all four system.activationScripts entries into one block and add etc = { deps = ["preserveSshHostKey"]; } and setupSecrets = { deps = ["restoreSshHostKey"]; } to enforce the correct save→etc→restore→sops chain. 2. No boot-time secrets: /run/secrets is a tmpfs cleared on every reboot, and sops-nix does NOT generate a boot-time service in this configuration (confirmed live: no sops-nix.service in systemctl list-unit-files). Add nixos-lxc-sops-reinstall.service, modelled after sops-nix's own service placement (wantedBy/before sysinit.target, DefaultDependencies=false), so secrets are reinstalled before basic.target on every non-first boot. ConditionPathExists skips it on first boot; nixos-lxc-first-boot-activate handles that case. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01B2EJ4qTsM5KUqhS5c3GAwx --- modules/platforms/lxc.nix | 95 ++++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 25 deletions(-) diff --git a/modules/platforms/lxc.nix b/modules/platforms/lxc.nix index 7d6cee4..0188cef 100644 --- a/modules/platforms/lxc.nix +++ b/modules/platforms/lxc.nix @@ -114,30 +114,39 @@ in # /etc/ssh/ssh_host_ed25519_key; deletion cascades into every sops secret # failing with "Error getting data key: 0 successful groups required, got 0". # - # Fix: two activation scripts that bracket the etc step. - # preserveSshHostKey — no deps, runs before etc — saves the live key to - # /run (tmpfs) before etc can delete it. - # restoreSshHostKey — deps=[etc], runs after etc — reinstalls the key via - # `install` (atomic, sets mode) if etc removed it. - # The resulting file is not registered in environment.etc - # for either the previous or current generation, so - # subsequent rebuilds leave it alone permanently. - system.activationScripts.preserveSshHostKey = '' - if [ -f /etc/ssh/ssh_host_ed25519_key ]; then - cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp - cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp - fi - ''; - - system.activationScripts.restoreSshHostKey = { - deps = [ "etc" ]; - text = '' - if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then - install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key - install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub + # Fix: activation scripts that bracket the etc step, with explicit deps + # to enforce the correct ordering. Without deps the topological sort places + # preserveSshHostKey AFTER etc (confirmed live on a deployed lxc-tor-relay: + # position 7 vs etc's position 5) -- the key is already gone by the time it + # tries to save it. The etc/setupSecrets entries ADD to existing deps + # (types.listOf concatenates across module definitions). + system.activationScripts = { + # Saves the live key to /run before etc can delete it. + preserveSshHostKey = '' + if [ -f /etc/ssh/ssh_host_ed25519_key ]; then + cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp + cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp fi - rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp ''; + + # Reinstalls the key after etc runs if it was removed as "obsolete". + # The resulting file is not registered in environment.etc for either + # generation, so subsequent rebuilds leave it alone permanently. + restoreSshHostKey = { + deps = [ "etc" ]; + text = '' + if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then + install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key + install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub + fi + rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp + ''; + }; + + # Force etc to wait until the key is saved, and sops to wait until the + # key is restored. Without these the topological sort breaks the chain. + etc = { deps = [ "preserveSshHostKey" ]; }; + setupSecrets = { deps = [ "restoreSshHostKey" ]; }; }; # virtualisation/proxmox-lxc.nix (imported above) registers the Nix @@ -146,9 +155,12 @@ in # sops-nix's "for users" secrets (password hashes -- installed by the # activation script itself, not a systemd service, since they need to # exist *before* user creation) nor the user-creation step that - # consumes them ever run on a real lxc-* boot. Regular secrets - # (nix-serve's key, beszel's token, etc.) work anyway because sops-nix - # provides its own systemd service for those. + # consumes them ever run on a real lxc-* boot. In this config sops-nix + # does NOT generate its own boot-time service (confirmed live: no + # sops-nix.service in systemctl list-unit-files on a deployed + # lxc-tor-relay container); /run/secrets is a tmpfs cleared on every + # reboot, so secrets must be reinstalled on each non-first boot by + # nixos-lxc-sops-reinstall (below). # # A systemd service, not boot.postBootCommands: tried that first (it's # a genuine, generally-invoked hook -- nixos/modules/system/boot/stage-2-init.sh, @@ -199,4 +211,37 @@ in touch /var/lib/nixos-lxc-first-boot-activated ''; }; + + # Reinstalls sops secrets on every non-first boot. /run/secrets is a + # tmpfs that is cleared on each reboot; without this service, secrets + # are permanently absent after the first boot and every service that + # reads from /run/secrets fails on start. + # + # wantedBy/before sysinit.target + DefaultDependencies=false mirrors how + # the sops-nix module places its own service when it generates one. This + # ensures secrets exist before basic.target (and thus before any user + # service) starts. DefaultDependencies=false is required to avoid a + # circular ordering: without it, systemd would add After=sysinit.target + # to a service that is itself part of sysinit.target. + # + # ConditionPathExists=... skips this service on the genuine first boot + # (the marker doesn't exist yet); nixos-lxc-first-boot-activate handles + # that case. On every subsequent boot the condition passes and secrets + # are reinstalled before user services start. + systemd.services.nixos-lxc-sops-reinstall = { + description = "Reinstall sops secrets on each non-first boot (LXC, /run is tmpfs)"; + wantedBy = [ "sysinit.target" ]; + before = [ "sysinit.target" ]; + unitConfig = { + DefaultDependencies = false; + ConditionPathExists = "/var/lib/nixos-lxc-first-boot-activated"; + }; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + /run/current-system/bin/switch-to-configuration test + ''; + }; }