nixos/modules/platforms/lxc.nix

{ config, lib, modulesPath, flakeTarget, ... }:

let
  # Bakes this exact flake target's pre-generated SSH host key straight
  # into /etc/ssh/ -- mirrors modules/installer/host-keys.nix's
  # builtins.getEnv pattern (impure and empty under normal `nix
  # build`/`nix eval`, so this is a no-op unless explicitly opted into
  # with NIXOS_HOST_KEYS_DIR=... --impure), but places the key directly
  # rather than staging it under /etc/host-keys/ for a later manual copy
  # -- this is the whole system for a `lxc-*` host, built straight to a
  # pct-restorable tarball with no install step, so there's no later copy
  # step to stage for.
  #
  # Without this, config.system.build.tarball's built-in system just
  # generates a fresh host key at first boot like any other host would --
  # but sops-nix derives its decryption key from *this* file, and
  # .sops.yaml only trusts whatever key scripts/secrets/sync-host-keys.sh already
  # registered for this exact target name. A freshly-generated key can
  # never match that, so every secret (including this host's own login)
  # permanently fails to decrypt. Confirmed live: sops-install-secrets
  # errored with "Error getting data key: 0 successful groups required,
  # got 0" -- the container's actual host key's age fingerprint didn't
  # match the one registered in .sops.yaml at all.
  hostKeysDirStr = builtins.getEnv "NIXOS_HOST_KEYS_DIR";
  hasHostKeysDir = hostKeysDirStr != "" && builtins.pathExists hostKeysDirStr;
  hostKeysDir = /. + hostKeysDirStr;

  # flakeTarget ("${platform}-${buildType}") comes in via specialArgs from
  # flake.nix's mkTarget -- exactly the name scripts/secrets/sync-host-keys.sh
  # registers keys under. Deliberately not read back from
  # config.environment.etc."flake-target" (which is set to the same value)
  # -- this module also *contributes* to environment.etc below, and a
  # module reading the merged value of an option it's still defining is a
  # circular dependency (confirmed: "infinite recursion encountered").
  privKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key";
  pubKeyFile = hostKeysDir + "/${flakeTarget}_ssh_host_ed25519_key.pub";
  hasKeyForThisTarget =
    hasHostKeysDir
    && builtins.pathExists privKeyFile
    && builtins.pathExists pubKeyFile;
in
{
  # LXC containers share the host kernel — Proxmox starts them by exec'ing
  # /sbin/init directly, no bootloader/initrd involved — and Proxmox has its
  # own container hostname/network provisioning outside Nix. nixpkgs' own
  # virtualisation/proxmox-lxc.nix module already handles all of this
  # correctly (boot.isContainer, loader.initScript, systemd-networkd) and,
  # critically, provides config.system.build.tarball — a directly
  # `pct restore`-able container image, no nixos-install/bind-mount needed
  # (nixos-install refuses to touch the filesystem it's currently running
  # on, which is exactly what bind-mounting / onto /mnt for an installer
  # LXC container does).
  imports = [
    (modulesPath + "/virtualisation/proxmox-lxc.nix")
  ];

  proxmoxLXC = {
    # host.nix declares each host's real hostname (networking.hostName);
    # keep that instead of letting Proxmox's ambient container config win.
    manageHostName = true;
    # Unprivileged by default -- matches how these containers are actually
    # created (scripts/proxmox/create-proxmox-resource.sh reads this value
    # back to decide `pct create`'s --unprivileged flag, so the two stay
    # in sync).
    #
    # lxc-docker and lxc-pxe-boot are the exceptions: the kernel's NFS client
    # doesn't set FS_USERNS_MOUNT, so mounting NFS from inside *any* non-init
    # user namespace -- which is exactly what an unprivileged container's
    # UID-mapped root runs in -- is rejected at the VFS layer with EPERM,
    # no matter what Proxmox's own `mount=nfs;nfs4` container feature
    # allows at the AppArmor layer (confirmed live: TCP to the NFS server
    # succeeds, the server's export table matches the container's IP, and
    # `mount.nfs: Operation not permitted` still fires immediately with no
    # corresponding denial anywhere in the server's logs -- a kernel-level
    # rejection, not a network or export-permission one). Keying off
    # hostName rather than something build-type-specific because
    # modules/build-types/{docker,pxe-boot}.nix are also composed for
    # linode-docker/proxmox-docker/proxmox-pxe-boot, which don't import
    # proxmox-lxc.nix at all -- setting this option there would break their
    # eval with "option does not exist" regardless of any mkIf guard, since
    # mkIf only makes a value conditional, not whether the option needs to
    # exist somewhere.
    privileged = config.networking.hostName == "docker"
      || config.networking.hostName == "pxe-boot";
  };

  boot.loader = {
    grub.enable = false;
    systemd-boot.enable = false;
  };

  # NetworkManager depends on a running udevd to enumerate/classify devices,
  # which boot.isContainer disables (see nixpkgs' container-config.nix) —
  # that's what broke DHCP-hostname registration in Pi-hole. The imported
  # proxmox-lxc.nix module already switches networking to systemd-networkd
  # for the same reason; it just doesn't disable NetworkManager itself,
  # which modules/common/configuration.nix enables for every host.
  networking.networkmanager.enable = lib.mkForce false;

  environment.etc = lib.mkIf hasKeyForThisTarget {
    "ssh/ssh_host_ed25519_key" = {
      source = privKeyFile;
      mode = "0600";
    };
    "ssh/ssh_host_ed25519_key.pub" = {
      source = pubKeyFile;
      mode = "0644";
    };
  };

  # NixOS's etc activation removes any /etc file that was in the previous
  # generation's environment.etc but is absent from the current one — even
  # real (non-symlink) copies.  On every routine nixos-rebuild switch/test that
  # lacks NIXOS_HOST_KEYS_DIR the key is absent from environment.etc, so it
  # gets removed as "obsolete".  sops-nix derives its age decryption key from
  # /etc/ssh/ssh_host_ed25519_key; deletion cascades into every sops secret
  # failing with "Error getting data key: 0 successful groups required, got 0".
  #
  # Fix: activation scripts that bracket the etc step, with explicit deps
  # to enforce the correct ordering. Without deps the topological sort places
  # preserveSshHostKey AFTER etc (confirmed live on a deployed lxc-tor-relay:
  # position 7 vs etc's position 5) -- the key is already gone by the time it
  # tries to save it. The etc/setupSecrets entries ADD to existing deps
  # (types.listOf concatenates across module definitions).
  system.activationScripts = {
    # Saves the live key to /run before etc can delete it.
    preserveSshHostKey = ''
      if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
        cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
        cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
      fi
    '';

    # Reinstalls the key after etc runs if it was removed as "obsolete".
    # The resulting file is not registered in environment.etc for either
    # generation, so subsequent rebuilds leave it alone permanently.
    restoreSshHostKey = {
      deps = [ "etc" ];
      text = ''
        if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
          install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
          install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
        fi
        rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
      '';
    };

    # Force etc to wait until the key is saved, and sops to wait until the
    # key is restored. Without these the topological sort breaks the chain.
    etc = { deps = [ "preserveSshHostKey" ]; };
    setupSecrets = { deps = [ "restoreSshHostKey" ]; };
  };

  # virtualisation/proxmox-lxc.nix (imported above) registers the Nix
  # store DB via a systemd service (register-nix-paths) -- it never runs
  # an activation script at all. Confirmed live this means neither
  # sops-nix's "for users" secrets (password hashes -- installed by the
  # activation script itself, not a systemd service, since they need to
  # exist *before* user creation) nor the user-creation step that
  # consumes them ever run on a real lxc-* boot. In this config sops-nix
  # does NOT generate its own boot-time service (confirmed live: no
  # sops-nix.service in systemctl list-unit-files on a deployed
  # lxc-tor-relay container); /run/secrets is a tmpfs cleared on every
  # reboot, so secrets must be reinstalled on each non-first boot by
  # nixos-lxc-sops-reinstall (below).
  #
  # A systemd service, not boot.postBootCommands: tried that first (it's
  # a genuine, generally-invoked hook -- nixos/modules/system/boot/stage-2-init.sh,
  # which becomes this container's actual /sbin/init, unconditionally
  # runs it) but switch-to-configuration behaves differently that early in
  # boot (raw stage-2-init.sh, before systemd itself has even started) --
  # confirmed live it silently failed to rewrite /etc/shadow from there
  # even in "test" mode, despite the exact same command working reliably
  # every time when run post-boot (i.e. as a normal systemd service, which
  # is what this is). Not fully root-caused why the early context
  # specifically breaks it; a real systemd service sidesteps needing to.
  #
  # /etc/shadow already has PLACEHOLDER entries for every declared user
  # baked in at build time (part of constructing the system closure).
  # update-users-groups.pl deliberately never overwrites an *existing*
  # shadow entry -- a correct safety property in general (don't clobber a
  # real user's real password on a config rebuild) -- but on a genuine
  # first boot that only means the real hashedPasswordFile-derived hash
  # never gets the chance to be applied either, since the placeholder is
  # already "seen". Safe to clear here specifically: there is no real
  # password yet to protect on a first boot.
  #
  # "test" mode, not "boot": confirmed live "boot" mode aborts partway
  # through (before rewriting /etc/shadow) on a warning that "/boot" is on
  # a different filesystem -- a real check for a host with a bootloader to
  # update, meaningless for a container that has none
  # (boot.loader.{grub,systemd-boot}.enable are both false above), but it
  # still aborts the script. "test" runs every activation step without
  # touching boot-loader state at all.
  #
  # ConditionPathExists (systemd-native, not a bash-level check) means
  # this only ever runs once, on the genuine first boot -- systemd itself
  # skips even starting it on every later boot once the marker exists.
  # switch-to-configuration is otherwise the operator's call per this
  # repo's own safety rules, not something to run on every boot.
  systemd.services.nixos-lxc-first-boot-activate = {
    description = "Complete first-boot NixOS activation (users, secrets) for this LXC container";
    wantedBy = [ "multi-user.target" ];
    unitConfig.ConditionPathExists = "!/var/lib/nixos-lxc-first-boot-activated";
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
    };
    script = ''
      rm -f /etc/shadow
      /run/current-system/bin/switch-to-configuration test
      mkdir -p /var/lib
      touch /var/lib/nixos-lxc-first-boot-activated
    '';
  };

  # Reinstalls sops secrets on every non-first boot. /run/secrets is a
  # tmpfs that is cleared on each reboot; without this service, secrets
  # are permanently absent after the first boot and every service that
  # reads from /run/secrets fails on start.
  #
  # wantedBy/before network.target: switch-to-configuration test requires
  # D-Bus to restart systemd targets after running activation scripts. D-Bus
  # is available once basic.target completes (the default After=basic.target
  # that DefaultDependencies would otherwise add). Placing the service before
  # network.target ensures secrets are ready before any network-dependent
  # service (including beszel-agent and nix-serve) starts, while running late
  # enough that D-Bus is already up.
  #
  # ConditionPathExists=... skips this service on the genuine first boot
  # (the marker doesn't exist yet); nixos-lxc-first-boot-activate handles
  # that case. On every subsequent boot the condition passes and secrets
  # are reinstalled before user services start.
  #
  # SuccessExitStatus=11: switch-to-configuration exits 11 when it cannot
  # acquire the activation lock (another switch is already in progress).
  # During a nixos-rebuild switch the activation already installs secrets, so
  # treating the lock-held case as success is correct.
  systemd.services.nixos-lxc-sops-reinstall = {
    description = "Reinstall sops secrets on each non-first boot (LXC, /run is tmpfs)";
    wantedBy = [ "network.target" ];
    before = [ "network.target" ];
    unitConfig.ConditionPathExists = "/var/lib/nixos-lxc-first-boot-activated";
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
      SuccessExitStatus = "11";
    };
    script = ''
      /run/current-system/bin/switch-to-configuration test
    '';
  };
}