diff --git a/modules/common/configuration.nix b/modules/common/configuration.nix index 0abadcc..716a848 100644 --- a/modules/common/configuration.nix +++ b/modules/common/configuration.nix @@ -13,24 +13,6 @@ networking.networkmanager.enable = true; # Easiest to use and most distros use this by default. - # No host declares a DNS search domain anywhere else, and cross-host - # references throughout this repo (vars.nfsServerHost, vars.nixCacheHost, - # vars.dockerHost, ...) are bare short names, not FQDNs -- resolving them - # depends entirely on whatever network stack happens to be in play - # picking up the DHCP-advertised domain as a search suffix. NetworkManager - # does that by default, which is why this went unnoticed on - # NetworkManager-managed hosts, but LXC containers (modules/platforms/lxc.nix - # force-disables NetworkManager and Proxmox writes their systemd-networkd - # config itself) never get one. Confirmed live on lxc-docker: systemd-resolved - # had no search domain for eth0, "server" failed to resolve - # ("Name or service not known") while "server.sweet.home" resolved fine via - # the same DNS server, so every NFS mount in modules/docker/mount-data.nix - # failed even after fixing the automount/mount=nfs bugs. This applies the - # search domain globally via systemd-resolved's own config rather than the - # per-link DHCP path, so it isn't at the mercy of whichever component owns - # a given host's interface file. - networking.search = [ vars.homeDomain ]; - # Recommended over the true default (bypasses ZFS's own import safeguards) # per the option's own docs; matches hosts/docker/host.nix and # modules/services/zfs/enable-service.nix, which already set this diff --git a/modules/docker/mount-data.nix b/modules/docker/mount-data.nix index c7677a5..267deae 100644 --- a/modules/docker/mount-data.nix +++ b/modules/docker/mount-data.nix @@ -9,11 +9,25 @@ let # (the VM platforms rely on automount itself to get that same # non-blocking behavior, so they don't need `nofail` too). automountOpts = if config.boot.isContainer then [ "nofail" ] else [ "x-systemd.automount" ]; + + # A bare hostname here never resolves reliably: systemd-resolved only + # ever tries LLMNR for single-label names (never DNS, regardless of any + # configured search domain), and a *global* search domain (the first fix + # attempted here) backfires worse -- confirmed live on lxc-docker, adding + # `networking.search` made systemd-resolved prioritize its domain-matched + # but server-less global scope over eth0's correctly-configured one for + # every "*.sweet.home" query, silently sending them to public fallback + # DNS instead. `resolvectl query --interface=eth0 server.sweet.home` + # resolved fine throughout, proving the LAN DNS server was never the + # problem -- only the ambient, unqualified device string was. Using the + # FQDN directly sidesteps all of that, matching the pattern + # ../raspi/mount-data.nix already uses for the same reason. + nfsServer = "${vars.nfsServerHost}.${vars.homeDomain}"; in { fileSystems = { ${vars.nfsShares.dockerConfig.mountpoint} = { - device = "${vars.nfsServerHost}:${vars.storageRoot}/${vars.nfsShares.dockerConfig.subpath}"; + device = "${nfsServer}:${vars.storageRoot}/${vars.nfsShares.dockerConfig.subpath}"; fsType = "nfs"; options = [ @@ -24,7 +38,7 @@ in }; ${vars.nfsShares.dockerDatabases.mountpoint} = { - device = "${vars.nfsServerHost}:${vars.storageRoot}/${vars.nfsShares.dockerDatabases.subpath}"; + device = "${nfsServer}:${vars.storageRoot}/${vars.nfsShares.dockerDatabases.subpath}"; fsType = "nfs"; options = [ @@ -35,7 +49,7 @@ in }; ${vars.nfsShares.dockerVolumes.mountpoint} = { - device = "${vars.nfsServerHost}:${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}"; + device = "${nfsServer}:${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}"; fsType = "nfs"; options = [ @@ -46,7 +60,7 @@ in }; ${vars.nfsShares.nextcloudData.mountpoint} = { - device = "${vars.nfsServerHost}:${vars.storageRoot}/${vars.nfsShares.nextcloudData.subpath}"; + device = "${nfsServer}:${vars.storageRoot}/${vars.nfsShares.nextcloudData.subpath}"; fsType = "nfs"; options = [ @@ -57,7 +71,7 @@ in }; ${vars.nfsShares.raspiVolumes.mountpoint} = { - device = "${vars.nfsServerHost}:${vars.storageRoot}/${vars.nfsShares.raspiVolumes.subpath}"; + device = "${nfsServer}:${vars.storageRoot}/${vars.nfsShares.raspiVolumes.subpath}"; fsType = "nfs"; options = [ diff --git a/modules/platforms/lxc.nix b/modules/platforms/lxc.nix index ab3d10f..ac24108 100644 --- a/modules/platforms/lxc.nix +++ b/modules/platforms/lxc.nix @@ -1,4 +1,4 @@ -{ lib, modulesPath, flakeTarget, ... }: +{ config, lib, modulesPath, flakeTarget, ... }: let # Bakes this exact flake target's pre-generated SSH host key straight @@ -58,8 +58,28 @@ in # host.nix declares each host's real hostname (networking.hostName); # keep that instead of letting Proxmox's ambient container config win. manageHostName = true; - # Unprivileged matches how these containers are actually created. - privileged = false; + # Unprivileged by default -- matches how these containers are actually + # created (scripts/proxmox/create-proxmox-resource.sh reads this value + # back to decide `pct create`'s --unprivileged flag, so the two stay + # in sync). + # + # lxc-docker is the one exception: the kernel's NFS client doesn't set + # FS_USERNS_MOUNT, so mounting NFS from inside *any* non-init user + # namespace -- which is exactly what an unprivileged container's + # UID-mapped root runs in -- is rejected at the VFS layer with EPERM, + # no matter what Proxmox's own `mount=nfs;nfs4` container feature + # allows at the AppArmor layer (confirmed live: TCP to the NFS server + # succeeds, the server's export table matches the container's IP, and + # `mount.nfs: Operation not permitted` still fires immediately with no + # corresponding denial anywhere in the server's logs -- a kernel-level + # rejection, not a network or export-permission one). Keying off + # hostName rather than something docker-build-type-specific because + # modules/build-types/docker.nix is also composed for linode-docker/ + # proxmox-docker, which don't import proxmox-lxc.nix at all --setting + # this option there would break their eval with "option does not + # exist" regardless of any mkIf guard, since mkIf only makes a value + # conditional, not whether the option needs to exist somewhere. + privileged = config.networking.hostName == "docker"; }; boot.loader = { diff --git a/scripts/lib/nix-eval.sh b/scripts/lib/nix-eval.sh index cca380e..931b7bf 100644 --- a/scripts/lib/nix-eval.sh +++ b/scripts/lib/nix-eval.sh @@ -34,3 +34,21 @@ flake_target_hostname() { nix eval --raw "${NIX_EVAL_FLAGS[@]}" \ "${flake_ref}#nixosConfigurations.${target}.config.networking.hostName" 2>/dev/null } + +# flake_target_lxc_privileged +# Prints "true" or "false" for one lxc-* target's config.proxmoxLXC.privileged +# (modules/platforms/lxc.nix is the single source of truth -- e.g. +# lxc-docker sets this true so it can NFS-mount; every other lxc-* host +# stays unprivileged). Only meaningful for lxc-* targets -- the option +# doesn't exist for linode-*/proxmox-* (nixpkgs' proxmox-lxc.nix, which +# declares it, is only ever imported by modules/platforms/lxc.nix). Empty +# (not an error under set -e) if the eval fails. +flake_target_lxc_privileged() { + local flake_ref="$1" target="$2" + # Not --raw: the option is a Nix boolean, and --raw can only coerce + # strings ("cannot coerce a Boolean to a string"). Plain `nix eval` + # prints a bare `true`/`false` for a boolean, which is exactly the + # string this needs. + nix eval "${NIX_EVAL_FLAGS[@]}" \ + "${flake_ref}#nixosConfigurations.${target}.config.proxmoxLXC.privileged" 2>/dev/null +} diff --git a/scripts/proxmox/create-proxmox-resource.sh b/scripts/proxmox/create-proxmox-resource.sh index 1726ee3..6de5c40 100755 --- a/scripts/proxmox/create-proxmox-resource.sh +++ b/scripts/proxmox/create-proxmox-resource.sh @@ -766,13 +766,22 @@ if [[ "$type" == "lxc" ]]; then # 512M default otherwise (confirmed live: --memory 2048 left swap at # 512). Default to matching whatever --memory resolved to above. local_swap="${swap:-$memory}" - # --unprivileged 1: modules/platforms/lxc.nix sets proxmoxLXC.privileged - # = false, so the NixOS config inside the image assumes it's running as - # an unprivileged container (cgroup/capability/mount expectations baked - # in at boot). `pct create`'s own CLI default for this flag is - # privileged (unlike the web UI, which defaults its checkbox the other - # way) -- leaving it unset creates a privileged container running a - # NixOS config that assumes unprivileged, a real mismatch. + # --unprivileged: read back from modules/platforms/lxc.nix's own + # proxmoxLXC.privileged (via flake_target_lxc_privileged) rather than + # hardcoded, since that's no longer the same for every lxc-* target -- + # lxc-docker sets it true so the container's NFS mounts work at all (the + # kernel's NFS client can't mount from inside any unprivileged + # container's user namespace, no matter what AppArmor allows -- see that + # option's own comment). The NixOS config inside the image bakes in + # cgroup/capability/mount expectations matching whichever value it was + # built with, so this must stay in sync with it -- `pct create`'s own + # CLI default for this flag is privileged (unlike the web UI, which + # defaults its checkbox the other way), so leaving it unset would create + # a privileged container running a NixOS config that assumes + # unprivileged for every target except lxc-docker, a real mismatch. + privileged_eval="$(flake_target_lxc_privileged "$repo_root" "$flake_target")" + unprivileged_flag=1 + [[ "$privileged_eval" == "true" ]] && unprivileged_flag=0 # # --features nesting=1,keyctl=1: required for a modern (v247+) systemd # guest to actually boot unprivileged -- confirmed live: without this, @@ -790,7 +799,7 @@ if [[ "$type" == "lxc" ]]; then # hands the whole string to `ssh` as a single command for the *remote* # shell to parse -- unquoted, that `;` would be read as a remote # command separator and silently truncate this into two commands. - create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --unprivileged 1 --features '${PROXMOX_DEFAULT_LXC_FEATURES}' --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --swap ${local_swap} --net0 name=eth0,bridge=${bridge},ip=dhcp" + create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --unprivileged ${unprivileged_flag} --features '${PROXMOX_DEFAULT_LXC_FEATURES}' --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --swap ${local_swap} --net0 name=eth0,bridge=${bridge},ip=dhcp" remote "$create_cmd" remote "pct start ${vmid}" else