Merge pull request 'Worktree fix lxc docker nfs mounts' (#14) from worktree-fix-lxc-docker-nfs-mounts into main

Reviewed-on: #14

modules/docker/mount-data.nix

@@ -1,5 +1,15 @@
 { config, lib, pkgs, vars, ... }:
 
+let
+  # `x-systemd.automount` never works inside a Linux container (LXC
+  # included, regardless of privilege) -- confirmed live on lxc-docker:
+  # systemd logs "Starting of <unit>.automount unsupported" for every
+  # share and never mounts them. Mount eagerly there instead, with
+  # `nofail` so a boot with the NFS server unreachable doesn't hang
+  # (the VM platforms rely on automount itself to get that same
+  # non-blocking behavior, so they don't need `nofail` too).
+  automountOpts = if config.boot.isContainer then [ "nofail" ] else [ "x-systemd.automount" ];
+in
 {
   fileSystems = {
     ${vars.nfsShares.dockerConfig.mountpoint} = {
@@ -9,9 +19,8 @@
       options = [
         "nfsvers=4.2"
         "_netdev"
-        "x-systemd.automount"
         "noatime"
-      ];
+      ] ++ automountOpts;
     };
 
     ${vars.nfsShares.dockerDatabases.mountpoint} = {
@@ -21,9 +30,8 @@
       options = [
         "nfsvers=4.2"
         "_netdev"
-        "x-systemd.automount"
         "noatime"
-      ];
+      ] ++ automountOpts;
     };
 
     ${vars.nfsShares.dockerVolumes.mountpoint} = {
@@ -33,9 +41,8 @@
       options = [
         "nfsvers=4.2"
         "_netdev"
-        "x-systemd.automount"
         "noatime"
-      ];
+      ] ++ automountOpts;
     };
 
     ${vars.nfsShares.nextcloudData.mountpoint} = {
@@ -45,9 +52,8 @@
       options = [
         "nfsvers=4.2"
         "_netdev"
-        "x-systemd.automount"
         "noatime"
-      ];
+      ] ++ automountOpts;
     };
 
     ${vars.nfsShares.raspiVolumes.mountpoint} = {
@@ -57,9 +63,8 @@
       options = [
         "nfsvers=4.2"
         "_netdev"
-        "x-systemd.automount"
         "noatime"
-      ];
+      ] ++ automountOpts;
     };
   };
 }

modules/raspi/mount-data.nix

@@ -1,4 +1,4 @@
-{ vars, ... }:
+{ config, lib, vars, ... }:
 
 {
   fileSystems.${vars.raspiMountpoint} = {
@@ -9,6 +9,15 @@
       "_netdev"
       "noatime"
 
+      # Explicitly use NFSv4.2 if supported
+      "nfsvers=4.2"
+    ] ++ lib.optionals (!config.boot.isContainer) [
+      # `x-systemd.automount` never works inside a Linux container (LXC
+      # included) -- confirmed live on lxc-docker: systemd logs "Starting
+      # of <unit>.automount unsupported" and never mounts it. `nofail`
+      # above already keeps boot non-blocking there, so plain eager
+      # mounting is fine.
+
       # Don't mount until first access
       "x-systemd.automount"
 
@@ -17,9 +26,6 @@
 
       # Give the Pi/Tailscale a little time to appear
       "x-systemd.device-timeout=10s"
-
-      # Explicitly use NFSv4.2 if supported
-      "nfsvers=4.2"
     ];
   };
 

scripts/create-proxmox-resource.sh

@@ -13,12 +13,21 @@
 #     refuses to run if the target VMID already exists on the node, or if
 #     a VM/CT identified as --host already exists under any other VMID
 #     (checked live against the node; --allow-duplicate-host overrides).
+#   - --allow-duplicate-host distinguishes an exact match (same --type
+#     *and* --host, e.g. re-running --type lxc --host docker while an
+#     lxc-docker container already exists -- almost always a redeploy of
+#     the same target to pick up a rebuilt image) from a cross-type match
+#     (a different platform sharing the same host identity, e.g. a
+#     proxmox-docker VM coexisting with lxc-docker). Only the exact match
+#     is destroyed and replaced, after typing the hostname back to
+#     confirm (outside --dry-run) -- a cross-type match is always left
+#     untouched, matching-or-not.
 #   - --modify only ever touches a resource you name explicitly via
 #     --vmid, shows exactly what will change first, and (outside
 #     --dry-run) always requires typing that VMID back to confirm before
 #     anything is sent to the node. There is no bulk/implicit modify.
-#   - Neither mode can start/stop/delete a resource. Not implemented on
-#     purpose -- ask before adding it.
+#   - Outside of --allow-duplicate-host's exact-match replace above,
+#     neither mode can start/stop/delete a resource.
 #
 # See --help for the full option list.
 set -euo pipefail
@@ -64,7 +73,13 @@ Create mode (default):
                                 already exists on the node (checked live via
                                 qm/pct, not any file in this repo) --
                                 otherwise refused, since it'd share that
-                                host's hostName/hostId.
+                                host's hostName/hostId. An existing resource
+                                of this *same* --type (e.g. re-running --type
+                                lxc --host docker over an existing lxc-docker)
+                                is destroyed and replaced, after confirming --
+                                a different --type sharing the same --host
+                                (e.g. a proxmox-docker VM) is always left
+                                untouched.
 
 Modify mode (reconfigure an EXISTING resource -- requires --modify):
   --modify                Switch to modify mode.
@@ -304,12 +319,13 @@ fi
 # this script itself always uses unless --name is overridden) -- a guest
 # manually renamed on the node afterwards wouldn't match, but nothing here
 # creates guests that way.
-if [[ "$allow_duplicate_host" -eq 1 ]]; then
-  echo
-  echo "--allow-duplicate-host: skipping the check for an existing '${host}' on ${node}."
-elif [[ "$dry_run" -eq 1 ]]; then
+if [[ "$dry_run" -eq 1 ]]; then
   echo
   echo "[dry-run] would check ${node} for an existing VM/CT identified as '${host}'"
+  if [[ "$allow_duplicate_host" -eq 1 ]]; then
+    echo "[dry-run] --allow-duplicate-host: an existing ${type} named '${host}' would be" \
+      "destroyed and replaced; a different-type match would be left untouched"
+  fi
 else
   echo
   echo "==> Checking ${node} for an existing VM/CT identified as '${host}'..."
@@ -334,17 +350,90 @@ REMOTE_SCRIPT
     echo "check entirely)." >&2
     exit 1
   fi
+
+  # Split into "exact" (same resource kind as --type -- i.e. literally this
+  # same host+platform combo already exists, almost always a redeploy of
+  # the same target to test a rebuilt image) vs "cross-type" (a different
+  # platform sharing this host identity, e.g. a stopped proxmox-docker VM
+  # coexisting with an lxc-docker container -- a deliberate, valid setup
+  # this script has never managed and still won't). Read via a herestring
+  # (not a pipe) so the appends below survive outside the loop.
+  this_kind="$type"
+  exact_matches=""
+  cross_matches=""
   if [[ -n "$existing" ]]; then
-    echo "ERROR: '${host}' already exists on ${node}:" >&2
-    echo "$existing" | while read -r kind id n; do
+    while read -r kind id n; do
+      [[ -z "$kind" ]] && continue
+      if [[ "$kind" == "$this_kind" ]]; then
+        exact_matches+="${kind} ${id} ${n}"$'\n'
+      else
+        cross_matches+="${kind} ${id} ${n}"$'\n'
+      fi
+    done <<<"$existing"
+  fi
+
+  if [[ -n "$exact_matches" && "$allow_duplicate_host" -ne 1 ]]; then
+    echo "ERROR: '${host}' already exists on ${node} as this same resource type:" >&2
+    echo "$exact_matches" | while read -r kind id n; do
+      [[ -z "$kind" ]] && continue
+      echo "  - ${kind} VMID ${id} (${n})" >&2
+    done
+    echo "Refusing to create a second ${this_kind} sharing this identity. Pass" >&2
+    echo "--allow-duplicate-host to destroy it and create a fresh one in its place" >&2
+    echo "(after confirming), or use --modify to reconfigure the existing one instead." >&2
+    exit 1
+  fi
+
+  if [[ -n "$cross_matches" && "$allow_duplicate_host" -ne 1 ]]; then
+    echo "ERROR: '${host}' already exists on ${node} as a different resource type:" >&2
+    echo "$cross_matches" | while read -r kind id n; do
+      [[ -z "$kind" ]] && continue
       echo "  - ${kind} VMID ${id} (${n})" >&2
     done
     echo "Refusing to create a second resource sharing this identity. Pass" >&2
     echo "--allow-duplicate-host to create one anyway (it gets its own distinct" >&2
-    echo "sops key and VMID -- the existing resource(s) above are left untouched)," >&2
+    echo "sops key and VMID -- the existing resource above is left untouched)," >&2
     echo "or use --modify to reconfigure the existing one instead." >&2
     exit 1
   fi
+
+  if [[ -n "$cross_matches" ]]; then
+    echo "--allow-duplicate-host: '${host}' also exists on ${node} as a different resource" \
+      "type -- leaving it untouched:"
+    echo "$cross_matches" | while read -r kind id n; do
+      [[ -z "$kind" ]] && continue
+      echo "  - ${kind} VMID ${id} (${n})"
+    done
+  fi
+
+  if [[ -n "$exact_matches" ]]; then
+    echo "--allow-duplicate-host: '${host}' already exists on ${node} as this same resource" \
+      "type -- it will be destroyed and replaced:"
+    echo "$exact_matches" | while read -r kind id n; do
+      [[ -z "$kind" ]] && continue
+      echo "  - ${kind} VMID ${id} (${n})"
+    done
+    echo
+    read -rp "Type the hostname (${host}) to confirm destroying the above and replacing it: " confirm
+    if [[ "$confirm" != "$host" ]]; then
+      echo "Cancelled -- input didn't match ${host}." >&2
+      exit 1
+    fi
+    echo "$exact_matches" | while read -r kind id n; do
+      [[ -z "$kind" ]] && continue
+      echo "==> Destroying ${kind} VMID ${id} (${n})..."
+      if [[ "$kind" == "vm" ]]; then
+        # qm destroy has no --force to stop-then-destroy in one call (pct's
+        # does) -- stop explicitly first if it's running.
+        if ssh "$ssh_target" "qm status ${id}" 2>/dev/null | grep -q running; then
+          ssh "$ssh_target" "qm stop ${id}"
+        fi
+        ssh "$ssh_target" "qm destroy ${id} --purge 1"
+      else
+        ssh "$ssh_target" "pct destroy ${id} --force 1 --purge 1"
+      fi
+    done
+  fi
 fi
 
 echo "Target: ${flake_target} (host=${host}, type=${type}) -> Proxmox resource '${name}'"
@@ -521,7 +610,17 @@ if [[ "$type" == "lxc" ]]; then
   # systemd routinely uses (even plain getty units), and every getty
   # crash-loops on a denied mount every ~3s (visible as garbage on the
   # console) while core services like nsncd fail the same way.
-  create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --unprivileged 1 --features ${PROXMOX_DEFAULT_LXC_FEATURES} --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --swap ${local_swap} --net0 name=eth0,bridge=${bridge},ip=dhcp"
+  #
+  # ...,mount=nfs;nfs4: without it AppArmor blanket-denies the `nfs`/
+  # `rpc_pipefs` mount syscalls any NFS client share needs -- confirmed
+  # live on lxc-docker: `mount: /var/lib/nfs/rpc_pipefs: permission
+  # denied`. The value's `;` (Proxmox's own multi-fstype separator for
+  # this one feature, per PVE::LXC's use of PVE::ParseUtils::split_list)
+  # must stay single-quoted here: create_cmd is sent to `remote()`, which
+  # hands the whole string to `ssh` as a single command for the *remote*
+  # shell to parse -- unquoted, that `;` would be read as a remote
+  # command separator and silently truncate this into two commands.
+  create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --unprivileged 1 --features '${PROXMOX_DEFAULT_LXC_FEATURES}' --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --swap ${local_swap} --net0 name=eth0,bridge=${bridge},ip=dhcp"
   remote "$create_cmd"
   remote "pct start ${vmid}"
 else

scripts/env.sh

@@ -45,7 +45,15 @@
 # crash-loops on a denied `/run/credentials/*` mount every ~3s (visible
 # as garbage on the console) and core services like nsncd fail the same
 # way on userns_create; system.build.tarball never finishes activating.
-: "${PROXMOX_DEFAULT_LXC_FEATURES:=nesting=1,keyctl=1}"
+#
+# mount=nfs;nfs4: without it, AppArmor blanket-denies the `nfs`/
+# `rpc_pipefs` mount syscalls any NFS client share needs -- confirmed
+# live on lxc-docker (which mounts several, see modules/docker/mount-data.nix
+# and modules/raspi/mount-data.nix): `mount: /var/lib/nfs/rpc_pipefs:
+# permission denied`. Harmless to grant on lxc targets that don't mount
+# NFS at all -- it only widens what the container is *allowed* to mount,
+# nothing here forces a mount to happen.
+: "${PROXMOX_DEFAULT_LXC_FEATURES:=nesting=1,keyctl=1,mount=nfs;nfs4}"
 
 export PROXMOX_HOST PROXMOX_SSH_USER PROXMOX_STORAGE PROXMOX_ISO_STORAGE \
   PROXMOX_BRIDGE PROXMOX_DEFAULT_CORES PROXMOX_DEFAULT_MEMORY_MB \