Archived
Worktree fix lxc docker nfs mounts #14
@@ -1,5 +1,15 @@
|
|||||||
{ config, lib, pkgs, vars, ... }:
|
{ config, lib, pkgs, vars, ... }:
|
||||||
|
|
||||||
|
let
|
||||||
|
# `x-systemd.automount` never works inside a Linux container (LXC
|
||||||
|
# included, regardless of privilege) -- confirmed live on lxc-docker:
|
||||||
|
# systemd logs "Starting of <unit>.automount unsupported" for every
|
||||||
|
# share and never mounts them. Mount eagerly there instead, with
|
||||||
|
# `nofail` so a boot with the NFS server unreachable doesn't hang
|
||||||
|
# (the VM platforms rely on automount itself to get that same
|
||||||
|
# non-blocking behavior, so they don't need `nofail` too).
|
||||||
|
automountOpts = if config.boot.isContainer then [ "nofail" ] else [ "x-systemd.automount" ];
|
||||||
|
in
|
||||||
{
|
{
|
||||||
fileSystems = {
|
fileSystems = {
|
||||||
${vars.nfsShares.dockerConfig.mountpoint} = {
|
${vars.nfsShares.dockerConfig.mountpoint} = {
|
||||||
@@ -9,9 +19,8 @@
|
|||||||
options = [
|
options = [
|
||||||
"nfsvers=4.2"
|
"nfsvers=4.2"
|
||||||
"_netdev"
|
"_netdev"
|
||||||
"x-systemd.automount"
|
|
||||||
"noatime"
|
"noatime"
|
||||||
];
|
] ++ automountOpts;
|
||||||
};
|
};
|
||||||
|
|
||||||
${vars.nfsShares.dockerDatabases.mountpoint} = {
|
${vars.nfsShares.dockerDatabases.mountpoint} = {
|
||||||
@@ -21,9 +30,8 @@
|
|||||||
options = [
|
options = [
|
||||||
"nfsvers=4.2"
|
"nfsvers=4.2"
|
||||||
"_netdev"
|
"_netdev"
|
||||||
"x-systemd.automount"
|
|
||||||
"noatime"
|
"noatime"
|
||||||
];
|
] ++ automountOpts;
|
||||||
};
|
};
|
||||||
|
|
||||||
${vars.nfsShares.dockerVolumes.mountpoint} = {
|
${vars.nfsShares.dockerVolumes.mountpoint} = {
|
||||||
@@ -33,9 +41,8 @@
|
|||||||
options = [
|
options = [
|
||||||
"nfsvers=4.2"
|
"nfsvers=4.2"
|
||||||
"_netdev"
|
"_netdev"
|
||||||
"x-systemd.automount"
|
|
||||||
"noatime"
|
"noatime"
|
||||||
];
|
] ++ automountOpts;
|
||||||
};
|
};
|
||||||
|
|
||||||
${vars.nfsShares.nextcloudData.mountpoint} = {
|
${vars.nfsShares.nextcloudData.mountpoint} = {
|
||||||
@@ -45,9 +52,8 @@
|
|||||||
options = [
|
options = [
|
||||||
"nfsvers=4.2"
|
"nfsvers=4.2"
|
||||||
"_netdev"
|
"_netdev"
|
||||||
"x-systemd.automount"
|
|
||||||
"noatime"
|
"noatime"
|
||||||
];
|
] ++ automountOpts;
|
||||||
};
|
};
|
||||||
|
|
||||||
${vars.nfsShares.raspiVolumes.mountpoint} = {
|
${vars.nfsShares.raspiVolumes.mountpoint} = {
|
||||||
@@ -57,9 +63,8 @@
|
|||||||
options = [
|
options = [
|
||||||
"nfsvers=4.2"
|
"nfsvers=4.2"
|
||||||
"_netdev"
|
"_netdev"
|
||||||
"x-systemd.automount"
|
|
||||||
"noatime"
|
"noatime"
|
||||||
];
|
] ++ automountOpts;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
{ vars, ... }:
|
{ config, lib, vars, ... }:
|
||||||
|
|
||||||
{
|
{
|
||||||
fileSystems.${vars.raspiMountpoint} = {
|
fileSystems.${vars.raspiMountpoint} = {
|
||||||
@@ -9,6 +9,15 @@
|
|||||||
"_netdev"
|
"_netdev"
|
||||||
"noatime"
|
"noatime"
|
||||||
|
|
||||||
|
# Explicitly use NFSv4.2 if supported
|
||||||
|
"nfsvers=4.2"
|
||||||
|
] ++ lib.optionals (!config.boot.isContainer) [
|
||||||
|
# `x-systemd.automount` never works inside a Linux container (LXC
|
||||||
|
# included) -- confirmed live on lxc-docker: systemd logs "Starting
|
||||||
|
# of <unit>.automount unsupported" and never mounts it. `nofail`
|
||||||
|
# above already keeps boot non-blocking there, so plain eager
|
||||||
|
# mounting is fine.
|
||||||
|
|
||||||
# Don't mount until first access
|
# Don't mount until first access
|
||||||
"x-systemd.automount"
|
"x-systemd.automount"
|
||||||
|
|
||||||
@@ -17,9 +26,6 @@
|
|||||||
|
|
||||||
# Give the Pi/Tailscale a little time to appear
|
# Give the Pi/Tailscale a little time to appear
|
||||||
"x-systemd.device-timeout=10s"
|
"x-systemd.device-timeout=10s"
|
||||||
|
|
||||||
# Explicitly use NFSv4.2 if supported
|
|
||||||
"nfsvers=4.2"
|
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -13,12 +13,21 @@
|
|||||||
# refuses to run if the target VMID already exists on the node, or if
|
# refuses to run if the target VMID already exists on the node, or if
|
||||||
# a VM/CT identified as --host already exists under any other VMID
|
# a VM/CT identified as --host already exists under any other VMID
|
||||||
# (checked live against the node; --allow-duplicate-host overrides).
|
# (checked live against the node; --allow-duplicate-host overrides).
|
||||||
|
# - --allow-duplicate-host distinguishes an exact match (same --type
|
||||||
|
# *and* --host, e.g. re-running --type lxc --host docker while an
|
||||||
|
# lxc-docker container already exists -- almost always a redeploy of
|
||||||
|
# the same target to pick up a rebuilt image) from a cross-type match
|
||||||
|
# (a different platform sharing the same host identity, e.g. a
|
||||||
|
# proxmox-docker VM coexisting with lxc-docker). Only the exact match
|
||||||
|
# is destroyed and replaced, after typing the hostname back to
|
||||||
|
# confirm (outside --dry-run) -- a cross-type match is always left
|
||||||
|
# untouched, matching-or-not.
|
||||||
# - --modify only ever touches a resource you name explicitly via
|
# - --modify only ever touches a resource you name explicitly via
|
||||||
# --vmid, shows exactly what will change first, and (outside
|
# --vmid, shows exactly what will change first, and (outside
|
||||||
# --dry-run) always requires typing that VMID back to confirm before
|
# --dry-run) always requires typing that VMID back to confirm before
|
||||||
# anything is sent to the node. There is no bulk/implicit modify.
|
# anything is sent to the node. There is no bulk/implicit modify.
|
||||||
# - Neither mode can start/stop/delete a resource. Not implemented on
|
# - Outside of --allow-duplicate-host's exact-match replace above,
|
||||||
# purpose -- ask before adding it.
|
# neither mode can start/stop/delete a resource.
|
||||||
#
|
#
|
||||||
# See --help for the full option list.
|
# See --help for the full option list.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
@@ -64,7 +73,13 @@ Create mode (default):
|
|||||||
already exists on the node (checked live via
|
already exists on the node (checked live via
|
||||||
qm/pct, not any file in this repo) --
|
qm/pct, not any file in this repo) --
|
||||||
otherwise refused, since it'd share that
|
otherwise refused, since it'd share that
|
||||||
host's hostName/hostId.
|
host's hostName/hostId. An existing resource
|
||||||
|
of this *same* --type (e.g. re-running --type
|
||||||
|
lxc --host docker over an existing lxc-docker)
|
||||||
|
is destroyed and replaced, after confirming --
|
||||||
|
a different --type sharing the same --host
|
||||||
|
(e.g. a proxmox-docker VM) is always left
|
||||||
|
untouched.
|
||||||
|
|
||||||
Modify mode (reconfigure an EXISTING resource -- requires --modify):
|
Modify mode (reconfigure an EXISTING resource -- requires --modify):
|
||||||
--modify Switch to modify mode.
|
--modify Switch to modify mode.
|
||||||
@@ -304,12 +319,13 @@ fi
|
|||||||
# this script itself always uses unless --name is overridden) -- a guest
|
# this script itself always uses unless --name is overridden) -- a guest
|
||||||
# manually renamed on the node afterwards wouldn't match, but nothing here
|
# manually renamed on the node afterwards wouldn't match, but nothing here
|
||||||
# creates guests that way.
|
# creates guests that way.
|
||||||
if [[ "$allow_duplicate_host" -eq 1 ]]; then
|
if [[ "$dry_run" -eq 1 ]]; then
|
||||||
echo
|
|
||||||
echo "--allow-duplicate-host: skipping the check for an existing '${host}' on ${node}."
|
|
||||||
elif [[ "$dry_run" -eq 1 ]]; then
|
|
||||||
echo
|
echo
|
||||||
echo "[dry-run] would check ${node} for an existing VM/CT identified as '${host}'"
|
echo "[dry-run] would check ${node} for an existing VM/CT identified as '${host}'"
|
||||||
|
if [[ "$allow_duplicate_host" -eq 1 ]]; then
|
||||||
|
echo "[dry-run] --allow-duplicate-host: an existing ${type} named '${host}' would be" \
|
||||||
|
"destroyed and replaced; a different-type match would be left untouched"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
echo
|
echo
|
||||||
echo "==> Checking ${node} for an existing VM/CT identified as '${host}'..."
|
echo "==> Checking ${node} for an existing VM/CT identified as '${host}'..."
|
||||||
@@ -334,17 +350,90 @@ REMOTE_SCRIPT
|
|||||||
echo "check entirely)." >&2
|
echo "check entirely)." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Split into "exact" (same resource kind as --type -- i.e. literally this
|
||||||
|
# same host+platform combo already exists, almost always a redeploy of
|
||||||
|
# the same target to test a rebuilt image) vs "cross-type" (a different
|
||||||
|
# platform sharing this host identity, e.g. a stopped proxmox-docker VM
|
||||||
|
# coexisting with an lxc-docker container -- a deliberate, valid setup
|
||||||
|
# this script has never managed and still won't). Read via a herestring
|
||||||
|
# (not a pipe) so the appends below survive outside the loop.
|
||||||
|
this_kind="$type"
|
||||||
|
exact_matches=""
|
||||||
|
cross_matches=""
|
||||||
if [[ -n "$existing" ]]; then
|
if [[ -n "$existing" ]]; then
|
||||||
echo "ERROR: '${host}' already exists on ${node}:" >&2
|
while read -r kind id n; do
|
||||||
echo "$existing" | while read -r kind id n; do
|
[[ -z "$kind" ]] && continue
|
||||||
|
if [[ "$kind" == "$this_kind" ]]; then
|
||||||
|
exact_matches+="${kind} ${id} ${n}"$'\n'
|
||||||
|
else
|
||||||
|
cross_matches+="${kind} ${id} ${n}"$'\n'
|
||||||
|
fi
|
||||||
|
done <<<"$existing"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$exact_matches" && "$allow_duplicate_host" -ne 1 ]]; then
|
||||||
|
echo "ERROR: '${host}' already exists on ${node} as this same resource type:" >&2
|
||||||
|
echo "$exact_matches" | while read -r kind id n; do
|
||||||
|
[[ -z "$kind" ]] && continue
|
||||||
|
echo " - ${kind} VMID ${id} (${n})" >&2
|
||||||
|
done
|
||||||
|
echo "Refusing to create a second ${this_kind} sharing this identity. Pass" >&2
|
||||||
|
echo "--allow-duplicate-host to destroy it and create a fresh one in its place" >&2
|
||||||
|
echo "(after confirming), or use --modify to reconfigure the existing one instead." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$cross_matches" && "$allow_duplicate_host" -ne 1 ]]; then
|
||||||
|
echo "ERROR: '${host}' already exists on ${node} as a different resource type:" >&2
|
||||||
|
echo "$cross_matches" | while read -r kind id n; do
|
||||||
|
[[ -z "$kind" ]] && continue
|
||||||
echo " - ${kind} VMID ${id} (${n})" >&2
|
echo " - ${kind} VMID ${id} (${n})" >&2
|
||||||
done
|
done
|
||||||
echo "Refusing to create a second resource sharing this identity. Pass" >&2
|
echo "Refusing to create a second resource sharing this identity. Pass" >&2
|
||||||
echo "--allow-duplicate-host to create one anyway (it gets its own distinct" >&2
|
echo "--allow-duplicate-host to create one anyway (it gets its own distinct" >&2
|
||||||
echo "sops key and VMID -- the existing resource(s) above are left untouched)," >&2
|
echo "sops key and VMID -- the existing resource above is left untouched)," >&2
|
||||||
echo "or use --modify to reconfigure the existing one instead." >&2
|
echo "or use --modify to reconfigure the existing one instead." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$cross_matches" ]]; then
|
||||||
|
echo "--allow-duplicate-host: '${host}' also exists on ${node} as a different resource" \
|
||||||
|
"type -- leaving it untouched:"
|
||||||
|
echo "$cross_matches" | while read -r kind id n; do
|
||||||
|
[[ -z "$kind" ]] && continue
|
||||||
|
echo " - ${kind} VMID ${id} (${n})"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$exact_matches" ]]; then
|
||||||
|
echo "--allow-duplicate-host: '${host}' already exists on ${node} as this same resource" \
|
||||||
|
"type -- it will be destroyed and replaced:"
|
||||||
|
echo "$exact_matches" | while read -r kind id n; do
|
||||||
|
[[ -z "$kind" ]] && continue
|
||||||
|
echo " - ${kind} VMID ${id} (${n})"
|
||||||
|
done
|
||||||
|
echo
|
||||||
|
read -rp "Type the hostname (${host}) to confirm destroying the above and replacing it: " confirm
|
||||||
|
if [[ "$confirm" != "$host" ]]; then
|
||||||
|
echo "Cancelled -- input didn't match ${host}." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "$exact_matches" | while read -r kind id n; do
|
||||||
|
[[ -z "$kind" ]] && continue
|
||||||
|
echo "==> Destroying ${kind} VMID ${id} (${n})..."
|
||||||
|
if [[ "$kind" == "vm" ]]; then
|
||||||
|
# qm destroy has no --force to stop-then-destroy in one call (pct's
|
||||||
|
# does) -- stop explicitly first if it's running.
|
||||||
|
if ssh "$ssh_target" "qm status ${id}" 2>/dev/null | grep -q running; then
|
||||||
|
ssh "$ssh_target" "qm stop ${id}"
|
||||||
|
fi
|
||||||
|
ssh "$ssh_target" "qm destroy ${id} --purge 1"
|
||||||
|
else
|
||||||
|
ssh "$ssh_target" "pct destroy ${id} --force 1 --purge 1"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "Target: ${flake_target} (host=${host}, type=${type}) -> Proxmox resource '${name}'"
|
echo "Target: ${flake_target} (host=${host}, type=${type}) -> Proxmox resource '${name}'"
|
||||||
@@ -521,7 +610,17 @@ if [[ "$type" == "lxc" ]]; then
|
|||||||
# systemd routinely uses (even plain getty units), and every getty
|
# systemd routinely uses (even plain getty units), and every getty
|
||||||
# crash-loops on a denied mount every ~3s (visible as garbage on the
|
# crash-loops on a denied mount every ~3s (visible as garbage on the
|
||||||
# console) while core services like nsncd fail the same way.
|
# console) while core services like nsncd fail the same way.
|
||||||
create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --unprivileged 1 --features ${PROXMOX_DEFAULT_LXC_FEATURES} --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --swap ${local_swap} --net0 name=eth0,bridge=${bridge},ip=dhcp"
|
#
|
||||||
|
# ...,mount=nfs;nfs4: without it AppArmor blanket-denies the `nfs`/
|
||||||
|
# `rpc_pipefs` mount syscalls any NFS client share needs -- confirmed
|
||||||
|
# live on lxc-docker: `mount: /var/lib/nfs/rpc_pipefs: permission
|
||||||
|
# denied`. The value's `;` (Proxmox's own multi-fstype separator for
|
||||||
|
# this one feature, per PVE::LXC's use of PVE::ParseUtils::split_list)
|
||||||
|
# must stay single-quoted here: create_cmd is sent to `remote()`, which
|
||||||
|
# hands the whole string to `ssh` as a single command for the *remote*
|
||||||
|
# shell to parse -- unquoted, that `;` would be read as a remote
|
||||||
|
# command separator and silently truncate this into two commands.
|
||||||
|
create_cmd="pct create ${vmid} ${iso_storage}:vztmpl/${remote_filename} --unprivileged 1 --features '${PROXMOX_DEFAULT_LXC_FEATURES}' --rootfs ${storage}:${local_disk_size} --hostname ${name} --cores ${cores} --memory ${memory} --swap ${local_swap} --net0 name=eth0,bridge=${bridge},ip=dhcp"
|
||||||
remote "$create_cmd"
|
remote "$create_cmd"
|
||||||
remote "pct start ${vmid}"
|
remote "pct start ${vmid}"
|
||||||
else
|
else
|
||||||
|
|||||||
+9
-1
@@ -45,7 +45,15 @@
|
|||||||
# crash-loops on a denied `/run/credentials/*` mount every ~3s (visible
|
# crash-loops on a denied `/run/credentials/*` mount every ~3s (visible
|
||||||
# as garbage on the console) and core services like nsncd fail the same
|
# as garbage on the console) and core services like nsncd fail the same
|
||||||
# way on userns_create; system.build.tarball never finishes activating.
|
# way on userns_create; system.build.tarball never finishes activating.
|
||||||
: "${PROXMOX_DEFAULT_LXC_FEATURES:=nesting=1,keyctl=1}"
|
#
|
||||||
|
# mount=nfs;nfs4: without it, AppArmor blanket-denies the `nfs`/
|
||||||
|
# `rpc_pipefs` mount syscalls any NFS client share needs -- confirmed
|
||||||
|
# live on lxc-docker (which mounts several, see modules/docker/mount-data.nix
|
||||||
|
# and modules/raspi/mount-data.nix): `mount: /var/lib/nfs/rpc_pipefs:
|
||||||
|
# permission denied`. Harmless to grant on lxc targets that don't mount
|
||||||
|
# NFS at all -- it only widens what the container is *allowed* to mount,
|
||||||
|
# nothing here forces a mount to happen.
|
||||||
|
: "${PROXMOX_DEFAULT_LXC_FEATURES:=nesting=1,keyctl=1,mount=nfs;nfs4}"
|
||||||
|
|
||||||
export PROXMOX_HOST PROXMOX_SSH_USER PROXMOX_STORAGE PROXMOX_ISO_STORAGE \
|
export PROXMOX_HOST PROXMOX_SSH_USER PROXMOX_STORAGE PROXMOX_ISO_STORAGE \
|
||||||
PROXMOX_BRIDGE PROXMOX_DEFAULT_CORES PROXMOX_DEFAULT_MEMORY_MB \
|
PROXMOX_BRIDGE PROXMOX_DEFAULT_CORES PROXMOX_DEFAULT_MEMORY_MB \
|
||||||
|
|||||||
Reference in New Issue
Block a user