fix(ha): resolve data disk device via by-id symlink even in dry-run

The by-id lookup is read-only so it's safe to run in dry-run mode.
Previously it was gated behind `if ! $DRY_RUN`, which always triggered
the sdb fallback warning in dry-run — making it look like the device
path wasn't reliable when the symlink actually exists on both servers.

Now the lookup always runs and the script errors out with a clear message
if the by-id symlink is genuinely missing, instead of silently falling
back to a guessed /dev/sd* name.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.gitignore

@@ -23,3 +23,5 @@ host-keys/
 # Temporary Milestone 1 audit checklist (remove-sensetive-info-refactor.md)
 # - working notes only, never committed, deleted once every row is rotated.
 secrets-inventory.md
+.claude/worktrees/                                                                                                  
+.claude/settings.local.json 

.sops.yaml

@@ -22,6 +22,8 @@ keys:
   - &proxmox-pxe-boot age1ug787sgt6st6k82fgkrug2lzltw4qsukrrqqs3w27ewwqj8rg4hsxcmylz
   - &proxmox-server age1529taqdwr6t0w7cvzmty0d5y5593wffl0krt48j6uc4u39k56g2qf6ywtp
   - &proxmox-tailscale-router age1zhfyuzlq40reuqlr34gf77852nhs3t6mqfzrqmas8z6sxk7tcfhsungrm0
+  - &proxmox-ha-server-1 age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6
+  - &proxmox-ha-server-2 age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r
 
 creation_rules:
   # Shared across every currently-deployed host: root/nixos password hash,
@@ -54,6 +56,8 @@ creation_rules:
           - *proxmox-pxe-boot
           - *proxmox-server
           - *proxmox-tailscale-router
+          - *proxmox-ha-server-1
+          - *proxmox-ha-server-2
 
   - path_regex: secrets/nix-cache\.yaml$
     key_groups:
@@ -77,6 +81,44 @@ creation_rules:
           - *admin
           - *lxc-tor-relay
 
+  - path_regex: secrets/tailscale-router\.yaml$
+    key_groups:
+      - age:
+          - *admin
+          - *linode-tailscale-router
+          - *lxc-tailscale-router
+          - *proxmox-tailscale-router
+
+  # HA file server per-node secrets (beszel-token).
+  # proxmox-ha-server-1 / proxmox-ha-server-2 keys are added automatically
+  # by scripts/secrets/sync-host-keys.sh once the hosts are provisioned;
+  # until then only the admin key can decrypt these files.
+  - path_regex: secrets/ha-server-1\.yaml$
+    key_groups:
+      - age:
+          - *admin
+          - *proxmox-ha-server-1
+          # proxmox-ha-server-1 added by sync-host-keys.sh
+
+  - path_regex: secrets/ha-server-2\.yaml$
+    key_groups:
+      - age:
+          - *admin
+          - *proxmox-ha-server-2
+          # proxmox-ha-server-2 added by sync-host-keys.sh
+
+  # Shared HA cluster corosync authkey (binary sops file).
+  # Encrypted for both HA nodes so either can decrypt on boot.
+  # Both host keys added by sync-host-keys.sh; admin key allows initial creation.
+  - path_regex: secrets/ha-corosync-authkey$
+    key_groups:
+      - age:
+          - *admin
+          - *proxmox-ha-server-1
+          - *proxmox-ha-server-2
+          # proxmox-ha-server-1 added by sync-host-keys.sh
+          # proxmox-ha-server-2 added by sync-host-keys.sh
+
   # gui-host-specific secrets (currently: wifi-password, see
   # modules/networking/wifi.nix). Only *lxc-gui has a registered key today
   # -- proxmox-gui/linode-gui/baremetal-gui haven't been provisioned via
@@ -91,3 +133,91 @@ creation_rules:
           - *baremetal-gui
           - *linode-gui
           - *proxmox-gui
+
+  # IPA host keytabs (binary sops files).
+  # Each keytab is encrypted for all platform variants of that host so any
+  # deployed variant can decrypt it at boot. Run
+  # scripts/ipa/create-nixos-ipa-host-account.sh <hostname> to enroll a new
+  # host and produce the keytab; this section is updated by that script.
+
+  - path_regex: secrets/nix-cache\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *linode-nix-cache
+          - *lxc-nix-cache
+          - *proxmox-nix-cache
+
+  - path_regex: secrets/tailscale-router\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *linode-tailscale-router
+          - *lxc-tailscale-router
+          - *proxmox-tailscale-router
+
+  - path_regex: secrets/pxe-boot\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *lxc-pxe-boot
+          - *proxmox-pxe-boot
+
+  # nixos = the workstation (hosts/nixos/host.nix). All gui platform variants
+  # share the hostname "nixos" and must be able to decrypt at boot.
+  - path_regex: secrets/nixos\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *baremetal-gui
+          - *lxc-gui
+          - *proxmox-gui
+          - *linode-gui
+
+  - path_regex: secrets/server\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *linode-server
+          - *lxc-server
+          - *proxmox-server
+
+  - path_regex: secrets/docker\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *linode-docker
+          - *lxc-docker
+          - *proxmox-docker
+
+  - path_regex: secrets/tor-relay\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *lxc-tor-relay
+
+  - path_regex: secrets/nix-minimal\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *lxc-minimal
+          - *proxmox-minimal
+          - *linode-minimal
+
+  # Host keytab for ha-server-1 FreeIPA enrollment (binary sops file).
+  # Generated by scripts/ipa/create-nixos-ipa-host-account.sh.
+  - path_regex: secrets/ha-server-1\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *proxmox-ha-server-1
+          # proxmox-ha-server-1 added by sync-host-keys.sh
+
+  # Host keytab for ha-server-2 FreeIPA enrollment (binary sops file).
+  # Generated by scripts/ipa/create-nixos-ipa-host-account.sh.
+  - path_regex: secrets/ha-server-2\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *proxmox-ha-server-2
+          # proxmox-ha-server-2 added by sync-host-keys.sh

AGENTS.md

@@ -6,12 +6,14 @@ This repository contains flake-based NixOS configurations for Wayne's LAN
 servers and workstation.
 
 The flake exposes NixOS configurations named `<platform>-<buildtype>`
-(platforms: `linode`, `proxmox`, `lxc`; build types: `minimal`, `nix-cache`,
+(platforms: `linode`, `proxmox`, `lxc`, `baremetal`; build types: `minimal`,
-`server`, `docker`, `gui`, `pxe-boot`, `tailscale-exit-node`, `tor-relay`), generated from `modules/platforms/*`
+`nix-cache`, `server`, `docker`, `gui`, `pxe-boot`, `tailscale-router`,
-and `modules/build-types/*` by the `mkTarget` function in `flake.nix`. Not
+`tor-relay`, `ha-server`), generated from `modules/platforms/*` and
-every combination is built — `pxe-boot` has no `linode` variant. See
+`modules/build-types/*` by the `mkTarget` function in `flake.nix`. Not every
-`README.md` for the full current target list; treat `flake.nix` as the
+combination is built — `pxe-boot` has no `linode` variant, `ha-server` only
-source of truth since this list can drift.
+exists on `proxmox`, and `tor-relay` only exists on `lxc`. See `README.md`
+for the full current target list; treat `flake.nix` as the source of truth
+since this list can drift.
 
 Do not deploy, switch, reboot, repartition, format disks, or run destructive
 install commands from this repository unless explicitly asked.

CLAUDE.md

@@ -21,9 +21,7 @@ machines when deployed.
   `modules/installer/common.nix` (the auto-installer's own root/nixos login —
   a deliberate, documented choice, see `docs/auto-installer.md`, not
   accidental tech debt) and **SSH public keys** in `variables.nix`
-  (`vars.adminSshKey`, `vars.remoteBuilderAuthorizedKeys`) plus a couple of
+  (`vars.adminSshKey`, `vars.remoteBuilderAuthorizedKeys`, `vars.beszelHubKey`). Don't use the installer's hardcoded hash as a
-  per-host `KEY` values for beszel-agent auth (`hosts/server/host.nix`,
-  `hosts/nix-cache/host.nix`). Don't use the installer's hardcoded hash as a
   template for a *real* host — every other host uses sops-nix
   (`hashedPasswordFile`, see "Security Notes" in `README.md`). Flag any *new*
   secret-like string you encounter instead of committing it.
@@ -375,9 +373,8 @@ removing a host.
 
 - `hosts/<name>/host.nix` — per-machine identity **only**: hostname, hostId,
   per-machine secrets, `system.stateVersion`. These files carry no `imports`
-  of their own beyond narrow parameterized helpers (see
+  of their own — all shared behavior comes from the platform/build-type modules
-  `modules/beszel/host-token.nix` below) — all shared behavior comes from the
+  composed in `flake.nix`, not from the host file.
-  platform/build-type modules composed in `flake.nix`, not from the host file.
 - `modules/platforms/{linode,proxmox,lxc,baremetal}.nix` — platform-specific
   config: boot method, guest tooling, and the hardware config, imported
   directly by the platform module itself — **not** wired in from
@@ -434,10 +431,10 @@ removing a host.
   substituter + SSH remote-builder wiring; see `docs/nix-cache.md` for the
   full design (per-host local stores, no shared `/nix/store`, and how the
   `nixremote` signing/SSH keys fit together).
-- `modules/beszel/host-token.nix` — parameterized helper module
+- `modules/beszel/enable-agent.nix` — enables beszel-agent, sets `HUB_URL`,
-  (`{ name, sopsFile }`) that wires a host's beszel-agent sops secret/template
+  fixes the upstream `StateDirectory` bug, and wires the universal
-  and `environmentFile`; used by `hosts/server/host.nix` and
+  `beszel-token` sops secret (from `secrets/common.yaml`) into the agent's
-  `hosts/nix-cache/host.nix` to avoid duplicating that boilerplate.
+  `environmentFile`; see `docs/beszel.md` for the full setup guide.
 - `modules/tailscale/`, `modules/docker/`, `modules/networking/`,
   `modules/traefik/`, `modules/tor/`, `modules/services/*` — single-purpose,
   single-host

README.md

@@ -10,13 +10,13 @@ pieces composed in `flake.nix`:
 
 - **Platforms** (what it runs on): `linode`, `proxmox`, `lxc`, `baremetal`
 - **Build types** (what it's for): `minimal`, `nix-cache`, `server`, `docker`,
-  `gui`, `pxe-boot`, `tailscale-exit-node`, `tor-relay`
+  `gui`, `pxe-boot`, `tailscale-router`, `tor-relay`, `ha-server`
 
 Not every combination exists — `pxe-boot` has no `linode` variant, since
 PXE/DHCP/TFTP need LAN L2 adjacency that a Linode VPS doesn't have,
-`tor-relay` currently only exists as `lxc-tor-relay`, and `baremetal`
+`tor-relay` and `ha-server` currently only exist on `lxc`/`proxmox`, and
-currently only exists as `baremetal-gui` (the real gui-host hardware). The
+`baremetal` currently only exists as `baremetal-gui` (the real gui-host
-full list:
+hardware). The full list:
 
 | Target | Purpose |
 | --- | --- |
@@ -29,8 +29,9 @@ full list:
 | `linode-gui` / `proxmox-gui` / `lxc-gui` | Cinnamon desktop workstation — previously the flat `nixos` target |
 | `baremetal-gui` | Same Cinnamon desktop workstation, on the real gui-host hardware — ZFS RAID0 root, systemd-boot |
 | `proxmox-pxe-boot` / `lxc-pxe-boot` | HTTP/iPXE boot asset host — previously the flat `pxe-boot` target |
-| `linode-tailscale-exit-node` / `proxmox-tailscale-exit-node` / `lxc-tailscale-exit-node` | Tailscale exit node |
+| `linode-tailscale-router` / `proxmox-tailscale-router` / `lxc-tailscale-router` | Tailscale subnet router + MagicDNS forwarder for the LAN |
 | `lxc-tor-relay` | Tor middle relay |
+| `proxmox-ha-server-1` / `proxmox-ha-server-2` | HA file-server cluster nodes — DRBD + XFS + iSCSI + NFS, managed by Corosync + Pacemaker |
 
 Which variant of a given buildtype is actually deployed isn't tracked
 anywhere in this repo — that's live infrastructure state, not something a
@@ -47,8 +48,7 @@ section for which is which.
 
 Each buildtype's `hosts/<name>/host.nix` carries the per-machine identity
 (hostname, hostId, per-machine secrets, `system.stateVersion`) that must stay
-fixed regardless of which platform it's built for — see
+fixed regardless of which platform it's built for. Every deployed host
-`flake-target-refactor-spec.md` for the full rationale. Every deployed host
 stamps its own active target name into `/etc/flake-target` at build time, so
 `nixos-rebuild switch --flake .#$(cat /etc/flake-target)` always picks up the
 right one even after a platform migration changes the flake attribute name.
@@ -167,7 +167,6 @@ per-boot host key for sops-nix to derive from on ephemeral media) — see
 (`vars/per-machine/<target>/openssh/`, committed and sops-encrypted) for
 their SSH host keys.
 
-This repository's git *history* still contains secrets committed before this
+This repository's git *history* still contains secrets committed before the
-migration (see `remove-sensetive-info-refactor.md`) — those are being
+sops-nix migration — those are being scrubbed and rotated separately; don't
-scrubbed and rotated separately; don't treat the repo as safe to make public
+treat the repo as safe to make public until that's finished.
-until that's finished.

certs/ipa-ca.crt

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIESDCCArCgAwIBAgIBATANBgkqhkiG9w0BAQsFADA1MRMwEQYDVQQKDApTV0VF
+VC5IT01FMR4wHAYDVQQDDBVDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwHhcNMjYwNzI2
+MjExMzQxWhcNNDYwNzI2MjExMzQxWjA1MRMwEQYDVQQKDApTV0VFVC5IT01FMR4w
+HAYDVQQDDBVDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggGiMA0GCSqGSIb3DQEBAQUA
+A4IBjwAwggGKAoIBgQCzljYktbHdMGVJ6Wq0XQJuHLN6dkCSOgtoIzQtriPQkkNI
+uo28LwobaiQQ8sX4kGRH/BTKnH8QlId/jug4Uc+sDHnABYu++AiOhPbBX8gCpRQ0
+hebBjZiktHSBUEJR31siWOVdBoKBDJEoxehx7XUXvcxIJcaRN+LHYjO86nJN55HB
+VwFU2JcYDk98c+144dFJxXdr++MjWe4Z/oVVU8JHIOtNtKhVhvij6oOSWxcYoJO/
+S80LRj1vx/o6o/3G6bYug7PjY7JjZk/Oj61whijZkcsoO1MXSYI6UywJZGflv+ZB
+7HyufdYAsK3WhE8O2FX3/kq64Ol83HNtoR8Dt68rTg1xpW6K45jS6iDPKueYGkb0
+oSx7e++90VAW2PDhj6QQ3JJ4O5VQwrrecekJzUrAean0FOEbmgyi4PsEp1Vk6LDQ
+SsIn1x0euyxVivQMlzNX2XrZL3urn1BNPqAdntXQMkR0Wl8sbUiJPe0kxG52CGXs
+6yfNEXbPmVGcC0TBdGECAwEAAaNjMGEwHQYDVR0OBBYEFLh5QbI1UWMH0WR4z8bG
+lhrOX3X5MB8GA1UdIwQYMBaAFLh5QbI1UWMH0WR4z8bGlhrOX3X5MA8GA1UdEwEB
+/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgHGMA0GCSqGSIb3DQEBCwUAA4IBgQCVodVN
+owwo53OQe02QhtEbIur2PL7zIfvhvCTRD4J8gwpbMIqT7JQK0tV6Mvsg2L8yTb2O
+KjrWeLKHGWaZZlhGSPTbkMFdb/Ls8M9FSnkc2bwcdWW3Z1lOiCjBYYqwLCG6JhvB
+5SXVwWNJwXeasL2m7oFTSwhsqPpARJ2t25u2N35o+tqIoCjijKwkmEOT66N9EAbu
+2VQjtYZWPkBtP4YCe0Ey6u4oy7sy8ThNAjOylZok+J4JW7QEFjK4Q/emhA4aQq5H
+gg9qgMuG+5oi6D1g2Wy+fMTRBaukJtLYZbBpQMQhMYWg44uPp/2bbNPTID/nV1KB
+GcPyHaskcVxPdYWxAPMwk3AeJXWyOq7atAPTF5sbk0kQQf2m+vyOqcli5CxRMUgV
+rcyi9l6+dZW4U+38Q0ET5M3OuxNI4hA7kVY2cfTakXWNqh97+TIHnstblDhAxECK
+6ZLMJQYUy7LqJTX84H27CBWLexEMjXwdr5HCV88Fj6mAK0fRufnIw5FeneA=
+-----END CERTIFICATE-----

docs/auto-installer.md

@@ -22,7 +22,7 @@ see "LXC hosts" immediately below for why those are different.**
 ## LXC hosts
 
 `lxc-*` targets (`lxc-minimal`, `lxc-nix-cache`, `lxc-server`, `lxc-docker`,
-`lxc-gui`, `lxc-pxe-boot`, `lxc-tailscale-exit-node`, `lxc-tor-relay`) are **not** installed via `auto-install.sh` — the
+`lxc-gui`, `lxc-pxe-boot`, `lxc-tailscale-router`, `lxc-tor-relay`) are **not** installed via `auto-install.sh` — the
 interactive menu deliberately excludes them. Don't try to select one there;
 `nixos-install` would bind-mount `/` onto `/mnt` (LXC containers have no raw
 disk to partition) and then refuse to touch the filesystem it's currently
@@ -134,12 +134,14 @@ Flake outputs:
 nixosConfigurations.installer   # ISO/netboot installer image
 
 packages.x86_64-linux.iso       # installer ISO/netboot image
-packages.x86_64-linux.pxe   # netboot-ipxe + netboot-initrd + netboot-kernel, bundled
+packages.x86_64-linux.pxe       # auto-installer netboot bundle (kernel + initrd + ipxe script)
+packages.x86_64-linux.pxe-minimal  # vanilla NixOS minimal netboot bundle (no installer wiring)
 ```
 
 ```sh
 nix build .#iso
 nix build .#pxe
+nix build .#pxe-minimal
 ```
 
 There's no `nixosConfigurations.proxmox-lxc` (installer-boots-as-an-LXC-

docs/beszel.md

@@ -0,0 +1,113 @@
+# Beszel agent
+
+[Beszel](https://github.com/henrygd/beszel) is the monitoring dashboard used
+in this LAN. The hub runs as a Docker container on `docker.sweet.home` (port
+`vars.ports.beszelHub`, 8090). Each monitored NixOS host runs a
+`beszel-agent` that connects back to the hub.
+
+---
+
+## How it works
+
+Everything is handled by a single module:
+
+**`modules/beszel/enable-agent.nix`** — imported by a build type. It:
+- Enables `beszel-agent`
+- Sets `HUB_URL` to `docker.sweet.home:8090`
+- Sets `KEY` from `vars.beszelHubKey` (`variables.nix`) — the hub's SSH
+  public key, shared by every agent. Update `beszelHubKey` if the docker
+  host is ever rebuilt and the hub generates a new keypair.
+- Reads the universal `beszel-token` from `secrets/common.yaml` via sops
+  and passes it to the agent as `TOKEN` in an env file
+- Fixes an upstream bug where the agent couldn't persist its hub-pairing
+  fingerprint across restarts (adds a real `StateDirectory`)
+
+A host file needs no beszel configuration at all — just import the module
+in the build type and add the system in the hub UI.
+
+---
+
+## Adding beszel to a new build type
+
+Add `../beszel/enable-agent.nix` to the `imports` list in
+`modules/build-types/<type>.nix`:
+
+```nix
+imports = [
+  ../beszel/enable-agent.nix
+  # ... other imports
+];
+```
+
+That's the only change required. The host file needs nothing.
+
+---
+
+## Adding a new system to the hub
+
+1. Rebuild and deploy the host with its build type importing `enable-agent.nix`.
+2. Open the beszel hub (`http://docker.sweet.home:8090`).
+3. Go to **Systems → Add system**, enter the host's IP and the default port
+   (45876). The agent will connect and the system will appear as active.
+
+---
+
+## One-time setup: add the token to `secrets/common.yaml`
+
+The universal token is stored once in the common secrets file, shared by all
+agents. Only needed once, not per-host:
+
+```sh
+sops secrets/common.yaml
+```
+
+Add:
+```yaml
+beszel-token: <token from the beszel hub Settings → Keys>
+```
+
+`secrets/common.yaml` is already a sops recipient for every host via their
+SSH host keys, so no additional sops recipient setup is needed.
+
+---
+
+## Optional: monitoring extra filesystems
+
+To report disk usage for a mount beyond the root filesystem, add
+`EXTRA_FILESYSTEMS` in the host file:
+
+```nix
+services.beszel.agent.environment = {
+  EXTRA_FILESYSTEMS = "/mnt/data";   # colon-separated for multiple paths
+};
+```
+
+The `server` host uses this to expose its ZFS data pool:
+
+```nix
+services.beszel.agent.environment = {
+  EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
+  LOG_LEVEL = "debug";
+};
+```
+
+---
+
+## Optional: monitoring Docker containers
+
+`enable-agent.nix` has a commented-out line for Docker monitoring:
+
+```nix
+#DOCKER_HOST = "tcp://docker-socket-proxy:2375";
+```
+
+Uncomment it if the host runs docker-socket-proxy and you want per-container
+stats. Hosts without Docker should leave it commented out.
+
+---
+
+## If the hub key changes
+
+If the docker host is ever rebuilt and beszel generates a new SSH keypair,
+update `beszelHubKey` in `variables.nix` and rebuild all beszel-enabled hosts.
+The new key is visible in the beszel hub under **Settings → Keys**.

docs/ip-addressing.md

@@ -0,0 +1,128 @@
+# IP Addressing Scheme
+
+## Subnets
+
+| Subnet | CIDR | Purpose | Routed? |
+|---|---|---|---|
+| LAN | `192.168.2.0/24` | General LAN — clients and infrastructure | Yes (gateway .254) |
+| Storage | `192.168.4.0/29` | HA file server DRBD replication | No — internal `vmbr1` only, no uplink |
+
+The storage subnet never leaves pve1. `vmbr1` is a Proxmox Linux bridge with no physical port
+attached; traffic between the two HA file server VMs stays in-kernel.
+
+The host octet is consistent across subnets for any host that has multiple interfaces — e.g.
+ha-node1 is always `.228` (LAN: `192.168.2.228`, storage: `192.168.4.228`).
+
+---
+
+## LAN — 192.168.2.0/24
+
+### Address map
+
+| Range | Purpose |
+|---|---|
+| .1–.9 | Reserved, never assign |
+| .10–.59 | Client DHCP pool (router-assigned) |
+| .60–.219 | Unallocated buffer |
+| .220–.229 | Virtual nodes (VMs / LXC containers) |
+| .230–.239 | Expansion buffer (reserved, unallocated) |
+| .240–.249 | Physical nodes (bare-metal hosts) |
+| .250–.253 | Network services |
+| .254 | Router / gateway |
+
+### Network services (.250–.253)
+
+| IP | Hostname | Role |
+|---|---|---|
+| `192.168.2.254` | router | Gateway (TP-Link) |
+| `192.168.2.253` | domain-controller | FreeIPA — authoritative DNS for `sweet.home`, Kerberos, LDAP |
+| `192.168.2.250`–`.252` | — | Reserved for future network services |
+
+### Physical nodes (.240–.249)
+
+| IP | Hostname | Role |
+|---|---|---|
+| `192.168.2.245` | pve1 | Proxmox VE hypervisor |
+| `192.168.2.244` | pbs | Proxmox Backup Server |
+| `192.168.2.243` | nixos | Bare-metal workstation (`baremetal-gui`) |
+| `192.168.2.246`–`.249` | — | Reserved — second Proxmox node and associated services |
+| `192.168.2.240`–`.242` | — | Reserved |
+
+pve1 sits mid-range deliberately so a second Proxmox node can slot in on either side.
+
+### Virtual nodes (.220–.229)
+
+All VMs and LXC containers run on pve1.
+
+| IP | Hostname | Role | Status |
+|---|---|---|---|
+| `192.168.2.229` | ha-vip | HA file server iSCSI floating VIP (Pacemaker) | Future |
+| `192.168.2.228` | ha-node1 | HA file server node 1 (DRBD + XFS + iSCSI) | Future |
+| `192.168.2.227` | ha-node2 | HA file server node 2 (DRBD + XFS + iSCSI) | Future |
+| `192.168.2.226` | server | Current NFS/ZFS file server — retires when HA is live | Retiring |
+| `192.168.2.225` | docker | Docker / Traefik stack | Active |
+| `192.168.2.224` | nix-cache | Nix binary cache + remote builder | Active |
+| `192.168.2.223` | pxe-boot | PXE / TFTP / HTTP netboot server | Active |
+| `192.168.2.222` | tailscale-router | Tailscale exit node / router | Active |
+| `192.168.2.221` | tor-relay | Tor relay | Active |
+| `192.168.2.220` | pdm | Proxmox Deploy Manager | Active |
+
+### Client DHCP pool (.10–.59)
+
+Assigned by the router. DNS option points to `192.168.2.253` (domain-controller).
+
+Devices in this range: phones, laptops, IoT, Canon printer, any non-infrastructure host.
+No static reservations for infrastructure hosts — all infra uses static IP configuration
+on the guest itself (not DHCP reservations), so IPs survive VM recreation regardless of
+MAC address churn.
+
+---
+
+## Storage network — 192.168.4.0/29
+
+Internal to pve1 only. Proxmox bridge `vmbr1`, no physical NIC attached.
+
+| IP | Hostname | Interface role |
+|---|---|---|
+| `192.168.4.228` | ha-node1 | DRBD replication NIC |
+| `192.168.4.227` | ha-node2 | DRBD replication NIC |
+| — | no gateway | Isolated — not routed to LAN or internet |
+
+---
+
+## Migration reference
+
+Current → target IP for every host being renumbered.
+
+| Host | Current IP | New IP | Config location |
+|---|---|---|---|
+| router | `192.168.2.254` | `192.168.2.254` | unchanged |
+| domain-controller | `192.168.2.138` | `192.168.2.253` | `/etc/sysconfig/network-scripts/ifcfg-eth0` on guest |
+| pve1 | `192.168.2.250` | `192.168.2.245` | `/etc/network/interfaces` on Proxmox host |
+| pbs | `192.168.2.108` | `192.168.2.244` | static config on PBS host |
+| nixos workstation | `192.168.2.119` | `192.168.2.243` | `networking.interfaces` / NetworkManager on guest |
+| ha-node1 | — | `192.168.2.228` | future |
+| ha-node2 | — | `192.168.2.227` | future |
+| ha-vip | — | `192.168.2.229` | future (Pacemaker resource) |
+| server | `192.168.2.252` | `192.168.2.226` | static config on guest |
+| docker | `192.168.2.249` | `192.168.2.225` | static config on guest |
+| nix-cache | `192.168.2.120` | `192.168.2.224` | static config on guest |
+| pxe-boot | `192.168.2.247` | `192.168.2.223` | static config on guest; update `vars.pxeServerIp` in `variables.nix` ✓ |
+| tailscale-router | `192.168.2.121` | `192.168.2.222` | static config on guest |
+| tor-relay | `192.168.2.107` | `192.168.2.221` | static config on guest |
+| pdm | `192.168.2.248` | `192.168.2.220` | static config on guest |
+
+### Cutover notes
+
+- **Do domain-controller first** — it becomes the DNS server; everything else depends on it
+  having its new IP and FreeIPA DNS configured before Pi-hole is retired.
+- **pve1 last among physical hosts** — changing the Proxmox management IP drops the web UI
+  briefly; all guests keep running.
+- **Update Pi-hole custom.list / FreeIPA DNS A records** to new IPs before flipping any host,
+  so name resolution stays valid throughout the migration.
+- **variables.nix already updated** for `pxeServerIp` (.247→.223), `pbsIp` (.108→.244), and
+  new `domainControllerIp` (.253). Rebuild affected hosts after renumbering.
+- **Router DHCP**: once domain-controller is at .253 and FreeIPA DNS is serving `sweet.home`,
+  switch router DHCP on with pool .10–.59 and DNS option pointing to .253; retire Pi-hole CT.
+- **Pi-hole's iPXE dnsmasq config** (`99-ipxe-chainload.conf`) moves to the pxe-boot CT as a
+  dnsmasq proxy-mode config before Pi-hole is decommissioned.

docs/network-cutover.md

@@ -0,0 +1,366 @@
+# Network Cutover Plan
+
+Moves the LAN from the current flat/Pi-hole-managed state to the new IP scheme
+defined in `docs/ip-addressing.md`. Works in five independent stages — each
+stage is safe to pause after and resume later. Rollback steps are given at
+every point where something can break.
+
+**Before starting anything:** confirm you have
+- SSH access to `192.168.2.138` (domain-controller, current IP)
+- SSH access to `192.168.2.250` (pve1)
+- Browser access to Pi-hole admin at `http://192.168.2.253`
+- Browser access to router admin at `http://192.168.2.254`
+- The FreeIPA `admin` password to hand
+
+---
+
+## Stage 1 — Prepare FreeIPA DNS (zero downtime)
+
+Everything here is additive. Pi-hole keeps running. Nothing breaks if you stop
+mid-stage.
+
+### 1a. Add NextDNS forwarders
+
+```bash
+ssh wayne@192.168.2.138
+kinit admin          # enter FreeIPA admin password when prompted
+ipa dnsconfig-mod \
+  --forwarder=45.90.28.142 \
+  --forwarder=45.90.30.142 \
+  --forward-policy=only
+```
+
+**Verify external resolution works through FreeIPA before continuing:**
+```bash
+dig @127.0.0.1 google.com +short   # must return an IP, not SERVFAIL
+```
+
+### 1b. Add A records for every host at their CURRENT IPs
+
+These represent the live state now. You'll update each record to the new IP
+when you renumber that host in Stage 5.
+
+```bash
+ipa dnsrecord-add sweet.home pve1             --a-rec 192.168.2.250
+ipa dnsrecord-add sweet.home pbs              --a-rec 192.168.2.108
+ipa dnsrecord-add sweet.home nixos            --a-rec 192.168.2.119
+ipa dnsrecord-add sweet.home server           --a-rec 192.168.2.252
+ipa dnsrecord-add sweet.home docker           --a-rec 192.168.2.249
+ipa dnsrecord-add sweet.home nix-cache        --a-rec 192.168.2.120
+ipa dnsrecord-add sweet.home pxe-boot         --a-rec 192.168.2.247
+ipa dnsrecord-add sweet.home tailscale-router --a-rec 192.168.2.121
+ipa dnsrecord-add sweet.home tor-relay        --a-rec 192.168.2.107
+ipa dnsrecord-add sweet.home pdm              --a-rec 192.168.2.248
+ipa dnsrecord-add sweet.home router           --a-rec 192.168.2.254
+```
+
+### 1c. Clean up stale reverse-zone PTR records
+
+FreeIPA already has PTR records from an earlier import but some are wrong.
+Fix them now so reverse DNS is accurate from day one.
+
+```bash
+# Remove stale "win11" entry at .250 (should be pve1)
+ipa dnsrecord-del 2.168.192.in-addr.arpa 250 --ptr-rec win11.
+ipa dnsrecord-add 2.168.192.in-addr.arpa 250 --ptr-rec pve1.sweet.home.
+
+# Fix unqualified PTR records (missing .sweet.home. suffix)
+ipa dnsrecord-mod 2.168.192.in-addr.arpa 108 --ptr-rec pbs.sweet.home.
+ipa dnsrecord-mod 2.168.192.in-addr.arpa 248 --ptr-rec pdm.sweet.home.
+ipa dnsrecord-mod 2.168.192.in-addr.arpa 249 --ptr-rec docker.sweet.home.
+ipa dnsrecord-mod 2.168.192.in-addr.arpa 252 --ptr-rec server.sweet.home.
+
+# Add any missing PTR records
+ipa dnsrecord-add 2.168.192.in-addr.arpa 119 --ptr-rec nixos.sweet.home.
+ipa dnsrecord-add 2.168.192.in-addr.arpa 120 --ptr-rec nix-cache.sweet.home.
+ipa dnsrecord-add 2.168.192.in-addr.arpa 121 --ptr-rec tailscale-router.sweet.home.
+ipa dnsrecord-add 2.168.192.in-addr.arpa 247 --ptr-rec pxe-boot.sweet.home.
+ipa dnsrecord-add 2.168.192.in-addr.arpa 254 --ptr-rec router.sweet.home.
+```
+
+### 1d. Point domain-controller's own DNS at itself
+
+```bash
+sudo nmcli connection modify "System eth0" ipv4.dns "127.0.0.1"
+sudo nmcli connection up "System eth0"
+```
+
+**Verify:**
+```bash
+dig pve1.sweet.home +short          # must return 192.168.2.250
+dig google.com +short               # must return an IP (NextDNS forwarding)
+```
+
+**Rollback 1d:** `sudo nmcli connection modify "System eth0" ipv4.dns "192.168.2.253" && sudo nmcli connection up "System eth0"`
+
+---
+
+## Stage 2 — Move pxe-boot DHCP options off Pi-hole (zero downtime)
+
+Pi-hole's dnsmasq currently serves the iPXE boot options via
+`99-ipxe-chainload.conf`. Before Pi-hole is retired, that config must move to
+the pxe-boot CT running dnsmasq in proxy mode so PXE boot keeps working.
+
+### 2a. Add dnsmasq proxy config to the pxe-boot NixOS module
+
+In `modules/build-types/pxe-boot.nix`, add:
+
+```nix
+services.dnsmasq = {
+  enable = true;
+  settings = {
+    # Proxy mode: respond only to PXE DHCP requests, leave normal leases to router
+    dhcp-range = [ "192.168.2.0,proxy" ];
+    # iPXE client detection
+    dhcp-match = [
+      "set:ipxe,175"
+      "set:efi64,option:client-arch,7"
+      "set:efi64,option:client-arch,9"
+    ];
+    dhcp-userclass = "set:ipxe,iPXE";
+    # Boot file selection
+    dhcp-boot = [
+      "tag:ipxe,tag:efi64,http://${vars.pxeServerIp}/boot.ipxe"
+      "tag:ipxe,http://${vars.pxeServerIp}/boot.ipxe"
+      "tag:efi64,ipxe.efi,,${vars.pxeServerIp}"
+      "undionly.kpxe,,${vars.pxeServerIp}"
+    ];
+  };
+};
+```
+
+### 2b. Rebuild and deploy the pxe-boot CT
+
+```bash
+# On pve1 — build the new tarball
+nix build .#lxc-pxe-boot.config.system.build.tarball
+
+# Verify dnsmasq starts correctly in the CT after deploy
+ssh nixos@192.168.2.247 systemctl status dnsmasq
+```
+
+### 2c. Remove the iPXE config from Pi-hole
+
+In the Pi-hole CT, remove `/etc/dnsmasq.d/99-ipxe-chainload.conf` and
+restart the FTL service:
+
+```bash
+ssh wayne@pve1.sweet.home \
+  "sudo pct exec 100 -- bash -c 'rm /etc/dnsmasq.d/99-ipxe-chainload.conf && systemctl restart pihole-FTL'"
+```
+
+**Verify:** PXE boot a test machine — it should still get an iPXE response and
+reach the boot menu.
+
+**Rollback 2c:** restore the file from the Pi-hole config backup at
+`/etc/pihole/config_backups/` and restart pihole-FTL.
+
+---
+
+## Stage 3 — DHCP migration: Pi-hole → router (brief maintenance window)
+
+**Do this in the evening.** Existing DHCP leases stay valid during the
+switchover so connected devices don't drop — only new lease requests fail
+during the gap, which is under 60 seconds if you follow the steps in order.
+
+The key: configure the router's DHCP DNS option to point at `.253` (Pi-hole's
+current IP). This way, all new leases issued by the router still get the same
+DNS server address — clients never need to change their DNS config. When Pi-hole
+is retired and the DC takes `.253` in Stage 4, `.253` just starts answering
+differently. No client reconfiguration.
+
+### 3a. Pre-configure router DHCP (do not enable yet)
+
+Log into `http://192.168.2.254`, find the DHCP settings and fill in — but
+leave DHCP **disabled** until step 3b:
+
+| Setting | Value |
+|---|---|
+| Start IP | 192.168.2.10 |
+| End IP | 192.168.2.59 |
+| Subnet mask | 255.255.255.0 |
+| Gateway | 192.168.2.254 |
+| Primary DNS | 192.168.2.253 |
+| Secondary DNS | *(leave blank)* |
+| Lease time | 24h |
+
+Save without enabling.
+
+### 3b. Switchover (do steps in quick succession)
+
+1. **Disable Pi-hole DHCP:** Pi-hole admin UI → Settings → DHCP → uncheck
+   "DHCP server enabled" → Save
+2. **Enable router DHCP** immediately after step 1
+
+### 3c. Verify router DHCP is working
+
+On a phone or laptop, disconnect from WiFi and reconnect (or run
+`sudo dhclient -r && sudo dhclient` on a Linux host):
+
+```bash
+ip addr show        # IP should be in 192.168.2.10–59 range
+dig google.com      # should resolve (Pi-hole DNS still running at .253)
+dig pve1.sweet.home # should resolve via FreeIPA at .138 (relayed via Pi-hole)
+```
+
+Wait 10–15 minutes for the most active devices to renew their leases. There's
+no need to wait for all leases to expire before proceeding.
+
+**Rollback 3b:** Re-enable Pi-hole DHCP. Disable router DHCP. Done — existing
+leases remain valid so most devices are unaffected.
+
+---
+
+## Stage 4 — Move domain-controller from .138 to .253
+
+Pi-hole lives at `.253`. The DC must take `.253` the moment Pi-hole stops so
+clients that still have `.253` as their DNS server don't notice the change.
+Script these commands in advance and run them in rapid succession.
+
+**Pre-stage: have this SSH command ready before running step 4a:**
+```bash
+ssh wayne@192.168.2.138 "
+  sudo nmcli connection modify 'System eth0' \
+    ipv4.addresses '192.168.2.253/24' \
+    ipv4.gateway '192.168.2.254' \
+    ipv4.dns '127.0.0.1' \
+    ipv4.method manual && \
+  sudo nmcli connection up 'System eth0'
+"
+```
+
+**Also update the Proxmox VM config to match (run from pve1):**
+```bash
+sudo qm set 108 \
+  --ipconfig0 ip=192.168.2.253/24,gw=192.168.2.254 \
+  --nameserver 192.168.2.253
+```
+
+### 4a. Stop Pi-hole
+
+```bash
+ssh wayne@pve1.sweet.home "sudo pct stop 100"
+```
+
+### 4b. Immediately: change DC's IP to .253
+
+Run the pre-staged SSH command from above. You have ~30 seconds before any
+client notices Pi-hole is gone. If SSH to `.138` refuses (the IP is already
+changing), open a Proxmox console to VM 108 and run the `nmcli` commands
+there.
+
+### 4c. Update Proxmox VM config
+
+Run the pre-staged `qm set 108` command from above.
+
+### 4d. Verify
+
+```bash
+ssh wayne@192.168.2.253   # must connect (new DC IP)
+dig @192.168.2.253 pve1.sweet.home +short   # must return 192.168.2.250
+dig @192.168.2.253 google.com +short        # must return an IP
+```
+
+From a client device that renewed its DHCP lease in Stage 3:
+```bash
+cat /etc/resolv.conf    # should show 192.168.2.253
+dig pve1.sweet.home     # should resolve
+```
+
+**Rollback 4:** `ssh wayne@pve1.sweet.home "sudo pct start 100"`. Change DC IP
+back to .138 via Proxmox console. This restores full Pi-hole DNS/DHCP service.
+Leave Pi-hole CT stopped-but-intact for 48 hours before deleting it.
+
+---
+
+## Stage 5 — Host renumbering (one at a time, any order)
+
+For each host:
+1. Update FreeIPA DNS A record and PTR record to the new IP
+2. Change the static IP on the host itself
+3. Verify SSH to new IP
+4. Update `variables.nix` if that host has an IP variable (pxe-boot, pbs — already done in this PR)
+
+**FreeIPA record update template** (run as admin on domain-controller):
+```bash
+ipa dnsrecord-mod sweet.home <hostname> --a-rec <new-ip>
+ipa dnsrecord-del 2.168.192.in-addr.arpa <old-last-octet> --ptr-rec <hostname>.sweet.home.
+ipa dnsrecord-add 2.168.192.in-addr.arpa <new-last-octet> --ptr-rec <hostname>.sweet.home.
+```
+
+### Renumbering order
+
+| # | Host | Old IP | New IP | How to change IP |
+|---|---|---|---|---|
+| 1 | nixos workstation | .119 | .243 | NetworkManager on guest; or `nmcli connection modify` |
+| 2 | nix-cache | .120 | .224 | `pct set 102 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.224/24,gw=192.168.2.254` then `pct reboot 102` |
+| 3 | tailscale-router | .121 | .222 | Static config on guest; check Tailscale ACLs if IP is referenced there |
+| 4 | tor-relay | .107 | .221 | `pct set 104 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.221/24,gw=192.168.2.254` then `pct reboot 104` |
+| 5 | pdm | .248 | .220 | `pct set 106 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.220/24,gw=192.168.2.254` then `pct reboot 106` |
+| 6 | pxe-boot | .247 | .223 | `pct set 103 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.223/24,gw=192.168.2.254` then rebuild NixOS (already updated in variables.nix) |
+| 7 | server | .252 | .226 | Static config on guest; NFS clients (docker) lose mounts briefly — they remount automatically |
+| 8 | docker | .249 | .225 | Static config on guest; do this after server is at .226 |
+| 9 | pbs | .108 | .244 | Static config on PBS host itself; update in `pbsIp` already done in variables.nix |
+| 10 | pve1 | .250 | .245 | Edit `/etc/network/interfaces` on the Proxmox host — see below |
+
+### pve1 renumber (step 10 — do last)
+
+All guests keep running; only the Proxmox web UI is briefly unreachable.
+
+```bash
+ssh wayne@pve1.sweet.home
+
+# Edit /etc/network/interfaces: change address from .250 to .245
+sudo nano /etc/network/interfaces
+# Change:  address 192.168.2.250/24
+# To:      address 192.168.2.245/24
+
+sudo systemctl restart networking
+# SSH will drop here — reconnect to new IP
+```
+
+```bash
+ssh wayne@192.168.2.245    # verify
+```
+
+Update FreeIPA DNS:
+```bash
+ipa dnsrecord-mod sweet.home pve1 --a-rec 192.168.2.245
+ipa dnsrecord-del 2.168.192.in-addr.arpa 250 --ptr-rec pve1.sweet.home.
+ipa dnsrecord-add 2.168.192.in-addr.arpa 245 --ptr-rec pve1.sweet.home.
+```
+
+**Rollback any step 5 host:** change the IP back on the guest and update the
+FreeIPA record back to the old IP. The old IP is unoccupied so you can
+temporarily use either.
+
+---
+
+## Stage 6 — Final cleanup
+
+Once all hosts are at their new IPs and verified:
+
+```bash
+# Delete the Pi-hole CT (already stopped since Stage 4)
+ssh wayne@pve1.sweet.home "sudo pct destroy 100"
+
+# Remove stale FreeIPA records for retired addresses
+ipa dnsrecord-del sweet.home pihole --del-all
+ipa dnsrecord-del 2.168.192.in-addr.arpa 253 --ptr-rec pihole.sweet.home.
+
+# Rebuild any NixOS hosts that reference pbsIp or pxeServerIp to pick up
+# the updated variables.nix values (pxe-boot mandatory; others as convenient)
+```
+
+---
+
+## Rollback summary
+
+| What broke | How to roll back |
+|---|---|
+| FreeIPA DNS not resolving | Check `systemctl status named` on DC; restart if failed |
+| FreeIPA DNS unreachable | `pct start 100` on pve1 (restores Pi-hole) |
+| Router DHCP not handing out leases | Re-enable Pi-hole DHCP; disable router DHCP |
+| DC unreachable after IP change | Proxmox console on VM 108 → `nmcli connection up "System eth0"` with old IP |
+| Host unreachable after renumber | Proxmox console → revert IP; or `pct set <id> --net0 ...` old IP and reboot CT |
+| pve1 web UI gone after renumber | SSH to .245 and check `/etc/network/interfaces`; if wrong, fix and restart networking |

docs/pxe-boot.md

@@ -15,6 +15,7 @@ rescue/inspection use.
 - TFTP root for first-stage bootloaders: `/srv/pxe/tftp`
 - iPXE entry script: `/srv/pxe/http/boot.ipxe`
 - Generated iPXE menu: `/srv/pxe/http/menu.ipxe`
+- Debian Minimal iPXE script: `/srv/pxe/http/debian.ipxe`
 - SystemRescue iPXE script: `/srv/pxe/http/systemrescue.ipxe`
 - TFTP fallback script: `/srv/pxe/tftp/autoexec.ipxe`
 - Boot binaries copied from the Nix `ipxe` package:
@@ -28,32 +29,51 @@ The host creates these directories with systemd tmpfiles:
 ```text
 /srv/pxe
 /srv/pxe/http
-/srv/pxe/http/images
+/srv/pxe/http/images  -> /mnt/pxe-images  (symlink to NFS share)
 /srv/pxe/http/auto-installer
 /srv/pxe/http/nixos-minimal
+/srv/pxe/http/debian
 /srv/pxe/http/systemrescue
 /srv/pxe/http/ubuntu
 /srv/pxe/http/rescue
 /srv/pxe/tftp
 ```
 
-Mount shared image storage under `/srv/pxe/http`, preferably
+`/srv/pxe/http/images` is a symlink to `/mnt/pxe-images`, which is an NFS
-`/srv/pxe/http/images` unless a menu entry expects files in a specific
+mount of `server.sweet.home:/tank/pxe-boot/images`
-directory such as `/srv/pxe/http/auto-installer`.
+(`modules/pxe-boot/mount-pxe-images.nix`). Place large images there (ISOs,
+disk images) rather than on the pxe-boot host's own root disk. For an LXC
+pxe-boot container the mount uses NFSv3+nolock with `nofail` (eager,
+non-blocking on server unavailability); for a Proxmox VM it uses NFSv4.2
+with `x-systemd.automount` (lazy, triggered on first access).
+
+When running as `lxc-pxe-boot`, the Proxmox container must have
+`features: nesting=1,mount=nfs` (at minimum) in its Proxmox config. `nesting=1`
+is required by systemd 260+ for credential isolation (user namespace creation
+and internal move-mounts); without it, AppArmor denies both, and every
+systemd service that uses `PrivateUsers`, `PrivateDevices`, or credential
+passing fails on boot. `mount=nfs` allows the NFSv3 mount. Both are set
+automatically by `scripts/proxmox/create-proxmox-resource.sh` (via
+`PROXMOX_DEFAULT_LXC_FEATURES` in `scripts/env.sh` which defaults to
+`nesting=1,keyctl=1,mount=nfs;nfs4`). If you ever change these features
+manually via `pct set`, be sure to include both — `pct set` replaces the
+entire features string, it does not append to it.
 
 The HTTP iPXE chain is:
 
 ```text
 undionly.kpxe or ipxe.efi
   -> autoexec.ipxe from the TFTP root, when iPXE requests it
-  -> http://192.168.2.247/boot.ipxe
+  -> http://192.168.2.223/boot.ipxe
-  -> http://192.168.2.247/menu.ipxe
+  -> http://192.168.2.223/menu.ipxe
 ```
 
 The generated menu currently exposes entries for:
 
 - NixOS Auto-Installer
 - NixOS Minimal
+- Debian Minimal
+- FreeIPA Server (Rocky Linux 9)
 - SystemRescue environment
 - iPXE shell
 - Reboot
@@ -84,19 +104,82 @@ directory name (`auto-installer` / `nixos-minimal`), so each one's
 generated system name (`nixos-system-<name>-*`) is self-describing rather
 than the nixpkgs default of `nixos-system-nixos-*` for both.
 
+The Debian Minimal entry chains `http://<pxeServerIp>/debian.ipxe`, which loads
+the Debian bookworm netboot kernel and initrd from `/srv/pxe/http/debian/`. The
+`fetch-debian-netboot.service` oneshot downloads these files from
+`deb.debian.org` on first boot (idempotent — skips if files are already
+present):
+
+```text
+/srv/pxe/http/debian/linux      (Debian bookworm netboot kernel)
+/srv/pxe/http/debian/initrd.gz  (Debian bookworm netboot initrd)
+```
+
+The service requires outbound internet access on the pxe-boot host. To
+re-download (e.g. after a Debian point release), delete the files and restart
+the service:
+
+```bash
+rm /srv/pxe/http/debian/linux /srv/pxe/http/debian/initrd.gz
+systemctl restart fetch-debian-netboot.service
+```
+
+To update to a different Debian release, change `debianRelease` in
+`modules/build-types/pxe-boot.nix` and redeploy.
+
+The **FreeIPA Server (Rocky Linux 9)** entry chains
+`http://<pxeServerIp>/rocky-freeipa.ipxe`, which boots the Rocky Linux 9
+Anaconda installer with a Kickstart file (`rocky-freeipa.ks`) hosted on the
+same server. The `fetch-rocky-pxeboot.service` oneshot downloads the pxeboot
+kernel and initrd from the Rocky Linux mirror on first boot (idempotent):
+
+```text
+/srv/pxe/http/rocky/vmlinuz     (Rocky Linux 9 Anaconda pxeboot kernel)
+/srv/pxe/http/rocky/initrd.img  (Rocky Linux 9 Anaconda pxeboot initrd)
+```
+
+The Kickstart file is generated from the NixOS module and staged at
+`/srv/pxe/http/rocky-freeipa.ks`. It performs a fully unattended install:
+
+1. Installs Rocky Linux 9 with `ipa-server` + `ipa-server-dns` packages
+2. Configures static IP `192.168.2.138`, hostname `domain-controller.sweet.home`
+3. Creates user `wayne` with the `adminSshKey` from `variables.nix`
+4. Generates random IPA passwords and writes them to `/root/ipa-credentials.txt`
+5. Creates a `freeipa-first-boot.service` oneshot that runs `ipa-server-install`
+   on first reboot (~20 minutes)
+
+After the install completes:
+- SSH in as `wayne@domain-controller` using the admin key
+- Monitor FreeIPA install progress: `sudo tail -f /root/freeipa-install.log`
+- Retrieve credentials: `sudo cat /root/ipa-credentials.txt` (save to password manager)
+- Configure Pi-hole: `server=/sweet.home/192.168.2.138` in dnsmasq
+
+To refresh the pxeboot files (e.g. after a Rocky point release):
+
+```bash
+rm /srv/pxe/http/rocky/vmlinuz /srv/pxe/http/rocky/initrd.img
+systemctl restart fetch-rocky-pxeboot.service
+```
+
+To update to a different Rocky release, change `rockyRelease` in
+`modules/build-types/pxe-boot.nix` and redeploy.
+
 The SystemRescue entry expects the source ISO at:
 
 ```text
 /srv/pxe/http/images/systemrescue.iso
 ```
 
+Since `/srv/pxe/http/images` is the NFS-backed symlink, place the ISO on the
+NFS share at `server.sweet.home:/tank/pxe-boot/images/systemrescue.iso`.
+
 The `stage-systemrescue.service` oneshot extracts that ISO into:
 
 ```text
 /srv/pxe/http/systemrescue
 ```
 
-The rescue menu entry then chains `http://192.168.2.247/systemrescue.ipxe`,
+The rescue menu entry then chains `http://192.168.2.223/systemrescue.ipxe`,
 which loads the SystemRescue kernel and initramfs from the extracted tree and
 uses `archiso_http_srv` to fetch the squashfs payload over HTTP.
 
@@ -113,6 +196,13 @@ After deployment by an operator, basic service checks are:
 ```bash
 curl http://pxe-boot/boot.ipxe
 curl http://pxe-boot/menu.ipxe
+curl http://pxe-boot/debian.ipxe
+curl -I http://pxe-boot/debian/linux
+curl -I http://pxe-boot/debian/initrd.gz
+curl http://pxe-boot/rocky-freeipa.ipxe
+curl http://pxe-boot/rocky-freeipa.ks
+curl -I http://pxe-boot/rocky/vmlinuz
+curl -I http://pxe-boot/rocky/initrd.img
 curl http://pxe-boot/systemrescue.ipxe
 curl -I http://pxe-boot/systemrescue/sysresccd/boot/x86_64/vmlinuz
 curl -I http://pxe-boot/systemrescue/sysresccd/boot/x86_64/sysresccd.img

flake.lock

@@ -173,11 +173,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1784350909,
+        "lastModified": 1785119570,
-        "narHash": "sha256-ZWyzLbS1yKUTeFJLmdVuWNnHttL333/ldJbEE+KzCrM=",
+        "narHash": "sha256-Rgs2xKnGLFWQscxUaXX07oyZeuMDOHEbqDOsgliLFGM=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "4ce190229c73d44536caa7072f6308fb2d8feeb3",
+        "rev": "d4fd24667c8cbef124bb70a20380cab75ec8474d",
         "type": "github"
       },
       "original": {
@@ -259,11 +259,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1784432872,
+        "lastModified": 1785133411,
-        "narHash": "sha256-n3gKTBIV4ZA5VQpUakffBe3KGu4+mhPoA34rrqS0GkA=",
+        "narHash": "sha256-Yjv0WEg39KRYS0rBdTbu6Fc/or/ihAKk13W9sQ6VWd0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "fd1462031fdee08f65fd0b4c6b64e22239a77870",
+        "rev": "2f5a153c270b70cb0f8c11f46d96d6d3bc39f4e3",
         "type": "github"
       },
       "original": {

flake.nix

@@ -130,6 +130,9 @@
         lxc-tailscale-router = mkTarget { platform = "lxc"; buildType = "tailscale-router"; hostPath = ./hosts/tailscale-router/host.nix; };
 
         lxc-tor-relay = mkTarget { platform = "lxc"; buildType = "tor-relay"; hostPath = ./hosts/tor-relay/host.nix; };
+
+        proxmox-ha-server-1 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-1/host.nix; };
+        proxmox-ha-server-2 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-2/host.nix; };
       };
 
       # Auto-install environments (migrated from the former nix-auto-installer

hosts/docker/host.nix

@@ -1,10 +1,24 @@
-_:
+{ vars, ... }:
 
 {
-  networking.hostName = "docker";
+  networking = {
-  networking.hostId = "007f0200";
+    hostName = "docker";
+    hostId = "007f0200";
+    useDHCP = false;
+    interfaces.${vars.vmLanInterface}.ipv4.addresses = [{
+      address = vars.dockerIp;
+      prefixLength = vars.lanPrefixLength;
+    }];
+    defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; };
+    nameservers = [ vars.domainControllerIp ];
+  };
   boot.zfs.forceImportRoot = false;
 
+  # Only advertise the LAN interface to IPA DNS. Without this, SSSD registers
+  # every Docker bridge (172.x.x.x) as an A record for docker.sweet.home —
+  # the default dyndns.interface = "*" catches them all.
+  security.ipa.dyndns.interface = vars.lxcLanInterface; # eth0
+
   # Preserved from the pre-refactor `docker` target — stateVersion must never
   # be bumped on an already-installed machine.
   system.stateVersion = "25.05";

hosts/ha-server-1/host.nix

@@ -0,0 +1,20 @@
+{ vars, ... }:
+{
+  networking = {
+    hostName = vars.haServer1Host;
+    hostId = "3a4b5c6d";
+    useDHCP = false;
+    interfaces.${vars.vmLanInterface}.ipv4.addresses = [{
+      address = vars.haServer1Ip;
+      prefixLength = vars.lanPrefixLength;
+    }];
+    interfaces.${vars.vmStorageInterface}.ipv4.addresses = [{
+      address = vars.haServer1StorageIp;
+      prefixLength = vars.haStoragePrefixLength;
+    }];
+    defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; };
+    nameservers = [ vars.domainControllerIp ];
+  };
+
+  system.stateVersion = "26.05";
+}

hosts/ha-server-2/host.nix

@@ -0,0 +1,20 @@
+{ vars, ... }:
+{
+  networking = {
+    hostName = vars.haServer2Host;
+    hostId = "7e8f9a0b";
+    useDHCP = false;
+    interfaces.${vars.vmLanInterface}.ipv4.addresses = [{
+      address = vars.haServer2Ip;
+      prefixLength = vars.lanPrefixLength;
+    }];
+    interfaces.${vars.vmStorageInterface}.ipv4.addresses = [{
+      address = vars.haServer2StorageIp;
+      prefixLength = vars.haStoragePrefixLength;
+    }];
+    defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; };
+    nameservers = [ vars.domainControllerIp ];
+  };
+
+  system.stateVersion = "26.05";
+}

hosts/nix-cache/host.nix

@@ -1,18 +1,15 @@
 { vars, ... }:
 
 {
-  imports = [
+  networking = {
-    (import ../../modules/beszel/host-token.nix {
+    hostName = vars.nixCacheHost;
-      name = "nix-cache";
+    useDHCP = false;
-      sopsFile = ../../secrets/nix-cache.yaml;
+    interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{
-    })
+      address = vars.nixCacheIp;
-  ];
+      prefixLength = vars.lanPrefixLength;
-
+    }];
-  networking.hostName = vars.nixCacheHost;
+    defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; };
-
+    nameservers = [ vars.domainControllerIp ];
-  services.beszel.agent.environment = {
-    #DOCKER_HOST = "tcp://docker-socket-proxy:2375";
-    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
   };
 
   # Preserved from the pre-refactor `nix-cache` target — stateVersion must

hosts/nixos/home.nix

@@ -26,8 +26,8 @@
 
     # Optional: set environment vars
     sessionVariables = {
-      EDITOR = "vim";
+      EDITOR = "nano";
-      SOPS_AGE_KEY_FILE = "~/.config/sops/age/keys.txt";
+      SOPS_AGE_KEY_FILE = "${config.home.homeDirectory}/.config/sops/age/keys.txt";
     };
 
     file = {

hosts/pxe-boot/host.nix

@@ -1,8 +1,19 @@
-_:
+{ vars, ... }:
 
 {
-  networking.hostName = "pxe-boot";
+  networking = {
-
+    hostName = "pxe-boot";
+    useDHCP = false;
+    interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{
+      address = vars.pxeServerIp;
+      prefixLength = vars.lanPrefixLength;
+    }];
+    defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; };
+    nameservers = [ vars.domainControllerIp ];
+  };
+services.beszel.agent.environment = {
+#    KEY = "";
+  };
   # Preserved from the pre-refactor `pxe-boot` target — stateVersion must
   # never be bumped on an already-installed machine.
   system.stateVersion = "25.05";

hosts/server/host.nix

@@ -1,19 +1,19 @@
 { vars, ... }:
 
 {
-  imports = [
+  networking = {
-    (import ../../modules/beszel/host-token.nix {
+    hostName = vars.nfsServerHost;
-      name = "server";
+    hostId = "6689f93e";
-      sopsFile = ../../secrets/server.yaml;
+    useDHCP = false;
-    })
+    interfaces.${vars.vmLanInterface}.ipv4.addresses = [{
-  ];
+      address = vars.serverIp;
-
+      prefixLength = vars.lanPrefixLength;
-  networking.hostName = vars.nfsServerHost;
+    }];
-  networking.hostId = "6689f93e";
+    defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; };
+    nameservers = [ vars.domainControllerIp ];
+  };
 
   services.beszel.agent.environment = {
-    #DOCKER_HOST = "tcp://docker-socket-proxy:2375";
-    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
     EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
     LOG_LEVEL = "debug";
   };

hosts/tailscale-router/host.nix

@@ -1,7 +1,16 @@
-_:
+{ vars, ... }:
 
 {
-  networking.hostName = "tailscale-router";
+  networking = {
+    hostName = "tailscale-router";
+    useDHCP = false;
+    interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{
+      address = vars.tailscaleRouterIp;
+      prefixLength = vars.lanPrefixLength;
+    }];
+    defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; };
+    nameservers = [ vars.domainControllerIp ];
+  };
 
   # No networking.hostId: only ZFS-touching hosts (server, docker) need one
   # for pool-import safety, and this host does neither.

hosts/tor-relay/host.nix

@@ -1,22 +1,20 @@
-{ ... }:
+{ vars, ... }:
 
 {
-  imports = [
+  networking = {
-    (import ../../modules/beszel/host-token.nix {
+    hostName = "tor-relay";
-      name = "tor-relay";
+    useDHCP = false;
-      sopsFile = ../../secrets/tor-relay.yaml;
+    interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{
-    })
+      address = vars.torRelayIp;
-  ];
+      prefixLength = vars.lanPrefixLength;
-
+    }];
-  networking.hostName = "tor-relay";
+    defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; };
-
+    nameservers = [ vars.domainControllerIp ];
-  # No networking.hostId: only ZFS-touching hosts (server, docker) need one
-  # for pool-import safety, and this host does neither.
-
-  services.beszel.agent.environment = {
-    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
   };
 
+  # No networking.hostId: only ZFS-touching hosts need one for pool-import
+  # safety, and this host does neither.
+
   # A genuinely new host (not a pre-refactor carry-over), so it tracks the
   # flake's current nixpkgs release rather than being pinned to an older one.
   system.stateVersion = "26.05";

modules/beszel/enable-agent.nix

@@ -1,10 +1,23 @@
-{ vars, ... }:
+{ config, vars, ... }:
 
 {
-  services.beszel.agent.enable = true;
+  # Universal token shared by all beszel agents. Add to secrets/common.yaml:
-  services.beszel.agent.environment = {
+  #   sops secrets/common.yaml
+  #   beszel-token: <value from the beszel hub UI>
+  sops.secrets."beszel-token" = { };
+
+  sops.templates."beszel.env".content = ''
+    TOKEN=${config.sops.placeholder."beszel-token"}
+  '';
+
+  services.beszel.agent = {
+    enable = true;
+    environmentFile = config.sops.templates."beszel.env".path;
+    environment = {
       #DOCKER_HOST = "tcp://docker-socket-proxy:2375";
       HUB_URL = "http://${vars.dockerHost}.${vars.homeDomain}:${toString vars.ports.beszelHub}";
+      KEY = vars.beszelHubKey;
+    };
   };
 
   # The upstream module runs beszel-agent under DynamicUser with

modules/beszel/host-token.nix

@@ -1,11 +0,0 @@
-{ name, sopsFile }:
-
-{ config, ... }:
-
-{
-  sops.secrets."beszel-token".sopsFile = sopsFile;
-  sops.templates."${name}-beszel.env".content = ''
-    TOKEN=${config.sops.placeholder."beszel-token"}
-  '';
-  services.beszel.agent.environmentFile = config.sops.templates."${name}-beszel.env".path;
-}

modules/build-types/docker.nix

@@ -15,7 +15,6 @@
     ../docker/enable-service.nix
     ../docker/nextcloud-cron-job.nix
     ../docker/docker-health-to-gotify.nix
-    ../tailscale/enable-service.nix
     ../traefik/rotate-logs.nix
     ../raspi/mount-data.nix
     ../services/enable-rpcbind.nix

modules/build-types/gui.nix

@@ -1,6 +1,17 @@
 { config, pkgs, lib, inputs, vars, ... }:
 
 {
+  imports = [
+    ../docker/enable-service.nix
+  ];
+
+  nixpkgs.overlays = [
+    (final: prev: {
+      docker = prev.docker_29;
+      docker_cli = prev.docker_29;
+    })
+  ];
+
   environment.systemPackages = with pkgs; [
     inputs.nixos-conf-editor.packages.${pkgs.stdenv.hostPlatform.system}.nixos-conf-editor
     nodejs
@@ -70,4 +81,70 @@ programs.direnv.enable = true;
   programs.firefox.enable = true;
 
   nixpkgs.config.allowUnfree = true;
+
+  # GUI-specific Home Manager additions for the IPA primary user, extending
+  # the baseline in modules/ipa/client.nix with desktop apps and services
+  # that only make sense on a graphical workstation.
+  home-manager.users.${vars.ipaUser} = { pkgs, ... }: {
+    home = {
+      packages = with pkgs; [
+        git
+        vim
+        nextcloud-client
+        chromium
+        claude-code
+        fish
+        sops
+      ];
+      sessionVariables = {
+        EDITOR = "vim";
+        SOPS_AGE_KEY_FILE = "/home/${vars.ipaUser}/.config/sops/age/keys.txt";
+      };
+      file = {
+        ".local/share/applications/proxmox-chromium-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox (Chromium)
+          Exec=chromium --app=https://pve.${vars.homeDomain}:${toString vars.ports.pveWeb} --window-size=1920,1080 --window-position=0,0
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=Hypervisor;
+          StartupWMClass=PVE
+        '';
+        ".local/share/applications/pbs-chromium-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox Backup Server (Chromium)
+          Exec=chromium --app=https://${vars.pbsIp}:${toString vars.ports.pbsWeb} --window-size=1920,1080 --window-position=0,0
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=backup;
+        '';
+        ".local/share/applications/proxmox-firefox-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox (Firefox)
+          Exec=firefox --new-instance https://pve.${vars.homeDomain}:${toString vars.ports.pveWeb} --profile ProxmoxWebApp --window-size=1920,1080 --class ProxmoxWebApp
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=Hypervisor;
+          StartupWMClass=PVE
+        '';
+        ".local/share/applications/pbs-firefox-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox Backup Server (Firefox)
+          Exec=firefox --new-window https://${vars.pbsIp}:${toString vars.ports.pbsWeb} --profile PbsWebApp --window-size=1920,1080 --class PbsWebApp
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=backup;
+          StartupWMClass=PBS
+        '';
+      };
+    };
+    services.nextcloud-client = {
+      enable = true;
+      startInBackground = true;
+    };
+  };
 }

modules/build-types/ha-server.nix

@@ -0,0 +1,51 @@
+# HA file server build type: DRBD + XFS + LIO iSCSI + NFS, managed by
+# Corosync + Pacemaker.  Both ha-server-1 and ha-server-2 use this type.
+#
+# NFS start/stop:
+#   services.nfs.server.enable = true configures /etc/exports, wires up
+#   rpcbind, and loads kernel modules — but nfs-server.service.wantedBy is
+#   force-cleared so systemd does NOT auto-start it at boot.  Pacemaker's
+#   ha-group resource group (configured by scripts/ha/cluster-init.sh)
+#   starts and stops nfs-server as part of the failover sequence after the
+#   XFS mount and iSCSI target are brought up on the new Active node.
+#
+# Beszel agent:
+#   Enabled here via enable-agent.nix.  The agent KEY (used to pair with
+#   the Beszel hub) is not set yet — add it to hosts/ha-server-{1,2}/host.nix
+#   under services.beszel.agent.environment.KEY once the hub accepts the
+#   new agents, following the pattern in hosts/server/host.nix.
+{ lib, pkgs, vars, ... }:
+
+let
+  # Generates /etc/exports lines for all nfsShares data entries. Shared
+  # pattern with modules/build-types/server.nix — both export the same
+  # set of shares, differing only in the storage root they serve from.
+  mkNfsExports = storageRoot:
+    lib.concatMapStrings
+      (share: "  ${storageRoot}/${share.subpath} ${vars.lanCidr}${vars.nfsShares.options}\n")
+      (lib.filter builtins.isAttrs (lib.attrValues vars.nfsShares));
+in
+{
+  imports = [
+    ../ha/pacemaker-stack.nix
+    ../ha/iscsi-target.nix
+    ../ha/cluster-config.nix
+    ../beszel/enable-agent.nix
+  ];
+
+  # xfsprogs: mkfs.xfs/xfs_info needed by cluster-init.sh.
+  # openiscsi: iscsiadm needed by acceptance-tests.sh T4 (iSCSI discovery check).
+  environment.systemPackages = [ pkgs.xfsprogs pkgs.openiscsi ];
+
+  services.nfs.server = {
+    enable = true;
+    exports = mkNfsExports vars.haStorageRoot;
+  };
+
+  # Pacemaker controls nfs-server — prevent systemd from starting it at boot
+  # on both nodes (only the Active node should be serving NFS).
+  systemd.services.nfs-server.wantedBy = lib.mkForce [ ];
+
+  # Same reason as server.nix: exports use standard auth, not Kerberos.
+  systemd.services.rpc-svcgssd.enable = false;
+}

modules/build-types/pxe-boot.nix

@@ -21,6 +21,216 @@ let
     chain ${pxeBaseUrl}/boot.ipxe
   '';
 
+  debianRelease = "bookworm";
+  debianMirror = "https://deb.debian.org/debian";
+  debianNetbootBase = "${debianMirror}/dists/${debianRelease}/main/installer-amd64/current/images/netboot/debian-installer/amd64";
+
+  rockyRelease = "9";
+  rockyArch = "x86_64";
+  rockyMirror = "https://dl.rockylinux.org/pub/rocky/${rockyRelease}";
+  rockyPxebootBase = "${rockyMirror}/BaseOS/${rockyArch}/os/images/pxeboot";
+
+  debianIpxe = pkgs.writeText "debian.ipxe" ''
+    #!ipxe
+
+    set base ${pxeBaseUrl}
+
+    kernel ''${base}/debian/linux
+    initrd ''${base}/debian/initrd.gz
+    boot
+  '';
+
+  fetchDebianNetboot = pkgs.writeShellScript "fetch-debian-netboot" ''
+    set -eu
+
+    dir="${httpRoot}/debian"
+    mirror="${debianNetbootBase}"
+
+    if [ -f "$dir/linux" ] && [ -f "$dir/initrd.gz" ]; then
+      echo "Debian ${debianRelease} netboot files already present; skipping download."
+      exit 0
+    fi
+
+    echo "Downloading Debian ${debianRelease} netboot kernel and initrd from $mirror ..."
+    ${pkgs.curl}/bin/curl -fsSL -o "$dir/linux.tmp" "$mirror/linux"
+    ${pkgs.curl}/bin/curl -fsSL -o "$dir/initrd.gz.tmp" "$mirror/initrd.gz"
+    mv "$dir/linux.tmp" "$dir/linux"
+    mv "$dir/initrd.gz.tmp" "$dir/initrd.gz"
+    echo "Debian ${debianRelease} netboot files staged."
+  '';
+
+  # Rocky Linux 9 iPXE script — boots vmlinuz+initrd.img from the staged
+  # /rocky/ directory and hands Anaconda the hosted Kickstart URL.
+  # net.ifnames=0 biosdevname=0 ensures the NIC is eth0 in both the
+  # installer and the installed system (matches the Kickstart NM config).
+  rockyFreeIpaIpxe = pkgs.writeText "rocky-freeipa.ipxe" ''
+    #!ipxe
+
+    set base ${pxeBaseUrl}
+
+    kernel ''${base}/rocky/vmlinuz inst.ks=''${base}/rocky-freeipa.ks inst.repo=${rockyMirror}/BaseOS/${rockyArch}/os/ net.ifnames=0 biosdevname=0 ip=dhcp quiet
+    initrd ''${base}/rocky/initrd.img
+    boot
+  '';
+
+  # Kickstart file for domain-controller.sweet.home.
+  # Installs Rocky Linux 9, sets a static IP, creates wayne with the
+  # admin SSH key, then on first reboot runs ipa-server-install via a
+  # systemd oneshot service.  Passwords are generated at %post time,
+  # written to /root/ipa-credentials.txt (chmod 600), and read back by
+  # the first-boot script — never hardcoded here or in the repo.
+  rockyFreeIpaKs = pkgs.writeText "rocky-freeipa.ks" ''
+    #version=RHEL9
+    # Unattended Rocky Linux 9 + FreeIPA install
+    # Target: domain-controller.${vars.homeDomain}  ${vars.domainControllerIp}
+
+    url --url=${rockyMirror}/BaseOS/${rockyArch}/os/
+    repo --name=appstream --baseurl=${rockyMirror}/AppStream/${rockyArch}/os/
+
+    lang en_US.UTF-8
+    keyboard us
+    timezone UTC --utc
+
+    # DHCP during install; static IP configured in %post via NM config file
+    network --bootproto=dhcp --device=link --activate
+    network --hostname=domain-controller.sweet.home
+
+    selinux --enforcing
+    firewall --enabled --service=ssh
+
+    rootpw --lock
+    user --name=wayne --groups=wheel --shell=/bin/bash
+    sshkey --username=wayne "${vars.adminSshKey}"
+
+    zerombr
+    clearpart --all --initlabel --drives=sda
+    # Keep net.ifnames=0 biosdevname=0 in the installed GRUB so the NIC
+    # stays eth0 after reboot (matches the NM connection file below).
+    bootloader --location=mbr --boot-drive=sda --append="net.ifnames=0 biosdevname=0"
+
+    part /boot --fstype=xfs --size=1024 --ondisk=sda
+    part swap  --fstype=swap --size=2048 --ondisk=sda
+    part /     --fstype=xfs  --grow --size=1 --ondisk=sda --asprimary
+
+    %packages
+    @^minimal-environment
+    ipa-server
+    ipa-server-dns
+    %end
+
+    reboot
+
+    %post --log=/root/ks-post.log
+    set -euo pipefail
+
+    # -- Static IP: write NM connection file directly (NM not running in chroot) --
+    mkdir -p /etc/NetworkManager/system-connections
+    cat > /etc/NetworkManager/system-connections/eth0.nmconnection << 'NMCONN'
+    [connection]
+    id=eth0
+    type=ethernet
+    interface-name=eth0
+    autoconnect=true
+
+    [ethernet]
+
+    [ipv4]
+    method=manual
+    addresses=${vars.domainControllerIp}/${toString vars.lanPrefixLength}
+    gateway=${vars.lanGateway}
+    dns=${vars.domainControllerIp};
+    dns-search=${vars.homeDomain};
+
+    [ipv6]
+    method=auto
+    NMCONN
+    chmod 600 /etc/NetworkManager/system-connections/eth0.nmconnection
+
+    # -- /etc/hosts: FQDN must resolve to the real IP (not loopback) for IPA --
+    sed -i '/domain-controller/d' /etc/hosts
+    echo '${vars.domainControllerIp} domain-controller.${vars.homeDomain} domain-controller' >> /etc/hosts
+
+    # -- Generate IPA passwords and store securely --
+    DM_PASS=$(openssl rand -base64 24 | tr -dc 'A-Za-z0-9' | head -c 24)
+    ADMIN_PASS=$(openssl rand -base64 24 | tr -dc 'A-Za-z0-9' | head -c 24)
+    printf 'Directory Manager: %s\nIPA Admin: %s\n' "$DM_PASS" "$ADMIN_PASS" \
+      > /root/ipa-credentials.txt
+    chmod 600 /root/ipa-credentials.txt
+
+    # -- First-boot script: reads passwords back, runs ipa-server-install --
+    cat > /usr/local/sbin/freeipa-first-boot.sh << 'FIRSTBOOT'
+    #!/bin/bash
+    set -euo pipefail
+    exec >> /root/freeipa-install.log 2>&1
+    echo "=== FreeIPA first-boot install started at $(date) ==="
+
+    DM_PASS=$(grep '^Directory Manager:' /root/ipa-credentials.txt | awk '{print $NF}')
+    ADMIN_PASS=$(grep '^IPA Admin:' /root/ipa-credentials.txt | awk '{print $NF}')
+
+    ipa-server-install \
+      --realm=SWEET.HOME \
+      --domain=sweet.home \
+      --hostname=domain-controller.sweet.home \
+      --ds-password="$DM_PASS" \
+      --admin-password="$ADMIN_PASS" \
+      --setup-dns \
+      --forwarder=192.168.2.253 \
+      --no-dnssec-validation \
+      --no-ntp \
+      --unattended
+
+    echo "=== FreeIPA install complete at $(date) ==="
+    echo "Credentials: /root/ipa-credentials.txt (save to password manager)"
+    echo "CA backup:   /root/cacert.p12 (encrypted with Directory Manager password)"
+    systemctl disable freeipa-first-boot.service
+    FIRSTBOOT
+    chmod 700 /usr/local/sbin/freeipa-first-boot.sh
+
+    # -- Systemd oneshot service: runs freeipa-first-boot.sh on first real boot --
+    cat > /etc/systemd/system/freeipa-first-boot.service << 'UNIT'
+    [Unit]
+    Description=FreeIPA first-boot installation
+    After=network-online.target
+    Wants=network-online.target
+    ConditionPathExists=/root/ipa-credentials.txt
+
+    [Service]
+    Type=oneshot
+    ExecStart=/usr/local/sbin/freeipa-first-boot.sh
+    TimeoutStartSec=1800
+    RemainAfterExit=yes
+
+    [Install]
+    WantedBy=multi-user.target
+    UNIT
+
+    mkdir -p /etc/systemd/system/multi-user.target.wants
+    ln -sf /etc/systemd/system/freeipa-first-boot.service \
+      /etc/systemd/system/multi-user.target.wants/freeipa-first-boot.service
+
+    echo "Kickstart %post complete. FreeIPA installs on first reboot (~20 min)."
+    %end
+  '';
+
+  fetchRockyPxeboot = pkgs.writeShellScript "fetch-rocky-pxeboot" ''
+    set -eu
+
+    dir="${httpRoot}/rocky"
+    base="${rockyPxebootBase}"
+
+    if [ -f "$dir/vmlinuz" ] && [ -f "$dir/initrd.img" ]; then
+      echo "Rocky Linux ${rockyRelease} pxeboot files already present; skipping download."
+      exit 0
+    fi
+
+    echo "Downloading Rocky Linux ${rockyRelease} pxeboot kernel and initrd from $base ..."
+    ${pkgs.curl}/bin/curl -fsSL -o "$dir/vmlinuz.tmp"   "$base/vmlinuz"
+    ${pkgs.curl}/bin/curl -fsSL -o "$dir/initrd.img.tmp" "$base/initrd.img"
+    mv "$dir/vmlinuz.tmp"   "$dir/vmlinuz"
+    mv "$dir/initrd.img.tmp" "$dir/initrd.img"
+    echo "Rocky Linux ${rockyRelease} pxeboot files staged."
+  '';
+
   systemRescueIpxe = pkgs.writeText "systemrescue.ipxe" ''
     #!ipxe
 
@@ -70,6 +280,8 @@ let
     menu PXE Boot Menu
     item auto-installer    NixOS Auto-Installer
     item nixos-minimal     NixOS Minimal
+    item debian            Debian Minimal
+    item rocky-freeipa     FreeIPA Server (Rocky Linux 9)
     item rescue            Rescue Environment
     item shell             iPXE Shell
     item reboot            Reboot
@@ -82,6 +294,12 @@ let
     :nixos-minimal
     chain ''${base}/nixos-minimal/netboot.ipxe
 
+    :debian
+    chain ''${base}/debian.ipxe
+
+    :rocky-freeipa
+    chain ''${base}/rocky-freeipa.ipxe
+
     :rescue
     chain ''${base}/systemrescue.ipxe
 
@@ -95,6 +313,8 @@ in
 {
   imports = [
     ../pxe-boot/stage-installer-artifacts.nix
+    ../pxe-boot/mount-pxe-images.nix
+    ../beszel/enable-agent.nix
   ];
 
   environment.systemPackages = with pkgs; [
@@ -129,25 +349,64 @@ in
     openssh.settings.PermitRootLogin = "yes";
   };
 
-  systemd.tmpfiles.rules = [
+  systemd = {
+    tmpfiles.rules = [
       "d ${pxeRoot} 0755 root root -"
       "d ${httpRoot} 0755 root root -"
-    "d ${httpRoot}/images 0755 root root -"
+      "L+ ${httpRoot}/images - - - - ${vars.nfsShares.pxebootImages.mountpoint}"
       "d ${httpRoot}/auto-installer 0755 root root -"
       "d ${httpRoot}/nixos-minimal 0755 root root -"
       "d ${httpRoot}/systemrescue 0755 root root -"
+      "d ${httpRoot}/debian 0755 root root -"
       "d ${httpRoot}/ubuntu 0755 root root -"
       "d ${httpRoot}/rescue 0755 root root -"
+      "d ${httpRoot}/rocky 0755 root root -"
       "d ${tftpRoot} 0755 root root -"
       "C+ ${httpRoot}/boot.ipxe 0644 root root - ${bootIpxe}"
       "C+ ${httpRoot}/menu.ipxe 0644 root root - ${menuIpxe}"
+      "C+ ${httpRoot}/debian.ipxe 0644 root root - ${debianIpxe}"
+      "C+ ${httpRoot}/rocky-freeipa.ipxe 0644 root root - ${rockyFreeIpaIpxe}"
+      "C+ ${httpRoot}/rocky-freeipa.ks 0644 root root - ${rockyFreeIpaKs}"
       "C+ ${httpRoot}/systemrescue.ipxe 0644 root root - ${systemRescueIpxe}"
       "C+ ${tftpRoot}/autoexec.ipxe 0644 root root - ${autoexecIpxe}"
       "C+ ${tftpRoot}/ipxe.efi 0644 root root - ${pkgs.ipxe}/ipxe.efi"
       "C+ ${tftpRoot}/undionly.kpxe 0644 root root - ${pkgs.ipxe}/undionly.kpxe"
     ];
 
-  systemd.services.stage-systemrescue = {
+    services = {
+      fetch-debian-netboot = {
+        description = "Download Debian ${debianRelease} netboot kernel and initrd for HTTP PXE boot";
+        after = [
+          "local-fs.target"
+          "systemd-tmpfiles-setup.service"
+          "network-online.target"
+        ];
+        wants = [ "network-online.target" ];
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = fetchDebianNetboot;
+          RemainAfterExit = true;
+        };
+      };
+
+      fetch-rocky-pxeboot = {
+        description = "Download Rocky Linux ${rockyRelease} pxeboot kernel and initrd for HTTP PXE boot";
+        after = [
+          "local-fs.target"
+          "systemd-tmpfiles-setup.service"
+          "network-online.target"
+        ];
+        wants = [ "network-online.target" ];
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = fetchRockyPxeboot;
+          RemainAfterExit = true;
+        };
+      };
+
+      stage-systemrescue = {
         description = "Stage SystemRescue ISO contents for HTTP PXE boot";
         after = [
           "local-fs.target"
@@ -159,7 +418,32 @@ in
           ExecStart = stageSystemRescue;
         };
       };
+    };
+  };
+
+  services.dnsmasq = {
+    enable = true;
+    settings = {
+      # Disable DNS listener — only proxy DHCP is needed here.
+      # Without this dnsmasq tries to bind port 53 which systemd-resolved
+      # already owns, causing startup failure.
+      port = 0;
+      dhcp-range = [ "192.168.2.0,proxy" ];
+      dhcp-match = [
+        "set:ipxe,175"
+        "set:efi64,option:client-arch,7"
+        "set:efi64,option:client-arch,9"
+      ];
+      dhcp-userclass = "set:ipxe,iPXE";
+      dhcp-boot = [
+        "tag:ipxe,tag:efi64,http://${vars.pxeServerIp}/boot.ipxe"
+        "tag:ipxe,http://${vars.pxeServerIp}/boot.ipxe"
+        "tag:efi64,ipxe.efi,,${vars.pxeServerIp}"
+        "undionly.kpxe,,${vars.pxeServerIp}"
+      ];
+    };
+  };
 
   networking.firewall.allowedTCPPorts = [ vars.ports.pxeBootHttp ];
-  networking.firewall.allowedUDPPorts = [ vars.ports.pxeBootTftp ];
+  networking.firewall.allowedUDPPorts = [ vars.ports.pxeBootTftp 67 ];
 }

modules/build-types/server.nix

@@ -10,8 +10,18 @@ let
     in lib.imap1 (i: _: lib.concatStringsSep "/" (lib.take i parts)) parts;
 
   poolDatasets = lib.unique (
-    lib.concatMap (share: ancestors share.subpath) (lib.attrValues vars.nfsShares)
+    lib.concatMap (share: ancestors share.subpath)
+      (lib.filter builtins.isAttrs (lib.attrValues vars.nfsShares))
   );
+
+  # Generates /etc/exports lines for all nfsShares data entries (every
+  # attrset value — excludes the bare `options` string). Both server and
+  # ha-server export the same share set from different storage roots, so
+  # this helper is the single source of truth for the export line format.
+  mkNfsExports = storageRoot:
+    lib.concatMapStrings
+      (share: "  ${storageRoot}/${share.subpath} ${vars.lanCidr}${vars.nfsShares.options}\n")
+      (lib.filter builtins.isAttrs (lib.attrValues vars.nfsShares));
 in
 {
   imports = [
@@ -43,14 +53,8 @@ in
         exit 0
       fi
 
-      # Pool exists on a device but not yet imported — let the standard
+      # Locate the data disk first — used for both the fallback import
-      # zfs-import-${poolName}.service handle it normally.
+      # attempt and, only if the disk is genuinely blank, pool creation.
-      if zpool import -d /dev/disk/by-id -N "${poolName}" 2>/dev/null; then
-        exit 0
-      fi
-
-      # No pool found at all.  Create it on the Proxmox data disk (scsi1),
-      # which appears as /dev/disk/by-id/scsi-*drive-scsi1 inside the VM.
       DATA_DISK=""
       for candidate in /dev/disk/by-id/scsi-*drive-scsi1; do
         [[ "$candidate" == *-part* ]] && continue
@@ -62,8 +66,31 @@ in
         exit 1
       fi
 
+      # Try importing via the by-id symlink directory first (normal path),
+      # then fall back to scanning the disk directly.  The two-step exists
+      # because of a udev race: systemd-udev-settle.service can clear before
+      # /dev/disk/by-id/ entries are fully populated, causing the first
+      # import to fail even when the pool is intact on the disk.
+      if zpool import -d /dev/disk/by-id -N "${poolName}" 2>/dev/null; then
+        exit 0
+      fi
+      if zpool import -d "$DATA_DISK" -N "${poolName}" 2>/dev/null; then
+        exit 0
+      fi
+
+      # Both import attempts failed.  Before creating a new pool, verify the
+      # disk is genuinely blank — if ZFS label metadata is present the import
+      # failed for some other reason and we must not clobber existing data.
+      if zdb -l "$DATA_DISK" 2>/dev/null | grep -q "name: '${poolName}'"; then
+        echo "zfs-init-${poolName}: $DATA_DISK has ZFS pool '${poolName}' metadata but import failed — refusing to overwrite existing data. Run 'zpool import -d $DATA_DISK ${poolName}' manually to investigate." >&2
+        exit 1
+      fi
+
+      # Disk is genuinely blank: create the pool.  -f is intentionally
+      # omitted so that if we somehow reach this point with an existing pool
+      # on the disk, zpool refuses rather than silently destroying data.
       echo "zfs-init-${poolName}: creating pool on $DATA_DISK"
-      zpool create -f "${poolName}" "$DATA_DISK"
+      zpool create "${poolName}" "$DATA_DISK"
       ${lib.concatMapStrings (ds: ''
         zfs create "${poolName}/${ds}"
       '') poolDatasets}
@@ -75,16 +102,20 @@ in
     requires = [ "zfs-mount.service" ];
   };
 
+  # rpc-svcgssd handles Kerberos/GSS-API for NFS. Not needed: exports use
+  # standard auth, not sec=krb5. On IPA-joined hosts the keytab exists (host/
+  # principal only) but has no nfs/ principal, causing spurious failure.
+  # Mask it so nfs-server's Wants= can't pull it in.
+  systemd.services.rpc-svcgssd.enable = false;
+
   services.nfs.server = {
     enable = true;
-    exports = ''
+    exports = mkNfsExports vars.storageRoot;
-      ${vars.storageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash)
-      ${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash)
-      ${vars.storageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash)
-      ${vars.storageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash)
-      ${vars.storageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash)
-    '';
   };
 
-  networking.firewall.allowedTCPPorts = [ vars.ports.nfsRpcbind vars.ports.nfsd ];
+  # mountd (20048) is needed for showmount/NFSv3 mount protocol — without it
+  # clients can reach portmapper (111) and get the mountd port back, then
+  # time out trying to connect to it. All three ports need TCP and UDP.
+  networking.firewall.allowedTCPPorts = [ vars.ports.nfsRpcbind vars.ports.nfsd vars.ports.nfsMountd ];
+  networking.firewall.allowedUDPPorts = [ vars.ports.nfsRpcbind vars.ports.nfsd vars.ports.nfsMountd ];
 }

modules/build-types/tailscale-router.nix

@@ -1,8 +1,10 @@
-{ ... }:
+{ vars, ... }:
 
 {
   imports = [
     ../tailscale/subnet-router.nix
+    ../tailscale/ts-dns-forwarder.nix
+    ../beszel/enable-agent.nix
   ];
 
   # "server", not "both": this build type advertises LAN subnet routes but
@@ -12,8 +14,31 @@
   # own setting) so the intent is clear at the build-type level.
   services.tailscale.useRoutingFeatures = "server";
 
+  # Advertise the LAN subnet so Tailscale peers can route back to LAN machines.
+  # Must also be approved in the Tailscale admin console (Machines → Edit route settings).
+  services.tailscale.extraUpFlags = [ "--advertise-routes=${vars.lanCidr}" ];
+
+  networking.firewall = {
     # Forwarded subnet-router traffic arrives on tailscale0 already
     # tailscale-authenticated -- the firewall's normal per-port allow-list
     # would otherwise drop it. Standard NixOS/Tailscale subnet-router guidance.
-  networking.firewall.trustedInterfaces = [ "tailscale0" ];
+    trustedInterfaces = [ "tailscale0" ];
+
+    # SNAT LAN traffic going into Tailscale so the remote peer sees it as
+    # coming from this router's Tailscale IP rather than a raw LAN IP.
+    # Without this, Tailscale drops forwarded packets whose source is not a
+    # recognised Tailscale address.
+    #
+    # We target POSTROUTING directly (always-existing built-in chain) rather
+    # than nixos-nat-post: extraCommands runs after the old nixos-nat-post is
+    # deleted but before the new one is created, so -A nixos-nat-post silently
+    # fails.  The -C check makes the rule idempotent across firewall reloads.
+    extraCommands = ''
+      iptables -t nat -C POSTROUTING -s ${vars.lanCidr} -o tailscale0 -j MASQUERADE 2>/dev/null || \
+        iptables -t nat -A POSTROUTING -s ${vars.lanCidr} -o tailscale0 -j MASQUERADE
+    '';
+    extraStopCommands = ''
+      iptables -t nat -D POSTROUTING -s ${vars.lanCidr} -o tailscale0 -j MASQUERADE 2>/dev/null || true
+    '';
+  };
 }

modules/common/aliases.nix

@@ -1,50 +1,7 @@
-{ config, pkgs, lib, vars, ... }:
+_:
 
-let
-  # Flake attribute names are now <platform>-<buildtype> (e.g. proxmox-docker)
-  # and no longer match networking.hostName, since a host's hostname stays
-  # fixed while the platform backing it can change. Each nixosConfiguration
-  # stamps its own active target name into /etc/flake-target at build time.
-  mySwitchCmd = ''
-    sudo nixos-rebuild switch \
-      --no-write-lock-file \
-      --refresh \
-      --flake git+https://${vars.lanDomain}/beatzaplenty/nixos.git#$(cat /etc/flake-target)
-  '';
-  myTestCmd = ''
-    sudo nixos-rebuild test \
-      --no-write-lock-file \
-      --refresh \
-      --flake git+https://${vars.lanDomain}/beatzaplenty/nixos.git#$(cat /etc/flake-target)
-  '';
-
-  # lxc-* hosts pre-seed their SSH host key at build time (see
-  # modules/platforms/lxc.nix) so sops-nix's .sops.yaml recipient matches on
-  # first boot -- without it, secrets permanently fail to decrypt (see that
-  # file's comment for the confirmed failure). That requires --impure plus
-  # NIXOS_HOST_KEYS_DIR pointing at the repo's host-keys/ dir, same pattern
-  # docs/auto-installer.md uses for the installer ISO. A function, not a
-  # shellAlias, since the target name has to interpolate into the middle of
-  # the flake attribute path, not just append after it. Must be run from the
-  # repo root, same as every other host-keys/ command in this repo.
-  buildImageFn = ''
-    buildImage() {
-      if [ -z "$1" ]; then
-        echo "usage: buildImage <flake-target>  (e.g. lxc-docker)" >&2
-        return 1
-      fi
-      NIXOS_HOST_KEYS_DIR="$(pwd)/host-keys" nix build --impure \
-        ".#nixosConfigurations.$1.config.system.build.tarball"
-    }
-  '';
-in
 {
-  programs.bash = {
+  # Switch-nix, Test-nix, and buildImage are defined system-wide in
-    enable = true;
+  # modules/common/configuration.nix so all users (including IPA accounts)
-    shellAliases = {
+  # get them. Add any Home-Manager-only per-user shell config here.
-      "Switch-nix" = mySwitchCmd;
-      "Test-nix" = myTestCmd;
-    };
-    initExtra = buildImageFn;
-  };
 }

modules/common/configuration.nix

@@ -1,17 +1,45 @@
 { config, lib, pkgs, vars, ... }:
 
+let
+  switchCmd = ''
+    sudo nixos-rebuild switch \
+      --no-write-lock-file \
+      --refresh \
+      --flake git+https://${vars.lanDomain}/beatzaplenty/nixos.git#$(cat /etc/flake-target)
+  '';
+  testCmd = ''
+    sudo nixos-rebuild test \
+      --no-write-lock-file \
+      --refresh \
+      --flake git+https://${vars.lanDomain}/beatzaplenty/nixos.git#$(cat /etc/flake-target)
+  '';
+  buildImageFn = ''
+    buildImage() {
+      if [ -z "$1" ]; then
+        echo "usage: buildImage <flake-target>  (e.g. lxc-docker)" >&2
+        return 1
+      fi
+      NIXOS_HOST_KEYS_DIR="$(pwd)/host-keys" nix build --impure \
+        ".#nixosConfigurations.$1.config.system.build.tarball"
+    }
+  '';
+in
 {
-  imports =
+  imports = [
-    [
-      # Include the results of the hardware scan.
-      #      ./hardware-configuration.nix
     ./set-locale.nix
+    ../ipa/client.nix
   ];
-  # Use the GRUB 2 boot loader.
-  # boot.loader.grub.enable = true;
-  #boot.loader.grub.device = "/dev/sda"; # or "nodev" for efi only
 
-  networking.networkmanager.enable = true; # Easiest to use and most distros use this by default.
+  # System-wide shell config so all users (including IPA accounts) get the
+  # same management aliases as the local nixos user's Home Manager provides.
+  programs.bash = {
+    shellAliases = {
+      "Switch-nix" = switchCmd;
+      "Test-nix" = testCmd;
+    };
+    interactiveShellInit = buildImageFn;
+  };
+  networking.networkmanager.enable = true;
 
   # Recommended over the true default (bypasses ZFS's own import safeguards)
   # per the option's own docs; matches hosts/docker/host.nix and

modules/common/preserve-ssh-host-key.nix

@@ -0,0 +1,35 @@
+# Shared activation-script logic to preserve the SSH host key across
+# nixos-rebuild on platforms that embed the key via environment.etc (lxc and
+# proxmox).  When NIXOS_HOST_KEYS_DIR is not set the key is absent from
+# environment.etc, and NixOS's etc activation removes any /etc file not in
+# the new generation — which would destroy the live key and break sops-nix
+# decryption permanently.  These scripts save the key to /run before etc
+# removes it, then restore it afterward.
+#
+# Explicit deps enforce the correct ordering: without them the topological
+# sort places preserveSshHostKey after etc (confirmed live on lxc-tor-relay:
+# position 7 vs etc's position 5), so the key is gone before it can be saved.
+_: {
+  system.activationScripts = {
+    preserveSshHostKey = ''
+      if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
+        cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
+        cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
+      fi
+    '';
+
+    restoreSshHostKey = {
+      deps = [ "etc" ];
+      text = ''
+        if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
+          install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
+          install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
+        fi
+        rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
+      '';
+    };
+
+    etc = { deps = [ "preserveSshHostKey" ]; };
+    setupSecrets = { deps = [ "restoreSshHostKey" ]; };
+  };
+}

modules/docker/enable-service.nix

@@ -1,23 +1,45 @@
-{ pkgs, ... }:
+{ lib, pkgs, vars, ... }:
 
+let
+  gid = toString vars.dockerAccessGid;
+in
 {
-  #  virtualisation.docker.enable = true;
   virtualisation.docker = {
     enable = true;
     package = pkgs.docker;
-    #  listenOptions = [
-    #  "unix:///var/run/docker.sock"
-    #  "tcp://0.0.0.0:2375"
-    #];
-
-    #  daemon.settings = {
-    #    metrics-addr = "0.0.0.0:9323";
-    #    experimental = true;
-    #  };
   };
-
+  # Pin the docker group GID to match the IPA "docker-access" group so that
+  # IPA group membership alone grants access to the Docker socket. Any user
+  # whose supplementary groups (resolved by SSSD from IPA) include GID
+  # vars.dockerAccessGid will pass the socket group-permission check without
+  # any per-host users.groups.docker.members entry.
+  users.groups.docker.gid = lib.mkForce vars.dockerAccessGid;
+  users.users.${vars.primaryUser}.extraGroups = [ "docker" ];
   environment.systemPackages = with pkgs; [
     docker-compose
     docker-buildx
   ];
+
+  # NixOS's group activation uses plain `groupmod` without --non-unique.
+  # When SSSD is active it exposes the IPA "docker-access" group at
+  # vars.dockerAccessGid via NSS, so groupmod sees that GID as already in
+  # use and silently skips the change (warning: "not applying GID change").
+  # This script runs after the normal "groups" step and applies the change
+  # with --non-unique (which lets the local docker group share the GID with
+  # the SSSD-provided IPA group). If the GID actually changed it also
+  # restarts docker.socket so the socket is recreated with the new GID.
+  system.activationScripts.docker-group-gid = {
+    deps = [ "groups" ];
+    text = ''
+      current=$(grep "^docker:" /etc/group | cut -d: -f3)
+      if [ "$current" != "${gid}" ]; then
+        ${pkgs.shadow}/bin/groupmod --non-unique -g ${gid} docker
+        if ${pkgs.systemd}/bin/systemctl is-active --quiet docker.socket; then
+          ${pkgs.systemd}/bin/systemctl stop docker.service docker.socket
+          rm -f /var/run/docker.sock
+          ${pkgs.systemd}/bin/systemctl start docker.socket docker.service
+        fi
+      fi
+    '';
+  };
 }

modules/ha/cluster-config.nix

@@ -0,0 +1,154 @@
+# Cluster-wide HA config shared by both ha-server nodes.
+#
+# Covers everything that is identical on both nodes and references cluster
+# topology (node IPs, hostnames, DRBD resource).  Per-node identity
+# (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix.
+#
+# Corosync authkey:
+#   /etc/corosync/authkey (mode 0400) is managed by sops-nix below.
+#   Bootstrap: run scripts/ha/cluster-init.sh on node1 to generate the key,
+#   then encrypt it with: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey
+#   Both host keys must be registered via sync-host-keys.sh first so both nodes can decrypt it.
+#
+# DRBD fencing:
+#   resource-only with crm-fence-peer.sh: DRBD calls the Pacemaker-aware
+#   crm-fence-peer.sh handler before promoting.  The handler checks the CIB
+#   to confirm the peer's DRBD resource is stopped and returns 7 (successfully
+#   fenced), allowing safe promotion without requiring power-fencing (STONITH).
+#   The unfence handler crm-unfence-peer.sh clears the outdate flag when the
+#   peer reconnects.  This is the correct setting for Pacemaker+DRBD clusters
+#   with STONITH disabled; crm-fence-peer.sh replaces the need for a separate
+#   STONITH device during the testing phase.  Switch to resource-and-stonith
+#   once the fence_pve_ssh STONITH resource is active (see
+#   scripts/ha/cluster-enable-stonith.sh).
+#
+#   PATH wrapper: when the DRBD kernel module invokes the fence-peer handler
+#   via the UMH (User Mode Helper) mechanism it provides a minimal PATH that
+#   omits /run/current-system/sw/bin.  crm-fence-peer.sh calls cibadmin,
+#   crm_mon etc.; if those aren't found a pipeline in the script breaks with
+#   SIGPIPE.  A process killed by signal has WEXITSTATUS() == 0, so the kernel
+#   sees exit code 0 and logs "fence-peer helper broken, returned 0", looping
+#   forever.  The writeShellScript wrappers below prepend the NixOS sw path
+#   before exec-ing the real handler, giving it a working Pacemaker toolchain.
+{ lib, pkgs, vars, ... }:
+let
+  fencePeerWrapper = pkgs.writeShellScript "drbd-fence-peer" ''
+    export PATH="/run/current-system/sw/bin:/run/current-system/sw/sbin:$PATH"
+    exec /run/current-system/sw/lib/drbd/crm-fence-peer.sh "$@"
+  '';
+  unfencePeerWrapper = pkgs.writeShellScript "drbd-unfence-peer" ''
+    export PATH="/run/current-system/sw/bin:/run/current-system/sw/sbin:$PATH"
+    exec /run/current-system/sw/lib/drbd/crm-unfence-peer.sh "$@"
+  '';
+in
+{
+  # Root SSH access — same key set as nixos user so all admin keys can reach root.
+  users.users.root.openssh.authorizedKeys.keys = [
+    vars.adminSshKey
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
+  ];
+
+  # Passwordless sudo for wheel — operator SSHes as nixos and uses sudo for
+  # cluster management commands (drbdadm, crm*, pcs, etc.)
+  security.sudo.wheelNeedsPassword = lib.mkForce false;
+
+  # DRBD lock-file directory (drbd-utils checks for it; missing → harmless but noisy warnings).
+  systemd.tmpfiles.rules = [ "d /var/lib/drbd 0750 root root -" ];
+
+  # Prevent drbd.service from auto-starting at boot / nixos-rebuild switch.
+  # Pacemaker's OCF drbd agent calls drbdadm up/down directly when managing
+  # the resource.  If drbd.service also runs drbdadm up all while DRBD is
+  # already Primary under Pacemaker, apply-al fails with "device busy" (exit 20).
+  systemd.services.drbd.wantedBy = lib.mkForce [];
+
+  services.drbd = {
+    enable = true;
+    config = ''
+      global {
+        usage-count yes;
+      }
+
+      common {
+        net {
+          protocol C;
+          ping-int 1;
+          verify-alg sha256;
+          after-sb-0pri discard-zero-changes;
+          after-sb-1pri discard-secondary;
+        }
+        disk {
+          fencing resource-only;
+        }
+        handlers {
+          fence-peer   "${fencePeerWrapper}";
+          unfence-peer "${unfencePeerWrapper}";
+        }
+      }
+
+      resource ha-data {
+        volume 0 {
+          device    /dev/drbd0;
+          disk      ${vars.haServerDrbdDisk};
+          meta-disk internal;
+        }
+
+        on ${vars.haServer1Host} {
+          address ${vars.haServer1StorageIp}:${toString vars.ports.haServerDrbd};
+        }
+
+        on ${vars.haServer2Host} {
+          address ${vars.haServer2StorageIp}:${toString vars.ports.haServerDrbd};
+        }
+      }
+    '';
+  };
+
+  # /etc/corosync/authkey — sops binary secret, identical on both nodes.
+  # Decryptable by both ha-server host keys (added by sync-host-keys.sh).
+  sops.secrets.corosync_authkey = {
+    sopsFile = ../../secrets/ha-corosync-authkey;
+    format = "binary";
+    path = "/etc/corosync/authkey";
+    mode = "0400";
+    restartUnits = [ "corosync.service" ];
+  };
+
+  # NixOS common config enables NetworkManager by default; HA cluster nodes
+  # need stable static IPs with predictable interface names — NM is not suitable.
+  networking.networkmanager.enable = lib.mkForce false;
+
+  # services.corosync.enable is set by modules/ha/pacemaker-stack.nix.
+  services.corosync = {
+    clusterName = "ha-cluster";
+    nodelist = [
+      { nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1StorageIp ]; }
+      { nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2StorageIp ]; }
+    ];
+  };
+
+  networking.firewall = {
+    allowedTCPPorts = [
+      vars.ports.haServerIscsi
+      vars.ports.haServerPacemakerRemoted
+      vars.ports.haServerPcsd
+      vars.ports.haServerDrbd
+      vars.ports.nfsRpcbind
+      vars.ports.nfsd
+      vars.ports.nfsMountd
+    ];
+    allowedUDPPorts = [
+      vars.ports.haServerCorosync1
+      vars.ports.haServerCorosync2
+      vars.ports.haServerCorosyncCrypto
+      vars.ports.nfsRpcbind
+      vars.ports.nfsd
+      vars.ports.nfsMountd
+    ];
+    extraCommands = ''
+      iptables -A INPUT -s ${vars.haServer1Ip}/32 -j ACCEPT
+      iptables -A INPUT -s ${vars.haServer2Ip}/32 -j ACCEPT
+      iptables -A INPUT -s ${vars.haStorageCidr} -j ACCEPT
+    '';
+  };
+}

modules/ha/iscsi-target.nix

@@ -0,0 +1,99 @@
+# LIO iSCSI target service (targetctl) for NixOS HA clusters.
+#
+# Provides the targetctl.service that saves/restores LIO configuration from
+# /etc/target/saveconfig.json.  Pacemaker manages this service via its
+# systemd resource agent (class="systemd" type="targetctl").
+#
+# Why ExecStop is not simply "targetctl save":
+#   targetctl save writes the LIO config to JSON but does NOT remove the LIO
+#   target from the kernel's configfs.  As a result, any fileio backing store
+#   that LIO has open (e.g. iscsi-lun.img on an XFS-over-DRBD filesystem)
+#   stays referenced in the kernel.  The subsequent XFS umount from the
+#   Filesystem OCF resource then returns EBUSY and either hangs for the full
+#   op-stop timeout or fails outright, blocking the entire failover.
+#
+#   The ExecStop script here additionally tears down the kernel LIO state
+#   via rtslib_fb after saving, so the backing-store file descriptor is
+#   released and umount succeeds immediately.
+#
+# Empty-config guard:
+#   The save step is skipped when no iSCSI targets are currently active.
+#   This prevents the secondary node (where LIO was never started) from
+#   overwriting a valid saveconfig.json with an empty one when Pacemaker
+#   stops the iscsi-target resource as part of a failover or cleanup.
+{ pkgs, ... }:
+
+let
+  python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]);
+  targetctl = "${python3}/bin/targetctl";
+
+  targetctlStop = pkgs.writeScript "targetctl-stop" ''
+    #!${python3}/bin/python3
+    import subprocess, sys
+    import rtslib_fb
+
+    root = rtslib_fb.RTSRoot()
+    targets = list(root.targets)
+    if targets:
+        subprocess.run(
+            ["${targetctl}", "save", "/etc/target/saveconfig.json"],
+            capture_output=True,
+        )
+        print(f"saved {len(targets)} iSCSI target(s)")
+    else:
+        print("no active LIO targets — saveconfig.json unchanged")
+
+    for target in targets:
+        try:
+            for tpg in list(target.tpgs):
+                tpg.enable = False
+            target.delete()
+        except Exception as e:
+            print(f"warn (target): {e}", file=sys.stderr)
+    for so in list(root.storage_objects):
+        try:
+            so.delete()
+        except Exception as e:
+            print(f"warn (backstore): {e}", file=sys.stderr)
+    print("LIO kernel target cleared")
+  '';
+in
+{
+  boot.kernelModules = [
+    "target_core_mod"
+    "iscsi_target_mod"
+    "target_core_file"
+    "target_core_pscsi"
+    "target_core_user"
+    "configfs"
+  ];
+
+  systemd = {
+    mounts = [{
+      where = "/sys/kernel/config";
+      what = "configfs";
+      type = "configfs";
+      wantedBy = [ "multi-user.target" ];
+      before = [ "targetctl.service" ];
+    }];
+    services.targetctl = {
+      description = "LIO iSCSI target config save/restore";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "sys-kernel-config.mount" "network.target" ];
+      requires = [ "sys-kernel-config.mount" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = "${targetctl} restore /etc/target/saveconfig.json";
+        ExecStop = "${targetctlStop}";
+      };
+      unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
+    };
+    tmpfiles.rules = [
+      "d /etc/target 0750 root root -"
+      "f /etc/target/saveconfig.json 0640 root root -"
+    ];
+  };
+
+  environment.systemPackages = [ pkgs.targetcli-fb ];
+}

modules/ha/pacemaker-stack.nix

@@ -0,0 +1,94 @@
+# Pacemaker + Corosync HA stack for NixOS with known-good workarounds.
+#
+# Issues fixed here (confirmed through live testing on NixOS 25.11):
+#
+# 1. StateDirectory ownership reset: systemd's StateDirectory=pacemaker
+#    creates /var/lib/pacemaker owned root:root.  pacemaker-based (the CIB
+#    daemon) runs as the hacluster user and calls pcmk__daemon_can_write,
+#    which requires the CIB directory to be owned by hacluster or be
+#    group-writable by haclient.  Workaround: remove StateDirectory and let
+#    ExecStartPre create every required subdirectory with correct ownership.
+#
+# 2. HA_SBIN_DIR wrong path: ocf-shellfuncs sets HA_SBIN_DIR to the Nix
+#    store path of the resource-agents derivation's /sbin, which doesn't
+#    exist.  The DRBD OCF agent uses ${HA_SBIN_DIR}/crm_master, so it exits
+#    127 without this override.  Fix: export HA_SBIN_DIR=/run/current-system/sw/bin.
+#
+# 3. Broad PATH for OCF agents: the resource executor (pacemaker-execd) runs
+#    OCF agent scripts as children.  NixOS provides no implicit PATH for
+#    system services; without an explicit PATH the agents can't find ip, ss,
+#    mount, umount, drbdadm, etc.
+#
+# 4. FUSER=true: the Filesystem OCF agent calls check_binary $FUSER (default:
+#    fuser from psmisc), which is not installed.  Setting FUSER=true makes
+#    check_binary succeed (true is always in PATH) and the subsequent
+#    "$FUSER -km $mountpoint" becomes a no-op.  Pair with force_unmount=false
+#    on each Filesystem resource unless you want lazy unmount behaviour.
+{ lib, pkgs, ... }:
+
+let
+  ocfBinPath = lib.concatStringsSep ":" [
+    "${pkgs.iproute2}/bin"
+    "${pkgs.iproute2}/sbin"
+    "${pkgs.iputils}/bin"
+    "${pkgs.util-linux}/bin"
+    "${pkgs.util-linux}/sbin"
+    "${pkgs.gawk}/bin"
+    "${pkgs.gnugrep}/bin"
+    "${pkgs.gnused}/bin"
+    "${pkgs.coreutils}/bin"
+    "${pkgs.bash}/bin"
+    "${pkgs.procps}/bin"
+    "${pkgs.xfsprogs}/bin"
+    "${pkgs.drbd}/bin"
+    "${pkgs.python3}/bin"
+    "/run/current-system/sw/bin"
+    "/run/current-system/sw/sbin"
+    "/usr/local/sbin"
+    "/usr/local/bin"
+    "/usr/sbin"
+    "/usr/bin"
+    "/sbin"
+    "/bin"
+  ];
+
+  # Single pre-start script: schemas symlink + directory ownership.
+  # Runs before pacemakerd so pacemaker-based finds hacluster-owned dirs.
+  preStartCmd = "${pkgs.bash}/bin/bash -c '"
+    + "ln -sfn ${pkgs.pacemaker}/share/pacemaker /var/lib/pacemaker/schemas; "
+    + "for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores "
+    + "/var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox "
+    + "/var/lib/pacemaker/hostcache; do "
+    + "mkdir -p \"\\$d\" && chown hacluster:pacemaker \"\\$d\" && chmod 2770 \"\\$d\"; "
+    + "done'";
+
+  ocfEnv = {
+    PATH = lib.mkForce ocfBinPath;
+    OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
+    HA_SBIN_DIR = "/run/current-system/sw/bin";
+    FUSER = "true";
+  };
+in
+{
+  users.groups.haclient = { };
+
+  services.corosync.enable = true;
+  services.pacemaker.enable = true;
+
+  systemd.services = {
+    pacemaker = {
+      serviceConfig = {
+        StateDirectory = lib.mkForce "";
+        ExecStartPre = lib.mkBefore [ preStartCmd ];
+      };
+      environment = ocfEnv;
+    };
+    pacemaker-execd.environment = ocfEnv;
+  };
+
+  environment.systemPackages = with pkgs; [
+    corosync
+    pacemaker
+    ocf-resource-agents
+  ];
+}

modules/ipa/client.nix

@@ -0,0 +1,210 @@
+# Fully declarative FreeIPA domain membership.
+#
+# Imported by modules/common/configuration.nix — no per-host wiring needed.
+# Enables itself automatically on any host that has a sops-encrypted keytab
+# at secrets/<hostname>.keytab; is a no-op for all other hosts.
+#
+# To enroll a new host:
+#   0. scripts/secrets/sync-host-keys.sh <flake-target>
+#   1. scripts/ipa/create-nixos-ipa-host-account.sh [--ip <addr>] <hostname>
+#      (adds .sops.yaml rule, runs ipa host-add, encrypts keytab in one step)
+#   2. git add secrets/<hostname>.keytab .sops.yaml && git commit
+#   3. Deploy — no further steps required.
+#
+# Manual fallback (if the script isn't usable):
+#   a. On the FreeIPA server: ipa host-add <fqdn> [--ip-address=<ip>] --force
+#   b. On the FreeIPA server: ipa-getkeytab -s <ipa-server> -p host/<fqdn> -k /tmp/<host>.keytab
+#   c. From the repo root (path must match for sops creation rule to apply):
+#        cp /tmp/<host>.keytab secrets/<host>.keytab
+#        sops -e --input-type binary -i secrets/<host>.keytab
+#   d. Commit secrets/<host>.keytab and the updated .sops.yaml, then deploy.
+#
+# vars dependencies: homeDomain, ipaServer, domainControllerIp, ipaUser
+
+{ config, lib, pkgs, vars, ... }:
+
+let
+  keytabPath = ../../secrets + "/${config.networking.hostName}.keytab";
+  enabled = builtins.pathExists keytabPath;
+
+  realm = lib.strings.toUpper vars.homeDomain;
+  fqdn = "${config.networking.hostName}.${vars.homeDomain}";
+  # "sweet.home" -> "dc=sweet,dc=home"
+  basedn = lib.strings.concatMapStringsSep "," (c: "dc=${c}") (lib.strings.splitString "." vars.homeDomain);
+  # security.ipa.certificate expects a derivation (package), not a raw path.
+  caCertPkg = pkgs.writeText "ipa-ca.crt" (builtins.readFile ../../certs/ipa-ca.crt);
+in
+lib.mkIf enabled {
+  networking.domain = lib.mkDefault vars.homeDomain;
+  networking.nameservers = lib.mkDefault [ vars.domainControllerIp ];
+
+  security = {
+    ipa = {
+      enable = true;
+      domain = vars.homeDomain;
+      inherit realm;
+      server = vars.ipaServer;
+      certificate = caCertPkg;
+      inherit basedn;
+      ipaHostname = fqdn;
+      offlinePasswords = true;
+      cacheCredentials = true;
+    };
+
+    # Create the home directory on first login if it doesn't exist yet.
+    # IPA users have no pre-created home on the host; without this sshd
+    # opens a session to a non-existent directory and resets the connection.
+    # lightdm also needs this so the GUI login path can create the home dir
+    # if it was not pre-seeded by the tmpfiles rule above (e.g. on first boot
+    # before SSSD has resolved the user).
+    pam.services = {
+      sshd.makeHomeDir = true;
+      lightdm.makeHomeDir = true;
+
+      # pam_unix returns PAM_AUTHINFO_UNAVAIL without prompting when the local
+      # stub has "!" in shadow (account locked), so PAM_AUTHTOK is never set
+      # and pam_sss's use_first_pass fails with "No authentication token".
+      # Changing to try_first_pass makes pam_sss prompt independently when no
+      # prior module has set the token, restoring IPA password login via
+      # LightDM and su.
+      login.rules.auth.sss.settings = lib.mkForce { try_first_pass = true; };
+      su.rules.auth.sss.settings = lib.mkForce { try_first_pass = true; };
+    };
+
+    # HM with useUserPackages = true (flake.nix) sets users.users.${ipaUser}.packages,
+    # which forces the stub into /etc/passwd. pam_sss.so with the "localusers" flag
+    # (added by NixOS when SSSD is enabled) then skips SSSD for any user it finds in
+    # local /etc/passwd — including this stub — falling through to pam_unix, which has
+    # no password for the stub → sudo auth always fails.
+    #
+    # Fix: NOPASSWD for the IPA user. The IPA user already authenticated to reach a
+    # shell (SSH public key from IPA or Kerberos), so re-prompting via a broken PAM
+    # path is security theater on a single-admin homelab.
+    sudo.extraRules = [{
+      users = [ vars.ipaUser ];
+      commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
+    }];
+  };
+
+  systemd = {
+    # Fetch SSH public keys from IPA so users can log in with the key stored
+    # in their IPA profile rather than needing ~/.ssh/authorized_keys on every
+    # host. sss_ssh_authorizedkeys queries SSSD (which queries IPA LDAP).
+    #
+    # /nix/store is 1775 (group-writable by nixbld). OpenSSH 10.0+ rejects
+    # AuthorizedKeysCommand binaries whose path contains any group-writable
+    # component, silently skipping the command. Copy to /usr/local/bin (all
+    # components root-owned, 755) so the path passes sshd's safety check.
+    tmpfiles.rules = [
+      "d /usr/local 0755 root root - -"
+      "d /usr/local/bin 0755 root root - -"
+      "C+ /usr/local/bin/sss_ssh_authorizedkeys 0555 root root - ${pkgs.sssd}/bin/sss_ssh_authorizedkeys"
+      # Pre-create the IPA user's home dir so Home Manager activation succeeds
+      # even before their first login. On a fresh system SSSD may not have
+      # resolved the user yet — tmpfiles warns and skips in that case (non-fatal),
+      # and pam_mkhomedir covers the first-login path as a fallback.
+      "d /home/${vars.ipaUser} 0700 ${vars.ipaUser} ${vars.ipaUser} - -"
+    ];
+
+    # security.ipa enables Kerberos (security.krb5) which causes systemd to
+    # start auth-rpcgss-module.service and rpc-gssd.service for Kerberos NFS
+    # authentication. LXC containers can't load the auth_rpcgss kernel module
+    # and don't have /var/lib/nfs/rpc_pipefs, so both services fail.
+    #
+    # The NixOS IPA module already adds a drop-in for auth-rpcgss-module.service
+    # with ConditionPathExists=/etc/krb5.keytab. We use lib.mkForce to win the
+    # text conflict and add ConditionVirtualization=!container alongside it so
+    # the service is skipped (not failed) in containers that do have a keytab.
+    # Same fix for rpc-gssd.service which also fails in containers.
+    units = lib.mkIf config.boot.isContainer {
+      "auth-rpcgss-module.service" = {
+        overrideStrategy = "asDropinIfExists";
+        text = lib.mkForce ''
+          [Unit]
+          ConditionPathExists=
+          ConditionPathExists=/etc/krb5.keytab
+          ConditionVirtualization=!container
+        '';
+      };
+      # rpc-gssd also has ConditionPathExists from the NixOS IPA module (and an
+      # X-Restart-Triggers store path from systemd.nix). Use mkForce to win;
+      # omit X-Restart-Triggers since this service is skipped in containers anyway.
+      "rpc-gssd.service" = {
+        overrideStrategy = "asDropinIfExists";
+        text = lib.mkForce ''
+          [Unit]
+          ConditionPathExists=
+          ConditionPathExists=/etc/krb5.keytab
+          ConditionVirtualization=!container
+        '';
+      };
+    };
+
+    # home-manager-<user>.service fails on first enrollment because /home/wayne
+    # doesn't exist until the user's first login (pam_mkhomedir creates it then).
+    # ConditionPathExists makes systemd skip the service (exit 0, condition not
+    # met) instead of failing. After first login the dir exists and subsequent
+    # rebuilds activate HM normally.
+    services."home-manager-${vars.ipaUser}".unitConfig.ConditionPathExists =
+      "/home/${vars.ipaUser}";
+  };
+
+  services.openssh.extraConfig = ''
+    AuthorizedKeysCommand /usr/local/bin/sss_ssh_authorizedkeys %u
+    AuthorizedKeysCommandUser nobody
+  '';
+
+  # Host keytab: pre-provisioned on the IPA server, sops-encrypted binary.
+  # Placed at /etc/krb5.keytab before SSSD starts so the host authenticates
+  # to IPA without running ipa-client-install.
+  sops.secrets."ipa-host-keytab" = {
+    sopsFile = keytabPath;
+    format = "binary";
+    path = "/etc/krb5.keytab";
+    owner = "root";
+    group = "root";
+    mode = "0600";
+    restartUnits = [ "sssd.service" ];
+  };
+
+  # NixOS requires isNormalUser/isSystemUser + group on any entry in
+  # users.users. HM with useUserPackages = true (set in flake.nix) adds a stub
+  # entry for each HM user so it can install packages to
+  # /etc/profiles/per-user/<name>/. This definition satisfies those assertions.
+  # With security.ipa setting "passwd: sss files" in nsswitch, SSSD's IPA entry
+  # takes priority for NSS lookups — this local stub is only a fallback when
+  # SSSD is unreachable (at which point auth fails anyway).
+  users.users.${vars.ipaUser} = {
+    isNormalUser = true;
+    group = "users";
+    extraGroups = [ "wheel" ];
+    createHome = false;
+    # "!" is not a password hash — it is the standard "account locked" marker.
+    # It cannot authenticate anyone locally. It exists solely so NixOS generates
+    # a shadow entry for this stub user; without one pam_unix returns
+    # PAM_AUTHINFO_UNAVAIL before prompting, which means PAM_AUTHTOK is never
+    # set and the subsequent pam_sss use_first_pass call has nothing to work
+    # with — blocking LightDM and su logins even when IPA/SSSD auth succeeds.
+    hashedPassword = "!";
+  };
+
+  # Home Manager config for the IPA primary user, applied on every enrolled
+  # host. Manages what IPA doesn't: dotfiles, user-scoped packages, session
+  # variables. Switch-nix/Test-nix/buildImage are system-wide (configuration.nix)
+  # so they don't need to be repeated here.
+  #
+  # homeDirectory uses mkForce because HM's NixOS integration module sets it to
+  # "/var/empty" for users not found in config.users.users at eval time (SSSD
+  # users aren't visible there).
+  home-manager.users.${vars.ipaUser} = { pkgs, ... }: {
+    home = {
+      username = vars.ipaUser;
+      homeDirectory = lib.mkForce "/home/${vars.ipaUser}";
+      stateVersion = "26.05";
+      packages = with pkgs; [ tmux sshfs ];
+      sessionVariables.EDITOR = lib.mkDefault "nano";
+    };
+    programs.home-manager.enable = true;
+    programs.bash.enable = true;
+  };
+}

modules/platforms/lxc.nix

@@ -52,6 +52,7 @@ in
   # LXC container does).
   imports = [
     (modulesPath + "/virtualisation/proxmox-lxc.nix")
+    ../common/preserve-ssh-host-key.nix
   ];
 
   proxmoxLXC = {
@@ -63,23 +64,22 @@ in
     # back to decide `pct create`'s --unprivileged flag, so the two stay
     # in sync).
     #
-    # lxc-docker is the one exception: the kernel's NFS client doesn't set
+    # Any lxc-* host with an NFS fileSystem must be privileged: the kernel's
-    # FS_USERNS_MOUNT, so mounting NFS from inside *any* non-init user
+    # NFS client doesn't set FS_USERNS_MOUNT, so mounting NFS from inside
-    # namespace -- which is exactly what an unprivileged container's
+    # *any* non-init user namespace -- which is exactly what an unprivileged
-    # UID-mapped root runs in -- is rejected at the VFS layer with EPERM,
+    # container's UID-mapped root runs in -- is rejected at the VFS layer
-    # no matter what Proxmox's own `mount=nfs;nfs4` container feature
+    # with EPERM, no matter what Proxmox's own `mount=nfs;nfs4` container
-    # allows at the AppArmor layer (confirmed live: TCP to the NFS server
+    # feature allows at the AppArmor layer (confirmed live: TCP to the NFS
-    # succeeds, the server's export table matches the container's IP, and
+    # server succeeds, the server's export table matches the container's IP,
-    # `mount.nfs: Operation not permitted` still fires immediately with no
+    # and `mount.nfs: Operation not permitted` still fires immediately with
-    # corresponding denial anywhere in the server's logs -- a kernel-level
+    # no corresponding denial anywhere in the server's logs -- a kernel-level
-    # rejection, not a network or export-permission one). Keying off
+    # rejection, not a network or export-permission one). Deriving this from
-    # hostName rather than something docker-build-type-specific because
+    # fileSystems rather than a per-host override keeps it self-consistent:
-    # modules/build-types/docker.nix is also composed for linode-docker/
+    # any new lxc-* host that declares an NFS mount automatically gets the
-    # proxmox-docker, which don't import proxmox-lxc.nix at all --setting
+    # privilege level it needs without a separate manual flag.
-    # this option there would break their eval with "option does not
+    privileged = builtins.any
-    # exist" regardless of any mkIf guard, since mkIf only makes a value
+      (fs: fs.fsType == "nfs" || fs.fsType == "nfs4")
-    # conditional, not whether the option needs to exist somewhere.
+      (builtins.attrValues config.fileSystems);
-    privileged = config.networking.hostName == "docker";
   };
 
   boot.loader = {
@@ -106,49 +106,6 @@ in
     };
   };
 
-  # NixOS's etc activation removes any /etc file that was in the previous
-  # generation's environment.etc but is absent from the current one — even
-  # real (non-symlink) copies.  On every routine nixos-rebuild switch/test that
-  # lacks NIXOS_HOST_KEYS_DIR the key is absent from environment.etc, so it
-  # gets removed as "obsolete".  sops-nix derives its age decryption key from
-  # /etc/ssh/ssh_host_ed25519_key; deletion cascades into every sops secret
-  # failing with "Error getting data key: 0 successful groups required, got 0".
-  #
-  # Fix: activation scripts that bracket the etc step, with explicit deps
-  # to enforce the correct ordering. Without deps the topological sort places
-  # preserveSshHostKey AFTER etc (confirmed live on a deployed lxc-tor-relay:
-  # position 7 vs etc's position 5) -- the key is already gone by the time it
-  # tries to save it. The etc/setupSecrets entries ADD to existing deps
-  # (types.listOf concatenates across module definitions).
-  system.activationScripts = {
-    # Saves the live key to /run before etc can delete it.
-    preserveSshHostKey = ''
-      if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
-        cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
-        cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
-      fi
-    '';
-
-    # Reinstalls the key after etc runs if it was removed as "obsolete".
-    # The resulting file is not registered in environment.etc for either
-    # generation, so subsequent rebuilds leave it alone permanently.
-    restoreSshHostKey = {
-      deps = [ "etc" ];
-      text = ''
-        if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
-          install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
-          install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
-        fi
-        rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
-      '';
-    };
-
-    # Force etc to wait until the key is saved, and sops to wait until the
-    # key is restored. Without these the topological sort breaks the chain.
-    etc = { deps = [ "preserveSshHostKey" ]; };
-    setupSecrets = { deps = [ "restoreSshHostKey" ]; };
-  };
-
   # virtualisation/proxmox-lxc.nix (imported above) registers the Nix
   # store DB via a systemd service (register-nix-paths) -- it never runs
   # an activation script at all. Confirmed live this means neither

modules/platforms/proxmox.nix

@@ -34,6 +34,7 @@ in
     ../hardware-configuration/vm/proxmox.nix
     ../boot/efi.nix
     ../disko/proxmox.nix
+    ../common/preserve-ssh-host-key.nix
   ];
 
   environment.etc = lib.mkIf hasKeyForThisTarget {
@@ -46,36 +47,4 @@ in
       mode = "0644";
     };
   };
-
-  # NixOS's etc activation removes any /etc file that was in the previous
-  # generation's environment.etc but is absent from the current one. Since
-  # the SSH key is only in environment.etc during the --impure build (when
-  # NIXOS_HOST_KEYS_DIR is set), normal rebuilds would remove it as
-  # "obsolete". These scripts mirror lxc.nix's approach: save the live key
-  # before etc runs, restore it after. Without the explicit deps, the
-  # topological sort places preserveSshHostKey after etc (confirmed live on
-  # lxc-tor-relay: position 7 vs etc's position 5), so the key is gone
-  # before it can be saved.
-  system.activationScripts = {
-    preserveSshHostKey = ''
-      if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
-        cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
-        cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
-      fi
-    '';
-
-    restoreSshHostKey = {
-      deps = [ "etc" ];
-      text = ''
-        if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
-          install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
-          install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
-        fi
-        rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
-      '';
-    };
-
-    etc = { deps = [ "preserveSshHostKey" ]; };
-    setupSecrets = { deps = [ "restoreSshHostKey" ]; };
-  };
 }

modules/pxe-boot/mount-pxe-images.nix

@@ -0,0 +1,42 @@
+{ config, lib, vars, ... }:
+
+let
+  # Use the same FQDN approach as docker/mount-data.nix — a bare hostname is
+  # unreliable: systemd-resolved only tries LLMNR for single-label names, and
+  # a global search domain causes it to skip the interface-scoped LAN DNS.
+  nfsServer = "${vars.nfsServerHost}.${vars.homeDomain}";
+in
+{
+  fileSystems.${vars.nfsShares.pxebootImages.mountpoint} = {
+    device = "${nfsServer}:${vars.storageRoot}/${vars.nfsShares.pxebootImages.subpath}";
+    fsType = "nfs";
+    options = [
+      "_netdev"
+      "noatime"
+    ] ++ (if config.boot.isContainer
+    # NFSv4 requires rpc_pipefs (sunrpc filesystem), which Proxmox LXC
+    # containers block unless `features: mount=nfs` is set. Use NFSv3+nolock
+    # instead: no rpc_pipefs dependency at the protocol level, and rpcbind
+    # on the server handles port resolution without needing client-side
+    # sunrpc infrastructure. nofail keeps boot clean if server is unreachable.
+    then [ "nfsvers=3" "proto=tcp" "nolock" "nofail" ]
+    else [ "nfsvers=4.2" "x-systemd.automount" ]);
+  };
+
+  # NixOS pulls var-lib-nfs-rpc_pipefs.mount (the sunrpc filesystem) into
+  # nfs-client.target for any nfs fileSystems entry. In LXC containers the
+  # sunrpc mount is blocked by Proxmox's AppArmor profile, causing it to fail
+  # and the activation to report an error even though our mount uses nofail.
+  # Add ConditionVirtualization=!container via drop-in so systemd skips the
+  # unit entirely in containers (skip = inactive, not failed), which keeps
+  # nfs-client.target green and activation clean.
+  systemd.units = lib.mkIf config.boot.isContainer {
+    "var-lib-nfs-rpc_pipefs.mount" = {
+      overrideStrategy = "asDropin";
+      text = ''
+        [Unit]
+        ConditionVirtualization=!container
+      '';
+    };
+  };
+}

modules/tailscale/subnet-router.nix

@@ -1,4 +1,4 @@
-_:
+{ pkgs, ... }:
 
 {
   imports = [ ./enable-service.nix ];
@@ -12,4 +12,24 @@ _:
     # instead of relaying through DERP.
     openFirewall = true;
   };
+
+  # Tailscale recommends these ethtool flags on the uplink interface to get
+  # full UDP GRO throughput on subnet routers (https://tailscale.com/s/ethtool-config-udp-gro).
+  # The interface is derived from the default route so it works regardless of
+  # what the NIC is named on a given host.
+  systemd.services.tailscale-udp-gro = {
+    description = "Enable UDP GRO forwarding on uplink for Tailscale subnet router";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    path = [ pkgs.ethtool pkgs.iproute2 ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      ExecStart = pkgs.writeShellScript "tailscale-udp-gro" ''
+        NETDEV=$(ip -o route get 8.8.8.8 | cut -f 5 -d " ")
+        ethtool -K "$NETDEV" rx-udp-gro-forwarding on rx-gro-list off
+      '';
+    };
+  };
 }

modules/tailscale/ts-dns-forwarder.nix

@@ -0,0 +1,60 @@
+{ vars, ... }:
+
+{
+  # Run dnsmasq on the LAN interface as a forwarding-only resolver for
+  # *.ts.net (Tailscale MagicDNS names). FreeIPA's bind-dyndb-ldap
+  # cannot reach 100.100.100.100 (Tailscale's internal resolver) directly
+  # because the DC is not a Tailscale node. This host IS a Tailscale node
+  # and can reach 100.100.100.100 via its tailscale0 interface, so it
+  # acts as an intermediary: FreeIPA has a conditional forward zone for
+  # ts.net pointing here (vars.tailscaleRouterIp), and this dnsmasq
+  # instance forwards those queries onward to Tailscale's resolver.
+  #
+  # Configure FreeIPA once after deploying this host:
+  #   kinit admin
+  #   ipa dnsforwardzone-add ${vars.tailnetDomain} \
+  #     --forwarder=${vars.tailscaleRouterIp} \
+  #     --forward-policy=only
+  # Note: IPA refuses to shadow ts.net (a real public TLD); use the
+  # tailnet-specific subdomain (vars.tailnetDomain) instead.
+  services.dnsmasq = {
+    enable = true;
+    # NixOS's dnsmasq module defaults resolveLocalQueries to true, which adds
+    # 127.0.0.1 to networking.nameservers and makes dnsmasq bind to
+    # listen-address=127.0.0.1. This instance is not the host's local
+    # resolver — it only serves IPA's conditional forwarder for tailnet names.
+    # The host uses domainControllerIp directly (networking.nameservers in
+    # host.nix). Without this, all host DNS goes through dnsmasq, which has
+    # no upstream for general queries (no-resolv=true), breaking resolution.
+    resolveLocalQueries = false;
+    settings = {
+      # Listen only on the LAN interface — not tailscale0 or loopback.
+      # bind-interfaces prevents dnsmasq from binding to 0.0.0.0 and
+      # then filtering by interface later; combined with `interface` this
+      # ensures it genuinely listens only on eth0.
+      bind-interfaces = true;
+      interface = [ vars.lxcLanInterface ];
+
+      # Forward-only: no local /etc/hosts or /etc/resolv.conf reading,
+      # no negative caching of NXDOMAIN for names this instance doesn't
+      # serve. All ts.net queries come from FreeIPA's conditional forwarder
+      # and must be answered by Tailscale's resolver.
+      no-hosts = true;
+      no-resolv = true;
+
+      # Tailscale's internal "Quad100" resolver — reachable from any
+      # Tailscale node via the tailscale0 interface. Scoped to the
+      # specific tailnet subdomain (vars.tailnetDomain) rather than
+      # all of ts.net: FreeIPA refuses to shadow ts.net (a real public
+      # TLD with DNSimple nameservers) so the conditional forward zone
+      # in FreeIPA must use the tailnet-specific subdomain instead:
+      #   ipa dnsforwardzone-add ${vars.tailnetDomain} \
+      #     --forwarder=${vars.tailscaleRouterIp} \
+      #     --forward-policy=only
+      server = [ "/${vars.tailnetDomain}/100.100.100.100" ];
+    };
+  };
+
+  networking.firewall.allowedUDPPorts = [ 53 ];
+  networking.firewall.allowedTCPPorts = [ 53 ];
+}

scripts/env.sh

@@ -88,6 +88,17 @@ export NIX_CACHE_HOST
 : "${LAN_DOMAIN:=gitea.lan.ddnsgeek.com}"
 export LAN_DOMAIN
 
+# Matches variables.nix's homeDomain -- the base LAN domain for service
+# subdomains, FreeIPA Kerberos realm, and host FQDNs.
+: "${HOME_DOMAIN:=sweet.home}"
+export HOME_DOMAIN
+
+# Matches variables.nix's ipaServer -- the FreeIPA server hostname.
+# scripts/ipa/create-nixos-ipa-host-account.sh SSHes here to run
+# ipa host-add and ipa-getkeytab.
+: "${IPA_SERVER:=domain-controller.sweet.home}"
+export IPA_SERVER
+
 # nix_extra_opts: call as a plain statement (NOT inside $(...)/<(...) --
 # that forks a subshell, and the whole point is exporting a decision back
 # into *this* shell) to populate the global NIX_OPTS array with whatever

scripts/gc-hosts.sh

@@ -0,0 +1,227 @@
+#!/usr/bin/env bash
+# gc-hosts.sh — Run nix-collect-garbage -d on all live NixOS hosts.
+#
+# The host list is rebuilt on every run:
+#   1. This workstation (nixos) — always first
+#   2. pve1 — always second (non-NixOS Proxmox node with Nix installed)
+#   3. Every NixOS guest currently running on pve1 (discovered via pct/qm list)
+#
+# nix-cache is excluded: gc-ing the shared binary cache evicts store paths
+# that other hosts depend on for substitution.
+#
+# NixOS hosts: tries "sudo -n nix-collect-garbage -d" first (works when
+# wheelNeedsPassword = false, e.g. the HA cluster).  Falls back to user-level
+# "nix-collect-garbage -d" if sudo needs a password — still collects
+# unreferenced store paths and old nixos-user profile generations, but leaves
+# old system generations in place.
+# pve1: runs "nix-collect-garbage -d" as the login user (no system generations
+# on a non-NixOS host).
+#
+# Usage (from repo root):
+#   bash scripts/gc-hosts.sh [--dry-run]
+
+set -euo pipefail
+cd "$(dirname "$0")/.."
+source scripts/env.sh 2>/dev/null || true
+source scripts/lib/nix-eval.sh 2>/dev/null || true
+
+# ── config ────────────────────────────────────────────────────────────────────
+
+: "${MAX_JOBS:=8}"
+: "${NIXOS_USER:=nixos}"
+: "${PVE1_SSH_USER:=${PROXMOX_SSH_USER:-wayne}}"
+SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10)
+
+DRY_RUN=0
+for arg in "$@"; do
+  case "$arg" in
+    --dry-run) DRY_RUN=1 ;;
+    *) echo "Unknown option: $arg" >&2; exit 1 ;;
+  esac
+done
+
+# ── build the host list ───────────────────────────────────────────────────────
+
+# ORDERED_HOSTS: names in display/execution order.
+# HOST_TARGET[name]: SSH target string (user@host).
+# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (user gc only).
+declare -a  ORDERED_HOSTS=()
+declare -A  HOST_TARGET=()
+declare -A  HOST_TYPE=()
+declare -A  _SEEN_HOSTNAMES=()   # dedup tracker
+
+_add_host() {
+  local name="$1" target="$2" type="$3"
+  if [[ -n "${_SEEN_HOSTNAMES[$name]+_}" ]]; then return; fi
+  _SEEN_HOSTNAMES[$name]=1
+  ORDERED_HOSTS+=("$name")
+  HOST_TARGET[$name]="$target"
+  HOST_TYPE[$name]="$type"
+}
+
+# 1. Workstation (hard-wired first)
+_add_host "nixos" "${NIXOS_USER}@nixos" "nixos"
+
+# 2. pve1 (hard-wired second; non-NixOS, no system generations)
+_add_host "pve1" "${PVE1_SSH_USER}@${PVE1_HOST}" "nix"
+
+# 3. Dynamically discover running NixOS guests on pve1
+echo "Discovering running guests on ${PVE1_HOST}..."
+
+# Evaluate the full flake hostname map in one shot.
+hostname_map="{}"
+if ! hostname_map="$(
+  nix eval --json "${NIX_EVAL_FLAGS[@]}" .#nixosConfigurations \
+    --apply 'cfgs: builtins.mapAttrs (_: cfg: cfg.config.networking.hostName) cfgs' \
+    2>/dev/null
+)"; then
+  echo "  warning: flake eval failed — skipping dynamic host discovery" >&2
+fi
+
+# Get names of all currently running guests from pve1.
+if ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
+  running_guests="$(
+    ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash <<'REMOTE'
+      { sudo pct list 2>/dev/null | awk 'NR>1 && $2=="running" { print $NF }';
+        sudo qm  list 2>/dev/null | awk 'NR>1 && $3=="running" { print $2  }'; } | sort -u
+REMOTE
+  )" || running_guests=""
+
+  while IFS= read -r guest; do
+    [[ -z "$guest" ]] && continue
+
+    # Resolve flake target name → NixOS hostname.
+    hostname="$(printf '%s' "$hostname_map" \
+      | jq -r --arg g "$guest" '.[$g] // empty' 2>/dev/null || true)"
+    [[ -z "$hostname" ]] && continue
+
+    # Exclude nix-cache and any target whose hostname is already in our list.
+    case "$hostname" in nix-cache) continue ;; esac
+    if [[ -n "${_SEEN_HOSTNAMES[$hostname]+_}" ]]; then continue; fi
+
+    echo "  + $guest → $hostname"
+    _add_host "$hostname" "${NIXOS_USER}@${hostname}" "nixos"
+  done <<< "$running_guests"
+else
+  echo "  warning: ${PVE1_HOST} unreachable — skipping dynamic host discovery" >&2
+fi
+
+echo ""
+echo "Hosts: ${ORDERED_HOSTS[*]}"
+echo ""
+
+# ── dry-run ───────────────────────────────────────────────────────────────────
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  echo "[dry-run] commands that would run:"
+  for host in "${ORDERED_HOSTS[@]}"; do
+    target="${HOST_TARGET[$host]}"
+    type="${HOST_TYPE[$host]}"
+    if [[ "$type" == "nixos" ]]; then
+      echo "  ssh ${SSH_OPTS[*]} $target 'sudo -n nix-collect-garbage -d'"
+      echo "  # fallback: ssh ... $target 'nix-collect-garbage -d'"
+    else
+      echo "  ssh ${SSH_OPTS[*]} $target '. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d'"
+    fi
+  done
+  exit 0
+fi
+
+# ── gc worker ─────────────────────────────────────────────────────────────────
+
+gc_one() {
+  local host="$1" target="${HOST_TARGET[$1]}" type="${HOST_TYPE[$1]}" logfile="$2"
+
+  if ! ssh "${SSH_OPTS[@]}" "$target" "true" 2>>"$logfile"; then
+    echo "unreachable"; return
+  fi
+
+  if [[ "$type" == "nixos" ]]; then
+    if ssh "${SSH_OPTS[@]}" "$target" "sudo -n nix-collect-garbage -d" \
+        >>"$logfile" 2>>"$logfile"; then
+      echo "ok(sudo)"; return
+    fi
+    echo "[sudo needs password — falling back to user-level gc]" >>"$logfile"
+    if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \
+        >>"$logfile" 2>>"$logfile"; then
+      echo "ok(user)"; return
+    fi
+  else
+    # Non-NixOS node: BatchMode SSH doesn't source the Nix daemon profile, so
+    # nix-collect-garbage won't be on PATH unless we source it explicitly.
+    local nix_profile='. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null || true'
+    if ssh "${SSH_OPTS[@]}" "$target" "$nix_profile && nix-collect-garbage -d" \
+        >>"$logfile" 2>>"$logfile"; then
+      echo "ok"; return
+    fi
+  fi
+
+  echo "failed:$?"
+}
+
+# ── parallel execution ────────────────────────────────────────────────────────
+
+echo "Running gc on ${#ORDERED_HOSTS[@]} hosts (up to ${MAX_JOBS} parallel)..."
+echo ""
+
+TMPDIR_GC="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR_GC"' EXIT
+
+declare -A LOGS=()
+job_count=0
+
+for host in "${ORDERED_HOSTS[@]}"; do
+  logfile="${TMPDIR_GC}/${host}.log"
+  resultfile="${TMPDIR_GC}/${host}.result"
+  LOGS[$host]="$logfile"
+  : > "$logfile"
+
+  ( result="$(gc_one "$host" "$logfile")"; echo "$result" > "$resultfile" ) &
+
+  (( job_count++ )) || true
+  if [[ "$job_count" -ge "$MAX_JOBS" ]]; then
+    wait -n 2>/dev/null || wait
+    (( job_count-- )) || true
+  fi
+done
+
+wait
+
+# ── summary ───────────────────────────────────────────────────────────────────
+
+echo "Results:"
+echo "──────────────────────────────"
+
+ok_hosts=()
+warn_hosts=()
+fail_hosts=()
+
+for host in "${ORDERED_HOSTS[@]}"; do
+  result="$(cat "${TMPDIR_GC}/${host}.result" 2>/dev/null || echo "failed:missing")"
+  case "$result" in
+    ok|"ok(sudo)"|"ok(user)")
+      printf "  %-22s  %s\n" "$host" "$result"
+      ok_hosts+=("$host") ;;
+    unreachable)
+      printf "  %-22s  UNREACHABLE\n" "$host"
+      warn_hosts+=("$host") ;;
+    *)
+      printf "  %-22s  FAILED (%s)\n" "$host" "$result"
+      fail_hosts+=("$host") ;;
+  esac
+done
+
+echo ""
+echo "  ${#ok_hosts[@]} succeeded, ${#warn_hosts[@]} unreachable, ${#fail_hosts[@]} failed"
+
+for host in "${warn_hosts[@]+"${warn_hosts[@]}"}" "${fail_hosts[@]+"${fail_hosts[@]}"}"; do
+  logfile="${LOGS[$host]}"
+  if [[ -s "$logfile" ]]; then
+    echo ""
+    echo "── $host ──"
+    cat "$logfile"
+  fi
+done
+
+echo ""
+[[ "${#fail_hosts[@]}" -eq 0 ]]

scripts/ha/acceptance-tests.sh

@@ -0,0 +1,231 @@
+#!/usr/bin/env bash
+# acceptance-tests.sh — HA cluster acceptance tests (T1–T7)
+#
+# Run from a host with SSH access to both HA nodes (or from node1 itself).
+# All 7 tests must pass before considering the cluster production-ready.
+# Test values below must match variables.nix haServer* values.
+set -euo pipefail
+
+# ── Configuration ─────────────────────────────────────────────────────────
+# All values override-able via environment variables; defaults match variables.nix.
+NODE1="${NODE1:-ha-server-1}"
+NODE2="${NODE2:-ha-server-2}"
+NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
+NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
+VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
+ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}"  # vars.haIscsiIqn
+# ──────────────────────────────────────────────────────────────────────────
+
+PASS=0
+FAIL=0
+RESULTS=()
+
+# Use PASS=$((PASS+1)) instead of ((PASS++)) — the latter evaluates to 0 when
+# PASS=0, which triggers set -e and kills the script after the very first PASS.
+pass() { echo "  PASS: $1"; PASS=$((PASS+1)); RESULTS+=("PASS $1"); }
+fail() { echo "  FAIL: $1"; FAIL=$((FAIL+1)); RESULTS+=("FAIL $1"); }
+
+HA_USER="nixos"
+n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
+
+echo "════════════════════════════════════════════════════"
+echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
+echo "════════════════════════════════════════════════════"
+
+# ── Pre-flight: DRBD sync must be complete ────────────────────────────────
+# Tests that check disk state, XFS mount, and iSCSI will fail or give false
+# results while the initial full sync is in progress.  Block until done.
+echo ""
+echo "Pre-flight: verifying DRBD sync is complete..."
+DRBD_PREFLIGHT=$(n1 "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
+if ! echo "$DRBD_PREFLIGHT" | grep -q "^UpToDate/UpToDate$"; then
+  echo ""
+  echo "  ERROR: DRBD initial sync not complete."
+  echo "  Current dstate on $NODE1: $DRBD_PREFLIGHT"
+  echo ""
+  echo "  Monitor progress:"
+  echo "    ssh nixos@$NODE1_IP 'sudo watch -n3 cat /proc/drbd'"
+  echo ""
+  echo "  Re-run this script once dstate shows UpToDate/UpToDate."
+  exit 1
+fi
+echo "  dstate: $DRBD_PREFLIGHT — ready."
+
+# ── Detect Active/Standby nodes ────────────────────────────────────────────
+# Use crm_mon to detect which node holds the Promoted (Primary) DRBD resource.
+# Pacemaker is authoritative; DRBD role can briefly read as Secondary while
+# Pacemaker is mid-transition, giving a false Active/Standby swap.
+# Wait up to 90 s for Pacemaker to settle before giving up.
+echo ""
+echo "Detecting Active/Standby nodes (waiting for Pacemaker to settle)..."
+ACTIVE_NODE=""
+for i in $(seq 1 30); do
+  # crm_mon -1 output contains "Promoted: [ <node> ]" for the DRBD master.
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || n2 "crm_mon -1" 2>/dev/null || true)
+  # crm_mon 2.x formats Promoted lines as "    * Promoted: [ node ]" — the * bullet
+  # means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
+  ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | grep -E '(Promoted|Masters):' | grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+  [[ -n "$ACTIVE_NODE" ]] && break
+  sleep 3
+done
+
+if [[ -z "$ACTIVE_NODE" ]]; then
+  echo "  WARNING: could not determine Active node from crm_mon after 90 s — defaulting to $NODE1"
+  ACTIVE_NODE="$NODE1"
+fi
+
+if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
+  ACTIVE_IP="$NODE1_IP"
+  STANDBY_NODE="$NODE2"; STANDBY_IP="$NODE2_IP"
+  na() { n1 "$@"; }
+  ns() { n2 "$@"; }
+else
+  ACTIVE_IP="$NODE2_IP"
+  STANDBY_NODE="$NODE1"; STANDBY_IP="$NODE1_IP"
+  na() { n2 "$@"; }
+  ns() { n1 "$@"; }
+fi
+echo "  Active:  $ACTIVE_NODE ($ACTIVE_IP)"
+echo "  Standby: $STANDBY_NODE ($STANDBY_IP)"
+
+# ── T1: Corosync quorum established ──────────────────────────────────────
+echo ""
+echo "[T1] Corosync quorum"
+if na "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
+  pass "cluster has quorum"
+else
+  fail "cluster does not have quorum — check corosync on both nodes"
+fi
+
+# ── T2: DRBD Primary on Active node, Secondary on Standby ────────────────
+echo ""
+echo "[T2] DRBD roles"
+DRBD_ROLE=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
+if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then
+  pass "DRBD Primary on $ACTIVE_NODE ($DRBD_ROLE)"
+else
+  fail "unexpected DRBD role on $ACTIVE_NODE: $DRBD_ROLE (expected Primary/Secondary)"
+fi
+
+DRBD_DSTATE=$(na "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
+if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then
+  pass "DRBD disk state UpToDate ($DRBD_DSTATE)"
+else
+  fail "DRBD disk not UpToDate: $DRBD_DSTATE"
+fi
+
+# ── T3: XFS mounted at haStorageRoot on the Active node ──────────────────
+echo ""
+echo "[T3] XFS mount"
+if na "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+  pass "XFS mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
+else
+  fail "XFS not mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
+fi
+
+if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+  fail "XFS unexpectedly mounted on $STANDBY_NODE (should only be on Active node)"
+else
+  pass "XFS not mounted on $STANDBY_NODE (correct — Standby)"
+fi
+
+# ── T4: iSCSI target visible on Active node ───────────────────────────────
+echo ""
+echo "[T4] iSCSI target"
+IQN_COUNT=$(na "bash -c 'ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn || true'" 2>/dev/null || echo "0")
+if [[ "$IQN_COUNT" -ge 1 ]]; then
+  pass "iSCSI IQN active on $ACTIVE_NODE ($IQN_COUNT target(s))"
+else
+  fail "no iSCSI IQN active on $ACTIVE_NODE"
+fi
+
+# iSCSI port reachable from Standby node via VIP.
+# Use bash TCP probe (no iscsiadm needed — just checks port 3260 is open).
+if ns "bash -c 'echo >/dev/tcp/${VIP}/3260' 2>/dev/null"; then
+  pass "iSCSI port 3260 reachable from $STANDBY_NODE via VIP ${VIP}"
+else
+  fail "iSCSI port 3260 not reachable from $STANDBY_NODE via ${VIP}"
+fi
+
+# ── T5: Failover — standby Active node, verify resources move to Standby ──
+echo ""
+echo "[T5] Failover (standby $ACTIVE_NODE)"
+ACTIVE_CRMD_NAME=$(na "crm_node -n" 2>/dev/null || echo "")
+na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v on" 2>/dev/null || true
+echo "  Waiting up to 120 s for resources to move to $STANDBY_NODE..."
+MOVED=false
+for i in $(seq 1 120); do
+  if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+    MOVED=true
+    echo "  Resources moved in ${i}s"
+    break
+  fi
+  sleep 1
+done
+
+if $MOVED; then
+  pass "XFS mounted on $STANDBY_NODE after failover"
+  IQN_ON_STANDBY=$(ns "bash -c 'ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn || true'" 2>/dev/null || echo "0")
+  [[ "$IQN_ON_STANDBY" -ge 1 ]] \
+    && pass "iSCSI target active on $STANDBY_NODE after failover" \
+    || fail "iSCSI target NOT active on $STANDBY_NODE after failover"
+else
+  fail "XFS did not mount on $STANDBY_NODE within 120 s — failover incomplete"
+fi
+
+# ── T6: Data integrity — file written post-failover readable ─────────────
+echo ""
+echo "[T6] Data integrity"
+# Write a test file on the new Active (former Standby) and verify it.
+# Use `echo | sudo tee` for the write: "echo ... > file" via bash -c has the
+# redirect interpreted by the remote nixos shell (not sudo), so the file open
+# runs as nixos and fails with EACCES on the root-owned XFS mount.  Piping
+# through sudo tee lets tee (running as root) open the file instead.
+TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$"
+TEST_CONTENT="ha-acceptance-test-$(date +%s)"
+echo "${TEST_CONTENT}" | ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${STANDBY_IP}" sudo tee "${TEST_FILE}" > /dev/null 2>/dev/null || true
+READBACK=$(ns cat "${TEST_FILE}" 2>/dev/null || echo "")
+if [[ "$READBACK" == "$TEST_CONTENT" ]]; then
+  pass "test file written and read back correctly on $STANDBY_NODE"
+else
+  fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')"
+fi
+ns rm -f "${TEST_FILE}" 2>/dev/null || true
+
+# ── T7: Node rejoin — un-standby original Active, verify cluster is healthy ─
+echo ""
+echo "[T7] Node rejoin"
+na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
+na "crm_resource --cleanup" 2>/dev/null || true
+sleep 5
+
+if na "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
+  pass "$ACTIVE_NODE rejoined — cluster has quorum"
+else
+  fail "$ACTIVE_NODE did not rejoin with quorum"
+fi
+
+DRBD_ROLE_AFTER=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
+if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then
+  pass "$ACTIVE_NODE is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
+else
+  fail "unexpected DRBD role on $ACTIVE_NODE after rejoin: $DRBD_ROLE_AFTER"
+fi
+
+# ── Summary ───────────────────────────────────────────────────────────────
+echo ""
+echo "════════════════════════════════════════════════════"
+echo " Results: ${PASS} PASS, ${FAIL} FAIL"
+echo "════════════════════════════════════════════════════"
+for r in "${RESULTS[@]}"; do echo "  $r"; done
+echo ""
+
+if [[ "$FAIL" -eq 0 ]]; then
+  echo "ALL PASS — cluster is production-ready."
+  exit 0
+else
+  echo "SOME TESTS FAILED — investigate before deploying."
+  exit 1
+fi

scripts/ha/cluster-enable-stonith.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# cluster-enable-stonith.sh — enable STONITH fence agent after the fence SSH
+# key is deployed to both nodes and authorised on the Proxmox host.
+#
+# Run from ha-server-1 as root AFTER:
+#   - /etc/pacemaker/fence_pve_ssh exists on both nodes (chmod +x)
+#     (copy from scripts/ha/fence-pve-ssh.py)
+#   - /etc/fence-pve-ssh-key (SSH private key) exists on both nodes
+#   - The corresponding public key is in authorized_keys on PVE_HOST
+#   - VMID_NODE1 / VMID_NODE2 filled in below
+set -euo pipefail
+
+# ── Configuration ─────────────────────────────────────────────────────────
+NODE1="ha-server-1"
+NODE2="ha-server-2"
+VMID_NODE1=""          # FILL IN: Proxmox VMID for ha-server-1
+VMID_NODE2=""          # FILL IN: Proxmox VMID for ha-server-2
+PVE_HOST="pve1.sweet.home"
+PVE_USER="wayne"
+FENCE_KEY="/etc/fence-pve-ssh-key"
+FENCE_SCRIPT="/etc/pacemaker/fence_pve_ssh"
+# ──────────────────────────────────────────────────────────────────────────
+
+log() { echo "[stonith-setup] $*"; }
+die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; }
+
+[[ $(id -u) -eq 0 ]] || die "must run as root"
+[[ -n "$VMID_NODE1" ]] || die "VMID_NODE1 not set — edit this script"
+[[ -n "$VMID_NODE2" ]] || die "VMID_NODE2 not set — edit this script"
+[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY"
+[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT"
+
+log "Verifying fence agent can reach ${PVE_HOST}..."
+ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \
+    -o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" \
+    "sudo /usr/sbin/qm list" &>/dev/null \
+  || die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo"
+log "Fence agent SSH connectivity confirmed"
+
+log "Creating Pacemaker STONITH resources..."
+cibadmin --create --scope resources --xml-text "
+<primitive id=\"stonith-${NODE1}\" class=\"stonith\" type=\"external/fence_pve_ssh\">
+  <instance_attributes id=\"stonith-${NODE1}-attrs\">
+    <nvpair id=\"stonith-${NODE1}-plug\" name=\"plug\" value=\"${NODE1}\"/>
+    <nvpair id=\"stonith-${NODE1}-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
+    <nvpair id=\"stonith-${NODE1}-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
+    <nvpair id=\"stonith-${NODE1}-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
+    <nvpair id=\"stonith-${NODE1}-vmid1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
+    <nvpair id=\"stonith-${NODE1}-vmid2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
+    <nvpair id=\"stonith-${NODE1}-host-list\" name=\"pcmk_host_list\" value=\"${NODE1}\"/>
+  </instance_attributes>
+  <operations>
+    <op id=\"stonith-${NODE1}-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
+  </operations>
+</primitive>
+" 2>/dev/null || true
+
+cibadmin --create --scope resources --xml-text "
+<primitive id=\"stonith-${NODE2}\" class=\"stonith\" type=\"external/fence_pve_ssh\">
+  <instance_attributes id=\"stonith-${NODE2}-attrs\">
+    <nvpair id=\"stonith-${NODE2}-plug\" name=\"plug\" value=\"${NODE2}\"/>
+    <nvpair id=\"stonith-${NODE2}-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
+    <nvpair id=\"stonith-${NODE2}-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
+    <nvpair id=\"stonith-${NODE2}-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
+    <nvpair id=\"stonith-${NODE2}-vmid1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
+    <nvpair id=\"stonith-${NODE2}-vmid2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
+    <nvpair id=\"stonith-${NODE2}-host-list\" name=\"pcmk_host_list\" value=\"${NODE2}\"/>
+  </instance_attributes>
+  <operations>
+    <op id=\"stonith-${NODE2}-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
+  </operations>
+</primitive>
+" 2>/dev/null || true
+
+log "Enabling STONITH and restoring quorum policy..."
+crm_attribute -t crm_config -n stonith-enabled -v true
+crm_attribute -t crm_config -n no-quorum-policy -v stop
+
+log "DRBD fencing mode must also be updated to resource-only (already the"
+log "default in cluster-config.nix; confirm with: cat /etc/drbd.d/ha-data.conf)"
+
+log "Testing fence agent..."
+stonith_admin --list-devices && log "Fence devices listed successfully." \
+  || warn "stonith_admin --list-devices failed — check config"
+
+log "STONITH enabled. Cluster is now fully HA."

scripts/ha/cluster-init.sh

@@ -0,0 +1,467 @@
+#!/usr/bin/env bash
+# cluster-init.sh — one-time HA cluster initialisation script
+#
+# Run ONCE from ha-server-1 as root AFTER both VMs are booted and have SSH
+# access.  It:
+#   1. Generates and distributes the corosync authkey
+#   2. Waits for corosync quorum and pacemaker
+#   3. Initialises DRBD metadata, promotes node1 to primary
+#   4. Creates XFS on /dev/drbd0 and mounts it
+#   5. Creates the directory tree and iSCSI LUN backing file
+#   6. Configures LIO iSCSI target (file-backed LUN)
+#   7. Configures Pacemaker resources: DRBD → XFS → iSCSI → NFS → VIP
+#
+# Prerequisites:
+#   - Both VMs booted with the ha-server config (nixos-rebuild done)
+#   - SSH key access from node1 to root@NODE2_IP
+#   - VMID_NODE1 / VMID_NODE2 filled in below (needed for STONITH setup;
+#     cluster starts without STONITH, which you enable separately via
+#     scripts/ha/cluster-enable-stonith.sh)
+#   - Run as root on ha-server-1
+set -euo pipefail
+
+# ── Configuration ─────────────────────────────────────────────────────────
+# All values override-able via environment variables; defaults match variables.nix.
+NODE1="${NODE1:-ha-server-1}"
+NODE2="${NODE2:-ha-server-2}"
+NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
+NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
+VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
+ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}"  # vars.haIscsiIqn
+ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
+ISCSI_LUN_SIZE="10G"
+DRBD_DEVICE="/dev/drbd0"
+# DRBD backing disk — by-id path that resolves correctly on both nodes
+# regardless of whether the OS-level name is sda or sdb (Proxmox VM disk
+# ordering is not guaranteed). Matches haServerDrbdDisk in variables.nix.
+# Override DRBD_DISK if your hardware uses a different controller/slot path.
+DRBD_DISK="${DRBD_DISK:-/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1}"
+VMID_NODE1="${VMID_NODE1:-}"            # set by deploy.sh; needed for STONITH
+VMID_NODE2="${VMID_NODE2:-}"
+PVE_HOST="${PVE_HOST:-pve1.sweet.home}"
+PVE_USER="${PVE_USER:-wayne}"
+# Inter-node SSH: HA_USER is the user to SSH as on NODE2; HA_KEY is the private
+# key to use.  Default is root-to-root (no key arg).  deploy.sh sets HA_USER=nixos
+# and HA_KEY=/tmp/cluster-init-key so the script works even when root-to-root SSH
+# is not available.
+HA_USER="${HA_USER:-root}"
+HA_KEY="${HA_KEY:-}"
+
+# NFS dataset subdirectories to create under XFS_MOUNT.
+# Must mirror vars.nfsShares subpath values in variables.nix.
+NFS_SUBDIRS=(
+  "docker/config"
+  "docker/volumes"
+  "docker/databases"
+  "docker/nextcloud-data"
+  "raspi/volumes"
+  "proxmox/iso"
+  "proxmox/lxc"
+  "pxe-boot/images"
+)
+# ──────────────────────────────────────────────────────────────────────────
+
+log()  { echo "[cluster-init] $*"; }
+die()  { echo "[cluster-init] ERROR: $*" >&2; exit 1; }
+warn() { echo "[cluster-init] WARNING: $*" >&2; }
+
+[[ $(id -u) -eq 0 ]] || die "must run as root"
+[[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1"
+
+# NixOS may not include xfsprogs in root's PATH even when it's in the store.
+# If mkfs.xfs is missing, search the Nix store for it.
+if ! command -v mkfs.xfs &>/dev/null; then
+  _xfs_bin=$(find /nix/store -maxdepth 3 -name mkfs.xfs 2>/dev/null | head -1 | xargs dirname 2>/dev/null || true)
+  [[ -n "$_xfs_bin" ]] && export PATH="$_xfs_bin:$PATH" \
+    || die "mkfs.xfs not found — add xfsprogs to ha-server.nix environment.systemPackages and rebuild"
+fi
+
+# drbdmeta lives alongside drbdadm but may not be in PATH when run via sudo.
+if ! command -v drbdmeta &>/dev/null; then
+  _drbd_bin=$(dirname "$(command -v drbdadm)" 2>/dev/null || true)
+  [[ -n "$_drbd_bin" ]] && export PATH="$_drbd_bin:$PATH" \
+    || die "drbdmeta not found — is drbd-utils in ha-server environment.systemPackages?"
+fi
+
+# Portable 16-hex-char UUID generator (no openssl required).
+_rand_uuid() {
+  cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' | cut -c1-16 | tr '[:lower:]' '[:upper:]'
+}
+
+# Inter-node SSH/SCP helpers — abstract over root-to-root vs nixos+sudo.
+_SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10"
+[[ -n "$HA_KEY" ]] && _SSH_OPTS="-i $HA_KEY $_SSH_OPTS"
+if [[ "$HA_USER" == "root" ]]; then
+  n2_ssh() { ssh $_SSH_OPTS "root@${NODE2_IP}" "$@"; }
+  n2_scp() { scp $_SSH_OPTS "$1" "root@${NODE2_IP}:$2"; }
+else
+  # Non-root user with passwordless sudo; wrap each command with sudo.
+  n2_ssh() { ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@"; }
+  n2_scp() {
+    # SCP to a tmp path, then sudo-move to the real destination as the remote user.
+    local src="$1" dst="$2"
+    local tmp="/tmp/_cluster_init_scp_$$"
+    scp $_SSH_OPTS "$src" "${HA_USER}@${NODE2_IP}:${tmp}"
+    ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo mv "${tmp}" "${dst}"
+  }
+fi
+
+# ── 0. Corosync authkey ───────────────────────────────────────────────────
+AUTHKEY="/etc/corosync/authkey"
+mkdir -p /etc/corosync
+if [[ ! -f "$AUTHKEY" ]]; then
+  log "Generating corosync authkey..."
+  corosync-keygen -k "$AUTHKEY"
+  chmod 0400 "$AUTHKEY"
+fi
+log "Distributing authkey to $NODE2..."
+n2_ssh "mkdir -p /etc/corosync"
+n2_scp "$AUTHKEY" "$AUTHKEY"
+n2_ssh "chmod 0400 '${AUTHKEY}'"
+
+log "Restarting corosync and pacemaker on both nodes..."
+systemctl restart corosync
+n2_ssh "systemctl restart corosync"
+sleep 3
+
+log "Starting pacemaker on both nodes (may have failed at boot before authkey was placed)..."
+systemctl start pacemaker 2>/dev/null || systemctl restart pacemaker 2>/dev/null || true
+n2_ssh "systemctl start pacemaker 2>/dev/null || systemctl restart pacemaker 2>/dev/null || true"
+sleep 2
+
+# ── 1. Corosync quorum ────────────────────────────────────────────────────
+log "Waiting for corosync quorum..."
+for i in $(seq 1 30); do
+  if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then
+    log "Quorum established"
+    break
+  fi
+  [[ $i -eq 30 ]] && die "corosync quorum not established after 60 s"
+  sleep 2
+done
+
+log "Waiting for pacemaker..."
+for i in $(seq 1 30); do
+  if crm_mon -1 &>/dev/null; then
+    log "Pacemaker running"
+    break
+  fi
+  [[ $i -eq 30 ]] && die "pacemaker not running after 60 s"
+  sleep 2
+done
+
+# ── 2. DRBD initialisation ────────────────────────────────────────────────
+# Put both nodes in Pacemaker standby first so it stops managed resources
+# cleanly, then enable maintenance-mode so Pacemaker's monitor operations are
+# suspended.  Without maintenance-mode, Pacemaker keeps monitoring: when it
+# sees DRBD Primary on a standby node (that it didn't start), it triggers a
+# stop action — killing the initial sync after ~10 s.  Maintenance-mode
+# disables all start/stop/monitor actions for the duration of the sync; it is
+# cleared after UpToDate/UpToDate is confirmed.
+log "Setting both nodes to Pacemaker standby for DRBD metadata init..."
+crm_standby -N "$NODE1" -v on 2>/dev/null || true
+crm_standby -N "$NODE2" -v on 2>/dev/null || true
+
+# Wait for Pacemaker to actually stop DRBD (if it was managing it).
+log "Waiting for DRBD to stop under Pacemaker control..."
+for i in $(seq 1 30); do
+  n1_role=$(drbdadm role ha-data 2>/dev/null || echo "Unconfigured")
+  n2_role=$(n2_ssh "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "Unconfigured")
+  if [[ "$n1_role" == "Unconfigured" ]] && [[ "$n2_role" == "Unconfigured" ]]; then
+    log "DRBD stopped on both nodes"
+    break
+  fi
+  [[ $i -eq 30 ]] && warn "DRBD still active after 60s standby — forcing down anyway"
+  sleep 2
+done
+
+log "Enabling Pacemaker maintenance-mode (suspends monitor/start/stop during sync)..."
+crm_attribute -t crm_config -n maintenance-mode -v true 2>/dev/null || true
+
+log "Detaching DRBD on $NODE1 (belt-and-suspenders after standby)..."
+drbdadm down ha-data 2>/dev/null || true
+log "Detaching DRBD on $NODE2..."
+n2_ssh "drbdadm down ha-data 2>/dev/null || true"
+sleep 2
+
+# Ensure /etc/drbd.conf on both nodes points to DRBD_DISK (the stable by-id
+# path).  VMs built before this fix may have /dev/sda or /dev/sdb hardcoded.
+# NixOS makes /etc/drbd.conf a symlink into the read-only Nix store, so
+# sed -i on the symlink target would fail — we break the symlink first with
+# cp --remove-destination, creating a regular writable copy.
+# Rebuild+redeploy (--force-rebuild) to make this permanent.
+_PATCH_DRBD=$(mktemp)
+cat > "$_PATCH_DRBD" << 'PATCHEOF'
+#!/bin/bash
+WANT="$1"
+conf=/etc/drbd.conf
+if [[ -L "$conf" ]]; then
+  cp --remove-destination "$(readlink -f "$conf")" "$conf"
+fi
+cur=$(drbdadm sh-ll-dev ha-data 2>/dev/null | head -1 || true)
+if [[ -n "$cur" && "$cur" != "$WANT" ]]; then
+  echo "[cluster-init] WARNING: patching $conf: $cur → $WANT (rebuild to make permanent)"
+  sed -i "s,${cur},${WANT},g" "$conf"
+fi
+PATCHEOF
+chmod +x "$_PATCH_DRBD"
+bash "$_PATCH_DRBD" "$DRBD_DISK"
+n2_scp "$_PATCH_DRBD" "/tmp/patch-drbd-disk.sh"
+n2_ssh "bash /tmp/patch-drbd-disk.sh ${DRBD_DISK}"
+n2_ssh "rm -f /tmp/patch-drbd-disk.sh"
+rm -f "$_PATCH_DRBD"
+
+log "Initialising DRBD metadata on $NODE1..."
+# Use drbdmeta --force directly for BOTH create-md and write-dev-uuid.
+# drbdadm create-md --force passes --force to drbdmeta create-md but NOT to
+# the write-dev-uuid sub-call it makes internally, so write-dev-uuid fails when
+# the backing disk is still busy and stdin is not a TTY:
+# "stdin not a TTY, not waiting for confirmation" → exit 20.
+# Calling drbdmeta --force directly bypasses the exclusive-open confirmation on
+# both steps without needing a TTY, regardless of whether the device is busy.
+# Skip metadata creation only if DRBD is UP and fully synced (UpToDate/UpToDate).
+# When the resource is down, drbdadm dstate reads metadata and returns just
+# "UpToDate" (no slash) — that must not be treated as "already synced".
+# Mismatched UUIDs from an interrupted sync cause instant WFConnection→StandAlone,
+# so we always recreate metadata unless the sync is genuinely complete.
+if [[ "$(drbdadm dstate ha-data 2>/dev/null)" != "UpToDate/UpToDate" ]]; then
+  UUID1=$(_rand_uuid)
+  drbdmeta --force 0 v08 "${DRBD_DISK}" internal create-md
+  drbdmeta --force 0 v08 "${DRBD_DISK}" internal write-dev-uuid "$UUID1"
+fi
+
+log "Initialising DRBD metadata on $NODE2..."
+if [[ "$(n2_ssh "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null)" != "UpToDate/UpToDate" ]]; then
+  UUID2=$(n2_ssh "cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' | cut -c1-16 | tr '[:lower:]' '[:upper:]'")
+  n2_ssh "drbdmeta --force 0 v08 ${DRBD_DISK} internal create-md"
+  n2_ssh "drbdmeta --force 0 v08 ${DRBD_DISK} internal write-dev-uuid ${UUID2}"
+fi
+
+log "Bringing up DRBD on both nodes..."
+drbdadm up ha-data 2>/dev/null || true
+n2_ssh "drbdadm up ha-data" 2>/dev/null || true
+
+log "Forcing $NODE1 to DRBD Primary for initial sync..."
+drbdadm primary ha-data --force
+# NOTE: Pacemaker standby is intentionally kept ON until after the sync
+# completes.  Clearing it here races with the OCF DRBD agent: Pacemaker
+# sees DRBD in WFConnection/SyncSource and may call drbdadm-down thinking
+# something went wrong, killing the sync.  Standby is cleared below, after
+# UpToDate/UpToDate is confirmed.
+
+log "Waiting for DRBD initial sync to complete (32 GB may take 10–20 min)..."
+log "  (monitor with: watch -n3 cat /proc/drbd)"
+_sync_chars=('|' '/' '-' $'\\')
+_sync_iter=0
+while true; do
+  _dstate=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
+  if echo "$_dstate" | grep -q "UpToDate/UpToDate"; then
+    printf "\r%-80s\r" ""
+    log "DRBD initial sync complete (dstate: $_dstate)"
+    break
+  fi
+  # Parse connection state from /proc/drbd (cs:SyncSource, cs:Connected, cs:StandAlone …)
+  _cs=$(grep -oE 'cs:[A-Za-z]+' /proc/drbd 2>/dev/null | head -1 | sed 's/cs://' || echo "unknown")
+  # /proc/drbd uses variable whitespace: "sync'ed:  5.2%" (two spaces).
+  _pct=$(grep -oE "sync'ed:[[:space:]]+[0-9.]+" /proc/drbd 2>/dev/null | grep -oE "[0-9.]+" | head -1 || echo "")
+  _eta=$(grep -oE "finish:[[:space:]]+[0-9:]+" /proc/drbd 2>/dev/null | grep -oE "[0-9:]+$" | head -1 || echo "")
+  _spd=$(grep -oE "speed:[[:space:]]+[0-9,]+" /proc/drbd 2>/dev/null | grep -oE "[0-9,]+$" | head -1 || echo "")
+  _sync_iter=$(( _sync_iter + 1 ))
+  _sc="${_sync_chars[$_sync_iter % 4]}"
+  if [[ "$_cs" == "StandAlone" && $_sync_iter -gt 5 ]]; then
+    printf "\r%-80s\r" ""
+    die "DRBD is StandAlone after 15 s — peer connection lost (dstate: $_dstate). " \
+        "Check corosync/network and re-run cluster-init."
+  elif [[ -n "$_pct" ]]; then
+    printf "\r  [%s] syncing: %s%% done — ETA %s @ %s K/s     " \
+      "$_sc" "$_pct" "${_eta:-??:??:??}" "${_spd:-?}"
+  else
+    printf "\r  [%s] cs:%s dstate:%s — waiting for sync to start     " "$_sc" "$_cs" "$_dstate"
+  fi
+  sleep 3
+done
+
+log "Disabling Pacemaker maintenance-mode and clearing standby — handing DRBD back to Pacemaker..."
+crm_attribute -t crm_config -n maintenance-mode -v false 2>/dev/null || true
+crm_standby -N "$NODE1" -v off 2>/dev/null || true
+crm_standby -N "$NODE2" -v off 2>/dev/null || true
+
+# ── 3. XFS filesystem ─────────────────────────────────────────────────────
+log "Creating XFS on ${DRBD_DEVICE}..."
+if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
+  mkfs.xfs -f "${DRBD_DEVICE}"
+fi
+
+log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
+mkdir -p "${XFS_MOUNT}"
+mountpoint -q "${XFS_MOUNT}" || mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
+
+# ── 4. NFS dataset directories ────────────────────────────────────────────
+log "Creating NFS dataset directories..."
+for subdir in "${NFS_SUBDIRS[@]}"; do
+  mkdir -p "${XFS_MOUNT}/${subdir}"
+done
+
+# ── 5. iSCSI LUN backing file ─────────────────────────────────────────────
+log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..."
+if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then
+  fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}"
+fi
+
+# ── 6. LIO iSCSI target ───────────────────────────────────────────────────
+log "Configuring LIO iSCSI target via targetcli..."
+# Note: do NOT bind portal to ${VIP} here — the VIP isn't assigned yet (Pacemaker
+# creates it).  The default portal (all IPs, port 3260) is correct; Pacemaker's
+# VIP resource will make the target reachable at the VIP address.
+#
+# Clear any existing LIO state first (idempotent: re-run after a partial failure).
+# Use specific delete commands — clearconfig does not reliably clear kernel state.
+if ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -q "${ISCSI_IQN}"; then
+  log "Clearing existing LIO target ${ISCSI_IQN} before reconfiguration..."
+  targetcli "/iscsi delete ${ISCSI_IQN}" 2>/dev/null || true
+fi
+if ls /sys/kernel/config/target/core/ 2>/dev/null | grep -q "fileio"; then
+  log "Clearing existing LIO backstore ha-lun0 before reconfiguration..."
+  targetcli "/backstores/fileio delete ha-lun0" 2>/dev/null || true
+fi
+targetcli <<EOF
+/backstores/fileio create name=ha-lun0 file_or_dev=${ISCSI_LUN_FILE} size=0 write_back=false
+/iscsi create ${ISCSI_IQN}
+/iscsi/${ISCSI_IQN}/tpg1/luns create /backstores/fileio/ha-lun0
+/iscsi/${ISCSI_IQN}/tpg1 set attribute authentication=0
+/iscsi/${ISCSI_IQN}/tpg1 set attribute demo_mode_write_protect=0
+saveconfig /etc/target/saveconfig.json
+EOF
+
+log "Tearing down LIO kernel objects — Pacemaker will restore via targetctl on the Active node..."
+# LIO holds the backing file open; clear kernel state now so the XFS unmount succeeds.
+# Use specific delete commands (clearconfig does not reliably clear kernel configfs state).
+targetcli "/iscsi delete ${ISCSI_IQN}" 2>/dev/null || warn "LIO iscsi delete failed — umount may fail"
+targetcli "/backstores/fileio delete ha-lun0" 2>/dev/null || warn "LIO backstore delete failed"
+
+log "Distributing iSCSI saveconfig to $NODE2..."
+n2_scp /etc/target/saveconfig.json /etc/target/saveconfig.json
+
+
+log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..."
+umount "${XFS_MOUNT}" || { sync; umount -l "${XFS_MOUNT}"; }
+
+log "Demoting DRBD to Secondary — Pacemaker manages primary role..."
+drbdadm role ha-data 2>/dev/null | grep -q "^Primary" && drbdadm secondary ha-data || true
+
+# ── 7. Pacemaker resources ────────────────────────────────────────────────
+log "Configuring Pacemaker cluster properties..."
+crm_attribute -t crm_config -n stonith-enabled -v false
+crm_attribute -t crm_config -n no-quorum-policy -v ignore
+
+log "Creating Pacemaker resources via cibadmin..."
+# Use cibadmin --replace with pacemaker-4.0-compatible XML.
+# Key schema rules for pacemaker-4.0:
+#   - globally-unique must be in <meta_attributes>, not a direct <clone> attribute
+#   - promoted-max / promoted-node-max (not master-max / master-node-max)
+#   - constraint with-rsc-role="Promoted" (not "Master")
+cibadmin --replace --scope resources --xml-text '<resources>
+  <clone id="ms-drbd0">
+    <meta_attributes id="ms-drbd0-meta">
+      <nvpair id="ms-drbd0-globally-unique" name="globally-unique" value="false"/>
+      <nvpair id="ms-drbd0-promotable" name="promotable" value="true"/>
+      <nvpair id="ms-drbd0-promoted-max" name="promoted-max" value="1"/>
+      <nvpair id="ms-drbd0-promoted-node-max" name="promoted-node-max" value="1"/>
+      <nvpair id="ms-drbd0-clone-max" name="clone-max" value="2"/>
+      <nvpair id="ms-drbd0-clone-node-max" name="clone-node-max" value="1"/>
+      <nvpair id="ms-drbd0-notify" name="notify" value="true"/>
+      <nvpair id="ms-drbd0-interleave" name="interleave" value="true"/>
+    </meta_attributes>
+    <primitive id="drbd0" class="ocf" type="drbd" provider="linbit">
+      <instance_attributes id="drbd0-attrs">
+        <nvpair id="drbd0-resource" name="drbd_resource" value="ha-data"/>
+      </instance_attributes>
+      <operations>
+        <op id="drbd0-start" name="start" interval="0" timeout="240s"/>
+        <op id="drbd0-stop" name="stop" interval="0" timeout="120s"/>
+        <op id="drbd0-promote" name="promote" interval="0" timeout="240s"/>
+        <op id="drbd0-demote" name="demote" interval="0" timeout="90s"/>
+        <op id="drbd0-monitor-promoted" name="monitor" interval="20s" timeout="20s" role="Promoted"/>
+        <op id="drbd0-monitor-unpromoted" name="monitor" interval="30s" timeout="20s" role="Unpromoted"/>
+      </operations>
+    </primitive>
+  </clone>
+  <group id="ha-group">
+    <primitive id="xfs-data" class="ocf" type="Filesystem" provider="heartbeat">
+      <instance_attributes id="xfs-data-attrs">
+        <nvpair id="xfs-data-device" name="device" value="/dev/drbd0"/>
+        <nvpair id="xfs-data-directory" name="directory" value="/srv/ha-data"/>
+        <nvpair id="xfs-data-fstype" name="fstype" value="xfs"/>
+        <nvpair id="xfs-data-options" name="options" value="defaults"/>
+        <nvpair id="xfs-data-force_unmount" name="force_unmount" value="true"/>
+      </instance_attributes>
+      <operations>
+        <op id="xfs-data-start" name="start" interval="0" timeout="60s"/>
+        <op id="xfs-data-stop" name="stop" interval="0" timeout="60s"/>
+        <op id="xfs-data-monitor" name="monitor" interval="20s" timeout="40s"/>
+      </operations>
+    </primitive>
+    <primitive id="iscsi-target" class="systemd" type="targetctl">
+      <operations>
+        <op id="iscsi-start" name="start" interval="0" timeout="60s"/>
+        <op id="iscsi-stop" name="stop" interval="0" timeout="60s"/>
+        <op id="iscsi-monitor" name="monitor" interval="20s" timeout="40s"/>
+      </operations>
+    </primitive>
+    <primitive id="nfs-server" class="systemd" type="nfs-server">
+      <operations>
+        <op id="nfs-start" name="start" interval="0" timeout="60s"/>
+        <op id="nfs-stop" name="stop" interval="0" timeout="60s"/>
+        <op id="nfs-monitor" name="monitor" interval="30s" timeout="40s"/>
+      </operations>
+    </primitive>
+    <primitive id="vip" class="ocf" type="IPaddr2" provider="heartbeat">
+      <instance_attributes id="vip-attrs">
+        <nvpair id="vip-ip" name="ip" value="192.168.2.229"/>
+        <nvpair id="vip-cidr" name="cidr_netmask" value="24"/>
+      </instance_attributes>
+      <operations>
+        <op id="vip-start" name="start" interval="0" timeout="20s"/>
+        <op id="vip-stop" name="stop" interval="0" timeout="20s"/>
+        <op id="vip-monitor" name="monitor" interval="10s" timeout="20s"/>
+      </operations>
+    </primitive>
+  </group>
+</resources>'
+
+log "Adding Pacemaker ordering and colocation constraints..."
+cibadmin --replace --scope constraints --xml-text '<constraints>
+  <rsc_order id="order-drbd-group" first="ms-drbd0" first-action="promote" then="ha-group" then-action="start" kind="Mandatory"/>
+  <rsc_colocation id="coloc-group-with-drbd" score="INFINITY" rsc="ha-group" with-rsc="ms-drbd0" with-rsc-role="Promoted"/>
+</constraints>'
+
+log "Clearing stale Pacemaker failure history..."
+crm_resource --cleanup 2>/dev/null || true
+
+log "Waiting for resources to start..."
+for i in $(seq 1 60); do
+  if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then
+    log "VIP is up: $(crm_resource -r vip --locate)"
+    break
+  fi
+  [[ $i -eq 60 ]] && { warn "VIP not up after 120 s — check: crm_mon -1"; break; }
+  sleep 2
+done
+
+log ""
+log "═══════════════════════════════════════════════════════════════"
+log " HA cluster initialised."
+log ""
+log "  crm_mon -1                         — cluster status"
+log "  iscsiadm -m discovery -t st -p ${VIP}  — verify iSCSI target"
+log "  showmount -e ${VIP}                    — verify NFS exports"
+log ""
+log "  To enable STONITH (after deploying fence SSH key):"
+log "    1. Fill in VMID_NODE1 / VMID_NODE2 in cluster-enable-stonith.sh"
+log "    2. Copy scripts/ha/fence-pve-ssh.py to /etc/pacemaker/fence_pve_ssh"
+log "       on both nodes (chmod +x)"
+log "    3. Generate and distribute the fence SSH key"
+log "       (see docs or cluster-enable-stonith.sh header)"
+log "    4. bash scripts/ha/cluster-enable-stonith.sh"
+log "═══════════════════════════════════════════════════════════════"

scripts/ha/deploy.sh

@@ -0,0 +1,499 @@
+#!/usr/bin/env bash
+# deploy.sh — Full lifecycle management for the HA file-server cluster.
+#
+# Handles everything from zero (no VMs, no secrets) through a running,
+# tested cluster, and optionally tears it back down.
+#
+# Usage:
+#   scripts/ha/deploy.sh [options]
+#   scripts/ha/deploy.sh --destroy [options]
+#
+# Phases (all run by default; skip any with --skip-*):
+#   1. ensure-bridge   Create storage bridge (vmbr1) on the Proxmox node if absent.
+#   2. sync-keys       Generate SSH host keys and register age keys for both nodes.
+#   3. create-vms      Build disk images and create both VMs via create-proxmox-resource.sh.
+#   4. add-hardware    Attach storage NIC (vmbr1) and DRBD data disk to each VM.
+#   5. boot-wait       Start VMs, wait for SSH on both nodes.
+#   6. cluster-init    Form the cluster: DRBD, corosync, Pacemaker, NFS, VIP.
+#                      Also encrypts the generated corosync authkey into the repo.
+#   7. run-tests       Run acceptance tests (T1–T7).
+#
+# Options:
+#   --node <host>            Proxmox host to deploy on (default: pve1.sweet.home)
+#   --vmid1 <n>              VMID for ha-server-1 (default: 200)
+#   --vmid2 <n>              VMID for ha-server-2 (default: 201)
+#   --storage <pool>         Proxmox storage pool (default: local-zfs)
+#   --storage-bridge <br>    Bridge for HA storage network (default: vmbr1)
+#   --drbd-disk-gb <n>       DRBD data disk size in GB (default: 32)
+#   --memory <MB>            RAM per node (default: 4096)
+#   --cores <n>              vCPUs per node (default: 4)
+#   --skip-ensure-bridge     Skip storage bridge creation/check
+#   --skip-sync-keys         Skip sync-host-keys.sh (clan vars already exist)
+#   --skip-create-vms        Skip VM creation (VMs already exist)
+#   --skip-add-hardware      Skip net1/scsi1 attachment (already attached)
+#   --skip-boot-wait         Skip boot/SSH wait (VMs already running)
+#   --skip-refresh-sops-keys Skip scanning running VMs for fresh SSH host keys
+#   --skip-cluster-init      Skip cluster formation (cluster already configured)
+#   --skip-tests             Skip acceptance tests
+#   --force-rebuild          Pass --force-rebuild to create-proxmox-resource.sh
+#   --destroy                Stop and delete both VMs (skip all other phases)
+#   --dry-run                Print what would run without executing
+#   -h|--help                Show this message
+#
+# Prerequisites:
+#   - SSH access to the Proxmox node as $PROXMOX_SSH_USER (wayne).
+#   - sops age key in the standard location (used by sync-host-keys.sh).
+#   - For --skip-sync-keys: clan vars already in vars/per-machine/proxmox-ha-server-{1,2}/.
+#   - For full tests: secrets/common.yaml decryptable on both nodes (run
+#     `sops updatekeys secrets/common.yaml` after sync-keys).
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# shellcheck source=../env.sh
+source "${REPO_ROOT}/scripts/env.sh"
+
+# ── Defaults ──────────────────────────────────────────────────────────────────
+
+NODE="${PROXMOX_HOST:-$PVE1_HOST}"
+VMID1=200
+VMID2=201
+STORAGE="${PROXMOX_STORAGE:-local-zfs}"
+STORAGE_BRIDGE="vmbr1"
+DRBD_DISK_GB=32
+MEMORY_MB=4096
+CORES=4
+
+SKIP_ENSURE_BRIDGE=false
+SKIP_SYNC_KEYS=false
+SKIP_CREATE_VMS=false
+SKIP_ADD_HARDWARE=false
+SKIP_BOOT_WAIT=false
+SKIP_REFRESH_SOPS_KEYS=false
+SKIP_CLUSTER_INIT=false
+SKIP_TESTS=false
+FORCE_REBUILD=false
+DESTROY=false
+DRY_RUN=false
+
+# ── Variables from repo ───────────────────────────────────────────────────────
+
+NODE1_HOST="ha-server-1"
+NODE2_HOST="ha-server-2"
+NODE1_IP="192.168.2.228"
+NODE2_IP="192.168.2.227"
+STORAGE_IP1="192.168.4.228"
+STORAGE_IP2="192.168.4.227"
+STORAGE_CIDR="192.168.4.0/29"
+SSH_USER="${PROXMOX_SSH_USER:-wayne}"
+
+# ── Argument parsing ──────────────────────────────────────────────────────────
+
+usage() {
+  sed -n '/^# Usage:/,/^[^#]/{ /^#/{ s/^# \?//; p } }' "$0"
+  exit "${1:-0}"
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --node)               NODE="$2";           shift 2 ;;
+    --vmid1)              VMID1="$2";          shift 2 ;;
+    --vmid2)              VMID2="$2";          shift 2 ;;
+    --storage)            STORAGE="$2";        shift 2 ;;
+    --storage-bridge)     STORAGE_BRIDGE="$2"; shift 2 ;;
+    --drbd-disk-gb)       DRBD_DISK_GB="$2";   shift 2 ;;
+    --memory)             MEMORY_MB="$2";      shift 2 ;;
+    --cores)              CORES="$2";          shift 2 ;;
+    --skip-ensure-bridge) SKIP_ENSURE_BRIDGE=true; shift ;;
+    --skip-sync-keys)     SKIP_SYNC_KEYS=true; shift ;;
+    --skip-create-vms)    SKIP_CREATE_VMS=true; shift ;;
+    --skip-add-hardware)  SKIP_ADD_HARDWARE=true; shift ;;
+    --skip-boot-wait)         SKIP_BOOT_WAIT=true; shift ;;
+    --skip-refresh-sops-keys) SKIP_REFRESH_SOPS_KEYS=true; shift ;;
+    --skip-cluster-init)      SKIP_CLUSTER_INIT=true; shift ;;
+    --skip-tests)         SKIP_TESTS=true;     shift ;;
+    --force-rebuild)      FORCE_REBUILD=true;  shift ;;
+    --destroy)            DESTROY=true;        shift ;;
+    --dry-run)            DRY_RUN=true;        shift ;;
+    -h|--help)            usage 0 ;;
+    *) echo "Unknown option: $1" >&2; usage 1 ;;
+  esac
+done
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+log()  { echo "==> $*"; }
+logn() { echo "    $*"; }
+err()  { echo "ERROR: $*" >&2; exit 1; }
+
+run() {
+  if $DRY_RUN; then
+    echo "[dry-run] $*"
+  else
+    "$@"
+  fi
+}
+
+pve() {
+  # Run a command on the Proxmox node via SSH.
+  if $DRY_RUN; then
+    echo "[dry-run] ssh ${SSH_USER}@${NODE} sudo $*"
+  else
+    ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
+  fi
+}
+
+pve_check() {
+  # Run a read-only probe on the Proxmox node — always executes even in dry-run.
+  ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
+}
+
+HA_USER="nixos"
+
+n1() {
+  # Run a command on ha-server-1 via SSH as nixos with sudo.
+  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null
+}
+
+n2() {
+  # Run a command on ha-server-2 via SSH as nixos with sudo.
+  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null
+}
+
+wait_for_ssh() {
+  local ip="$1" label="$2"
+  if $DRY_RUN; then
+    logn "[dry-run] Skipping SSH wait for ${label} (${ip})"
+    return 0
+  fi
+  local deadline=$(( $(date +%s) + 300 ))
+  log "Waiting for SSH on ${label} (${ip}) — up to 5 min..."
+  while [[ $(date +%s) -lt $deadline ]]; do
+    if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=3 \
+        -o BatchMode=yes "${HA_USER}@${ip}" true 2>/dev/null; then
+      logn "${label} is up."
+      return 0
+    fi
+    sleep 5
+  done
+  err "Timed out waiting for SSH on ${label} (${ip})"
+}
+
+# ── Destroy mode ─────────────────────────────────────────────────────────────
+
+if $DESTROY; then
+  log "Destroying HA cluster VMs (${VMID1}=${NODE1_HOST}, ${VMID2}=${NODE2_HOST}) on ${NODE}"
+  for vmid in "$VMID1" "$VMID2"; do
+    STATUS=$(pve "qm status ${vmid} 2>/dev/null" 2>/dev/null || true)
+    if echo "$STATUS" | grep -q "running"; then
+      log "Stopping VMID ${vmid}..."
+      pve "qm stop ${vmid} --skiplock 1"
+      sleep 5
+    fi
+    if $DRY_RUN || pve "qm config ${vmid} >/dev/null 2>&1"; then
+      log "Deleting VMID ${vmid}..."
+      run pve "qm destroy ${vmid} --purge 1"
+    else
+      logn "VMID ${vmid} not found — already gone."
+    fi
+  done
+  log "Done — cluster VMs destroyed."
+  exit 0
+fi
+
+# ── Phase 1: Storage bridge ───────────────────────────────────────────────────
+
+if ! $SKIP_ENSURE_BRIDGE; then
+  log "Phase 1: Ensuring storage bridge ${STORAGE_BRIDGE} on ${NODE}"
+  if pve_check "test -d /sys/class/net/${STORAGE_BRIDGE}" &>/dev/null; then
+    logn "${STORAGE_BRIDGE} already exists — skipping."
+  else
+    logn "Creating isolated internal bridge ${STORAGE_BRIDGE} (no upstream port, ${STORAGE_CIDR})"
+    BRIDGE_CONF="auto ${STORAGE_BRIDGE}
+iface ${STORAGE_BRIDGE} inet manual
+    bridge-ports none
+    bridge-stp off
+    bridge-fd 0"
+    if $DRY_RUN; then
+      echo "[dry-run] Would write /etc/network/interfaces.d/${STORAGE_BRIDGE}.conf and ifup it"
+    else
+      ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
+        "echo '${BRIDGE_CONF}' | sudo tee /etc/network/interfaces.d/${STORAGE_BRIDGE}.conf > /dev/null && sudo ifup ${STORAGE_BRIDGE}"
+      logn "${STORAGE_BRIDGE} created and brought up."
+    fi
+  fi
+fi
+
+# ── Phase 2: Sync host keys ───────────────────────────────────────────────────
+
+if ! $SKIP_SYNC_KEYS; then
+  log "Phase 2: Syncing SSH host keys for both HA targets"
+  for target in proxmox-ha-server-1 proxmox-ha-server-2; do
+    CLAN_DIR="${REPO_ROOT}/vars/per-machine/${target}/openssh"
+    if [[ -d "$CLAN_DIR" ]]; then
+      logn "Clan vars for ${target} already exist — skipping."
+    else
+      logn "Generating host keys for ${target}..."
+      run bash "${REPO_ROOT}/scripts/secrets/sync-host-keys.sh" "$target"
+    fi
+  done
+fi
+
+# ── Phase 2.5: Prepare Proxmox node for building ─────────────────────────────
+if ! $SKIP_CREATE_VMS && ! $DRY_RUN; then
+  CURRENT_BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD)"
+
+  # Fix /nix ownership if it exists but belongs to a different UID.
+  # pve1's IPA-enrolled wayne (UID 50002) can't write to a store created by
+  # another UID — passwordless sudo corrects it once.
+  # Use direct SSH (no sudo) for the writability check so we test wayne's own
+  # access, not root's.
+  local_ssh() { ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "$*"; }
+  if local_ssh "test -d /nix" &>/dev/null && ! local_ssh "test -w /nix" &>/dev/null; then
+    logn "/nix exists but not writable by ${SSH_USER} — fixing ownership with sudo (one-time)..."
+    local_ssh "sudo chown -R ${SSH_USER} /nix"
+    logn "Done."
+  fi
+  unset -f local_ssh
+
+  # Ensure the remote clone is on the correct branch so create-proxmox-resource.sh
+  # builds from the same commits we're deploying.
+  REMOTE_REPO="/home/${SSH_USER}/nixos"
+  if pve_check "test -d ${REMOTE_REPO}/.git" &>/dev/null; then
+    REMOTE_BRANCH=$(ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
+      "cd ${REMOTE_REPO} && git rev-parse --abbrev-ref HEAD 2>/dev/null")
+    if [[ "$REMOTE_BRANCH" != "$CURRENT_BRANCH" ]]; then
+      logn "Remote clone is on '${REMOTE_BRANCH}', switching to '${CURRENT_BRANCH}'..."
+      ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
+        "cd ${REMOTE_REPO} && git fetch origin && git checkout '${CURRENT_BRANCH}' && git pull --ff-only"
+      logn "Done."
+    fi
+  fi
+fi
+
+# ── Phase 3: Create VMs ───────────────────────────────────────────────────────
+
+if ! $SKIP_CREATE_VMS; then
+  log "Phase 3: Building and creating VMs on ${NODE}"
+
+  CREATE="${REPO_ROOT}/scripts/proxmox/create-proxmox-resource.sh"
+
+  for spec in "${VMID1}:ha-server-1:proxmox-ha-server-1" "${VMID2}:ha-server-2:proxmox-ha-server-2"; do
+    IFS=: read -r vmid host_name flake_target <<< "$spec"
+    log "Creating ${flake_target} (VMID ${vmid}) on ${NODE}..."
+    # Always --force-rebuild: create-proxmox-resource.sh only calls
+    # sync_remote_host_keys (which populates host-keys/ for proxmox.nix to
+    # bake the clan-var SSH key into the disko image) when it actually builds.
+    # Reusing a cached image skips that step, so destroy+recreate would reuse
+    # an image with a stale/random key baked in → sops fails on first boot.
+    run bash "$CREATE" \
+      --type vm \
+      --host "$host_name" \
+      --vmid "$vmid" \
+      --node "$NODE" \
+      --storage "$STORAGE" \
+      --memory "$MEMORY_MB" \
+      --cores "$CORES" \
+      --force-rebuild
+  done
+fi
+
+# ── Phase 4: Add storage NIC and DRBD disk ────────────────────────────────────
+
+if ! $SKIP_ADD_HARDWARE; then
+  log "Phase 4: Attaching storage NIC (${STORAGE_BRIDGE}) and DRBD disk (${DRBD_DISK_GB}G) to each VM"
+  for vmid in "$VMID1" "$VMID2"; do
+    log "  VMID ${vmid}: stopping to add hardware..."
+    pve "qm stop ${vmid} --skiplock 1 2>/dev/null; sleep 3" || true
+
+    logn "Adding net1 (${STORAGE_BRIDGE})..."
+    pve "qm set ${vmid} --net1 virtio,bridge=${STORAGE_BRIDGE},firewall=0"
+
+    logn "Adding scsi1 (${STORAGE}:${DRBD_DISK_GB}G for DRBD)..."
+    pve "qm set ${vmid} --scsi1 ${STORAGE}:${DRBD_DISK_GB},format=raw"
+
+    logn "Starting VMID ${vmid}..."
+    pve "qm start ${vmid}"
+  done
+fi
+
+# ── Phase 5: Wait for SSH ─────────────────────────────────────────────────────
+
+if ! $SKIP_BOOT_WAIT; then
+  log "Phase 5: Waiting for both nodes to come up"
+  wait_for_ssh "$NODE1_IP" "$NODE1_HOST"
+  wait_for_ssh "$NODE2_IP" "$NODE2_HOST"
+  logn "Both nodes are SSHable."
+  # Give systemd a few seconds to settle after activation
+  sleep 10
+fi
+
+# ── Phase 5.5: Refresh sops host-key registrations ───────────────────────────
+#
+# Disko builds raw disk images; each new VM boots with a freshly-generated SSH
+# host key rather than the one pre-seeded in clan vars.  This phase scans the
+# actual running VMs, and if their ed25519 host keys differ from what clan vars
+# record: updates the clan var pub-key files, rewrites the .sops.yaml age-key
+# anchors, and re-encrypts all affected sops files so the nodes can decrypt
+# secrets on the next nixos-rebuild.  Safe no-op when keys haven't changed.
+
+if ! $SKIP_REFRESH_SOPS_KEYS; then
+  if $DRY_RUN; then
+    logn "[dry-run] Would scan VM host keys and refresh .sops.yaml / secrets if needed"
+  else
+    log "Phase 5.5: Refreshing sops host-key registrations (disko key drift fix)"
+    SOPS_UPDATED=false
+
+    for spec in \
+      "${NODE1_IP}:proxmox-ha-server-1:${NODE1_HOST}" \
+      "${NODE2_IP}:proxmox-ha-server-2:${NODE2_HOST}"; do
+      IFS=: read -r node_ip flake_target host_name <<< "$spec"
+      CLAN_PUB="${REPO_ROOT}/vars/per-machine/${flake_target}/openssh/ssh_host_ed25519_key.pub/value"
+
+      logn "Scanning ed25519 host key from ${host_name} (${node_ip})..."
+      RAW=$(ssh-keyscan -t ed25519 "${node_ip}" 2>/dev/null | grep -v "^#") || true
+      if [[ -z "$RAW" ]]; then
+        logn "WARNING: no ed25519 key returned by ssh-keyscan for ${node_ip} — skipping"
+        continue
+      fi
+      # ssh-keyscan returns:  <ip>  ssh-ed25519  <b64key>
+      SCANNED_TYPE=$(awk '{print $2}' <<< "$RAW")
+      SCANNED_KEY=$(awk  '{print $3}' <<< "$RAW")
+      SCANNED_PUBKEY="${SCANNED_TYPE} ${SCANNED_KEY} ${host_name}"
+
+      CURRENT=$(tr -d '\n' < "$CLAN_PUB" 2>/dev/null || true)
+      if [[ "$SCANNED_PUBKEY" == "$CURRENT" ]]; then
+        logn "${host_name}: clan var matches running key — no update needed"
+        continue
+      fi
+
+      logn "${host_name}: key drift detected — updating clan var"
+      logn "  old: ${CURRENT}"
+      logn "  new: ${SCANNED_PUBKEY}"
+      echo "$SCANNED_PUBKEY" > "$CLAN_PUB"
+      SOPS_UPDATED=true
+
+      # Rewrite the .sops.yaml anchor for this host with the new age key.
+      ANCHOR="${flake_target}"   # e.g. proxmox-ha-server-1
+      NEW_AGE=$(echo "$SCANNED_PUBKEY" | \
+        nix run --quiet --no-warn-dirty nixpkgs#ssh-to-age 2>/dev/null)
+      if [[ -z "$NEW_AGE" ]]; then
+        err "ssh-to-age produced no output for ${host_name} — check nixpkgs#ssh-to-age"
+      fi
+      logn "  new age key: ${NEW_AGE}"
+      sed -i "/&${ANCHOR} /s| age[a-z0-9]*$| ${NEW_AGE}|" "${REPO_ROOT}/.sops.yaml"
+    done
+
+    if $SOPS_UPDATED; then
+      logn "Running sops updatekeys on affected secrets..."
+      SOPS="nix run --quiet --no-warn-dirty nixpkgs#sops --"
+      (cd "${REPO_ROOT}" && \
+        $SOPS updatekeys -y secrets/common.yaml && \
+        $SOPS updatekeys -y secrets/ha-server-1.yaml && \
+        $SOPS updatekeys -y secrets/ha-server-2.yaml && \
+        $SOPS updatekeys -y secrets/ha-server-1.keytab && \
+        $SOPS updatekeys -y secrets/ha-server-2.keytab)
+      # Note: ha-corosync-authkey is re-generated and re-encrypted by cluster-init below.
+
+      logn "Committing refreshed host keys and re-encrypted secrets..."
+      (cd "${REPO_ROOT}" && \
+        git add \
+          vars/per-machine/proxmox-ha-server-1/openssh/ssh_host_ed25519_key.pub/value \
+          vars/per-machine/proxmox-ha-server-2/openssh/ssh_host_ed25519_key.pub/value \
+          .sops.yaml \
+          secrets/common.yaml \
+          secrets/ha-server-1.yaml \
+          secrets/ha-server-2.yaml \
+          secrets/ha-server-1.keytab \
+          secrets/ha-server-2.keytab && \
+        git commit -m "secrets(ha): refresh sops host-key registrations for new VM instances" || true)
+      logn "Sops keys refreshed and committed."
+    fi
+  fi
+fi
+
+# ── Phase 6: Cluster init ─────────────────────────────────────────────────────
+
+if ! $SKIP_CLUSTER_INIT; then
+  log "Phase 6: Initialising HA cluster"
+
+  CLUSTER_INIT="${REPO_ROOT}/scripts/ha/cluster-init.sh"
+  [[ -x "$CLUSTER_INIT" ]] || chmod +x "$CLUSTER_INIT"
+
+  if $DRY_RUN; then
+    logn "[dry-run] Would generate temp key, authorise on ${NODE2_HOST}, scp cluster-init.sh to ${NODE1_HOST}, and run it as root via sudo"
+  else
+    # Generate a temp keypair so cluster-init.sh can SSH node1→node2 as ${HA_USER}.
+    # Root on node1 has no keys; a temp key bridging node1→node2 nixos solves this.
+    TEMP_KEY="${REPO_ROOT}/.tmp-cluster-init-key"
+    TEMP_KEY_PUB="${TEMP_KEY}.pub"
+    rm -f "$TEMP_KEY" "$TEMP_KEY_PUB"
+    ssh-keygen -t ed25519 -f "$TEMP_KEY" -N "" -C "cluster-init-temp-$(date +%s)" -q
+    TEMP_PUBKEY=$(cat "$TEMP_KEY_PUB")
+
+    logn "Authorising temp key on ${NODE2_HOST} for ${HA_USER}..."
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE2_IP}" \
+      "mkdir -p ~/.ssh && chmod 700 ~/.ssh && echo '${TEMP_PUBKEY}' >> ~/.ssh/authorized_keys"
+
+    logn "Placing temp key on ${NODE1_HOST} for root..."
+    scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
+      "$TEMP_KEY" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init-key"
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo mkdir -p /root/.ssh && sudo cp /tmp/cluster-init-key /root/.ssh/cluster-init-key && \
+       sudo chmod 600 /root/.ssh/cluster-init-key && rm -f /tmp/cluster-init-key"
+
+    logn "Uploading cluster-init.sh to ${NODE1_HOST}..."
+    scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
+      "$CLUSTER_INIT" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init.sh"
+
+    logn "Running cluster-init.sh on ${NODE1_HOST}..."
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo env NODE1=${NODE1_HOST} NODE2=${NODE2_HOST} \
+       NODE1_IP=${NODE1_IP} NODE2_IP=${NODE2_IP} \
+       VIP=192.168.2.229 XFS_MOUNT=/srv/ha-data \
+       ISCSI_IQN=iqn.2026-01.home.sweet:ha-storage \
+       VMID_NODE1=${VMID1} VMID_NODE2=${VMID2} \
+       HA_USER=${HA_USER} HA_KEY=/root/.ssh/cluster-init-key \
+       bash /tmp/cluster-init.sh"
+
+    logn "Cleaning up temp key from both nodes..."
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE2_IP}" \
+      "sed -i '/cluster-init-temp/d' ~/.ssh/authorized_keys" 2>/dev/null || true
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo rm -f /root/.ssh/cluster-init-key" 2>/dev/null || true
+    rm -f "$TEMP_KEY" "$TEMP_KEY_PUB"
+
+    # Encrypt the corosync authkey generated by cluster-init and commit it.
+    log "  Encrypting corosync authkey into secrets/ha-corosync-authkey..."
+    AUTHKEY_TMP="${REPO_ROOT}/secrets/ha-corosync-authkey.tmp"
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo cat /etc/corosync/authkey" > "$AUTHKEY_TMP"
+    if [[ ! -s "$AUTHKEY_TMP" ]]; then
+      err "corosync authkey on node1 is empty — cluster-init may have failed."
+    fi
+    mv "$AUTHKEY_TMP" "${REPO_ROOT}/secrets/ha-corosync-authkey"
+    (cd "${REPO_ROOT}" && nix run nixpkgs#sops -- -e --input-type binary -i secrets/ha-corosync-authkey)
+    logn "Authkey encrypted. Committing..."
+    (cd "${REPO_ROOT}" && git add secrets/ha-corosync-authkey && \
+      git commit -m "secrets(ha): encrypt corosync authkey generated by cluster-init")
+    logn "Committed."
+  fi
+fi
+
+# ── Phase 7: Acceptance tests ─────────────────────────────────────────────────
+
+if ! $SKIP_TESTS; then
+  log "Phase 7: Running acceptance tests (T1–T7)"
+  if $DRY_RUN; then
+    logn "[dry-run] Would run acceptance-tests.sh against ${NODE1_HOST}/${NODE2_HOST}"
+  else
+    NODE1="$NODE1_HOST" NODE2="$NODE2_HOST" \
+    NODE1_IP="$NODE1_IP" NODE2_IP="$NODE2_IP" \
+    VIP="192.168.2.229" \
+    bash "${REPO_ROOT}/scripts/ha/acceptance-tests.sh"
+  fi
+fi
+
+log "Deploy complete."

scripts/ha/failover.sh

@@ -0,0 +1,256 @@
+#!/usr/bin/env bash
+# failover.sh — graceful HA cluster failover
+#
+# Detects which node is active and moves all resources to the other node by
+# putting the active node into Pacemaker standby.  Waits for the XFS mount to
+# appear on the target before returning.
+#
+# Usage:
+#   scripts/ha/failover.sh [--to node1|node2] [--force] [--timeout <s>] [--dry-run]
+#
+#   --to node1|node2  target node (default: the node that is NOT currently active)
+#   --force           skip the interactive confirmation prompt
+#   --timeout <s>     seconds to wait for resources to move (default: 120)
+#   --dry-run         show what would be done without changing anything
+set -euo pipefail
+
+# ── Configuration ─────────────────────────────────────────────────────────
+NODE1="${NODE1:-ha-server-1}"
+NODE2="${NODE2:-ha-server-2}"
+NODE1_IP="${NODE1_IP:-192.168.2.228}"
+NODE2_IP="${NODE2_IP:-192.168.2.227}"
+VIP="${VIP:-192.168.2.229}"
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"
+HA_USER="${HA_USER:-nixos}"
+# ──────────────────────────────────────────────────────────────────────────
+
+n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
+
+# ── Argument parsing ───────────────────────────────────────────────────────
+TARGET_NODE=""
+FORCE=false
+DRY_RUN=false
+TIMEOUT=120
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --to)
+      shift
+      case "${1:-}" in
+        node1|ha-server-1) TARGET_NODE="$NODE1" ;;
+        node2|ha-server-2) TARGET_NODE="$NODE2" ;;
+        *) echo "ERROR: --to must be node1, node2, ha-server-1, or ha-server-2"; exit 1 ;;
+      esac
+      ;;
+    --force)   FORCE=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --timeout) shift; TIMEOUT="${1:?--timeout requires a value}" ;;
+    *) echo "Unknown argument: $1"; echo "Usage: $0 [--to node1|node2] [--force] [--timeout <s>] [--dry-run]"; exit 1 ;;
+  esac
+  shift
+done
+
+DRY_PREFIX=""
+$DRY_RUN && DRY_PREFIX="[dry-run] "
+
+echo "════════════════════════════════════════════════════"
+echo " HA Cluster Failover — $(date '+%Y-%m-%d %H:%M:%S')"
+$DRY_RUN && echo " MODE: dry-run — no changes will be made"
+echo "════════════════════════════════════════════════════"
+
+# ── Detect active node ─────────────────────────────────────────────────────
+echo ""
+echo "Detecting active node..."
+
+CRM_OUT=""
+if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
+fi
+if [[ -z "$CRM_OUT" ]]; then
+  if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then
+    CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
+  fi
+fi
+
+# crm_mon 2.x formats Promoted lines as "    * Promoted: [ node ]" — the * bullet
+# means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
+ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
+  grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+
+if [[ -z "$ACTIVE_NODE" ]]; then
+  echo ""
+  echo "ERROR: could not determine active node from crm_mon."
+  echo "  Is Pacemaker still settling?  Try running scripts/ha/health.sh first."
+  echo "  If Pacemaker is down on both nodes, manual recovery is required."
+  exit 1
+fi
+
+if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
+  ACTIVE_IP="$NODE1_IP"
+  STANDBY_NODE="$NODE2"
+  STANDBY_IP="$NODE2_IP"
+  na() { n1 "$@"; }
+  ns() { n2 "$@"; }
+else
+  ACTIVE_IP="$NODE2_IP"
+  STANDBY_NODE="$NODE1"
+  STANDBY_IP="$NODE1_IP"
+  na() { n2 "$@"; }
+  ns() { n1 "$@"; }
+fi
+
+echo "  Active:  $ACTIVE_NODE ($ACTIVE_IP)"
+echo "  Standby: $STANDBY_NODE ($STANDBY_IP)"
+
+# ── Validate target ────────────────────────────────────────────────────────
+if [[ -n "$TARGET_NODE" ]]; then
+  if [[ "$TARGET_NODE" == "$ACTIVE_NODE" ]]; then
+    echo ""
+    echo "ERROR: $TARGET_NODE is already the active node — nothing to do."
+    exit 1
+  fi
+  echo "  Target:  $TARGET_NODE (as requested)"
+else
+  echo "  Target:  $STANDBY_NODE (auto — the other node)"
+fi
+
+# ── Pre-checks ─────────────────────────────────────────────────────────────
+echo ""
+echo "Pre-checks..."
+
+DRBD_DSTATE=$(na "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
+if ! echo "$DRBD_DSTATE" | grep -q "UpToDate/UpToDate"; then
+  echo ""
+  echo "  WARNING: DRBD dstate is '$DRBD_DSTATE' (not UpToDate/UpToDate)."
+  echo "  Failing over with a partially-synced disk risks split-brain."
+  if ! $FORCE; then
+    echo "  Use --force to proceed anyway (not recommended)."
+    exit 1
+  fi
+  echo "  --force specified — proceeding despite non-ideal DRBD state."
+else
+  echo "  DRBD dstate: $DRBD_DSTATE — OK"
+fi
+
+QUORUM_OK=$(na "corosync-quorumtool -s 2>/dev/null | grep -c 'Quorate:.*Yes'" 2>/dev/null || echo "0")
+if [[ "$QUORUM_OK" -lt 1 ]]; then
+  echo "  ERROR: cluster does not have quorum — failover would be unsafe."
+  exit 1
+fi
+echo "  Quorum: OK"
+
+# ── Confirm ────────────────────────────────────────────────────────────────
+if ! $FORCE && ! $DRY_RUN; then
+  echo ""
+  echo "  This will move all resources from $ACTIVE_NODE → $STANDBY_NODE."
+  echo "  VIP and services will be unreachable for ~10–30 seconds."
+  printf "  Proceed? [y/N] "
+  read -r ANSWER
+  [[ "${ANSWER,,}" == "y" || "${ANSWER,,}" == "yes" ]] || { echo "Aborted."; exit 0; }
+fi
+
+# ── Capture active node's crm_node name ───────────────────────────────────
+# crm_node -n returns the node name as registered in Pacemaker (may differ
+# from hostname if Pacemaker was configured with explicit node names).
+ACTIVE_CRMD_NAME=$(na "crm_node -n 2>/dev/null" 2>/dev/null || echo "$ACTIVE_NODE")
+
+# ── Perform failover ───────────────────────────────────────────────────────
+echo ""
+echo "${DRY_PREFIX}Putting $ACTIVE_NODE into standby (resources will migrate to $STANDBY_NODE)..."
+if ! $DRY_RUN; then
+  na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v on" 2>/dev/null || true
+fi
+
+# ── Wait for resources to move ─────────────────────────────────────────────
+echo "${DRY_PREFIX}Waiting up to ${TIMEOUT}s for XFS to mount on $STANDBY_NODE..."
+MOVED=false
+SPIN_CHARS=('|' '/' '-' '\')
+SPIN_I=0
+
+if $DRY_RUN; then
+  echo "  [dry-run] would wait for mountpoint $XFS_MOUNT on $STANDBY_NODE"
+  MOVED=true
+else
+  for i in $(seq 1 "$TIMEOUT"); do
+    if ns "mountpoint -q '${XFS_MOUNT}' 2>/dev/null" 2>/dev/null; then
+      printf "\r%-80s\r" ""
+      echo "  Resources moved in ${i}s"
+      MOVED=true
+      break
+    fi
+    SPIN_I=$(( SPIN_I + 1 ))
+    SC="${SPIN_CHARS[$((SPIN_I % 4))]}"
+    printf "\r  [%s] waiting... (%ds)     " "$SC" "$i"
+    sleep 1
+  done
+fi
+
+if ! $MOVED; then
+  echo ""
+  echo "ERROR: XFS did not mount on $STANDBY_NODE within ${TIMEOUT}s."
+  echo ""
+  echo "  Current resource state:"
+  na "crm_mon -1 2>/dev/null" 2>/dev/null | grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED' | sed 's/^/  /' || true
+  echo ""
+  echo "  Clearing standby to restore $ACTIVE_NODE (undo the failover attempt)..."
+  na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
+  na "crm_resource --cleanup" 2>/dev/null || true
+  exit 1
+fi
+
+# ── Clear failure history ──────────────────────────────────────────────────
+echo "${DRY_PREFIX}Clearing Pacemaker failure history..."
+if ! $DRY_RUN; then
+  ns "crm_resource --cleanup 2>/dev/null" 2>/dev/null || true
+fi
+
+# ── Re-enable original active node as standby ─────────────────────────────
+echo "${DRY_PREFIX}Re-enabling $ACTIVE_NODE (now standby — will not claim resources)..."
+if ! $DRY_RUN; then
+  na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
+fi
+
+# ── Wait briefly for DRBD resync to begin ─────────────────────────────────
+if ! $DRY_RUN; then
+  sleep 5
+fi
+
+# ── Final state ────────────────────────────────────────────────────────────
+echo ""
+echo "Failover complete.  Final state:"
+echo ""
+
+CRM_OUT_AFTER=""
+if ! $DRY_RUN; then
+  CRM_OUT_AFTER=$(ns "crm_mon -1" 2>/dev/null || na "crm_mon -1" 2>/dev/null || true)
+else
+  CRM_OUT_AFTER="$CRM_OUT"
+fi
+
+NEW_ACTIVE=$(echo "$CRM_OUT_AFTER" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
+  grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+
+if [[ -n "$NEW_ACTIVE" ]]; then
+  if [[ "$NEW_ACTIVE" == "$ACTIVE_NODE" ]]; then
+    echo "  WARNING: $ACTIVE_NODE is still showing as active in crm_mon."
+    echo "  Pacemaker may still be settling — check again in a few seconds."
+  else
+    echo "  Active:  $NEW_ACTIVE"
+    echo "  Standby: $ACTIVE_NODE"
+  fi
+fi
+
+echo ""
+RESOURCES_AFTER=$(echo "$CRM_OUT_AFTER" | awk '/Full List of Resources/,0' | tail -n +2 || true)
+[[ -z "$RESOURCES_AFTER" ]] && RESOURCES_AFTER=$(echo "$CRM_OUT_AFTER" | \
+  grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED|Master|Slave' || true)
+[[ -n "$RESOURCES_AFTER" ]] && echo "$RESOURCES_AFTER" | sed 's/^/  /'
+
+echo ""
+echo "  (DRBD resync of $ACTIVE_NODE may take a moment; monitor with:"
+echo "   ssh nixos@${ACTIVE_IP} 'sudo watch -n3 cat /proc/drbd')"
+echo ""
+echo "════════════════════════════════════════════════════"

scripts/ha/fence-pve-ssh.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""
+fence_pve_ssh - Proxmox VE SSH fence agent for Pacemaker.
+
+Uses SSH to reach the Proxmox host and run 'qm stop/start <vmid>'.
+Deploy to /etc/pacemaker/fence_pve_ssh on both HA nodes (chmod +x).
+
+Configuration (as pacemaker stonith resource attributes):
+  pve_host     Proxmox host to SSH to (default: pve1.sweet.home)
+  pve_user     SSH user (default: wayne)
+  key_file     SSH private key path (default: /etc/fence-pve-ssh-key)
+  vmid_node1   VMID for ha-server-1
+  vmid_node2   VMID for ha-server-2
+  plug         Node name to act on (set by pacemaker: ha-server-1 or ha-server-2)
+  action       Action: off|on|reboot|status|list|metadata
+"""
+
+import argparse
+import subprocess
+import sys
+import os
+
+
+METADATA = """<?xml version="1.0" ?>
+<resource-agent name="fence_pve_ssh" shortdesc="Proxmox VE SSH fence agent (test lab)">
+  <longdesc>Fences a VM on a Proxmox VE host by SSHing to the PVE host and
+  running qm stop/start.  For test use only.</longdesc>
+  <vendor-url>https://proxmox.com</vendor-url>
+  <parameters>
+    <parameter name="action" required="1" unique="0">
+      <getopt mixed="-a, --action=[action]"/>
+      <content type="string" default="reboot"/>
+      <shortdesc lang="en">Fencing action: off|on|reboot|status|list</shortdesc>
+    </parameter>
+    <parameter name="plug" required="0" unique="0">
+      <getopt mixed="-n, --plug=[nodename]"/>
+      <content type="string"/>
+      <shortdesc lang="en">Cluster node name to fence</shortdesc>
+    </parameter>
+    <parameter name="pve_host" required="0" unique="0">
+      <getopt mixed="--pve-host=[host]"/>
+      <content type="string" default="pve1.sweet.home"/>
+      <shortdesc lang="en">Proxmox VE host to SSH to</shortdesc>
+    </parameter>
+    <parameter name="pve_user" required="0" unique="0">
+      <getopt mixed="--pve-user=[user]"/>
+      <content type="string" default="wayne"/>
+      <shortdesc lang="en">SSH user on the Proxmox host</shortdesc>
+    </parameter>
+    <parameter name="key_file" required="0" unique="0">
+      <getopt mixed="--key-file=[path]"/>
+      <content type="string" default="/etc/fence-pve-ssh-key"/>
+      <shortdesc lang="en">SSH private key file path</shortdesc>
+    </parameter>
+    <parameter name="vmid_node1" required="1" unique="0">
+      <getopt mixed="--vmid-node1=[vmid]"/>
+      <content type="string"/>
+      <shortdesc lang="en">VMID for ha-test-node1</shortdesc>
+    </parameter>
+    <parameter name="vmid_node2" required="1" unique="0">
+      <getopt mixed="--vmid-node2=[vmid]"/>
+      <content type="string"/>
+      <shortdesc lang="en">VMID for ha-test-node2</shortdesc>
+    </parameter>
+  </parameters>
+  <actions>
+    <action name="off" timeout="60s"/>
+    <action name="on" timeout="60s"/>
+    <action name="reboot" timeout="60s"/>
+    <action name="status" timeout="30s"/>
+    <action name="list" timeout="10s"/>
+    <action name="metadata" timeout="5s"/>
+  </actions>
+</resource-agent>
+"""
+
+
+def parse_args():
+    p = argparse.ArgumentParser(add_help=False)
+    p.add_argument("-a", "--action", default="reboot")
+    p.add_argument("-n", "--plug")
+    p.add_argument("--pve-host", default="pve1.sweet.home")
+    p.add_argument("--pve-user", default="wayne")
+    p.add_argument("--key-file", default="/etc/fence-pve-ssh-key")
+    p.add_argument("--vmid-node1")
+    p.add_argument("--vmid-node2")
+    # Allow remaining unknown args (pacemaker may pass extra ones)
+    return p.parse_known_args()[0]
+
+
+def ssh(pve_host, pve_user, key_file, cmd):
+    result = subprocess.run(
+        [
+            "ssh",
+            "-i", key_file,
+            "-o", "StrictHostKeyChecking=no",
+            "-o", "BatchMode=yes",
+            "-o", "ConnectTimeout=10",
+            f"{pve_user}@{pve_host}",
+            cmd,
+        ],
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    return result
+
+
+def get_vmid(args):
+    node = args.plug
+    if not node:
+        print("ERROR: --plug not specified", file=sys.stderr)
+        sys.exit(1)
+    mapping = {
+        "ha-server-1": args.vmid_node1,
+        "ha-server-2": args.vmid_node2,
+    }
+    vmid = mapping.get(node)
+    if not vmid:
+        print(f"ERROR: unknown node '{node}'", file=sys.stderr)
+        sys.exit(1)
+    return vmid
+
+
+def main():
+    args = parse_args()
+    action = args.action.lower()
+
+    if action == "metadata":
+        print(METADATA)
+        sys.exit(0)
+
+    if action == "list":
+        if args.vmid_node1:
+            print("ha-server-1")
+        if args.vmid_node2:
+            print("ha-server-2")
+        sys.exit(0)
+
+    vmid = get_vmid(args)
+
+    if not os.path.exists(args.key_file):
+        print(f"ERROR: SSH key not found at {args.key_file}", file=sys.stderr)
+        sys.exit(1)
+
+    if action in ("off", "reboot"):
+        print(f"Stopping VM {vmid} ({args.plug}) on {args.pve_host}...")
+        r = ssh(args.pve_host, args.pve_user, args.key_file,
+                f"sudo /usr/sbin/qm stop {vmid}")
+        if r.returncode != 0:
+            print(f"ERROR stopping VM: {r.stderr}", file=sys.stderr)
+            sys.exit(1)
+        print(f"VM {vmid} stopped")
+
+    if action in ("on", "reboot"):
+        print(f"Starting VM {vmid} ({args.plug}) on {args.pve_host}...")
+        r = ssh(args.pve_host, args.pve_user, args.key_file,
+                f"sudo /usr/sbin/qm start {vmid}")
+        if r.returncode != 0:
+            print(f"ERROR starting VM: {r.stderr}", file=sys.stderr)
+            sys.exit(1)
+        print(f"VM {vmid} started")
+
+    if action == "status":
+        r = ssh(args.pve_host, args.pve_user, args.key_file,
+                f"sudo /usr/sbin/qm status {vmid}")
+        if r.returncode != 0:
+            print(f"ERROR querying VM status: {r.stderr}", file=sys.stderr)
+            sys.exit(1)
+        # qm status returns "status: running" or "status: stopped"
+        status_line = r.stdout.strip()
+        print(status_line)
+        if "stopped" in status_line:
+            sys.exit(2)   # pacemaker interprets exit 2 as "off"
+        sys.exit(0)       # running = exit 0
+
+
+if __name__ == "__main__":
+    main()

scripts/ha/health.sh

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+# health.sh — HA cluster health snapshot (read-only, non-destructive)
+#
+# Prints a compact status panel across both nodes: SSH reachability, quorum,
+# DRBD state, Pacemaker resources, and service ports via the VIP.
+# Run from any host with SSH access to the HA nodes.
+set -euo pipefail
+
+# ── Configuration ─────────────────────────────────────────────────────────
+NODE1="${NODE1:-ha-server-1}"
+NODE2="${NODE2:-ha-server-2}"
+NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
+NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
+VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
+HA_USER="${HA_USER:-nixos}"
+# ──────────────────────────────────────────────────────────────────────────
+
+REACHABLE_1=false
+REACHABLE_2=false
+
+n1()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
+
+probe_node() {
+  local ip=$1
+  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
+}
+
+section() { echo ""; echo "── $* ──"; }
+
+echo "════════════════════════════════════════════════════"
+echo " HA Cluster Health — $(date '+%Y-%m-%d %H:%M:%S')"
+echo "════════════════════════════════════════════════════"
+
+# ── Node reachability ──────────────────────────────────────────────────────
+section "Nodes"
+N1_STATUS=$(probe_node "$NODE1_IP")
+N2_STATUS=$(probe_node "$NODE2_IP")
+[[ "$N1_STATUS" == "ONLINE" ]] && REACHABLE_1=true
+[[ "$N2_STATUS" == "ONLINE" ]] && REACHABLE_2=true
+
+if ! $REACHABLE_1 && ! $REACHABLE_2; then
+  echo "  ERROR: both nodes unreachable — cannot continue."
+  exit 1
+fi
+
+# ── Detect active node ─────────────────────────────────────────────────────
+# crm_mon 2.x formats the Promoted line as "    * Promoted: [ node ]" — the
+# bullet * means ^\s*(Promoted|Masters): never matches.  Filter out Unpromoted
+# first, then match anywhere on the line.
+ACTIVE_NODE=""
+CRM_OUT=""
+if $REACHABLE_1; then
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
+elif $REACHABLE_2; then
+  CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
+fi
+ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
+  grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+
+STANDBY_NODE=""
+if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
+  STANDBY_NODE="$NODE2"
+elif [[ "$ACTIVE_NODE" == "$NODE2" ]]; then
+  STANDBY_NODE="$NODE1"
+fi
+
+n1_tag=""; n2_tag=""
+[[ "$ACTIVE_NODE" == "$NODE1" ]] && n1_tag=" [ACTIVE]"  || n1_tag=" [STANDBY]"
+[[ "$ACTIVE_NODE" == "$NODE2" ]] && n2_tag=" [ACTIVE]"  || n2_tag=" [STANDBY]"
+[[ -z "$ACTIVE_NODE" ]] && { n1_tag=""; n2_tag=""; }
+
+printf "  %-14s  [%s]%s\n" "$NODE1" "$N1_STATUS" "$n1_tag"
+printf "  %-14s  [%s]%s\n" "$NODE2" "$N2_STATUS" "$n2_tag"
+
+if [[ -z "$ACTIVE_NODE" ]]; then
+  echo ""
+  echo "  WARNING: could not determine active node from crm_mon."
+  echo "  Pacemaker may still be settling, or both nodes may be in standby."
+fi
+
+# ── Quorum ─────────────────────────────────────────────────────────────────
+section "Quorum"
+if $REACHABLE_1; then
+  QUORUM=$(n1 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
+elif $REACHABLE_2; then
+  QUORUM=$(n2 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
+fi
+
+if [[ -z "${QUORUM:-}" ]]; then
+  echo "  corosync-quorumtool: unavailable"
+else
+  QUORATE=$(echo "$QUORUM" | grep "Quorate:" | awk '{print $2}' || echo "?")
+  VOTES=$(echo "$QUORUM" | grep "Total votes:" | awk '{print $3}' || echo "?")
+  NEEDED=$(echo "$QUORUM" | grep "Quorum votes:" | awk '{print $3}' || echo "?")
+  echo "  Quorate: $QUORATE   Votes: $VOTES / Expected: $NEEDED"
+fi
+
+# ── DRBD ───────────────────────────────────────────────────────────────────
+section "DRBD (ha-data)"
+
+drbd_info_from() {
+  local node=$1 run=$2
+  local role dstate cs pct
+  role=$($run "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
+  dstate=$($run "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
+  cs=$($run "grep -oE 'cs:[A-Za-z]+' /proc/drbd 2>/dev/null | head -1 | sed 's/cs://'" 2>/dev/null || echo "unknown")
+  pct=$($run "grep -oE \"sync'ed:[[:space:]]+[0-9.]+\" /proc/drbd 2>/dev/null | grep -oE '[0-9.]+$' | head -1" 2>/dev/null || echo "")
+  printf "  %-14s  role: %-22s  dstate: %-25s  cs: %s" \
+    "$node" "${role:-unknown}" "${dstate:-unknown}" "${cs:-unknown}"
+  [[ -n "$pct" ]] && printf "  syncing: %s%%" "$pct"
+  echo ""
+}
+
+$REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo "  $NODE1  [OFFLINE]"
+$REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo "  $NODE2  [OFFLINE]"
+
+# ── Pacemaker (full crm_mon output) ───────────────────────────────────────
+section "Pacemaker"
+if [[ -n "${CRM_OUT:-}" ]]; then
+  echo "$CRM_OUT" | sed 's/^/  /'
+else
+  echo "  crm_mon returned no output — trying again without suppression:"
+  if $REACHABLE_1; then
+    n1 "crm_mon -1" || true
+  elif $REACHABLE_2; then
+    n2 "crm_mon -1" || true
+  fi
+fi
+
+# ── XFS mount ─────────────────────────────────────────────────────────────
+section "XFS Mount ($XFS_MOUNT)"
+check_mount() {
+  local node=$1 run=$2
+  local status
+  if $run "mountpoint -q '$XFS_MOUNT' 2>/dev/null" 2>/dev/null; then
+    local usage
+    usage=$($run "df -h '$XFS_MOUNT' 2>/dev/null | tail -1 | awk '{print \$3\"/\"\$2\" used (\"\$5\")\";}'" 2>/dev/null || echo "")
+    status="mounted"
+    [[ -n "$usage" ]] && status="mounted  $usage"
+  else
+    status="not mounted"
+  fi
+  printf "  %-14s  %s\n" "$node" "$status"
+}
+
+$REACHABLE_1 && check_mount "$NODE1" n1 || echo "  $NODE1  [OFFLINE]"
+$REACHABLE_2 && check_mount "$NODE2" n2 || echo "  $NODE2  [OFFLINE]"
+
+# ── Service ports via VIP ──────────────────────────────────────────────────
+section "Services via VIP ($VIP)"
+
+check_port() {
+  local name=$1 port=$2
+  if bash -c "echo >/dev/tcp/${VIP}/${port}" 2>/dev/null; then
+    printf "  %-10s  port %-5s  OK\n" "$name" "$port"
+  else
+    printf "  %-10s  port %-5s  UNREACHABLE\n" "$name" "$port"
+  fi
+}
+
+if ping -c1 -W2 "$VIP" >/dev/null 2>&1; then
+  echo "  Ping       OK"
+else
+  echo "  Ping       UNREACHABLE"
+fi
+check_port "NFS"    2049
+check_port "iSCSI"  3260
+
+echo ""
+echo "════════════════════════════════════════════════════"
+if [[ -n "$ACTIVE_NODE" ]]; then
+  echo " Active: $ACTIVE_NODE   Standby: $STANDBY_NODE"
+else
+  echo " Active: unknown (Pacemaker not settled)"
+fi
+echo "════════════════════════════════════════════════════"
+echo ""

scripts/ha/resize-data-disk.sh

@@ -0,0 +1,274 @@
+#!/usr/bin/env bash
+# resize-data-disk.sh — online resize of the HA cluster data disk
+#
+# Three-phase process (all online-safe, no downtime required):
+#   1. Proxmox: grow scsi1 on both HA VMs   (qm resize)
+#   2. Guest:   rescan block device on both nodes so the kernel sees the new size
+#   3. DRBD + XFS: drbdadm resize, then xfs_growfs — active node only
+#
+# Usage:
+#   scripts/ha/resize-data-disk.sh --size +20G [--force] [--dry-run]
+#
+#   --size +NNg    amount to grow scsi1 by, e.g. +20G, +50G (required)
+#                  XFS and DRBD cannot shrink; only positive deltas accepted
+#   --force        skip the interactive confirmation prompt
+#   --dry-run      show what would be done without changing anything
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=../env.sh
+source "${SCRIPT_DIR}/../env.sh"
+
+# ── Configuration ─────────────────────────────────────────────────────────
+NODE1="${NODE1:-ha-server-1}"
+NODE2="${NODE2:-ha-server-2}"
+NODE1_IP="${NODE1_IP:-192.168.2.228}"
+NODE2_IP="${NODE2_IP:-192.168.2.227}"
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"
+DRBD_RESOURCE="${DRBD_RESOURCE:-ha-data}"
+DATA_DISK_SLOT="${DATA_DISK_SLOT:-scsi1}"   # Proxmox disk name (scsi1 = data disk)
+HA_USER="${HA_USER:-nixos}"
+PVE_HOST="${PVE_HOST:-${PVE1_HOST}}"
+PVE_SSH_USER="${PVE_SSH_USER:-${PROXMOX_SSH_USER}}"
+PVE_SUDO=""
+[[ "$PVE_SSH_USER" != "root" ]] && PVE_SUDO="sudo"
+# By-id symlink for the data disk; basename resolves to the raw block device.
+# matches variables.nix's haServerDrbdDisk.
+DATA_DISK_BYID="${DATA_DISK_BYID:-scsi-0QEMU_QEMU_HARDDISK_drive-${DATA_DISK_SLOT}}"
+# ──────────────────────────────────────────────────────────────────────────
+
+pve() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=10 \
+          "${PVE_SSH_USER}@${PVE_HOST}" "$@"; }
+n1()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
+          "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
+          "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
+
+# ── Argument parsing ───────────────────────────────────────────────────────
+SIZE=""
+FORCE=false
+DRY_RUN=false
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --size)    shift; SIZE="${1:?--size requires a value (e.g. +20G)}" ;;
+    --force)   FORCE=true ;;
+    --dry-run) DRY_RUN=true ;;
+    *) echo "Unknown argument: $1"
+       echo "Usage: $0 --size +NNg [--force] [--dry-run]"
+       exit 1 ;;
+  esac
+  shift
+done
+
+if [[ -z "$SIZE" ]]; then
+  echo "ERROR: --size is required (e.g. --size +20G)"
+  echo "Usage: $0 --size +NNg [--force] [--dry-run]"
+  exit 1
+fi
+
+# Only positive deltas — qm resize, DRBD, and XFS all refuse to shrink.
+if [[ ! "$SIZE" =~ ^\+[0-9]+(G|M|T)$ ]]; then
+  echo "ERROR: --size must be a positive delta like +20G, +50G, +500M, +2T"
+  echo "       (qm resize, drbdadm resize, and xfs_growfs can only grow, not shrink)"
+  exit 1
+fi
+
+DRY_PREFIX=""
+$DRY_RUN && DRY_PREFIX="[dry-run] "
+
+echo "════════════════════════════════════════════════════"
+echo " HA Data Disk Resize — $(date '+%Y-%m-%d %H:%M:%S')"
+echo " Size delta: $SIZE  •  Proxmox: $PVE_HOST"
+$DRY_RUN && echo " MODE: dry-run — no changes will be made"
+echo "════════════════════════════════════════════════════"
+
+# ── Detect active node ─────────────────────────────────────────────────────
+echo ""
+echo "Detecting active node..."
+
+CRM_OUT=""
+if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
+     "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
+fi
+if [[ -z "$CRM_OUT" ]]; then
+  if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
+       "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then
+    CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
+  fi
+fi
+
+# crm_mon 2.x formats Promoted lines as "    * Promoted: [ node ]" (bullet *),
+# so ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
+ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
+  grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+
+if [[ -z "$ACTIVE_NODE" ]]; then
+  echo "ERROR: could not determine active node from crm_mon."
+  echo "  Is Pacemaker still settling?  Try running scripts/ha/health.sh first."
+  exit 1
+fi
+
+if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
+  ACTIVE_IP="$NODE1_IP"
+  na() { n1 "$@"; }
+else
+  ACTIVE_IP="$NODE2_IP"
+  na() { n2 "$@"; }
+fi
+
+echo "  Active node: $ACTIVE_NODE ($ACTIVE_IP)"
+
+# ── Pre-check DRBD state ───────────────────────────────────────────────────
+echo ""
+echo "Pre-checks..."
+
+DRBD_DSTATE=$(na "drbdadm dstate ${DRBD_RESOURCE} 2>/dev/null" 2>/dev/null || echo "unknown")
+if ! echo "$DRBD_DSTATE" | grep -q "UpToDate/UpToDate"; then
+  echo "  WARNING: DRBD dstate is '$DRBD_DSTATE' (expected UpToDate/UpToDate)."
+  echo "  Resizing with a partially-synced disk may cause issues."
+  if ! $FORCE; then
+    echo "  Use --force to proceed anyway."
+    exit 1
+  fi
+  echo "  --force specified — proceeding despite non-ideal DRBD state."
+else
+  echo "  DRBD dstate: $DRBD_DSTATE — OK"
+fi
+
+# ── Find VMIDs on Proxmox ─────────────────────────────────────────────────
+echo ""
+echo "Looking up VM IDs on ${PVE_HOST}..."
+
+QM_LIST=$(pve "$PVE_SUDO qm list 2>/dev/null" || true)
+VMID1=$(echo "$QM_LIST" | awk -v name="$NODE1" '$0 ~ name {print $1}' | head -1)
+VMID2=$(echo "$QM_LIST" | awk -v name="$NODE2" '$0 ~ name {print $1}' | head -1)
+
+if [[ -z "$VMID1" ]]; then
+  echo "  ERROR: could not find VMID for $NODE1 on $PVE_HOST"
+  echo "  qm list output:"
+  echo "$QM_LIST" | sed 's/^/    /'
+  exit 1
+fi
+if [[ -z "$VMID2" ]]; then
+  echo "  ERROR: could not find VMID for $NODE2 on $PVE_HOST"
+  echo "  qm list output:"
+  echo "$QM_LIST" | sed 's/^/    /'
+  exit 1
+fi
+
+echo "  $NODE1: VMID $VMID1"
+echo "  $NODE2: VMID $VMID2"
+
+# ── Confirm ────────────────────────────────────────────────────────────────
+if ! $FORCE && ! $DRY_RUN; then
+  echo ""
+  echo "  Plan:"
+  echo "    Phase 1 — qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}  (on $PVE_HOST)"
+  echo "              qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}  (on $PVE_HOST)"
+  echo "    Phase 2 — block device rescan on $NODE1 and $NODE2"
+  echo "    Phase 3 — drbdadm resize + xfs_growfs on $ACTIVE_NODE"
+  echo "  No downtime required (all operations are online-safe)."
+  printf "  Proceed? [y/N] "
+  read -r ANSWER
+  [[ "${ANSWER,,}" == "y" || "${ANSWER,,}" == "yes" ]] || { echo "Aborted."; exit 0; }
+fi
+
+# ═══════════════════════════════════════════════════════════════
+# Phase 1 — Resize both VM data disks in Proxmox
+# ═══════════════════════════════════════════════════════════════
+echo ""
+echo "── Phase 1 — Proxmox disk resize (${DATA_DISK_SLOT} ${SIZE} on both VMs) ──"
+
+echo "  ${DRY_PREFIX}qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}  ($NODE1 on ${PVE_HOST})"
+if ! $DRY_RUN; then
+  pve "$PVE_SUDO qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}"
+fi
+
+echo "  ${DRY_PREFIX}qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}  ($NODE2 on ${PVE_HOST})"
+if ! $DRY_RUN; then
+  pve "$PVE_SUDO qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}"
+fi
+
+echo "  Phase 1 done."
+
+# ═══════════════════════════════════════════════════════════════
+# Phase 2 — Rescan block device on both guest nodes
+# ═══════════════════════════════════════════════════════════════
+echo ""
+echo "── Phase 2 — Block device rescan (both nodes) ──"
+
+rescan_node() {
+  local node_name=$1 run_fn=$2
+
+  # Resolve block device name from the stable by-id symlink on the guest.
+  # Read-only lookup — safe to run even in dry-run so we show the real device.
+  local blk_dev=""
+  blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
+  if [[ -z "$blk_dev" ]]; then
+    echo "  ERROR: /dev/disk/by-id/${DATA_DISK_BYID} not found on $node_name" >&2
+    echo "         Check DATA_DISK_BYID or DATA_DISK_SLOT configuration." >&2
+    exit 1
+  fi
+
+  echo "  ${DRY_PREFIX}Rescanning /dev/${blk_dev} on ${node_name}..."
+  if ! $DRY_RUN; then
+    $run_fn "bash -c 'echo 1 > /sys/block/${blk_dev}/device/rescan'" 2>/dev/null
+    local new_size
+    new_size=$($run_fn "lsblk -nd -o SIZE /dev/${blk_dev} 2>/dev/null" 2>/dev/null || echo "unknown")
+    echo "    /dev/${blk_dev} on $node_name now reports: $new_size"
+  fi
+}
+
+rescan_node "$NODE1" n1
+rescan_node "$NODE2" n2
+
+echo "  Phase 2 done."
+
+# ═══════════════════════════════════════════════════════════════
+# Phase 3 — Grow DRBD metadata, then XFS (active node only)
+# ═══════════════════════════════════════════════════════════════
+echo ""
+echo "── Phase 3 — DRBD resize + XFS grow (on active node: $ACTIVE_NODE) ──"
+
+echo "  ${DRY_PREFIX}drbdadm resize ${DRBD_RESOURCE}"
+if ! $DRY_RUN; then
+  na "drbdadm resize ${DRBD_RESOURCE}"
+fi
+
+echo "  ${DRY_PREFIX}xfs_growfs ${XFS_MOUNT}"
+if ! $DRY_RUN; then
+  na "xfs_growfs ${XFS_MOUNT}"
+fi
+
+echo "  Phase 3 done."
+
+# ── Verify ────────────────────────────────────────────────────────────────
+echo ""
+echo "── Verify ──"
+
+if ! $DRY_RUN; then
+  DF_OUT=$(na "df -h '${XFS_MOUNT}' 2>/dev/null" 2>/dev/null || echo "")
+  if [[ -n "$DF_OUT" ]]; then
+    echo "  ${XFS_MOUNT}:"
+    echo "$DF_OUT" | sed 's/^/    /'
+  fi
+
+  DRBD_DSTATE_AFTER=$(na "drbdadm dstate ${DRBD_RESOURCE} 2>/dev/null" 2>/dev/null || echo "unknown")
+  echo "  DRBD dstate: $DRBD_DSTATE_AFTER"
+  if ! echo "$DRBD_DSTATE_AFTER" | grep -q "UpToDate/UpToDate"; then
+    echo "  NOTE: DRBD is resyncing — normal immediately after resize."
+    echo "        Monitor: ssh nixos@${ACTIVE_IP} 'sudo watch -n3 cat /proc/drbd'"
+  fi
+else
+  echo "  [dry-run] would verify df -h ${XFS_MOUNT} and drbdadm dstate on $ACTIVE_NODE"
+fi
+
+echo ""
+echo "════════════════════════════════════════════════════"
+echo " Resize complete."
+echo " Active node: $ACTIVE_NODE"
+echo "════════════════════════════════════════════════════"
+echo ""

scripts/ipa/create-nixos-ipa-host-account.sh

@@ -0,0 +1,303 @@
+#!/usr/bin/env bash
+# Add a NixOS host to the FreeIPA domain and produce a sops-encrypted keytab
+# at secrets/<hostname>.keytab, ready for modules/ipa/client.nix.
+#
+# One command replaces three error-prone manual steps:
+#   1. ipa host-add on the domain controller
+#   2. ipa-getkeytab on the domain controller + SCP back
+#   3. sops encrypt in-place (must be at secrets/<hostname>.keytab for
+#      the creation rule to match -- the common mistake that breaks sops)
+#
+# Usage:
+#   scripts/ipa/create-nixos-ipa-host-account.sh [options] <hostname>
+#
+# Arguments:
+#   <hostname>       Short hostname, e.g. "tailscale-router". The FQDN is
+#                    derived as <hostname>.<HOME_DOMAIN>.
+#
+# Options:
+#   --ip <addr>      Register this IP with the IPA host record (optional).
+#   --dc <host>      SSH to this host to run IPA commands.
+#                    Default: $IPA_SERVER (from env.sh / environment).
+#   --dc-user <u>    SSH user on the domain controller. Default: wayne.
+#   --dry-run        Print what would be done without making any changes.
+#   -h, --help       Show this message.
+#
+# Prereqs:
+#   1. Run from the repo root (so .sops.yaml and secrets/ are found).
+#   2. SSH access to the domain controller as --dc-user (default: wayne)
+#      with passwordless sudo (or sudo cached). IPA commands and kinit run
+#      as root via sudo so the Kerberos ticket is in root's cache where all
+#      ipa tools expect it. If there's no valid ticket, the script runs
+#      `sudo kinit admin` interactively — you'll be prompted for the IPA
+#      admin password once. The password never touches this script.
+#   3. The host's age key(s) must already be in .sops.yaml. Run
+#      scripts/secrets/sync-host-keys.sh <flake-target> first so the host
+#      can decrypt its own keytab on boot. This script adds the .sops.yaml
+#      creation rule for secrets/<hostname>.keytab automatically, but the
+#      host age key anchor (&lxc-<hostname> etc.) must already exist —
+#      otherwise only the admin key can decrypt the keytab and the deployed
+#      host will fail to read it.
+#   4. sops in PATH, or Nix available to run it via `nix run`.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# shellcheck source=../env.sh
+source "${SCRIPT_DIR}/../env.sh"
+
+# --- Argument parsing ---
+
+DC_HOST="${IPA_SERVER}"
+DC_USER="wayne"
+IP_ADDR=""
+DRY_RUN=false
+TARGET=""
+
+usage() {
+  sed -n '/^# Usage:/,/^[^#]/{ /^#/{ s/^# \?//; p } }' "$0"
+  exit "${1:-0}"
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --ip)      IP_ADDR="$2";  shift 2 ;;
+    --dc)      DC_HOST="$2";  shift 2 ;;
+    --dc-user) DC_USER="$2";  shift 2 ;;
+    --dry-run) DRY_RUN=true;  shift ;;
+    -h|--help) usage 0 ;;
+    -*)  echo "Unknown flag: $1" >&2; usage 1 ;;
+    *)
+      if [[ -n "${TARGET}" ]]; then echo "Unexpected argument: $1" >&2; usage 1; fi
+      TARGET="$1"; shift
+      ;;
+  esac
+done
+
+if [[ -z "${TARGET}" ]]; then
+  echo "Error: hostname required." >&2
+  usage 1
+fi
+
+# Reject FQDNs passed by mistake — the script appends HOME_DOMAIN itself.
+# "nixos.sweet.home" → FQDN would become "nixos.sweet.home.sweet.home".
+if [[ "${TARGET}" == *"."* ]]; then
+  echo "Error: <hostname> must be the short name (e.g. 'nixos'), not a FQDN." >&2
+  echo "       The FQDN is derived automatically as ${TARGET}.${HOME_DOMAIN}." >&2
+  exit 1
+fi
+
+FQDN="${TARGET}.${HOME_DOMAIN}"
+KEYTAB_SECRET="${REPO_ROOT}/secrets/${TARGET}.keytab"
+# Temp path on the domain controller — use a name that won't collide.
+DC_TMP="/tmp/nixos-keytab-${TARGET}-$$.keytab"
+
+# --- Helpers ---
+
+log()  { echo "==> $*"; }
+logn() { echo "    $*"; }
+
+run() {
+  if $DRY_RUN; then
+    echo "[dry-run] $*"
+  else
+    "$@"
+  fi
+}
+
+dc_run() {
+  # Run a command string on the domain controller via SSH.
+  if $DRY_RUN; then
+    echo "[dry-run] ssh ${DC_USER}@${DC_HOST} $*"
+  else
+    ssh "${DC_USER}@${DC_HOST}" "$@"
+  fi
+}
+
+# --- Locate sops ---
+
+if command -v sops &>/dev/null; then
+  SOPS_CMD=(sops)
+else
+  log "sops not in PATH — will use 'nix run nixpkgs#sops'"
+  SOPS_CMD=(nix run "nixpkgs#sops" --)
+fi
+
+# --- Preflight checks ---
+
+cd "${REPO_ROOT}"
+
+[[ -f .sops.yaml ]] || { echo "Error: .sops.yaml not found — run from repo root." >&2; exit 1; }
+[[ -d secrets  ]]   || { echo "Error: secrets/ not found — run from repo root."   >&2; exit 1; }
+
+# --- Step 1: Ensure .sops.yaml has a creation rule for this keytab ---
+#
+# sops matches creation rules against the PATH of the file being encrypted,
+# not the output path. To match secrets/<hostname>.keytab, the file must
+# already be at that path when sops -e -i is called. The creation rule must
+# also exist at that point or sops will refuse with "no matching creation
+# rules found."
+
+log "Checking .sops.yaml for creation rule: secrets/${TARGET}.keytab"
+
+RULE_EXISTS=false
+# Match "path_regex: secrets/<hostname>...keytab" — using .*keytab rather
+# than \.keytab because the file stores the regex verbatim (\.keytab = two
+# chars: backslash + dot), which a BRE \.  (= escaped literal dot) won't span.
+if grep -q "path_regex: secrets/${TARGET}.*keytab" .sops.yaml 2>/dev/null; then
+  RULE_EXISTS=true
+  logn "Rule already exists — skipping addition."
+fi
+
+if ! $RULE_EXISTS; then
+  # Collect which platform-variant age anchors exist in .sops.yaml for this
+  # hostname. The keytab is platform-agnostic (same FQDN regardless of
+  # whether lxc/proxmox/linode variant is deployed), so all platform anchors
+  # that have been registered get added as recipients.
+  RECIPIENTS=("*admin")
+  for platform in lxc proxmox linode; do
+    anchor="${platform}-${TARGET}"
+    if grep -q "^  - &${anchor} " .sops.yaml; then
+      RECIPIENTS+=("*${anchor}")
+    fi
+  done
+
+  if [[ ${#RECIPIENTS[@]} -eq 1 ]]; then
+    echo "Warning: no platform age keys found for '${TARGET}' in .sops.yaml." >&2
+    echo "         Run scripts/secrets/sync-host-keys.sh <flake-target> first," >&2
+    echo "         otherwise only the admin key can decrypt the keytab and the" >&2
+    echo "         deployed host won't be able to read it at boot." >&2
+    echo "         Continuing with admin-only encryption..." >&2
+  fi
+
+  # Build the indented recipient list for the YAML block.
+  RECIPIENT_YAML=""
+  for r in "${RECIPIENTS[@]}"; do
+    RECIPIENT_YAML+="          - ${r}"$'\n'
+  done
+  RECIPIENT_YAML="${RECIPIENT_YAML%$'\n'}"  # strip trailing newline
+
+  NEW_RULE="
+  # Host keytab for ${TARGET} FreeIPA enrollment (binary sops file).
+  # Generated by scripts/ipa/create-nixos-ipa-host-account.sh.
+  - path_regex: secrets/${TARGET}\\.keytab\$
+    key_groups:
+      - age:
+${RECIPIENT_YAML}"
+
+  if $DRY_RUN; then
+    echo "[dry-run] Would append to .sops.yaml:"
+    echo "${NEW_RULE}"
+  else
+    logn "Adding creation rule (recipients: ${RECIPIENTS[*]})"
+    printf '%s\n' "${NEW_RULE}" >> .sops.yaml
+    logn "Added."
+  fi
+fi
+
+# --- Step 2: Add IPA host account (idempotent) ---
+
+log "Adding FreeIPA host account: ${FQDN}"
+
+# Ensure there's a valid admin Kerberos ticket on the DC.
+# ipa host-add and ipa-getkeytab both need one. All IPA commands run via
+# sudo so the ticket must be in root's cache — check and refresh as root.
+# ssh -t allocates a PTY so kinit (and sudo if needed) can prompt normally;
+# no password ever touches this script or the shell history.
+if ! $DRY_RUN; then
+  if ! ssh "${DC_USER}@${DC_HOST}" "sudo klist -s" &>/dev/null; then
+    log "No valid Kerberos ticket on ${DC_HOST} — running sudo kinit admin"
+    ssh -t "${DC_USER}@${DC_HOST}" "sudo kinit admin"
+    if ! ssh "${DC_USER}@${DC_HOST}" "sudo klist -s" &>/dev/null; then
+      echo "Error: kinit admin failed or produced no valid ticket." >&2
+      exit 1
+    fi
+  else
+    logn "Kerberos ticket on ${DC_HOST} is valid."
+  fi
+fi
+
+IP_FLAG=""
+[[ -n "${IP_ADDR}" ]] && IP_FLAG="--ip-address=${IP_ADDR}"
+
+# --force: create the host record even if DNS doesn't resolve it yet.
+if $DRY_RUN; then
+  echo "[dry-run] ssh ${DC_USER}@${DC_HOST} sudo ipa host-add '${FQDN}' ${IP_FLAG} --force"
+else
+  HOST_ADD_OUT=$(ssh "${DC_USER}@${DC_HOST}" "sudo ipa host-add '${FQDN}' ${IP_FLAG} --force 2>&1") \
+    && HOST_ADD_RC=0 || HOST_ADD_RC=$?
+  if [[ $HOST_ADD_RC -eq 0 ]]; then
+    echo "${HOST_ADD_OUT}"
+  elif echo "${HOST_ADD_OUT}" | grep -q "already exists"; then
+    logn "(host already registered)"
+  else
+    echo "Error: ipa host-add failed (exit ${HOST_ADD_RC}):" >&2
+    echo "${HOST_ADD_OUT}" >&2
+    exit 1
+  fi
+fi
+
+# --- Step 3: Fetch the keytab from the domain controller ---
+
+log "Fetching keytab for host/${FQDN}"
+
+# Remove the plaintext keytab if the script aborts before encryption completes.
+# The trap is cleared at the end of step 4 once sops has encrypted it in-place.
+trap 'rm -f "${KEYTAB_SECRET}"' EXIT
+
+dc_run "sudo ipa-getkeytab -s '${IPA_SERVER}' -p 'host/${FQDN}' -k '${DC_TMP}'"
+
+if $DRY_RUN; then
+  echo "[dry-run] Would stream ${DC_USER}@${DC_HOST}:${DC_TMP} → secrets/${TARGET}.keytab"
+else
+  logn "Streaming keytab from ${DC_HOST}:${DC_TMP} → secrets/${TARGET}.keytab"
+  # scp can't read a root-owned temp file as ${DC_USER}; pipe through sudo cat instead.
+  ssh "${DC_USER}@${DC_HOST}" "sudo cat '${DC_TMP}'" > "${KEYTAB_SECRET}"
+
+  logn "Removing temp file on ${DC_HOST}"
+  dc_run "sudo rm -f '${DC_TMP}'"
+fi
+
+# --- Step 4: Encrypt in-place ---
+#
+# The file must already be at secrets/<hostname>.keytab (done above) so
+# sops matches the creation rule by path. Using -i (in-place) rather than
+# stdout redirect keeps the path intact through the encrypt call.
+
+log "Encrypting secrets/${TARGET}.keytab in-place with sops"
+run "${SOPS_CMD[@]}" -e --input-type binary -i "${KEYTAB_SECRET}"
+
+# Encryption succeeded — the file is now sops-encrypted; cancel the cleanup trap.
+trap - EXIT
+
+# --- Done ---
+
+if ! $DRY_RUN; then
+  echo ""
+  echo "Done. secrets/${TARGET}.keytab is sops-encrypted and ready."
+  echo ""
+  echo "Next steps:"
+  echo "  1. Verify: grep '\"data\": \"ENC' secrets/${TARGET}.keytab"
+  echo "  2. Stage and commit:"
+  echo "       git add secrets/${TARGET}.keytab .sops.yaml"
+  echo "       git commit -m 'secrets: add IPA keytab for ${TARGET}'"
+  echo "  3. Add to hosts/${TARGET}/host.nix (networking block and imports):"
+  echo ""
+  echo "       networking = {"
+  echo "         hostName = \"${TARGET}\";"
+  echo "         domain   = vars.homeDomain;          # required for Kerberos FQDN"
+  echo "         nameservers = [ vars.domainControllerIp ];  # IPA DNS"
+  echo "         ..."
+  echo "       };"
+  echo ""
+  echo "       imports = ["
+  echo "         (import ../../modules/ipa/client.nix {"
+  echo "           keytabSopsFile = ../../secrets/${TARGET}.keytab;"
+  echo "           caCertFile     = ../../certs/ipa-ca.crt;"
+  echo "         })"
+  echo "       ];"
+  echo ""
+  echo "  4. Deploy: nixos-rebuild switch (or create-proxmox-resource.sh)"
+fi

scripts/proxmox/create-proxmox-resource.sh

@@ -867,17 +867,17 @@ if [[ "$type" == "lxc" ]]; then
   local_swap="${swap:-$memory}"
   # --unprivileged: read back from modules/platforms/lxc.nix's own
   # proxmoxLXC.privileged (via flake_target_lxc_privileged) rather than
-  # hardcoded, since that's no longer the same for every lxc-* target --
+  # hardcoded. lxc.nix derives this automatically: any lxc-* host whose
-  # lxc-docker sets it true so the container's NFS mounts work at all (the
+  # config.fileSystems has an NFS entry gets privileged=true, because the
-  # kernel's NFS client can't mount from inside any unprivileged
+  # kernel's NFS client (FS_USERNS_MOUNT not set) rejects NFS mounts from
-  # container's user namespace, no matter what AppArmor allows -- see that
+  # inside any non-init user namespace -- exactly what an unprivileged
-  # option's own comment). The NixOS config inside the image bakes in
+  # container's UID-mapped root lives in -- with EPERM at the VFS layer,
-  # cgroup/capability/mount expectations matching whichever value it was
+  # regardless of AppArmor (see lxc.nix's own comment). The NixOS config
-  # built with, so this must stay in sync with it -- `pct create`'s own
+  # bakes in cgroup/capability/mount expectations matching whichever value
-  # CLI default for this flag is privileged (unlike the web UI, which
+  # it was built with, so this must stay in sync -- `pct create`'s CLI
-  # defaults its checkbox the other way), so leaving it unset would create
+  # default is privileged (unlike the web UI, which defaults the other
-  # a privileged container running a NixOS config that assumes
+  # way), so leaving it unset would create a privileged container running
-  # unprivileged for every target except lxc-docker, a real mismatch.
+  # a NixOS config that assumes unprivileged, a real mismatch.
   privileged_eval="$(flake_target_lxc_privileged "$repo_root" "$flake_target")"
   unprivileged_flag=1
   [[ "$privileged_eval" == "true" ]] && unprivileged_flag=0

scripts/secrets/sync-host-keys.sh

@@ -85,6 +85,10 @@ ensure_admin_decrypt_key() {
   fi
 
   local key_file="$DEFAULT_SOPS_AGE_KEY_FILE"
+  # Expand a leading ~ that survived variable substitution without tilde
+  # expansion (happens when SOPS_AGE_KEY_FILE or XDG_CONFIG_HOME is set with
+  # a literal ~ in the caller's environment).
+  key_file="${key_file/#~\//$HOME/}"
 
   if [[ -s "$key_file" ]]; then
     echo "Found existing sops age key at ${key_file}."
@@ -93,36 +97,23 @@ ensure_admin_decrypt_key() {
 
   if [[ "$dry_run" -eq 1 ]]; then
     echo "[dry-run] No sops age decryption key found (checked \$SOPS_AGE_KEY, \$SOPS_AGE_KEY_FILE, ${key_file})."
-    echo "[dry-run] Would generate a new one here -- continuing the dry run without one; any"
+    echo "[dry-run] Continuing dry run without one -- any 'would re-encrypt' output below"
-    echo "[dry-run] 'would re-encrypt' output below couldn't actually run for real yet."
+    echo "[dry-run] couldn't actually run for real until a key is present."
     return
   fi
 
-  echo "No sops age decryption key found (checked \$SOPS_AGE_KEY, \$SOPS_AGE_KEY_FILE, ${key_file})."
+  cat >&2 <<EOF
-  echo "Generating a new one at ${key_file}..."
+No sops age decryption key found (checked \$SOPS_AGE_KEY, \$SOPS_AGE_KEY_FILE, ${key_file}).
-  mkdir -p "$(dirname "$key_file")"
-  nix-shell "${NIX_OPTS[@]}" -p age --run "age-keygen -o '${key_file}'" 2>&1 | grep -v "^Public key:" || true
-  local new_pub
-  new_pub="$(age_pubkey_from_identity_file "$key_file")"
 
-  cat <<EOF
+Place your admin age private key at ${key_file}, or set SOPS_AGE_KEY (inline
+key) or SOPS_AGE_KEY_FILE (path to a different key file) and re-run.
 
-A brand-new age key was just generated -- it cannot decrypt anything that
+If the key is truly missing (not just mislocated), this is a manual recovery
-already exists in secrets/*.yaml, since nothing was ever encrypted for it.
+situation -- generating a brand-new admin key won't help, since it cannot
-That trust can't be bootstrapped automatically (nobody can decrypt a file
+decrypt anything already encrypted for the old one.  Each secrets/*.yaml is
-for a recipient that didn't exist when it was last encrypted).
+also encrypted for its respective host key(s), so a running deployed host can
-
+still decrypt what it needs -- but the admin key is required for re-encryption
-To actually use this key:
+(e.g. adding new recipients via sops updatekeys).
-  1. Have someone who currently CAN decrypt replace the &admin entry in
-     .sops.yaml with this public key:
-       ${new_pub}
-  2. They re-encrypt every secrets/*.yaml:
-       sops updatekeys --yes secrets/common.yaml
-       sops updatekeys --yes secrets/nix-cache.yaml
-       sops updatekeys --yes secrets/server.yaml
-  3. Re-run this script.
-
-Exiting without making any other changes.
 EOF
   exit 1
 }

secrets/common.yaml

@@ -1,216 +1,235 @@
 root-hashedPassword: ENC[AES256_GCM,data:Kp0nOZI7vDoLhJHiOJBwJn0rQZ5yhnwapGnAcA+qh8vlDETtFs/iQdetF/2ZxmANf62SviTNd+Ag0q5JIF1996x7onZGXqxgSMCuVzZLBdUlsO5IR0BslWWz47khYGTe4WkUg4NB1itBfQ==,iv:5Sra5vJ79V8hxQT3g9qJ+dOj2W2sumIhqpitqnHjJdk=,tag:3Igu0+8GeUZHqS3fKUVwog==,type:str]
 nixos-hashedPassword: ENC[AES256_GCM,data:pT7tVRN6X4a+DNUgB7fIUUE3CbnetkjxmoSL1PxSU+ktsFU+fB0mEvJjA1uujsGH5Rcztg7YM815+M0Z67ILmHaXbza5DtFacrqhi4/b277xly0SHRX4yOvBwQh6mJG1jn/0O/wvUUIYdw==,iv:bp2nfhC8nFbk6o5iWDAugvbzu7J/a1xayFnBEtkhNpE=,tag:HqWgkIpSrSM/K9OK2WO+VQ==,type:str]
 nix-github-token: ENC[AES256_GCM,data:k1vYz7SqVhzpWa6jTL6NUD8lKOCpHCgTm+HT4IcnbzbSTUZP/bJUYw==,iv:UqAULZnr/4+VcioUDfTwvOSuwM8K9JgGhiApvYQPyoc=,tag:1LKHXhWAO/AHPDIZFBb04A==,type:str]
+beszel-token: ENC[AES256_GCM,data:ds7OFjIXpOe/OIiEIydK9qsYkq3rMShK+jCTRRHKLzxUV0Bl,iv:nEt5FxkQaiTmAPFbj7vGJIMAEBjXmx+XYcdqaAxGzo8=,tag:+ql/UsgBR+TPySG9VDZi1g==,type:str]
 sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrMGd3ZVNlNXdmOUZMUzdQ
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiU2IxZ056SUs3SVZiUTlQ
-            TW1acEs0NFA2Q01rM2dkc1h0NHkzQmhiWFZZCjMyK202VWdlaGhsZW04MnVwUVdO
+            NXZEYmg3VzJzeWJPRnZXcUtOMGQ1MElBU0dVCm5xSURZZ0sxTjVxQnhpVVNEU092
-            alA0Q2FETThsYkhSS0hKdHBaS3VaY28KLS0tIG1Gdk8yalREOUtIZTUyY2p1UHlJ
+            bDBHbFAzZ3hzejc5NmM2U0FQTkRNSTQKLS0tIC9OOURPRlcvQXIxMVZXR1JjUWRX
-            eG5iQnJsaTJBY3Y1dkw1c0VEaDQwdDQKfV04fLy32Lp2ZQ2VnvQ0h/Vsf+qdaJiv
+            bE5ja3FoWXFIZ2ZLTklqWHhMRHBTRWcK2Z4sEcdyvx1rdBIGrbNw7bYvEPXdobvF
-            DnLXGZ9hE5yzpKWkQIRgqYGBkF8PkH0YC4OIaVkA53wrtjqS4ZHR9Q==
+            h0aNihn817DvtVr8eeucj22iYCS40etElQNcO5i68t5wab4/d3rUbQ==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjWFNRY2FiK3VkSm1RdHBn
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrN0lNNGtYbE8rRTB0U2hL
-            cWl1ZStLcTRFZWY5VVI5N0FhODZvR1ltM0FvCnBHZUtTUm9QeHNlbVBoZEx1V3Fa
+            eUNOZUNFRGk1RzE3R1VjV0Q3ckszcDNHWGxJCkhFN25JbDVSaWJIZ09uNzczbkM1
-            Nk9iMmJKVnhocEpERi9leE1ySUtNMFkKLS0tIDRRYkxnbU90S2RyMHdJNzRJNXBi
+            ZWZ1Y2wrTHZ6VzlnVWRoT2JOdDZHM0UKLS0tIHQ2ZE9BaFRCeDNqbldRQkpLclI4
-            QjRmZFhVakVic2tYODZHcWtJRmNQTDQK7G8eSJInt11P0DiL9uzNQ/ZHHLVNIYPe
+            dXZqOXNaR2Q2QWJtRlZUa0hLaTh2YlkKeCh3+yIt0sjh2Ig8m/YQifhMDHnTpuZE
-            bvlhuGkEuQ/+j5sVSKOfSI2Y7CvM7TpE3APyKBcLG3ajYg6F/Ev3SA==
+            zHBE74SUyWFmqLgtd1b8Rx5gn9vBcS/3SMv3yKWispg4DZ3bCOeNFA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age19m0m7vdfg86yqy8l5mmle5jdd0unrn3f55t232w8h5ey42cqw34sfpt32n
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1S3dOcFdwV2NoMW1oMnY1
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5c2luYmpYcVVxRHROQlpk
-            VGdxeXJVR1lsbzNVZHlGb0NGOVo0SndiakVjCmg0QlNnZDV5RlFja2hCbVRXV1VF
+            cTM4cm9uNEtVS21pK2pDRVRySmxYS3NwdGg0CnQvS3g4Q2tSdlpVeWZxbzArcTM1
-            S1ZtbC9KU0U1ZW9zeVoyR3hxNW1XTFUKLS0tIFUzWTJhTzM4QnpWV3h1OFU0N3BK
+            YWRMcDAxcWgySFRqRC9XSElsTVorVEEKLS0tIFd6YjdrY2ZXQVBaZ01wMTIvV2sv
-            RnF1N2k0S0lIVitoNDJLUmZqdHRzZVkKUfNg24p8zxb3749v/A1BOKCNw75AUKpf
+            ZEk0TG0ybENlTlhpZnovMXh0OGd5V1kKEbjrmKP6Su16KRDQ4PMqi0tHRS6+PfuT
-            RUmFCw5DDWF2aNM0mZqcjjVmJ/FRKV2HXwwUGsHPKSOTnKfOUlPNKA==
+            0v5DEyi8EDtIYSMzzdF7Jk2C0h5XuRux45Z6E6JhKW5c788dGQHz/A==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1rrxqea6q6pn39sw8y5te63h2py8jgjl9v0jyper86w3ggtn67upqg3ah39
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzazl5MUpUNEVNMTdLWXBp
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1eW5kd1FrSGJaSDkwTzdY
-            c1hkaVhjcTZXTUNsWS96QVVWb0RVQlZ4VlNvCnE3TFRySU5jTFk2WjBONUQyQUhl
+            SnJhR2hqRkNTTGtSTWsvYmZpSlVWV0xvYkRZCmtqR1FQOTRQR3dGYU8zNFpMdkUr
-            U1lNTDFTRmZhMUFyZmpVY2xpaUVxRW8KLS0tIE9MQ1M2U2ZXQmRCVll2UGRWL1RG
+            SWN2KzhXaE1rYWZYNHJsOVVvRDY3R00KLS0tIG1aUldSWHlHeHkrcjF2TENvazR3
-            K2tnU0NOKzU5dkpiekF0Vk1VM0ZZZDgKK13aFypGAqrKWPOr3UwtXI1EoXf1+UzS
+            YXJLUXV3VVNSWEdseFlZQUJPQUF6OFkKOPKH52muUha+MV2leWDiqpuTaIj0ZKzV
-            rBqcwnX6WPxSKUwWoins4Aojek4QhbhY4R5ei6rRS0KEQeryGxy8bg==
+            keEdyqkIck6KKfIjphNKTW5ujiuKjGQbRliquBknfCRGMnRlMNfmkA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1adur9g330gua4l6ndk8cqjg35qc8yxwgme6wrl2hpylcc7vxm38q05ejuy
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCZUpVT1p4TGZyNC9qMm5G
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmTDExNFpCZHZmWnpwYis4
-            UzNHV08rWURjY2lqS3FTWFdoc0FYYTFjb0NBCk1mZ2JzaXk5RmExNE9xWGZ5K0pv
+            bFJGZXV5UksyS01MT0d3cXE2Wno2Zzh6OFFnCm9hT3MyZ0hTbnkzdXk3NmN1TUVF
-            UEtqMkltV0dIWll5eVBUVVRNOUNDWUUKLS0tIGpKNVJudUM1UGNvaGl1UDBOeFA1
+            OGtCZnlJRmZJaWJWaE1pMGcrTzJzZm8KLS0tIFE1L3ZxeUl6WVA2UXN2b0lIcTBl
-            RjUyRlZ6a0Y4SXNsL21zSURVRk9KTFEKU1L6BQ6ZlYQQtqx3uF/uM5CQ1ercmvRT
+            YmtXZnZaMml6ejY1Y3lRUFFtTEVMS0UKZbcuunA9GRqctCfC0Fh2foKt7n3l8KkE
-            TL3r2/Y07gE7CjRn3pR9z0co8KndGzxV6YR+ubyWptwBS8KQh5stkw==
+            FNjjxUzL7OQjNG9M1aAtRMMO5KXL7YKXRUJWOPcpiLeLUNvDLeLXvA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age17e89ty6p0fw24daanen57wg8uald9s025t3wwxsw269svwpmgvrshfvfvt
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsaWxxWS9xQXViY3VnUEx2
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQYlp5U0FvVkdIKzBGZVRl
-            V09FbTI0WGtNbW0yclhOSGZDbG5NUTNaTkFFCitkcjJ3OE9BSnN4bjFWcE9nYVBk
+            UEtOOG5kVHVLQ0ptVTVpdG9Lb1Y3bzlOODNzCmNYN0Y5ZGNzTGFGYmxJcWFxaExn
-            ZzMyVHlJQ2wwdU5JOXdCQm9oNkhNd2MKLS0tIE9tRzFYS05vSkUwWFRkaTdtc0k0
+            NkkrT2RLSlZTdmFiNks3RVZXQlBEYnMKLS0tIDdCVlBxUG8vYU05d3c2MXhjQjhM
-            blVoMWV0QklBVkluT0Z4NHYyS1F0blUKO+Uc0of/V77ZUZOsxTzeH8/LmmAOQt+J
+            TTVJUGxlWUxWalBRYzFKQUQzd1R6dHcKPz1ciNICph7qnqSH5WYmOjUCNHsIAmTX
-            x/COHxnLCnZ4eWI6q1a0Qn5Br15OJYTxUI2QTV4goTnXBNUDo9wdpQ==
+            lRnbFCZdD/ZIFRK8OYAyCGfQ42UYSTf16kH2jDdTFUy7Fk2+O7lNIA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1hrx8qj02fj2ea6d4g9vqhyj9hl7fppkjqfdx2l37py3h6pdkr95s8n8rvs
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsLzFnZkdVdGQ2dHRwcnRt
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiemVqWW9LdlV0RjRuRHpF
-            Ui8rc1NoQzNKSHVzTk0vYU1vMlRxcjBKZ2hNCmhLYXVGSis4RU9HNGVxZkpvUkd5
+            U2MwN3RDeHU5SWhuQ2l4YStRVVE2eU9XSUc4ClNGMkNXQVdkWFkwcys0aFVhZnZv
-            L0xvalhDYTQ4N21OcHRheTlkaUxvTkEKLS0tIFVkRGxtLzhQT0paV2U3ZnNScVdn
+            Y3J6RmxwakdaZVo5VWZhb0lVTUdTYzQKLS0tIEQ2NW0yb2lGQ0NLZm1PSWNFVkQ1
-            alZnaVppeGI3OUVscGpONkk3YTRXd3MK61na8x5qX7+dyMHasDz2dj7yeaUlX8me
+            a0Y1NlYwZ0dnbHhsQVB5eS9kZVdqbmsKgFLUpZMqGywO9Ext5WRwco2bpenVNrzL
-            N4/SIk1JDBhv9G7mdKLbKhSF1UJrSY7TJqJqx8/dqEc0uG3vptA1ew==
+            XrGCqhn/7TW9DrJlPnFe8r4fl4DFP6GWByn7axDl58Qy1OzeBZwnNg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e7l8dusgmgfzd2cxrrzwepzjxt69hzqj4epee0cs27u6yg4kxcuqm34ncx
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4ZEtvOUhMU1FRWWpJQjF4
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLRzlzWEcwTUx6M1JNYWgr
-            NWphNXp5M0dLZXhkZndhT1Y3L09maytHazJVCm1MeEtMWXg1Zjg3bFVnZEorci9J
+            ZWxnVXhFWkhQVjBrSndGTU9oY2MrOTQxTGhFCjhUdERPc3hzQlkzMjBEa3AvaWVX
-            bkNZQU9Ta1dDTFFHaGFWQVBpK3pYRDQKLS0tIEhPVGliRDR3ZTF2aEl3ZnJEYWtR
+            T0RJWXFFZHdmUVBObGhpQktCQ2NPbWcKLS0tIDN2YkE4bEVtbjJlcndWVmlrQ3ZT
-            anh0SEpnVW8xdXNkZEZQSjcxU1BHMFEKVRJUA71fi1QawB2TnuTWMYhzQR18u4M2
+            dlVvRmt4RklOdUJyZ25ia0ZoYjU4ZHcK5fL1yvRcty6+EMzfSkwwJM3qvrr6h46d
-            s1V4j4TwYyyKZFoNvt8kOUayjC499c5OBUufYs6G2ciC6gK2A9E0EQ==
+            d7Zs+EElXrsGDQvL0fevB/zDcICAtudYnVv4nP3Jf8GQnL19zWlwbg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1jcx3yajjhghn8qh8za3yeu8nxykzlg3p4nrv03vnfvzl0mzayg2qmg940e
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTV2IzSjBEMjUwZjB0dCtj
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhZElRNXNLK0tVUjQ0ektQ
-            aTdZVlEvYkVRbTNjUC9ZRVdNZVhPQTJEd1ZZClc3NDJiR1BVYkZkdVVxMVZGc0VN
+            Tzc5NDJReldiYVFFZnRLS3l5dWw3UVQ0WWhJCm1DYVloZGpYeVJtNUNRcEJvVUZE
-            dWdSSXBFR2xxR0xrV2thRmUwSS96S0UKLS0tIHRRTXVlUi9UYnFRRlhsU21HZVY4
+            Q0xMcmR0WWdVVTBaWko0dGI5V1hqaUkKLS0tIFBBeXdNY1pFai9QT2t3NWFLV2sw
-            SDFYd0NwVEtVZXNsWUI1a1ZZU2xNRGMKuQUhOq2FRD+PGn5OkdODZItbxCzRKjne
+            cWpRU2RISSs1UXh3Wkl1dWI3VEk1bXMKP6v3Lm0XZkp3Gn3iJkB0K0Y0E5BeilqR
-            E60UOYtHjanuGjJ1svuR9cYsLZz7lLOwItklecYaQYpMRZEwzzBGCQ==
+            plmPh1rKbT/wkefFwpa8e2Wwmhx80YK+bjn633MH7mLQDf6AiK83Xw==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1sweerhrga9yf8x6sv0apz4ed4g48rnlcq34rpv20t0rcelwgpgeqwvndzz
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrSjJoUVUyd1JqRm1ZQzZx
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMVk8yT1FtMVoxYjF2YVBs
-            SFJ2cWNUY0E4b3FndVA5Y0hIOEZnZUlVWlhBCm1vY1luOXZBelRUTmF5Y1NMeDBn
+            UTFVbmhzblBCRno2V3YvaERRN0xWcHd2NVE4CmVyK3p1SWtodk5iWjdkQjJRc0dM
-            cE1BTDErc041UjJCWTBQbnk0Wk80dkEKLS0tIDVSZzd1UktvZGdyanFUMkVORUtl
+            Vm8zZHczYlFFSG91QThmUGNaWHBoN0UKLS0tIDZMMXoxbUM0TUVaNFUzRExLc2VM
-            eVB5TnJkMlp6dUpXSTlxRlplZ2NxUlEK0AYOxIbswjM0SUASDfmZ7PqcEU844fgI
+            OVFrNGZNL256cjg0Q3ZBQW9xUFcxSUEKv4HqaTcTzNYRJCmNEeUQLFfEoFXyOBEC
-            ycFWVSEPodwUZ6UFoYXhHlJzHFcgpLvwUd1PMktLHe1qrZ7GOQJIMA==
+            yZrg8D+ROBn98484bxNeoyOjQdcVn7OYiCv1y6utnyeA9sdMUB0ong==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1f7usptjx9rv4rxauasve200gxtdt9jkqhhdqstlf20wvlm7u75rsjfw50m
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPakhQcE92ZU0zYmk2QS9D
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1Rkl0UXduS3k3WVU1RzdG
-            TG0xc1JPZXZCZ0tZOXA3MGNLVnBlZGVtRFRzCmIvMHhQKzFVWCtpMTQrQUhGVGJp
+            N2IybE1TZ1kwVmM4Y2lvUS9DVG0zSkdpZGhRCnFkZnJVS2FBZm9Xa0V4SjRwaGdR
-            RU1jbldYckw3TXI2SlNpZVBIZHRsWWcKLS0tIGtJTUtJejFxem5jajFQUDFTQWU1
+            TWpaaTh4VEk4cUFrb2pURERzbWtSR0UKLS0tIEsybHdvUWg3UlUvL0lhdHIrUE0y
-            VnlxYmVlNG04ay9ETi9FRmVYQXVoRkUK9oFNolI7jRjo9RUs1g4ghrx7aYV4U/ce
+            R0pMdnI5QlpIcWVWdGRYQ3JCaElLWTgK5cTfvehj4BPU22tYiR5piZdouUdeh5Gr
-            ZTc2tFh57+7aKgrDi+2W3jwhfkjvBsThk//p5mLlqEEgw2lwlnhvPA==
+            x/xSbFWqYVauDB6rpzZvhGMI4KCJh2RmDdt7FrlCb/YCB3QoSgvl8w==
             -----END AGE ENCRYPTED FILE-----
           recipient: age17jqc66x9yeshfgd9v78mj483r4zzarqdtuxtrkxe4x5mw679gphshd94th
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoVnh6dFAwTkY4cHJpaEs4
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwL0NNbGlGS3lTZjEydGJm
-            QnZPeXZHK0tXYWZPNytmYXVsdGRWQVI3RlhRClFVb2I5OVZzZFNrRXFaa0JTUkRJ
+            MWZRK0Z5YktuNVNudk84MndLUUdXR0RlRFF3ClB4MWRUTFBWYUdwbityeUU2SlR4
-            OC9GQ1V5K0JhWlhkUjU1WStCa1lPV1kKLS0tIEhzdnBBZkRnK0NtV1FuTkVsNlgv
+            azZMb012YXQzWHUxZnZzeVp3MmdzcU0KLS0tIFJ2SUNyMTBEY1NWejdVQ3hyUytQ
-            QzVEcEVkQm5NL0Z5dUU1U0ZFaTJITnMKaWE9vlrOpQstr6FGP5ObdilsCYk4kYAj
+            VnlQWHRDYWZFaU1ON2RQTXBwNFNGYUEKwnRUWCdsiStia1jLHJ6UNP/kgEt4Aw8U
-            /phboR+Ym7QDTyUF9LZXJCU54YJp6vEWkRnlJFqC75UW/v/lgBhBMQ==
+            pA8PKKu6TZVPI8pw7svtI/uvJuW+8ARuNcyb1FdYmX+sOHnOEqHyjg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1px0h5l9zp2dww0m8fncrc82kfdmzplsfv2ltat7sna28xpg09pqqcl3s2k
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkUHAzWk1KblJxVVZZV3FM
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxM0FyaFlTR2FzMkhGTzZx
-            WndzRGRtbkw2azRVWnBuZmhKWElqRFk0RENzCk5DMHlMVWpwbXEwVUhkaFZUbkp2
+            d1d3dWNGOG1ESllETEVxRGJlYWkrTUNIdkYwCmI4andLMUUvNVU3R0N2WCsvWWhV
-            QXNlZFV4SjBEdmR6UEw0N1JOUnhNKzAKLS0tIHo5RkNDUk1ESWRHQmV6bzkvSTlP
+            UmlkcHdMSFA3bFp0ZG5OckE3YmJrejAKLS0tIDhEVEJYOFc5VWVHcGcvMm5yanBl
-            dk1GQ0Y3V0dTRlByb2xUOERVOTVwbVEKY4sAHyAhvGSYJzPuufWUIQD2xZcSt/nX
+            eGxaNEFxdEN4NWh3cnlGeElBUTJmUzAKvCSJF+LGf1JqEdkzT/Wk6hEzferseyWO
-            t2ZFXu891/QdEzyUXCIzdwAV+Y/LjvroIlCp5Hkbrk0s7N+ghqsB1A==
+            vAYR8Z+u6GDwQi7JdFz9iZKD21LAs0IMzd7cZqXKIXVedyXuaIhPtg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1ufg390ydrmma849t9xfkxxl5xvdkk6mngnlzhmy7mvuaje8sgcmsmnq6l7
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBYVkwYXpYSjdmM1FpbDl3
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGZ2M0cG5nMU5nYVliRzlx
-            QWpycXoyL1AyOEpZUmtpbjl3MFAwTkJoOWpVCmVDd0FBUWxaQmZCU2VmNkZGMk9o
+            TVpDcHpwclhXNkxuc2VKanBvcnRxRW04UXh3CmNJeEl3K0FrWWVHQWRwK3Q4Tzk2
-            TUdLNGtac2N4REg2eVF1eVh0WnNaTE0KLS0tIDJvcFErSjRiWmhPMmpadjROOHdt
+            MEZQbFV1QURLZEhCdEhaMjBRUzllaVUKLS0tIFhSUTJ4SXJwRHhVbkZFc2Rwb3Fh
-            NXp6Y1JpdHFlSlRoa3JTaEt3emdnalUKjoFfZAiKMPF3noX+K0+vc3+p/XUHnhic
+            QzAxTnpWSHZaWlF3NUlFMlBHUkRUa2sK3Kim0mp2adSvruI6Necd3UpZHow0EFmT
-            k888KdUwcZYl2/dAIc8UDSggbMnncJAJgoezoCHLkj97GNNAD7E+gQ==
+            JFOIpIK+3Pc06gsiqrDtDAx2Qtc10Ja4ILTutPlp4va6zzmzrPABiw==
             -----END AGE ENCRYPTED FILE-----
           recipient: age16j42pdc5dr6wnj7xayhkqdj2rny9u68fcqejs50hqq42scssh4gsnrrnlt
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqYnQyYzF5cktZNG5PcThy
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwTUZ1RzJKcVF5bnJGamlU
-            SEszVEwrUkQ1VVRsM3pSTlRQaHVLN0VuSHhrCmN1Z3pwNlFsbDN2UGI0KzYyallM
+            bWJvcU1sTTJuME94YjhMZWRWRWR0aGJ3UUNvCjh6U1BkaEg0TDlCeTNyUlNTRWV6
-            SHJ5eklQeEIxSlhiYW5PUlpJcG5KNjQKLS0tIEk0QkpMdlBlRjVYMmJaMzJUbDNm
+            Nnd0NjE3NmplSWFjVWpSRy9qWjBUbnMKLS0tIGdMZUFzbGdMejZoMlFBVUJXSE13
-            K25pZldwd3JoZi9vdURoa3Myb2RQNG8K6N6bO2YKooPfpKihgsYqilfz/yAYCLZD
+            TW4rUHFjZk5oVkw0UGtpaG5mOHRRbHcKFX4ZyNQ0TUeqwXEaz1qmAXQRIOctDkvz
-            XJ/THgT4URX2VNvSspvBtN8luOiJUVcchp5WtL2m9jARL5txEcDorA==
+            TIwxJseAjS3qVxCslnxzgaqmCh8DzCfjrRxLUS/HVTnwOaE1YAh0vg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1nruncs4l0ufk7yuc4des8p99c0alfndl0lhsws8tycl5pplfp56s30af5f
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvY0VpeW1nazVvRWJFaGNU
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTWStVRDhlblZDMDJ3ejIr
-            RGVzVjRmWWV4dnJzRHdsaW9ENVZYck5lWGtVClNtd3ppelowRjZpRFFSMC9EK09n
+            MkwyeHRwcnpWQVBmOXY0a3NCU1JnTjkvVGtVClVvVUpLTVhUNGNNLzY4U0o2SDRD
-            b2hqNnkzejdrTnhYNGNKblpteDNLRWcKLS0tIE02bWVjazRWNEVKbURITGlQODlR
+            bGZteUU1Uzlzc0NjTW56OVNCcnVpUUEKLS0tIC8wN1V2Nkc0cVU3OHNYa1NBVlhq
-            cTJJVnBVdGIrdzBoSXExelNrVk1XcEUKc77o2EX7PCm/HjUo5GsUiQdm488WB2mg
+            YTV5QUpsQWppNHRpT1dMb2JDQnpoR3MKCoBngnNg5qpktg5j4CcuTUKiooOVI8IN
-            wHd/qDbQhF1W75RrVTuIKgtEtrRjZqpmr8toe+aHJizPofcrToUfzw==
+            04plE+1lra8q5ZK8q4r1/K8hBM23poyfq+XushO1GZ17dPtWs9wITg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1k7d2du5mejsmv5rzavm4xwgpthqvcfsehduquv28nzs53zppa3kqngfxq2
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSMmZnd1pXRWh5NDVSN2xq
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUU2ZSazhZNWFZVXk0UXdx
-            MVhTN3N1OFpkcUh0a0tjYzJnWUlINGNVd3k0CmZkTnBac0l1dllxazdLY2l0Rzli
+            Mjh3R1NLMThlaHUvWVFMZEQ4Z0NYOW83dWgwCkVnd2dhdzQrZkxaL1NRbDJodERQ
-            bE9sNTBVSkJNaWF1T3c0WktoOHl0NU0KLS0tIHEyeWZTUjdQeUN6U2t0d3JwNTBX
+            SkM5S0kxcVBtY3laNWp5OXpXZHhqZ0UKLS0tIHVOT2VUZU5DZjJjVW1kcU5qS3FD
-            ZE1Za0tXb0gwc1FSakVYdU9OTHkyd28KkwmlzSYP8XofB0VGag+S18+S2TyQjLrM
+            YXNhRkpaREMzRDB5aWgrdVNvUk1qcWsKQnPqE+5xSBGeDMmLfKB0tEBt2eylBOxT
-            qaXtbBtLzJGNDhe9FhAKTPFcjTLWbohlG69vxcImyCyCns+QQ+gvug==
+            oi26mC+EoO5T2pCIpbD+DODAAlWKLattqCYHneUO9wRk9DnhuEXRBg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age16kqfmvz4e23hmdlqresnyw69ej604s320mmd49h4hm3fhqchtgyqrws0k2
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2Q2RMWmZVOEwrOC9VcUxp
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZUlFlenlOVDk1NmxQVGUx
-            VVE4R2NqYlZWZlhKSTBHRks4bDNoaUliMWlJCm5FbWdZS05GY0VLc0sxY2x1U21V
+            UHhVK3ZtMHdzaXdtWmE1TXhOekpsTmUwSlJ3CmdncjMxOUw0alBQZkhZZjA0YzJZ
-            eExwK29GVVBqYlRPZ0l5RWVXRFhRNlEKLS0tIDl6dVZJQndwVStFVEJnRHRyMW1W
+            bFNVOHUySXRuYW8wVjdHMDN4ZThuQmsKLS0tIGdvUUdHcXlpWlZkL0xBcVpMN1E1
-            NnFqc1F0SGJqT0xmREpaN21EdnlJK3MKRPE5rfFpVnH5wAOkuB5pNMlMd3omcpku
+            NGZCQURYQVY5ZzdDODNNY0o3SG04eDAKiYVuKsJLaSC7/bQRB51tX282/yqpAjWT
-            do2hFZwyI7t80jxF4+g3J7EolOx8AGjpc9Ba7Gj6IMDjye728q5N+g==
+            XNnqPQBQ7H3xuK0tDVdCAUMruOSlqAe9YRZwmiou1irpg9vZYwgTPA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1arhf2q45zw6wf2uevju4savp575x3m2tfvved5zzq3ay92ynua9s3cm92c
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhSG94VE96TVlTNkEyclFE
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnUUlPS3ZPS1lSVmxwV0lq
-            SmdCSkJpeWlVWDFIMDhwUEkxL1RUTDJ5UG5FCjhKRDB6VGtwTVozdUVzbUxGL3BW
+            WWE2QXltb1BzU3p3cFRuMzVieGJta05nWno4Cnk1RG1CUXZjM3dNQzlJK1ZGaHpZ
-            SnR3cmpSN2RxNnl4QmNvT2lkYmtoVFkKLS0tIHd2V2h2Wk5xOXlISzhjVzBsVkhz
+            b3drMTB2L3hidkpoTXdMQ05vOUtTb28KLS0tIHhyUllpQnlzL093MCs4YWx4VlVn
-            VVpRenVnSVpHUWJqV0JHNXNWWXJOdW8Kv7PJSTDbwFOAcl7pynALaJiTXU/87bSF
+            S3Q3QnpVNEx6ek54RGx5bmlqUk5hbVEKHVUvfk/xKtlOk2o73LodweIivSB4Oii/
-            F3HQllYOwOoibGzBCe18H2N+VxyNxoQL9OWe0TvOIR6bgHFIIF0/Dg==
+            6dwnE8wa+Qx/OU0vDJVelKOUE6JiLwckX/0u2YIUOybkx49n+tYs/Q==
             -----END AGE ENCRYPTED FILE-----
           recipient: age19mn8zrxl8zpps9yvrh4euquvygpp4fp8queg7xc6qhtnl4ng8c9qx02qwn
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsMEZPUExRVHFFb3pSVHNO
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3MHlrTTdxYit0MTkxa293
-            cGJFN1ZzTDFVNEdneVpMZ253ekFJNjVtYkNnCnRGQjU2Q3dsRGRFV25LQ3pCbTJE
+            eFFIcmRLMUpkZjd3eWdsOXYySEljNDB2NUdNCnAyanVkS0dmenZSa2dpSWZTSkZw
-            OTROaFBiT01xb200S1pUK0NYaTQ3R2sKLS0tIEdiQlZTbi9Vcm0zc2t6bHplZktF
+            Tm5LMkRoWEk3L2ExVktnZDZMM09hZ0EKLS0tIGVGcHFuMjh4OTUxdHNJQSsrR1pq
-            ekRySENXcjBuR2psdHZSSUJrR0xUdjgKvBsmnC+cbq5TUDFjXCyImIoPKvh8wsjE
+            WU1jVG5UN2hMbW9iQ3JmbHlwc2VveFUK4JC32glesv/9nMWBR0lFeX7MPgIt9sIo
-            7Shk7Act8Jayrhx0lXBDRmfpHRrB4L16rDSmqO0DTE48VhT3TiFyug==
+            9HsAkOMTxPTI0NZW8q79+G5hq8qG+QvQhwO6wQtwz/KEHPXntKDX+Q==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1jlltcv5jcnm40z5k0q6hv053k2rqpqvemtuecdwn527uw8uqz4es3x7m68
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxNDlya3RmYzNhU2p6RkVw
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRbEZGNmtlUjJVZ2dlVGZC
-            cHJVWWY0Yi93cW1uZi9FWkVONmtpRWh5SzFzCmtWYTRIY3BkTWU5R2Jsa0ZJK3kz
+            MGVEQXV1SFRpMjFqWks1Y2F6c29TY25oL1MwCjVFRk90Q2dJRXhUMmIrRVdHVFRD
-            SnZvZ3YwaGtoMVZ3V2laTlBBK3UyTmMKLS0tIFdpZWtqeGlacjlLbmFySHlSUUlj
+            TWgvQS91VWdFWERJWEYrL3VnRXFraUkKLS0tIExUU01VRWlENlpBSGt3V1Izc1dS
-            RmRqQWVHK0FUT3VDbFhLbXQ5WDhLeEEKcDkgV34lUFJRIHRoLB8F2IOvGAM93sM+
+            RFBhSTVKTGV2KzFidDQ3b3lmUHZ3d0UKHWvWO/Om6ahwBkONreGeENqdFG9UI6JB
-            AkmaM4+WRcGeYWQKMG2x6cYCUKFaT1lDXuWZ9kI8Fd7b9gTSnQMs6w==
+            fRA3gRHy98EorEiia1him6lGY+ewYkTtVNo6iEt0abncW5x5xMQywQ==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1ug787sgt6st6k82fgkrug2lzltw4qsukrrqqs3w27ewwqj8rg4hsxcmylz
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjeFM4ajhHeVd4bzlqZmJ1
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2dFFpeDU2Y2dtQU12cytZ
-            MUxYMUpZVjdjR0NGTzVteVB6WFBQeFI0ZkZrCk55TEt0Zjdwbk51RnhYclNzam1H
+            WUJjKzZ5S2RiZmtvYVJScStYVHI5d2FvcmpzCm9COGZmRllyaElqb2NsczhKRjhO
-            cExKZXQxWFVLa1pDNFpkcGZzcnl6a0kKLS0tIFJ4ZEdJc3JVaEc5RU1aZk1uYm1l
+            ZGhwZCtoaFcrWmNZbGVHMUNLMFQ3Ym8KLS0tIHJQR1JOblZlY0cydnhNUnBCT1Fh
-            d3RHS3hHSkRKRXFnN21FQmh0TlNtNmcKdc2G/1dhTJen6iT9kUWZM5OzCmDVprgx
+            aHJiZVpMU3Y3NXd0ajc0MGMxSGp4MmcKky1oaS8yFydszcvZE2pAL90ZUKoN9MnJ
-            WN1Bl3JzYhLsNKn794887bVAICVqbXqkdpEZztNIS5n/Rw6geKsNvQ==
+            1TdqatDSMtmo3PWkdFeHbWxPjTizQGIcKEcsv6GmO/5KCAJDCHDNMg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1529taqdwr6t0w7cvzmty0d5y5593wffl0krt48j6uc4u39k56g2qf6ywtp
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYa0gxNnBwNjNNTTgxKzgv
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrd3QwWE4yaVlIakdiYkNX
-            YkQwTjZxb25tQ053Ny9tSW4wNGVYYnlIaFZNCkVvOFNPOFkwQzhYZGxjb0FOZzJ5
+            bFVmR01yNjdWa0d6V29tcWh4M0hWRmQwR3pBCkFRV2gzSVdPQmQ5dXZYc0F1dW1r
-            ZjNXam1ZTWZrS1M4cGhhcHZaT1NjblkKLS0tIEdvL1dDTHpWcWF0S3ZqNkxrQW52
+            OFZNTkRwSzRFR0tlUXE2UzFxeU1Ld0kKLS0tIEZ6RlZkMFFiZytPZ05DSEI1Ni9X
-            VlYwa29sZVloOS9qajJWQWFzY2FKRmsKy074SLdttogXsWycaFX8xso4ek7Cbjph
+            WEtpUlMycVhkMERsWDRGQ2lrNHdQcDAKRqVAOzQY4ZM1uH7LyZSqX1T6+1PwNhBY
-            MMEhZd/svmnSiYM81nmeaze7qXEUcsZXuSmZCYATTBEGtx/Srll8aA==
+            5M0RxSuI43E8YKC9axcppvKFJMQI1GAGvgfQxg2J0CEpnmPSVsO5DA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1zhfyuzlq40reuqlr34gf77852nhs3t6mqfzrqmas8z6sxk7tcfhsungrm0
-    lastmodified: "2026-07-23T21:15:41Z"
+        - enc: |
-    mac: ENC[AES256_GCM,data:qFhnPra6IE3wyKQ4WKweON0S0YtD5I0adGZVfA0m6BVilN6bX5oC/1j5NK2oHrsz920hSl0SOF8LrpqOrUyGjSRkPsN4kq8qr9bJcrX4URiktP0oRden5LLt6hf+ZRP7WmRXFqixPkPHJnZIoAvkNnTFce7cDq5NEAHkKUEKG7k=,iv:nyblUDGeu3TUfFivYylOn3C/HITj99qiPI2+mh8AGh4=,tag:FrtRzSCylC4wlIoqZdfx7w==,type:str]
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvb0xNMloyRW5tTUUvZGph
+            YWxIc0ZhSWI5RHNmaEIvdjZJNHFDM2xBMkNzCldlM2RUeFZCNE9tMVp0MFlWM2sr
+            Sm44MjkrWlNkY0U0eUlyNlE2ODl5SFkKLS0tIEdISElQa2VBZlduajhsV0ZaU3dZ
+            ci9COWE4bldUMS9lSVB1eEk4TG1sVjQKYu9zby36DP+41dTV8sRdtDil3Az40pf0
+            vIhqhCTxQymnJHEwEukiEzM/w3Z85R/W/MbOKyzEZmxzG0a5O5svhg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCZlFtOHI0NnlMeWdjZEsr
+            UnQ0UFJXY1pHTm4yLyszY0dBaldtUkIxdzJvCkNQbk9mN0wzK3BtN2pEa3JSWWFk
+            RXNoemFFbDZ1SlUxOEZMUm9mY1RaZW8KLS0tIEk3bCtyaUVGM3MwNUwxS3QyTjA4
+            YVpTTFlOdzljbmI2cnZUNG1nd1NrVFkKxARm3bpBL685Wky4Y06eA18VvUZRkAl3
+            BBYPJMn3lfDgiQPvfXINfhJI6O+bWyjt0WnfodCGFM6EJLARXiTaqA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r
+    lastmodified: "2026-07-29T01:49:58Z"
+    mac: ENC[AES256_GCM,data:+m1nB00cOyr5IuxUMwvumkEIPKbYSw30UQVFyLPe+4VSOniKm8zxb52DTTaZyFyxUNAQOWewdZfvPRpTHBAgqt/HE8dMcALqPLAneNzpxIYr7oUh3TkRw9Qkk8NVsZnqTiar6C0n9xsA23tfVt5FSByqYzuAirNdwbKi2pkH59c=,iv:jonNGWQP0plAL/lrpiBKwpVcXULWl2+ZOnzBI47J1Ss=,tag:DjBN2/V5SUYq0zeYkdJylQ==,type:str]
     unencrypted_suffix: _unencrypted
-    version: 3.13.2
+    version: 3.13.3

secrets/docker.keytab

@@ -0,0 +1,26 @@
+{
+	"data": "ENC[AES256_GCM,data://9IEHIVCfHjyMNao5sCu2zNlZ/CaW+JyxVpGpD/vab2qvURunCUY7eMfSOyvOx/2WPXnWWlkoVJPCR1ec/yUg09EaSMfxrvqlu4UJI3Sxvu9xNuDszMwMMD/sCulJDiMWFNLp8qaYt7UGzexp4+GlGiqWDxk3ZEu/iLmSApzBrpciTF0lfehT4qblDovo9QXG2KDWFhCt2SwEKmHJ61Yl3pVAQnPLyTWaNhwWD/mYL76mIiDVKq7DJlvi9MBxzi3aYh/ttuHgRCvnCL0C9oUvOyT2cljwra0LXuOrum7FhPIXXboA7WTEbTHJm1LB5yLfxDrnWCrGxQzFQAwNixVHIcjuvjjvA/LifTvbj5FYM8x7an+DwXPfPhruFrDew1paAWsEGNNYXSmAlju2QLI/OwLHFFLuDyLnBKQ6RLtZbCEAOUKf2h4iZj7nH86eb/aGU=,iv:rj76+MJBCpiYyx2Ogut5UxJj3Gn+bygvu2mtuY5hFLA=,tag:yQmTvWVu8ZLug/7fo6RnwA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqcURBS2tIMGl1aWxLMHFQ\nS3ZGVTE0cnYzTmpnQ0daL1F3QnlpemRCMmtjClJiZzIrc2syN29sRXVoaXlPRUNF\nakw4RndmbEduTFg5ZVRKUTVwRWpGRjgKLS0tIHZ5ZFlyODlGWTJoNGpXR3RhY0ZH\nOGpPdDZQVmt6UWZXbHkxQTBoeW1JencKbsfH1V1lUj8mmHyLNj36VaRDgaBojcDU\ndoQWmSEXxjticJqdadbVKb3UABpvzAZxASCy81sa3wH0gT+7zLaNyQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrbFdaMmRjcHgyV3Y0eWZL\neG1uMEtpQVUxQmNTU0Z3aC95Zi9uQlgvMEhRCmJiVlRTR2NocGlnbVU1UmlLVVA2\neERiMXpiQlVPNVhRU09XVGxrY0Ixa2sKLS0tIFRuOHdoelJxOXNRRHBJNnlhSE9j\nUUZHZmQzOVFwRmhFV3pvdnpRMTMraGcKRHBuSUpbHaEzH2tuSBE5MsLJDCuH3vUx\nO0jnDldCWkCw7Wvr/tQAkaDI8axZcYVDUkCEk+xqAdxDozLCPhEO6g==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age17e89ty6p0fw24daanen57wg8uald9s025t3wwxsw269svwpmgvrshfvfvt"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZMW5taXh2QkJkT0JjaVpq\nQlBjVFpFYUhXTHhOT3prbFcvUFB0YnRGUzMwCjhHT01lY3dPWDE2dDZWZnJza1hN\nS1AvZWIvdjZoYmRjWlliK1hiOEdrT00KLS0tIDljWWY0NlQveS9VR2hOSUNyTUtH\nK2gvbEVibmZSdktPemdEQ2p5U0Y4d2MKKitoTi2vbxJ41IoWMlj4vO91Ahpj0hHP\nrKCPyx7ws/IUMmKGyvDLpZ3pYqHD9jl5pLfB05Hh4Emhv4lA1qtwwQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age17jqc66x9yeshfgd9v78mj483r4zzarqdtuxtrkxe4x5mw679gphshd94th"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3K2dVV1BHbDk2Z3FhNmdV\nZEN2Um94K0pma1k4b1V5ZUJxQ1hWU1grZjJNCi9iRVVqOVpEMmtBUi80NThlYldY\nTUtZOVErT2VSWk43NndpVzBlL1lQaTgKLS0tIEpCM3Q4NjM3N1lpUE56N04rTXN0\nZkZ6TkV1TU9OV3dpc25RNWRpVnprM00KItUzKBdShakOfX8Sr+k906nsvYPl8QLb\nge//1GA+ukGsaS9rcChOY89vFdm61JDmj1jXSJ0CN4wLMW9/eblZRw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1arhf2q45zw6wf2uevju4savp575x3m2tfvved5zzq3ay92ynua9s3cm92c"
+			}
+		],
+		"lastmodified": "2026-07-28T01:44:28Z",
+		"mac": "ENC[AES256_GCM,data:efFXIqbauOURR7lrVpK7kqRIPgYjgWfenBYoX7mUrQ/thFc+ApcAx58Fi1zg4biwIs7NarJAgDqAxZi+yKb2Sll8H/wlsEjWDU4iQlLJdIQw7wey9eDW8pgBLd+6E7FXrgn1Vh49dS5GXlC6clAYk1lCiB4NvfzPDy5Ktpl+Chs=,iv:+zCqj5I1MLJfRRiIrqgocYB49PZnlV45PUTH4gYlfrQ=,tag:WxY1sF4kFOTEzbSFcfJFbQ==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/ha-corosync-authkey

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:JwjmYg0qzoer+8/jv7KOfK/BxmpObHOn04GNsUFQd6hSFBWbQlRbODfluL54hnWQ4gsS9MHgwWh8nrLciBhye05kRQCqADB7Crm0CZYubNb65WzM/devh14jmerc3MYIp1M3VmDisML3x5IqhJkFMSqKM/VaNAC3f0otcVR8TTgZw/fJyvPlfqpPTjikGmo5xg++Yk7Cy8A1Dj70kYk10+EQjQ78jf4k/agLoaS+YvMmKXFk4EljtHg8Fe5H6Rn9nfMSSKGTZeAH3Riix2e+fIH+nWEZiPHC0UlPjJrB3SeBqqGRSUUJshVKHFIdKI1Tq2Ea+GDWrAyIRI0qO8e7WQ==,iv:mr39NxbZuYUDHDw0e+fJDshbt9R1hLgVqkUxUVX/+l0=,tag:u31xZRPfvH5j3Sw5U8/nqg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtVW5ENXlkUW83elc3ZnhE\nOEZ4MWIvQWZ4aXhuUmZGYXZNQnhJemsvOWlVCjIwNmFjS3pJOWEwRnhBOGNRamlM\ncXZid1RNYVBXOFZpMlpnU3ZnVE1scnMKLS0tIGFhWFZFd2JPTFNrV1VHSDVKVHRK\nbmhoeFh6YlUySU1ZRVhHbjZnM1IxMFkKo7aHkz2pEeV64m+OEkBZ2V1e+PUzoChu\nUc/Mnh37dNXSSJtg2KnocHdyzDGi1yQbA72xKTxx6QjYJWrAC/jurg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBReWRLb09reXUxS0YyMUVV\nQlR2bEdYY08rK0RoR2pIelpLVEtYL0pqNUVNCklRaDNoSjBWT20xbk1UdmNqWHNt\nR3JoSlVwODVMUVBzMERUNktHMVViZm8KLS0tIHc0SXRIS1IreDF1VCthb0FBR0Na\ncGFRWTROUHF3clFtTmY1azEwNUpPNzQKsgCND/BZcMTgBTAcnHunQcT6LG1jNrtO\n+W7Yx7bFtFBajWnRYiNpUZPibQJlv5SE9os47WDH1gs86xftgV+uyA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGbllNTlhxNVVkd01GSUNi\nR0lxQWZoRXI5NWxoTExRUnBSb2FPR1BONnpzCkxCN3pNOXVrTW5GR2J0WFhZUE9r\ncm1QTThINDVVVlZEalJiS1RXcHJRS28KLS0tIHV6aGh5QjB0M1VPWDVMT1k3cTBh\nNTNGY25NV0tWSUl3UHlJSExFQVdpVDQK3ARj8xhFRYU6oqYxNQ5+Ryza86ALNUwX\n7yK/8ATnquYC8/ZIYUbgTPxzsocmXX9lVT0+ktIALqjtG8PgNTlOow==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r"
+			}
+		],
+		"lastmodified": "2026-07-28T16:04:51Z",
+		"mac": "ENC[AES256_GCM,data:9X0dkAEGJiug16LDy/8//QOkHITTzt4zwTKiIU13tRgSRXRpt/7b27+3eDauscXDMamg5wbYo1YF0+VwLl21Ip4icnnwrZBHd6HWleP30HTgef8rmo21SstshnclZz6RH8SEGVDO/vjrMDChaZ3A2WD+nbcomw47DpK3cG3kBIk=,iv:G7CScSD5Z1Z9WMEMydyeGK/RD4W43xA0PlcpmTCxLOc=,tag:DZmby/xkxpinoJztZZRWwA==,type:str]",
+		"version": "3.13.3"
+	}
+}

secrets/ha-server-1.keytab

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:VpxnzrdX5FY2gfDcZpqg1f2wqL6mAawcrdE5A9A3fQOKr4BWXmCT/hjQEtt8qG+yVOTApdG+vF8SKDj1AKZ25KsZ253WS7W5C4j8oiT/xHo0CtvVani4fZqN60lB/9381h7aUZ+lcBMPaQcNn35zoQqYthuCGrC9PH1qQgD2VVHr0v8J6kceXKp0KTJldtDsQxky6ROC9Zyc44wwkrFPgPss4/yGaRWWqiojDvK5fK8238hmR8HyfD6Tf5KhgbqCsSGS2g6yt1muUhmegaZwtpi/KpoqBmf1XFaObuHGRPKvblq4Fa0x/sSjpPogBZyNuvRKIwRhla0FkKOH1bX4gC9bzOlgiUUfVcRJT6Nr9jocYuhIv9wvIT4TujGbgHyorYaDSQLFPJoJEUa3fgpGq+AC6p1DJJE3b/wqUSngNbdOtAPE/imaEuprOfnbS7cRx+Ti/Yu+ZYZuAlDtLkSVdXi1hQ4fLQ==,iv:rq/BzePXa/w/Gqewz8JfM/NdU+x4ShiH1OuQT7wRyCQ=,tag:IcO3ddoj4M57CkndHEmNkQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRdWFEUlRJMlZId2Yvc29F\nNnkwREkwQkllY25vZnRjV3Nob3BvNE1Zd21NCm56UVg5YUFGbEc2ak5MakxDbCs2\nTFVsRFhjVlRwMXdUbGpMKzQ1N2hBUUEKLS0tIEhmdm9hZEJqUEhRN0dsSk80bDY2\nY1JpajNOVFpEejhibjdKZlNHVjE1bGcKycyDfEBZ1WYc4EfAK2y5x/nKRqq7mnb4\nDlpR1Som4bSt7+B+OCZa48mC0Zv05HbP8PnoZdCe83swGNljvRONcQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0SGFVbXYycTE3amR3ZVhi\nVFJHVVhWNWVQQlJKZUJvZ2p4ditqdnYrWnd3ClZaaUVLUXFKaWlDdDQ4QW5SQUZY\nbG05c0RGNnZUenI1QTBlemVjT0szRncKLS0tIHBWTmNpd2FjWHc3MmR3cWt4SCtH\nWS8xZnVON1lGZVc4YzZFRmRBamdSVlUKdlQTlwiExTUVsiY05MXFG2/IQt1bZwKU\nlFDnuy6YCQzawBLQZuSUAp7WSGedupexlZKQKNUzPf+dis26XEYZoA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6"
+			}
+		],
+		"lastmodified": "2026-07-28T11:53:06Z",
+		"mac": "ENC[AES256_GCM,data:PMsPSffQkxRmqT6HwFezIoT4WKJ9WDlAcflWynsFLtzbqVTREN0YGOJgMwLd0yBR2sbGBWogG5xAuKjO16aiWUVpCFPRvP97um351679ZoXIlrOL+GlFn4NKwoLH9/eCp8BteB9mv5huhEEN631lqCr+jtYO6yS0ULk6QS4+7/c=,iv:oYWV4J1JUuGryvqntVXkj/HOzRDRr9E2C1RKNvr2bT4=,tag:e4p+I40y1LrgdlK9tLRicA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.13.3"
+	}
+}

secrets/ha-server-2.keytab

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:Ea5AGwloFMyRgmQhJaDNTW19mpDvx0+dSc3ibENEXEniEIBV4Wv135mAwE6jQPbN8wF6f5ki3B5mDXHOMuhDQEqUXg6jcs4uEm2nwz7wAKYXEX4T8p0gUj7FPISiSQrw8O3jwzt4qF7AVcPsNhMtZLgrx9ssMtVmGct5do+E61Y7dFe0XbpJLynfzeiMKYvaBngTbHwQjv1mn9AMpX98apsp3tyzg8avLDR+WXRcMusds2mdov0jCoKuWPLG/srhMwjKOp1Z1Tu+aZCr9lHTT76mytpGIWGSbrP5l3UTp3oPLmy87Kwi/FJStZYdYikMnvmSo5qovePiYsLKVD74dS+d3oFGARyW4TILd9OQdXEjzdDRak5HcPv2/HhR638kvukVMdEdH0hpeJcqcV0h+BoLyIHe1e037iF1ZkCjuXE+A8LtlzRR0wqP202uzBIyg21ca4X5bRiHkkAOX4P3riaEcMgKfw==,iv:IXq4sHHsjnK6maVd3RRVZzhnxx1hOnxfsvYX8IKLn+4=,tag:jRZinAVLH8nGVf3gE4aSKA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByOTkrVVd0TXlBRzdkTktO\nalU3WlVOb2ltc1k5RG5sRE9EN2M2WUlnQVRnClFGZFYvSHdpVlhtNXExSXZMZnU3\ncnl6aHRzU3k2V3R3dU9RVDA3VWp3VkkKLS0tIHcwREM1dDREK0h0UFp0bU5JTW9Q\ncWdyanNMVjY0SnZUWUl3Y2R2SXNyc1UKI1FPIE66to19oQK2TwM5B4snGOVMLFfx\nl6JMEilDVSgy2RR/tSiovmo4NwOsv0hmmF1t34547rbZb+dB5KrMEQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnbWp2UWphSnFibnA4N0ZV\nRHlNUnBNYUNzMzNZakFVc3JXT1d0R3Yyakg0CkowMFlOcEhPcHE5dXd4ZUd3cDAx\nbjQ2T1NqejNRMnFvbTdNRGFGdW1pM2sKLS0tIDhGb1JBQ0RFQ3NxbmpkdXg2WVF6\nS25uZW1hMVo4Q1JGdGNQWWRyWE84b0EK0FmFZB6sN7uZynhzYU932x461zSZIUWU\n8oyDEKNoUuPqpWK8RktgjWlKHtB0oLXC+SLAOJEhtnhWF+GSqloGfw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r"
+			}
+		],
+		"lastmodified": "2026-07-28T11:53:32Z",
+		"mac": "ENC[AES256_GCM,data:EhWEEIamG62xHI6SqO4tzdK6gcEXxUU/UGcF25m+X11kUVXzHd6yLcfiM2DvqcwGLmwmOOf3SpFLABlQUzTka4QRzTMFIAJi7x1rzQV4J0YIGITHk9rgvy0V50TI0loONBb2du+vDt8IlTBNvF9WMbNCki+fHPAjIemyoRBdubo=,iv:wfsx0H4fD3L7WeGzz57SzZtajBr/xTRDljSpz22DD7Y=,tag:naDyGH0WTKkqgd954Ya8jA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.13.3"
+	}
+}

secrets/nix-cache.keytab

@@ -0,0 +1,26 @@
+{
+	"data": "ENC[AES256_GCM,data:z2bazlADvtWkGcY4an7NgZgIDSDq8KmYpSq8tcBleW/33HlElMNhdmje8m8hd7WxgrVvCQDdlAfodBLj2mjbgSwBptlPX2zbwcmJdm0g0Ope+YyQZucIio7NnBaYa+OK8meWqmxO2WEeOk+hnNfNPbt8MkzF/FMMbP8xv9wSTKP9CDdmO7s8rdg8G1JxVfn9dyaSDUTuA2JGNtbh+osd4CtNCV/bc84MjXDZTgTfT0W/uv8VH3iPNT7BhQbU9FIY4dsFRYBf758VtJ+3mTBiQkR38IevJBvhFB7Wiqn118LSBgjdTKJnBwmGo6lMdo0LRYHEgUnb2ejhWMNie19x+LEAErDki0RBmjc2BbnOF908kUlW2fUPo5fz/hlfqlhDLS1uOxESKhfnKt7s5qzWiMX/nohpknVfyMbmu9j/M571z9sgFnJAylpW3NLp53lX8N9mINwRGXsyL/ijbNE=,iv:M/MNquAGN+lIIyVVvFRgKR6DKAhLclc/OY1HV5Xwoog=,tag:eEuURMErSqsoh+l9ZaHY9Q==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqZlUxRG0zR3IzMlpDL3Vl\nNmRWQXUvc1NDek5wbWtKVEFTcTlxL3ZoWldNCmJVMkFtbDkxN1FGQ2VLTlhBUW5Q\nZ3VzL0UydUQ2YnlUMmtqa1Z0d0FSVFUKLS0tIGJFelc4Rm84T3ZHSGZDdnNUbUFM\nRXZHV3N3aVlSOXhUZk04UFRMWU5TbFUKhPU80PVYuDFUCxu1CA8+W8bqkr+Ne2fh\n+nBUPJbGxfN9TyD9tUC77AMbcL1R2L5x+SSAh0bEgSWE24/RjsuVKw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3M1o3S2FibHV6N2tIOFEx\nRytJRDFIUU05azNDbWlpaDdFQXFRWlBUNnlFCnBwNVB0SEVwL3NWNS9aeG4xSUI1\nV3NOcUFxUUhaODYyWXdFMEhWeFpvancKLS0tIHY4NHFQRGJwUURQbWZ2cGlSdm9s\nOFJodGZkekk3UnBRRUwxb2YrQ1ZPcU0K0tCapb1hfVHFSNpmESXexYa5k9OE9Tha\n51QpU4mZHKGrnWPK/kyj7rHiz95TLsmwUdA8Q2hOiYNSxEaVBjYDQw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1jcx3yajjhghn8qh8za3yeu8nxykzlg3p4nrv03vnfvzl0mzayg2qmg940e"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkTmNrbUFoVFVGcytzQkk1\nMUxDdmJGL0xDcUNZaDFRQnUwNXoyMU90SEVvCi9EaTlVOGlMeUZtQzRJS1J2TTJY\neWdkMGNFMWZzQXVKOU5KcXVvalk1ekkKLS0tIDN2MlRQcG5aQnN3bUMrSDdvc09X\nM29hQ0pJRzk1amRHaS9mRlhrb2FMZG8KEWkSSP+MqGRU75qo7ctOCL7qHhyCM3l4\nL+ga1XOKxRsQkKQPozCi5Bm+k28W9hIDaC34Rw9difxICM9Z1rAbOA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1ufg390ydrmma849t9xfkxxl5xvdkk6mngnlzhmy7mvuaje8sgcmsmnq6l7"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1TUtxVjJ2STFpU3RNdWpJ\nVjcwUHVORVN6L1pKVC9TeWt4cTNoR2pPWVRBCkR4QVJ1bHFUa2Z5aWw1bG0vQWVy\ncjNGRDl5REZzbzdZWTFMRkU1eHFTc1kKLS0tIHNLVDFiRkRKTHhrSEx2S0J1R3cx\nSkdEQWJya1ptTTRqUjJZT3FFaFhkd0kKgecHrnzW9+Eb+b7c0z1yR+Y0Czsr9Kjh\nx1UUOfleHntJUCs8oYcOogKknEnhBJJoJ5oe/gWruRSwle1fs3cBqQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1jlltcv5jcnm40z5k0q6hv053k2rqpqvemtuecdwn527uw8uqz4es3x7m68"
+			}
+		],
+		"lastmodified": "2026-07-27T21:22:58Z",
+		"mac": "ENC[AES256_GCM,data:VG2ygV4X6yMxGlJgq3sN4GAzgVqiD06noyMxB2yb/FCmiFVYZAH/9LqF//G7MTInfIjFrTDyoHAny+m/ztUrJkHXI1Fzg0U8R/zYyQ6wj/GguL0gyhA70uriQhvHsRHefHPa7Km61Blo9cME6CBJWejtRvh1IMw9itAioVrSIZE=,iv:zjxNG6IYxkcNqLk08NADoDiIbS6at86y136SBzadW1U=,tag:sj4ZRR7ixU8h7aOUJmTCYg==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/nix-cache.yaml

@@ -1,4 +1,3 @@
-beszel-token: ENC[AES256_GCM,data:meuzUP/6wCssJDVTgbC0XwiLZPMGyDl55HEIiON9xOXCD9k6,iv:TDqWcp+8Mxd8wN09r5otQRQXq3XTeQphaTWxvvuLTAs=,tag:cRPZQGlwB/dTguBAheWPQg==,type:str]
 cache-priv-key: ENC[AES256_GCM,data:6vQKIf7eS0WNL2Eptoi4VWr18SRMZfN/H/aFUUtXdMYQY5LLyBp2EHRKqZcGFuh1nZhUdAxUztq/CVXx+QFxKW+ElHxCxUSp0QqI1fdSkBkKZb8hlit5SoX9JtLzZGg0HBNM3nJu,iv:0J+xmrPJhInHhFR/c41ACjuTfaIoMkQFSfbL2KkgFa8=,tag:f4s9Szs5oprVVRSyXaX48A==,type:str]
 sops:
     age:
@@ -38,7 +37,7 @@ sops:
             IqA477+noQ/Rjrszb2hEvxID7keogcDUWMWQzQvdMc22+3mvAr9qIg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1jlltcv5jcnm40z5k0q6hv053k2rqpqvemtuecdwn527uw8uqz4es3x7m68
-    lastmodified: "2026-07-19T23:30:21Z"
+    lastmodified: "2026-07-29T01:59:11Z"
-    mac: ENC[AES256_GCM,data:kLGE2xawQT7mx+sfw68hmGk5nCEGiEjZrqTEl9B1dtQmTrMwmoVr/1RISi4LfJrwxy31mDgff4lcIL4wIJuM373uk3X8j4RNyYQNTfKEkORT6r8NHeepNs267O77pKGd7OmcM4MT/BqOnB8ELS7Wlf2ect7CAlvUUVyc8icxgZE=,iv:EYLDsHYHZ1XOQXafOTqHHWpk/OBNq/R6IJnOBYV33E4=,tag:thxrCPC5oGvDjhK7Dz87YA==,type:str]
+    mac: ENC[AES256_GCM,data:/nbcfause6G6F8IvMoyPZtkWS1XRLAivhwTFu6y5P0Mm0eCcO6M7/rgioN9dngKzPXrCUl3Dx/EvhrrWKe2/Saq9WEOFgvS2V05pTRbQgYjuugVzW2paPq1fgmoDYNjHz2yFYWAovbIFtjVxMR9tmcASMTe6r/FocvMXbFCJh7Y=,iv:n3KoaPPtDhWK8mxJNJRk7WPVUUNR59nZSA8JyysWNDc=,tag:tYhkNXsj2ohVjHkvxife2w==,type:str]
     unencrypted_suffix: _unencrypted
-    version: 3.13.1
+    version: 3.13.3

secrets/nixos.keytab

@@ -0,0 +1,30 @@
+{
+	"data": "ENC[AES256_GCM,data:apiijrtrqd77CTizITg0R35BfCi8PBnufpxIyC+hLYqwoBzP//3z/yjFyHPLG98m/c/qywoi3Kn+zsaTT7MjP++9OMhhX94YKlSHV1/cHB76OkwsNc+ClqWxl6vpaFX29Qvh3gFX9c/NR3xvYQutYwrIrQ9NR+t/M52IMC8hvtR1LQy0ak3VIuXJlSnG2r4kF2Ym1iP7phjuq39Gd245Axzw8OB7yGvOjNxSdTPxW/qL0fMlzNcMrjr9hw15WlqnZfWPOsB1+gZjHXpGfPD5BCbAAMoTRJd75vhKKXP/ERhIffewuuH2x/QHfSFvXVB3QyhBQMxd2b8QEEE5cjvcExOST3tkj6QARkzoUpRT7AE3jhl3XZ0uA2qu9SwyrSvbr0tBRKxCdK0g2E2/hqwcK/Tck5GB1eKb4aN+UkqxOblNDH+B1RfDoyNAuN+KEg==,iv:0p+ScrKpP4kQvO52gBAlwAis6oAzZ0EHFnU74hYPrn4=,tag:ON7qOjztF52xsJWAou7ogg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlYXB3cVAzb2xEZ2pGa1RJ\nbS9ZVTc4Ums5eUZJUjEvd1g1aGVyNUNramowCnptZXFOZVB3MFRFcUtzSXBEZk1B\neEtKcDdLS0h0b1h3VjRjRXRvV3V5V3MKLS0tIHBDemkyUnV6ZXhTeE5VOVVOMlky\nWWMzVGVzZlAxMjZYUGpQUCs5QmxiYkkKcuBshCgWX4TwfVlQ5lHikzvwWdLEXWD1\n/uSiy0J6yMSiu8u6cg2SxeFrlKJ3j47dDlT6WHCxS0PfeEA0bJb3LA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGa2VTc2NWdkFRNUJxelVR\nWFk4RWxoelYzNHo1UFVhU2ZkLzEySlRWN2xNCmNmcmJod2crL3NMRlVsSmpmVkU2\nMjlXMktjc3piUVNhUXlTdnVGTWJkUTQKLS0tIGQrMUxrNDlNTkRCSUtFWkxRdXgw\nRlV4ZmtYSGhPQU84eWtiQXVqTmxUK3cKk5fn72UZPH68t5ZappfAhZJwzpLkfKmT\ny9TbUPIr4Pbrexau6YiH43QIbDQFdwYPfkBjGkd57zCg8AVo1+MBRw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1adur9g330gua4l6ndk8cqjg35qc8yxwgme6wrl2hpylcc7vxm38q05ejuy"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArNWkwQXFWT1RpaDNvbzYy\nQWc1aHhmNHFEUXVsQjZqb0EzM0wrV0dwN1hnCjFsUFJiT3REK05uSGRWTEw2SFE4\nY1FleE1XVjhBbndiMmZxTWNTYmhYeVEKLS0tIEJiZzJvS3BsYzB3cHIxa2k5N1Ro\nenFFZDVaODNnVGdBZTBOYWJwRjQzc1kKlXJgee8wTSN4Beq4P0t9cYbk0BWHCseQ\nyaWpiPT9aZBEGLFmuEd3zKABc8lrilX/ySTmOG49vRg6CPmr7cT0Wg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1rrxqea6q6pn39sw8y5te63h2py8jgjl9v0jyper86w3ggtn67upqg3ah39"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEa0JYenFMNmNzWnVmcXdz\ndG90ZUZ0WWlIU0FCZG9OaWpBM3ZDWnFhZFhNCjhuV1FTOTJ2WVJGa2RuNVV2MjR0\naVNXa3diaWxWUlJtdkNOQXZ2R2NsQkUKLS0tIDcyYXh3N3B2QmNiK3dzemFFMGV1\nNTZpTk5yNGV5YVo3cGswK0NLWFQxQlEKIe0N5OxooWXzt1cUViBmjihmGEe3G6/f\nkz2/IscnG78ZvNgYKjdoG1jlsyje/3zI4C8aWXLq2DnIyxUyAhPgsQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age19mn8zrxl8zpps9yvrh4euquvygpp4fp8queg7xc6qhtnl4ng8c9qx02qwn"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2b0NVVm9QNm9waUUzcXBi\nWnZWU0JETjZRZmprZVRUL1h6ZHB4cXkrdm5rCm9GZ0VnTXB3S1BYSmlGWFJVcDhJ\naUl3RjR0ak9BRmQvVk1GRnQxNmtYM00KLS0tIFlTU1p2OHhWUGlOWngwbE56NEhF\nRW5QSkVVUWZpdDZXWEIxZ1BkbzVwclEK2P25nBgf8255vaKW/+T97aNTecRgNjLu\nedIUiPdXbFATCe3v/YRo6sqzFwIsvM6Bl9yHh/SXo6Ftc7eWZZd8zQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hrx8qj02fj2ea6d4g9vqhyj9hl7fppkjqfdx2l37py3h6pdkr95s8n8rvs"
+			}
+		],
+		"lastmodified": "2026-07-28T01:44:24Z",
+		"mac": "ENC[AES256_GCM,data:J0D8bEs5mHLraLS6TvYuCgfiNU1xKM2Yfb5Y0f/q/4wM4LzXufNzv3+SWDHumTe328U8UnNXLqjNHEKL0bZi0coxpU5hVM+BvPcmqD72vscETzbQ2hnU05sfW+XjfZhcN8/ke0bpLt7nP0crD5hsZv3esV1E2UWvzjEiYtWzFHY=,iv:lcmXYG2H469UKBYDndWKMO+GP0mSGLztenm+kBaUdYI=,tag:ujiXbLM9CUsuoFwQQWI84Q==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/pxe-boot.keytab

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:Q++XWxg9tvY7ugT8+8FWCC5jgOfQ1+LYLsnqN0/WGxTf5XT2VmoXvszcE/ow2BUOBG3qBTXS3OHYFPwmj1GgaBHB8rpdXX6+LveSBE2gmx1VR+NUDTxy+0/DBq411MK9n/J9eIYcybdFIE11biFSAob9EgfxBF5roCDXIPDPyVCSe6LyhNvqYPnGQsfHCbejSewLTcRQEiguP9BX96CpMPIpmaB9fHN25t5RWCgMI7MtacrRxRsyKg44+2FZstXdZrp2Wv9u86BxqdAFqtZE8qpPeGdrdzluZx9jhnw0wZPzHdKg5wS7/UrLCb0UxIQiDxDMDuuBuLGkKXfAhb6SGZU41wWAWYk548iGRUaG+79BO4HhRZNObOFvfsjpMXEfcH/Vbv/wHVY4OpJUPe/ZWK4wpL9YrY4nT0t62HH7MirOy1uhwLE50h2+6dHKj6ur5G7thkSqgzVWXQ==,iv:zaRBwS+gfXLhH30havn6Q2+oPWuLV3qBfbOj00kewlQ=,tag:Vr5iEQ2u+9YNahryhgzwSw==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArTGJnaVhleUtsWnlITE1s\nNmRCaW1QWTFNSG5LRFdmb1lzRlV5NWl3YTBNCmF6eUN6RkJBZzc0MmJJa0dKTW01\ncVh4K1VLR2lURUpKQXpxNGpQNnpSUlUKLS0tIFJONnJFWkNCR3pqalRsUW9POVBj\nQ0pFQ3ltKzBETTVXTW5sV1ppWTFJc1kKzxUboNZO+Nwn2eTWy11VP9w1pRswCHaJ\nE2dYU0oUOClVzc0oSuIJxraG6TPj1N4WGC24gS+UmpkmSuCiOeZBsw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDSmxNR1I3dUFKT0xUVG9h\nWkRIakNVeWRQOEN3blNRVjZlWHF4K2NRa0hBClRiOXlmTTJ4T2JTUEw2c1l0R2N2\nMnRwdDA5bEZlQWJRTm9vUmNKclBSU1EKLS0tIFcyeDFjbTZyVEVDUjN1VzU1VHly\nNWNDMW9rTXY2bHNWYVR0SmtMckovUzQKhTWr6yFVW9am3okCiIswwqR5+/p9OLmB\nWCgPtwoFaBt1RjUXPK4/eS4LlucR2K6V/mNMn4xVsnkIl193U9632g==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age16j42pdc5dr6wnj7xayhkqdj2rny9u68fcqejs50hqq42scssh4gsnrrnlt"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2ME9CbmVGcENRWksyL0pW\nOEJyUGhpbGMwWlBhVXBSeXQ2MW1EWnFuR0E4CnMwU0pjdk1YMzF5ZEhTVFlBaHZq\nam94UWVGbjhZSEx6VHBmem9JRWgwYzAKLS0tIHFJZWRyRjRHNzhXdDJSYWN3bDlR\nYVp3eGJWWkh3Y09ZWElyclZQN1ZSVFkKjR32//EcFAdMjVlNgky5zvVkwXwEN68D\nrkTuHKjiO5aV7yAQGPkdNw0UM0oRGF0u4YF3oOUcZfSvnKgDeoi2Zw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1ug787sgt6st6k82fgkrug2lzltw4qsukrrqqs3w27ewwqj8rg4hsxcmylz"
+			}
+		],
+		"lastmodified": "2026-07-28T00:27:46Z",
+		"mac": "ENC[AES256_GCM,data:AbJIHYcFpeanQsJ3x7RPL9Yjlg5BJgkepKax0fL9L/PpA03Antab93iUNG95Mp6k/duovp8Jm445lbuppDZq1dh9ij/deBa8GbzJ50wwEe9zMc3EwRKScpqZEhRPF7KlJsIjsHJyd8NkcI5ji49XkHb4Ae1//8zG5HpVgy+3b04=,iv:uxQCbMwMIfP5S1dbsvIx3F79YWEguxwox8T0YZvUBdc=,tag:3utjmBGDmPc8q4JjaXvCkA==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/server.keytab

@@ -0,0 +1,26 @@
+{
+	"data": "ENC[AES256_GCM,data:aqlMnoVkGtH9z3fJRweeC0OYf7LGqJU2sWA9Q25dKK6NuNyJd4BvjPtpfeg/WhVtJsaOtcbwVm4WAhVK9FARE8g8j+vmq0f6BAU4s6mx0ZIhl+mP+/hIpt//LOdd+9YezelJxdpzUyZbdAngU99rsTluLRe2XmZ7Fquxd8yH/OHenSDY6dizp9+5jfEi8EU+EmuXvuWMPY59xnlnqYNPfSFxs43/pS402LzJoJ5H+cBPprddkUBVzy4cBQvMnrRFUSjnqp74ovZkfIWFqDWQ5YgSU2PjatBg18oulZ7wNRhQ6OLqj6gsu+xrMjNFwnp7rMlA3X//hIidxTkVcYITycXd8KzuMIaofUpnwoyT34fy6+H35/39iiEyG4LRTrOOKRDzXkY2rhJUxFSZ8GlhNhMd0RlkmLngVYrtjsswJ9meIwoAFLPYt7BC61PJf0TXdtk=,iv:mCA819J9LpAOj8QxFAkrHI9wFJIy8qVxv31D6IwWFnk=,tag:qqMtM0eyHbQEyl6ND/wf7g==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOZVpRYmNMQ1E4ZlFXMDVq\nbW1FcnYzWTBHQmdhSjlsZUtZK3dzNUZjY1FvCkV4VXU4MTcwNVZoZUpVVS9JeHE5\nbGk3UzE3YmpZQ1JISHV0RHJqSG9ZNXcKLS0tIHhBdnIvMjJkeWlVRE9Fb0FhWjNX\nUjBVWDltK2w0akkwOTJaTGNSWWNrRXMKqZRNnHiXvn1QBoSGdABp7vOqNlsEN6Xr\nDp3NByXow6PuRuWvQXHzd+WC+ADkwNaaiT6TUrbZcd/Pl8Ges9kcZg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2VFBxc2ZMUFp0YkJhVFhW\nc0ZNbnJUcGtkT1VLR2YrVGhRRFVTZFlyZXlJCmVnblNBNGxIdHh4Q0IweWR1b0dq\nRWFyOURuWkdkRE1RTnJRMEpXdk9HaUUKLS0tIFhKV3FoWW1zRk1pMlhuWWlhV2E5\nQzJTRHAyc0JtSjd2NHlJODZVbndaVDgKFA4565X/4FqNq/fZDZTg81/55hZi4c7b\nTti2AnyE3OcY/kurXJFHRinVMqURQf1fx9MxqUYRitiCz4qe5zFF+A==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1sweerhrga9yf8x6sv0apz4ed4g48rnlcq34rpv20t0rcelwgpgeqwvndzz"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwRWlKTFNocnBRN01pZ0hm\na3pyRUkxMWM3dE1iTkZZcGRYUFcyb1RDb2h3Ckt4blpGZHBjcnZ4SVE5UnF4cmY4\nWW0rZG5wUkZINVk5a2lmSzN2L3JJY1kKLS0tIExpWGlBRFZJVGpGbFhxRE5ZYjBo\nQVliTlFIZ1U5dE9xbDhHMUtxenpBSm8KY6sIFEfK8p+70IXsC4Jwb9Lm/pd9+V6K\n4JAzGrpA6mAuIwwSNnbdcA5j8FmBhCpK6nLBWmFhGm9Y+MRTaM7Jrw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1nruncs4l0ufk7yuc4des8p99c0alfndl0lhsws8tycl5pplfp56s30af5f"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoV0pIUXlyOFFTUHVTdllu\neG10VENOYWl5Wk0ycnpLUytTbnlrQ25pMkFRCnhyRU5xWW9vQXJJOHdFMGZqMkR1\nSktqT3lOdVVPRUN4YTlveGp0NXpqd0kKLS0tIGRaTktra0ZtSVpzSHBrY3VSSUph\nRVRZOSsxTTNmMmltMlJnVy9oT2VEWU0Krxf49B1BsrWn05fqg+cZ0k0PtfJJNfn0\nUL44RUWXWbK2igQHaIct9DfYe7DEonBJeROuxDYm8g7yNOv15S+P4Q==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1529taqdwr6t0w7cvzmty0d5y5593wffl0krt48j6uc4u39k56g2qf6ywtp"
+			}
+		],
+		"lastmodified": "2026-07-28T01:44:26Z",
+		"mac": "ENC[AES256_GCM,data:eyOSn03dzHSgkshPzLVwc95382eEFaDQarHs9l83dtcsb1Ui9CjkKipl2DVSUb6bdMUH1qKYmXqJhwFnAZbFZFjT4VTKtutNtM+OkhVXfT+fJs+1+u7k+ZUYFL9HuxKA6AWpwX3eJ8vmJDJZaAayJbm4PRzOyJywvKeneQqdUS0=,iv:QRKltR4qzVofo/Elt2Us/lrHlD7BenX605X31x+Ng78=,tag:SZ97CcfgMJeu3yqNk8Y/cA==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/server.yaml

@@ -1,43 +0,0 @@
-beszel-token: ENC[AES256_GCM,data:cbQOXhLzNk4g9d6hvm2DH7Q5ApTPCTzsW2txflDT2dD/UPIE,iv:V19MI1GEo5/0205Hrt7JImfkjduFiZ7f9aIkDVaI8mU=,tag:WCArgdrnIOudVe/Tw+oxRw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZeDQyNFh3dm1YZ2FTMjdK
-            cDBWcDE2a1c1S0s1enhWRnVuMlVJc0gyTXk0CmxWNkRJMDhkeGpRTGltaitnVkZS
-            a3Q5TW4zYm5Ja2FETEhJcGF0N2ZKbmcKLS0tIDN0a1FqRGNOY3Y1UWxvUU8zWU1m
-            bE9DVzZESG1HTEhVWUdJOTF0bDhRVGcKP6OoyDAGLB9jQ69jpFyho5eaeK9XtZgN
-            RlSJpBm2Jo19h/crpH9AWXUAIG0BWueyr8mwBu12cQdFIU3IyZT6gg==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwRVFQcHpkQkNlSDhhM1Zp
-            bUZyOFduQXg1cDd6ZlNObFgzL2hmUmJhM1JFCnprMldPQXJNVW10dVRqQTdWcGlv
-            RnBYWWFsaVNrMkJpS0pkOGlQQzlJVVUKLS0tIFNsMEEwZTREZ1lwWFJGdE5YSVVU
-            ZEZ1bVpFMEQ5N0g0L2RacUpLMWQrVDQKxPzq6f960purgAmUJw6IZnZSnhkzNE8r
-            CSrFDowKTZI2KRdCtQ5fGhEoWO0ZPgVNxYV0KH7JBttylcpRLm6r5w==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1sweerhrga9yf8x6sv0apz4ed4g48rnlcq34rpv20t0rcelwgpgeqwvndzz
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0czVoT3ZyYTZhOEQ3RWtR
-            bVl1MW5vNERxUEpmNXV0MGVBRE9ySnBjT3pzCkM2aUVpZjg0SkNVTnRRMlhyMTN6
-            NlFrZDVKV09Yc0tuKzFzR0ZtQ2t6WkkKLS0tIG1mbUNFdHBycS9UOGc2cjNpeHVm
-            NUd1NThRQlZXeG1WbmR5Y3pTYXRKc3MKwSnE+0bGmxOAQUje6jHxuzIIyD6ZAwVz
-            b5AAYwbGRagKj6fimsHBUmi4ohyG1huIGGOU8HiUYpu4PGJgOscztg==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1nruncs4l0ufk7yuc4des8p99c0alfndl0lhsws8tycl5pplfp56s30af5f
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlVGRDMWNDZUR6c1VpUjRK
-            WDY3L2lNcWFFcm1UV3RPMjlqYnBGVEJLcFFzCjFxck4wdlp0Wmtzc1RKNS82MXpK
-            ZHBzOGhkc3ZuZUE2UmpUSTgycWdLSGMKLS0tIFUyYjczeUFWU2FyMlBTdzAxMTBE
-            VzhaVzlSL05nZzNmR0ZjNEFPTXYycHcKfiJ0KjdxtLWsXxsWKzAL+H3hYYjHrYO9
-            BjKknq1ZQJM0sB/Tid+GLqDwKi966MQK+AwHF5MqbsHW7eE5bO1nwg==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1529taqdwr6t0w7cvzmty0d5y5593wffl0krt48j6uc4u39k56g2qf6ywtp
-    lastmodified: "2026-07-19T02:30:40Z"
-    mac: ENC[AES256_GCM,data:rKHZjU/MH08ASTlu32HZO9uWmsBYuMCEC6M8gwVhzuWvmablnP05tS2z13XfaWaCEUXk6kmGJKuU0zu5+IKVZgamCF6DAMtxQb6bVCaLsoAm/GSqWQ5VI9eHqgnSSdN/o3ul/33Rf8iBQo4aw8FFAmDVuNz8bfAn0QefFTj0ByI=,iv:JD2gtqRinOY77etg6PUmZNovkYl1Q3F6ZvRi4x7RznQ=,tag:/5IMpWKRVt+l1luCTQE0BA==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.1

secrets/tailscale-router.keytab

@@ -0,0 +1,26 @@
+{
+	"data": "ENC[AES256_GCM,data:+/1AAVha+86abQIX8tebqPJ6BIcCCiTJgPe0OfFOwA2Mcx0NJdiPtQgyeo3G8fAolIQOFkk5reL+GUhlOz4iEbPOCWwFCckdH0tXiCiVcD35qQNMRurY8HmpM5FWFnyzBkKuyNZ8okPuo2+fA3NQh3gs94rpm2kBsfS18xvrlv9b8u1JvAxlH6GRMN1uUgFGmOXlpGAVjDUFiKUHN8tRSZpsxG0aKKPBZ82JdKgYfCiCQifS1366gIFrsjGriUC+P8wkadRnD1JE2ZAGSL7wJLaRmzA4LAMGXFGbitOsFxKxX3q8VWEXaaL+9h4rTe0WCrvmgDM9NUeVHpfRiJrLvQgrFZMUxuZF66vQVsLybjIWpYYJcjC6nfR0U3lMa9gjzOTMPxm8HdGKC53FhLM1HiPoe1a2Nno283fV0uaByMPmo257O6l1CXqb6KoZOGFccm7fKR/VnFAS1AInF+szI7/r+UcaTW6LwYeV3OwLE4a8b4OiqTLgCu6v,iv:kUlVXF4Yl5HGvoLwu9loiuuDVtz0kARRiUU3K+BPL2A=,tag:pVaGzQAX9Etoqc5XMGmNeg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6OUVUdkoxM1M5THlYZ1ky\nYWJ5RjhtRGoxTldwRkI2T1p6dHR4Y3gwR1ZrCjRDS1hTM01lZ0FSalBYQ1B0bENs\nUytXNmlrUkV4TmJkMUM1QW1ZaExIcGMKLS0tIHdHKzdEcUxvM0ZYUEp0a2d6SHp5\nYXQydy9uNG15V2FhUUd4NEFTMTNNMEUKkIzKEoYzoVs+nhnpkHFgDkQqrWykatND\ntsNxcr1SXSKeEW1m/QpXZnn/aW3zSQR09PqOHf7PYU47/AdkwrUaAg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByZENrck42dWJjR2hYUVBl\ncERtRjR6dTdING5nWXlBc3o3eXFhaSs5NkFRCk43QXFUeTBJR0U5dG9YTDRmQ0Yx\naEo2blUvUXNZY3dpbEZRTUx1elNzdkkKLS0tIHJwNjRNM3V6WktWZ1BlUUtLRTNI\nMTVEbCtYbllIbTdxTGozWUluS3pIRXMKZCruDIkD/JofdAHWgPuaaKTDsz408ZkY\n77mhO8J+kd03qwt6qhFC5KF1lyjhwEnqrOE195+R/8Yl7hA/DsL2ZQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1k7d2du5mejsmv5rzavm4xwgpthqvcfsehduquv28nzs53zppa3kqngfxq2"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxaC95RjYybzVlU2tYdHdU\naGJ4M2IvNkJwVlNIU0VCZEFUZk5VaksrVnpFCmJXS3luTG9WejR0Rk9DbHA3dngy\nRElFWjU1N2xuc3JaKzQ0L0U2Yktib1kKLS0tIFlrK3F0ZFh5Ync3ejVWMzRKTUx2\nZDhxdEMwa3B4TFZUcWdTdktDeGt1QUEKfgYnK2lW3cZuJGaw+bAKDipLuC4S5vK2\nxK2eJB5TP/xXrp0F3lx9sc2b1FOY9Vt9IQ7zVlBqJFkzJrcw8zYJJw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1zhfyuzlq40reuqlr34gf77852nhs3t6mqfzrqmas8z6sxk7tcfhsungrm0"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwOGFkSXNxTUIzLzR1Skk5\nMGZveHFDN3YwVmpwT3VTYkppWEU1aGVpdkhRCjZtRC92bVphd0ZFbzFxVzRUcGk4\nMUd1aWdEbFRaM01FT25JSVRoMTRsNUUKLS0tIDAybmcrYjFVWkFYaGY5TFZjcjFO\nYlZoQjJjN2lFNzd4RlJuSHFlRlNCOFEKgUIPnL2/OJgz9oMYt86/llHa7adTkhs6\n8yGYGV1wtU9aYtUMIR907SfYyZ6M4z8jH2wwzpQLbwQQMLgkejl1jg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1f7usptjx9rv4rxauasve200gxtdt9jkqhhdqstlf20wvlm7u75rsjfw50m"
+			}
+		],
+		"lastmodified": "2026-07-27T23:14:51Z",
+		"mac": "ENC[AES256_GCM,data:CYhzR0Y29GXvWUBbb13s16v9hDQ4k4Xmd4FUpcyNtHe0L/IrDGkx1WHzAohNGZjGMNB9W3BgiEH85OoOo8r1F82El4/8s2prEJi8AARvQU3+2cmRjjhNCCYQWmJkoF/cZNpw6nVyWzZdfUpPvHjbuf6utlI3rtR0qiCpSo/32S0=,iv:XLKTU2Ute4jMkfRAbXiVGxqP9+fjSs6HwRv1JfTTe7w=,tag:oIqS/1HWQZ7mLWaO7GwLFA==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/tor-relay.keytab

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:QvViw/s1N7eIN3CoD27llEjriuSrohOou4Cv310nigcW8xMnA2SDN2id3H3AoKii1JlJ+qWpKn+gUmt5HOM0UlbeNe3os2RVwiX38O7eN9xFH9F5kA3TFs6Umqq3EoI586PwIVmB2LyxDnTeEEXVd7v5PFkBcfu7u8YIcNF7lpcj+6rOyHMB8uxPhrGep3yiKawFd9c9wWD0hlSSatV5tMHA1qmdK8VmDbCU/iuGwIoMzN1eZwGAXzG6LkCA63bUfdxU6yGuTboD+kN2Wbo+GZB0EACmiZoofl2wqlXuiw6qwTvlXkyauc9O5EG//PUkIECzDwiXcX+qSOM9DIBlZNth4ebhtic/PskyF09etL/gICz5YvV4ph5lyrWHq41KxljSU7QXOkGhzagruuMrYzhZb35wFH4Tie1ee2DXbGhreJr8V3Zse/zTMaD+iM57V8bvNcarTzOFXyKfp7Y=,iv:XBPhj2wT0k/yRCRHU4d+BQA/k00ZHWSKOucnZ5+PGys=,tag:Nh5MOxZIUXAYVZY5SZh/JA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0aUtETXlBMmRlTC9vK3NF\nRW4zS1FrSmo1dDN0OGN4SmV5ODd5MTRVejFJCnpPQkM2MVJ5WFlYRW1NVjduV1hv\nUWUzY0hHV05LV1BROVZVUlM2NldsTlEKLS0tIHFsTzI2SVZzYUtJWTM0MmFiUlVQ\neHEvUXgzc1pxSU1OZFo0cXhSZDdGUVEKpLVfzQEnntluUGsblnkHZJ9Jezu8tFte\nxEoV96GVHxUca6TFWpTLMqdR7NtuQGCkx295W3i1tkp58DP1OzRTUA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4ZW5ybVBvTkxibWJMWGRL\nZ1lnWEkxZ3l6d0FWbmxMY2VsTVlzOHJJM3owClNYd3BlQi9pL1lIdzJmekJUVEVi\ncWhSK3ZkZEMyeTNoZlpTT0NMUXZFWVEKLS0tIHVkVlhmQXdRQktTK3J5dXZ4aTNt\ncUw3WCt5dXJhTkdUbVpmeWoxWkoxNnMKj4XtdwmgFVOiVsIJs2Du7QJ09A9tv/Lf\nkFOq8y4tlZe0nCwRjq43sVz7hdCTdQ0rsaWjBGY90LLkJbOA+f+Wrw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age16kqfmvz4e23hmdlqresnyw69ej604s320mmd49h4hm3fhqchtgyqrws0k2"
+			}
+		],
+		"lastmodified": "2026-07-28T01:44:30Z",
+		"mac": "ENC[AES256_GCM,data:zR2WLWX7NaHA15gi4kX0jDvzUIe9jtz5bMCAggbPW+IXOEedPrddAHZ8OfPErVMfx8O1pJKkAKSzoPTAbEle54FisSLMHXp8fI0297MByJrF9pOsMFpVcDy/L4Q+pBzmB7aS9r7+u7KRVVTZT3QwG1rFWZaDs5dFTP80RhtCbWQ=,iv:ZzPoO+h7ebS+jsSH7tWMx6QK8umpa2/HFQmx9dnJN+Y=,tag:vG9+ifxw4HaBE6YsmOwXcg==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/tor-relay.yaml

@@ -1,25 +0,0 @@
-beszel-token: ENC[AES256_GCM,data:gjbT3uROiVKQOJaUeafTxjVknQO1Tvbyx/Pl2bTad7DezByX,iv:3ikf7OaT2omO8yd6G6UwYbaRBSzyvbn+NghxAe5bcgI=,tag:Zuc2EP8rUtdDhr5CzSW2Pw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJL25EUUZack1FMzlnMmdk
-            Z1hnejZRNnMwVHpmWkFNdGcyeHVsNnRsSGtZCk9OdDhhcnR1WW9ZMEZ1OUVYbm1n
-            RmZRVy8wb1J3emJBK3Rrd1d4U1dYUDAKLS0tIGJaaElvSk1sOTBOM0lKck16OUtu
-            NnRZb3U0ZndmaHBZTm8zczhWdE1oaEUKkf6fLomAHoKPhuM4e9q96YmmH+h4VrEj
-            2x0rnwBwOoRzYWutB2MVtlsphAZmZ/PK0tEecT2MM0XXayVG/33qdg==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpeEJvaHl4akc1TFdEVGV2
-            czg3OVprU3p4ejNpTktXZEpneDgrOEhrZGlZCmJwT1dhSkZneHE5UmR0WTd5UENq
-            VHJEWG1EekJLY2pRZldtVGtxTHlGaGMKLS0tIEYrWHE0WTgyUlIwdktmNzNIS3FW
-            ZVRvT1dHa1Vzc2RSakVISzdMTlpnVGsKeT+edn4+LUkVtpRUNd/gKX3H1HG2bvNo
-            c8iI6qr/l6oxfP85OrKYFDU9IGvDMxSSdbixHtojPEb5OKVurV0WPQ==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age16kqfmvz4e23hmdlqresnyw69ej604s320mmd49h4hm3fhqchtgyqrws0k2
-    lastmodified: "2026-07-23T23:32:57Z"
-    mac: ENC[AES256_GCM,data:l9a/yNRoxY1hvSkLuR4N7deeKue/1JPlSvZvJfCSNbQ21p1qR433BbSDYvfW+kXQXS8GVcfgXSd9ywNzgVvkA5lR1++uYsZBLbYxJ+s3TKWs6/yECAZ0eM1KBA0BEm7cLSsHTOwd+2WspvmCYir++FDO9XRuS3guiMnQBglDf/E=,iv:eE6GVqexQNSiLYfmTTUdUx5AO//wyjSIsr96xAX1pcI=,tag:CNsoyDZYLUt5Seu7W5wJrw==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

variables.nix

@@ -4,8 +4,20 @@
   homeDomain = "sweet.home"; # base LAN domain for service subdomains (pve., docker.)
   tailnetDomain = "tail13f623.ts.net"; # Tailscale MagicDNS suffix
   lanCidr = "192.168.2.0/24"; # LAN subnet
-  pxeServerIp = "192.168.2.247"; # pxe-boot host's LAN IP
+  lanGateway = "192.168.2.254"; # LAN default gateway (router)
-  pbsIp = "192.168.2.108"; # Proxmox Backup Server LAN IP
+  lanPrefixLength = 24; # LAN subnet prefix length (/24 = 255.255.255.0)
+  lxcLanInterface = "eth0"; # LAN NIC name in LXC containers (set by Proxmox --net0 name=eth0)
+  vmLanInterface = "ens18"; # LAN NIC name in Proxmox VMs (virtio, first NIC)
+  vmStorageInterface = "ens19"; # storage NIC name in HA server VMs (virtio, second NIC on vmbr1)
+  pxeServerIp = "192.168.2.223"; # pxe-boot LXC container LAN IP
+  nixCacheIp = "192.168.2.224"; # nix-cache LXC container LAN IP
+  tailscaleRouterIp = "192.168.2.222"; # tailscale-router LXC container LAN IP
+  torRelayIp = "192.168.2.221"; # tor-relay LXC container LAN IP
+  serverIp = "192.168.2.226"; # server (NFS/ZFS) Proxmox VM LAN IP
+  dockerIp = "192.168.2.225"; # docker Proxmox VM LAN IP
+  pbsIp = "192.168.2.244"; # Proxmox Backup Server LAN IP (not NixOS-managed)
+  domainControllerIp = "192.168.2.253"; # FreeIPA domain controller — authoritative DNS for sweet.home (not NixOS-managed)
+  ipaServer = "domain-controller.sweet.home"; # FreeIPA server hostname (used by security.ipa and Kerberos; must be a resolvable FQDN, not an IP)
 
   # Cross-host references (LAN hostnames/users other hosts reach over the network)
   nixCacheHost = "nix-cache"; # substituter/remote-builder hostname
@@ -28,17 +40,25 @@
   # from scratch.
   nixCacheHostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICuHUxGNH6ei3BZD+EfZs3l4X8uJNcjQiOsM/G4yo4O/ lxc-nix-cache";
 
+  # Beszel hub's SSH public key — used by every agent to authenticate the
+  # hub's incoming connection. Update if the docker host is ever rebuilt and
+  # the hub generates a new keypair.
+  beszelHubKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
+
   # Public keys authorized to SSH in as remoteBuilderUser on the nix-cache
   # host (modules/nix-cache/server.nix) — one per client host that's allowed
   # to use it as a distributed builder.
   remoteBuilderAuthorizedKeys = [
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFDEA1S2ikpObREgbP5uVBWMxIOGbY8B+Wx7VTZK1m6t root@server"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK+ioWPhHixlgCB9KIQ0QTHTz6A+Oo2F3uKiINLip5rO root@docker"
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPAYIT9ormlmxZ0SziyDQaUntnKI8HK9/s3Qac1ZKjP2 root@docker"
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKKKzoEPl/ZW9KBRHBcp6/ThOngGpwMv5EhkTlgC4aDf root@nixos"
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIGtOWOCS+ImHc7NehguoyD7PbonGosKMZqc9+QR3v/h root@nixos"
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHxXTQxFnArK5HXG7czeoybZebCGfxpUdusJkPn+BCSp root@server"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface"
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJEj26SL/emsVjW2YhRucJVp2kTz8WgcEQgjBEBLRikk root@claude"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII/rLceRhnDobVXQYiPceuhDHHvVjFQ1pc9A6un/eUlA root@server"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHj11bLPpRzH2oslnwFzEvY9cSgfEFtSZbLQaDm4nZMK root@pxe-boot"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH4/Sesm8NYpj73R0cbGhI0Ubvz73vIVWAnbEDTlBTdh root@tor-relay"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIBCFUUtePndW7pqtlawft1QCdHmBVs3O/c8EJO+RcXV root@tailscale-router"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZ9WKKAlP9Z7GQdgaZ1Xgw9C+vja2lqEZO5rJFpVqYN root@ha-server-1"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEGNKlaaMckd8nLWNGz4B2QokXjnnIvM+rEUv+R6h0sp root@ha-server-2"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK+XeMco7OxUpjrjZm54HogMs9QB5xlcKmElASRvrmlW root@nixos"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMseQwPpmaa6cgV5U8KhUsiVSYARG85zGa9rho0LJWks wayne@pve1"
   ];
 
   # Admin SSH public key, authorized on the primary user of every host and
@@ -68,6 +88,40 @@
   # one-line change.
   primaryUser = "nixos";
 
+  # Primary IPA/domain user. Home Manager is configured for this user on every
+  # IPA-enrolled host (see modules/ipa/client.nix) to manage the environment
+  # that IPA itself doesn't cover: dotfiles, user packages, session variables.
+  ipaUser = "wayne";
+
+  # GID of the IPA "docker-access" group (GID 50010 on the IPA server).
+  # The local "docker" group is pinned to this GID on every host that runs
+  # Docker so that IPA group membership alone grants docker socket access -
+  # no per-host users.groups.docker.members entry for the IPA user needed.
+  dockerAccessGid = 50010;
+
+  # HA file server cluster
+  # LAN IPs (vmbr0 / ens18) — client-facing: iSCSI initiators, NFS, management.
+  # Storage IPs (vmbr1 / ens19) — isolated internal bridge, used for DRBD
+  # replication and Corosync heartbeat only; never leaves pve1.
+  # haServerVip: floating virtual IP managed by Pacemaker's IPaddr2 resource;
+  # NFS and iSCSI clients connect here regardless of which node is Active.
+  haServer1Host = "ha-server-1";
+  haServer2Host = "ha-server-2";
+  haServer1Ip = "192.168.2.228"; # LAN IP, node 1
+  haServer2Ip = "192.168.2.227"; # LAN IP, node 2
+  haServerVip = "192.168.2.229"; # floating VIP (Pacemaker IPaddr2)
+  haServer1StorageIp = "192.168.4.228"; # storage-net IP, node 1 (vmbr1 / ens19)
+  haServer2StorageIp = "192.168.4.227"; # storage-net IP, node 2 (vmbr1 / ens19)
+  haStorageCidr = "192.168.4.0/29"; # storage subnet — internal to pve1 only
+  haStoragePrefixLength = 29; # storage subnet prefix length (/29)
+  haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node
+  haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage";
+  # DRBD backing disk — identified by SCSI controller path so it resolves to the
+  # correct block device regardless of OS-level naming (sda vs sdb can differ
+  # between Proxmox VMs depending on disk-add order). drive-scsi1 is always the
+  # dedicated data disk on all HA nodes; drive-scsi0 is the OS disk.
+  haServerDrbdDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1";
+
   # Storage
   storageRoot = "/tank"; # ZFS pool root on `server`
 
@@ -83,6 +137,7 @@
   # dataset or moving where it's mounted only needs changing it here — the
   # export and every client reference follow automatically.
   nfsShares = {
+    options = "(rw,sync,no_subtree_check,no_root_squash)";
     dockerConfig = {
       subpath = "docker/config";
       mountpoint = "/mnt/docker/config";
@@ -103,6 +158,18 @@
       subpath = "raspi/volumes";
       mountpoint = "/mnt/raspi-backup";
     };
+    proxmoxIsos = {
+      subpath = "proxmox/iso";
+      mountpoint = "/mnt/iso";
+    };
+    proxmoxLxcImages = {
+      subpath = "proxmox/lxc";
+      mountpoint = "/mnt/lxc";
+    };
+    pxebootImages = {
+      subpath = "pxe-boot/images";
+      mountpoint = "/mnt/pxe-images";
+    };
   };
 
   # The Raspberry Pi's own NFS export — not under storageRoot/nfsServerHost,
@@ -129,10 +196,25 @@
     # (modules/build-types/pxe-boot.nix).
     pxeBootTftp = 69;
 
-    # `server`'s NFS exports need both the portmapper (rpcbind) and the
+    # `server`'s NFS exports: portmapper (rpcbind), NFS data, and the
-    # NFS data port itself opened (modules/build-types/server.nix).
+    # mountd RPC service (used by showmount/NFSv3 mount protocol).
+    # Mountd listens on a fixed port so the firewall can whitelist it
+    # explicitly rather than opening all of rpcbind's dynamic range.
+    # All three need both TCP and UDP (modules/build-types/server.nix and
+    # modules/build-types/ha-server.nix).
     nfsRpcbind = 111;
     nfsd = 2049;
+    nfsMountd = 20048;
+
+    # HA cluster ports opened on ha-server-1 and ha-server-2
+    # (modules/build-types/ha-server.nix / modules/ha/cluster-config.nix).
+    haServerDrbd = 7789; # DRBD replication (TCP)
+    haServerIscsi = 3260; # iSCSI target (TCP)
+    haServerCorosync1 = 5404; # Corosync totem ring (UDP)
+    haServerCorosync2 = 5405; # Corosync totem ring (UDP)
+    haServerCorosyncCrypto = 5407; # Corosync crypto sync (UDP)
+    haServerPacemakerRemoted = 3121; # pacemaker-remoted (TCP)
+    haServerPcsd = 2224; # pcsd cluster daemon (TCP)
 
     # Opened on the docker host's firewall for the Traefik-fronted
     # container stack (docker-compose config lives in the separate

vars/per-machine/proxmox-ha-server-1/openssh/ssh_host_ed25519_key.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN7IABG7x9ejHSENmHsL/BF9MQadGuBnHZa1eUEFMYnH ha-server-1

vars/per-machine/proxmox-ha-server-1/openssh/ssh_host_ed25519_key/secret

@@ -0,0 +1,14 @@
+{
+	"data": "ENC[AES256_GCM,data:EhYMNPzlMDyby6g3hfvBKisKJnU+YAxZjf326tPdflQ2PcmPTzZ0GjHqJOPrOWyLU6IyvnsLMBV0JF5/UUi2c/H3p2wvoBgsxDMaaMW0YUK2IZFo+PZpWoYJSMrx40sH90W0qL0VBw17JLwPU6CjeUZrn5+sCN9EhQvncD8NoJ6QwZpWKhWLn9/7mxd/PF1uxjEQXzHYFeuUuVpQdKSUfV/BSJIT3pMM/VdH2203jTMtwUbK1/7UtGKtDWdfFbkEvJ6qsV7hnN3EYU16WtdyAiHEZvvZ3gqg6YK2DemojWG7z7VQkQGbGm40yA6/mYGW2dVK5ULzjxALl05G6lo1vzxODbMHhzsfJZVw6ocGx3r8xwaasiX8S3loq8WNUWJ9QgjbxuEX9uGZja6A3nPhcwVGGZExbUoU+syXUk2m+WfIQ0epplaPDyAxrYO5ZPWLdMYoPRxbDx94rx5fMxA3jst++IEL2kQMTieGQYJy3R8veQIe3Jb074zY+Y79EmRxfo8ruR6ZMqORtbZzJOBcd0OObjUVLUqB4rnC,iv:XkCfOKtmZNz+UTZdElLm8L0PpymkDzCM24VkAa9Y/Qo=,tag:hmr9ik0V7mxEKR0DcDB3/w==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqQ3J3VDVwUVBvUjVXSWNz\nU3YrRXZlT2NLRThCNzd3YWNHVTlHeS9xMHpzCi9HVW5kMXZrdjE4bXdML3NhRnI1\nL0NQQkNQUmg2M0NBYzh2cG1vdHA0clkKLS0tIDJZYUwzK0RGNzJWckJJekI0SFg1\nUXE3N05xWThtdjdCZkdJTFJ2YUx6cm8KVANudVL54WBNc9DK9s9h4WQRLMewUqgN\ntu4LdMOmi5oV3LX06lbxhBq79dmsV5uos/qszhJkVGhEZwD1RGBoZA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			}
+		],
+		"lastmodified": "2026-07-28T06:42:29Z",
+		"mac": "ENC[AES256_GCM,data:Xn3/CWhQJHWtd3QHoqpdwLuJdZTp+oxx43k38j4jspQSBPdVDSG6Ifi7Sb2r1g9YTxpjUto8mv7FH+BqI0wSYUkLfjaxI51xs3dq6e6kl5NWOsvWwIOGmBuhfIwusHGgsfBguxk0J6Ev8Irw674CRFc37jozXTVdpOOzFhPTvFs=,iv:/7OFE9NYkw2kYEetzeZRXah0KXuD5OXmfjQMIeqI75Q=,tag:VamuInZqYGYRwlyU8TtL9g==,type:str]",
+		"version": "3.13.3"
+	}
+}

vars/per-machine/proxmox-ha-server-2/openssh/ssh_host_ed25519_key.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINWuFwPKW2jwQEj6c//xb95DbtandPkJ7M0ceyTNd35G ha-server-2

vars/per-machine/proxmox-ha-server-2/openssh/ssh_host_ed25519_key/secret

@@ -0,0 +1,14 @@
+{
+	"data": "ENC[AES256_GCM,data:GEebcQrPPrQxLC6mWXLLtytGkIy6Q2FbcQ/Z+ImGa3gahI/nigdNyR/avIz1DAUsOibpoMVlrk0GQUBMuxiEXulIwUEzeTh7sMI3XTZiugujOFHPyJoFeynlgeRNBbl2EVVjGmEmOsR/cZJGyKaet9aNTHTPDJUhodY5d+D/sE3SgpHoch1DI02/DRnqlnlOYiBoujk++BW71EDHLVyz3n9NsurTDYDK4L0Ch5cMgdlvwIuxGuPKtkhU9z6PtGxPbNlYlNP8U+3jG4XWEEY5hZHBGTVHn+bd5FViGY+sMiigO6yaOmddbUqzJoblKeI6F/rfrL3kXETLgVP7uVm3gw9KPEl5JaTdVqkG4hQ9G9/LvRlyEh619mCp9IRXxabm5SQ4NWFzah1ZwqIKp6GoAtqeWRgZNVqijoF+bnb0YmNFmjwpSrSMZvKsrZb5geyGmqah+NJii0xqVebnufB5p8lW9eytBSNgC/XLjj0olwHMw7CWGmnY6Q7lxF1tNYAD/K3dkbxQYFM3kBnmAdweC8WYc+hDjtk0HJul,iv:H4BonLmf3VoW2S9+IUE+SzPb0qiWHuNuGpVIUgUYScs=,tag:5+bq7vC81DMmOk4jn3u8lg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrQitsaDlqTmlYM2RvR1Ew\nb3NzZTJzR3F2bjM5NHhsa3dCUzZ4TW1pa3pBCkxIeEJtbWdaUFMwMkFyRHpCZ1Bl\nbmM5dGV3b3BDcDEwVlJ4UWV5RFNvRkUKLS0tIFRDd0U3dHBHb0Y3WDhNV3hkd2Ew\nclRDVEZHYjZUSzNVZkFjWGt3SGVQTlUKOULuXiYD9k2uVUmhuC15Kgezrd69rc9P\n7SocPa8kBliffP9IuxxW2S/hPbK8rqEx/sh/Km85ZIi4pG0AUQJ3fA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			}
+		],
+		"lastmodified": "2026-07-28T06:45:10Z",
+		"mac": "ENC[AES256_GCM,data:ZnsqhgELqEdNzlTKJx2+xzcZRMNGTme/0a+PVvpbUv5IApimtif0zkTEC1LJsyLPut3MmO3QGprVlb8ihoRdm/IVvLaQonc36f6vN5cxrpQadru/RWEjLbnge1E8VkN/PVvHqRfLyIOPSq6wtGjAPaC/iHW1scBi50pWr0XO628=,iv:FlqAL1It6JuvoBhZotHWdSz17nckBf2rXnwovl6ZJBA=,tag:tNwcaL1sJakXtgejsB/Sbg==,type:str]",
+		"version": "3.13.3"
+	}
+}