feat(ha): add resize-data-disk.sh for online data disk expansion

Three-phase online resize: Proxmox qm resize → guest block device
rescan → drbdadm resize + xfs_growfs. No downtime required.
Detects active node via crm_mon, pre-checks DRBD UpToDate/UpToDate,
auto-discovers VMIDs from qm list, and resolves the block device name
from the stable scsi-0QEMU... by-id symlink with a slot-index fallback.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.sops.yaml

@@ -22,6 +22,8 @@ keys:
   - &proxmox-pxe-boot age1ug787sgt6st6k82fgkrug2lzltw4qsukrrqqs3w27ewwqj8rg4hsxcmylz
   - &proxmox-server age1529taqdwr6t0w7cvzmty0d5y5593wffl0krt48j6uc4u39k56g2qf6ywtp
   - &proxmox-tailscale-router age1zhfyuzlq40reuqlr34gf77852nhs3t6mqfzrqmas8z6sxk7tcfhsungrm0
+  - &proxmox-ha-server-1 age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6
+  - &proxmox-ha-server-2 age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r
 
 creation_rules:
   # Shared across every currently-deployed host: root/nixos password hash,
@@ -54,6 +56,8 @@ creation_rules:
           - *proxmox-pxe-boot
           - *proxmox-server
           - *proxmox-tailscale-router
+          - *proxmox-ha-server-1
+          - *proxmox-ha-server-2
 
   - path_regex: secrets/nix-cache\.yaml$
     key_groups:
@@ -63,16 +67,6 @@ creation_rules:
           - *lxc-nix-cache
           - *proxmox-nix-cache
 
-  # Host keytab for nix-cache FreeIPA enrollment (binary sops file).
-  # Generate with: sops -e --input-type binary /tmp/nix-cache.keytab > secrets/nix-cache.keytab
-  - path_regex: secrets/nix-cache\.keytab$
-    key_groups:
-      - age:
-          - *admin
-          - *linode-nix-cache
-          - *lxc-nix-cache
-          - *proxmox-nix-cache
-
   - path_regex: secrets/server\.yaml$
     key_groups:
       - age:
@@ -103,12 +97,14 @@ creation_rules:
     key_groups:
       - age:
           - *admin
+          - *proxmox-ha-server-1
           # proxmox-ha-server-1 added by sync-host-keys.sh
 
   - path_regex: secrets/ha-server-2\.yaml$
     key_groups:
       - age:
           - *admin
+          - *proxmox-ha-server-2
           # proxmox-ha-server-2 added by sync-host-keys.sh
 
   # Shared HA cluster corosync authkey (binary sops file).
@@ -118,6 +114,8 @@ creation_rules:
     key_groups:
       - age:
           - *admin
+          - *proxmox-ha-server-1
+          - *proxmox-ha-server-2
           # proxmox-ha-server-1 added by sync-host-keys.sh
           # proxmox-ha-server-2 added by sync-host-keys.sh
 
@@ -136,12 +134,90 @@ creation_rules:
           - *linode-gui
           - *proxmox-gui
 
-  # Host keytab for tailscale-router FreeIPA enrollment (binary sops file).
+  # IPA host keytabs (binary sops files).
-  # Generated by scripts/ipa/create-nixos-ipa-host-account.sh.
+  # Each keytab is encrypted for all platform variants of that host so any
+  # deployed variant can decrypt it at boot. Run
+  # scripts/ipa/create-nixos-ipa-host-account.sh <hostname> to enroll a new
+  # host and produce the keytab; this section is updated by that script.
+
+  - path_regex: secrets/nix-cache\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *linode-nix-cache
+          - *lxc-nix-cache
+          - *proxmox-nix-cache
+
   - path_regex: secrets/tailscale-router\.keytab$
     key_groups:
       - age:
           - *admin
+          - *linode-tailscale-router
           - *lxc-tailscale-router
           - *proxmox-tailscale-router
-          - *linode-tailscale-router
+
+  - path_regex: secrets/pxe-boot\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *lxc-pxe-boot
+          - *proxmox-pxe-boot
+
+  # nixos = the workstation (hosts/nixos/host.nix). All gui platform variants
+  # share the hostname "nixos" and must be able to decrypt at boot.
+  - path_regex: secrets/nixos\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *baremetal-gui
+          - *lxc-gui
+          - *proxmox-gui
+          - *linode-gui
+
+  - path_regex: secrets/server\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *linode-server
+          - *lxc-server
+          - *proxmox-server
+
+  - path_regex: secrets/docker\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *linode-docker
+          - *lxc-docker
+          - *proxmox-docker
+
+  - path_regex: secrets/tor-relay\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *lxc-tor-relay
+
+  - path_regex: secrets/nix-minimal\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *lxc-minimal
+          - *proxmox-minimal
+          - *linode-minimal
+
+  # Host keytab for ha-server-1 FreeIPA enrollment (binary sops file).
+  # Generated by scripts/ipa/create-nixos-ipa-host-account.sh.
+  - path_regex: secrets/ha-server-1\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *proxmox-ha-server-1
+          # proxmox-ha-server-1 added by sync-host-keys.sh
+
+  # Host keytab for ha-server-2 FreeIPA enrollment (binary sops file).
+  # Generated by scripts/ipa/create-nixos-ipa-host-account.sh.
+  - path_regex: secrets/ha-server-2\.keytab$
+    key_groups:
+      - age:
+          - *admin
+          - *proxmox-ha-server-2
+          # proxmox-ha-server-2 added by sync-host-keys.sh

AGENTS.md

@@ -6,12 +6,14 @@ This repository contains flake-based NixOS configurations for Wayne's LAN
 servers and workstation.
 
 The flake exposes NixOS configurations named `<platform>-<buildtype>`
-(platforms: `linode`, `proxmox`, `lxc`; build types: `minimal`, `nix-cache`,
+(platforms: `linode`, `proxmox`, `lxc`, `baremetal`; build types: `minimal`,
-`server`, `docker`, `gui`, `pxe-boot`, `tailscale-exit-node`, `tor-relay`), generated from `modules/platforms/*`
+`nix-cache`, `server`, `docker`, `gui`, `pxe-boot`, `tailscale-router`,
-and `modules/build-types/*` by the `mkTarget` function in `flake.nix`. Not
+`tor-relay`, `ha-server`), generated from `modules/platforms/*` and
-every combination is built — `pxe-boot` has no `linode` variant. See
+`modules/build-types/*` by the `mkTarget` function in `flake.nix`. Not every
-`README.md` for the full current target list; treat `flake.nix` as the
+combination is built — `pxe-boot` has no `linode` variant, `ha-server` only
-source of truth since this list can drift.
+exists on `proxmox`, and `tor-relay` only exists on `lxc`. See `README.md`
+for the full current target list; treat `flake.nix` as the source of truth
+since this list can drift.
 
 Do not deploy, switch, reboot, repartition, format disks, or run destructive
 install commands from this repository unless explicitly asked.

README.md

@@ -10,13 +10,13 @@ pieces composed in `flake.nix`:
 
 - **Platforms** (what it runs on): `linode`, `proxmox`, `lxc`, `baremetal`
 - **Build types** (what it's for): `minimal`, `nix-cache`, `server`, `docker`,
-  `gui`, `pxe-boot`, `tailscale-exit-node`, `tor-relay`
+  `gui`, `pxe-boot`, `tailscale-router`, `tor-relay`, `ha-server`
 
 Not every combination exists — `pxe-boot` has no `linode` variant, since
 PXE/DHCP/TFTP need LAN L2 adjacency that a Linode VPS doesn't have,
-`tor-relay` currently only exists as `lxc-tor-relay`, and `baremetal`
+`tor-relay` and `ha-server` currently only exist on `lxc`/`proxmox`, and
-currently only exists as `baremetal-gui` (the real gui-host hardware). The
+`baremetal` currently only exists as `baremetal-gui` (the real gui-host
-full list:
+hardware). The full list:
 
 | Target | Purpose |
 | --- | --- |
@@ -29,8 +29,9 @@ full list:
 | `linode-gui` / `proxmox-gui` / `lxc-gui` | Cinnamon desktop workstation — previously the flat `nixos` target |
 | `baremetal-gui` | Same Cinnamon desktop workstation, on the real gui-host hardware — ZFS RAID0 root, systemd-boot |
 | `proxmox-pxe-boot` / `lxc-pxe-boot` | HTTP/iPXE boot asset host — previously the flat `pxe-boot` target |
-| `linode-tailscale-exit-node` / `proxmox-tailscale-exit-node` / `lxc-tailscale-exit-node` | Tailscale exit node |
+| `linode-tailscale-router` / `proxmox-tailscale-router` / `lxc-tailscale-router` | Tailscale subnet router + MagicDNS forwarder for the LAN |
 | `lxc-tor-relay` | Tor middle relay |
+| `proxmox-ha-server-1` / `proxmox-ha-server-2` | HA file-server cluster nodes — DRBD + XFS + iSCSI + NFS, managed by Corosync + Pacemaker |
 
 Which variant of a given buildtype is actually deployed isn't tracked
 anywhere in this repo — that's live infrastructure state, not something a
@@ -47,8 +48,7 @@ section for which is which.
 
 Each buildtype's `hosts/<name>/host.nix` carries the per-machine identity
 (hostname, hostId, per-machine secrets, `system.stateVersion`) that must stay
-fixed regardless of which platform it's built for — see
+fixed regardless of which platform it's built for. Every deployed host
-`flake-target-refactor-spec.md` for the full rationale. Every deployed host
 stamps its own active target name into `/etc/flake-target` at build time, so
 `nixos-rebuild switch --flake .#$(cat /etc/flake-target)` always picks up the
 right one even after a platform migration changes the flake attribute name.
@@ -167,7 +167,6 @@ per-boot host key for sops-nix to derive from on ephemeral media) — see
 (`vars/per-machine/<target>/openssh/`, committed and sops-encrypted) for
 their SSH host keys.
 
-This repository's git *history* still contains secrets committed before this
+This repository's git *history* still contains secrets committed before the
-migration (see `remove-sensetive-info-refactor.md`) — those are being
+sops-nix migration — those are being scrubbed and rotated separately; don't
-scrubbed and rotated separately; don't treat the repo as safe to make public
+treat the repo as safe to make public until that's finished.
-until that's finished.

docs/auto-installer.md

@@ -22,7 +22,7 @@ see "LXC hosts" immediately below for why those are different.**
 ## LXC hosts
 
 `lxc-*` targets (`lxc-minimal`, `lxc-nix-cache`, `lxc-server`, `lxc-docker`,
-`lxc-gui`, `lxc-pxe-boot`, `lxc-tailscale-exit-node`, `lxc-tor-relay`) are **not** installed via `auto-install.sh` — the
+`lxc-gui`, `lxc-pxe-boot`, `lxc-tailscale-router`, `lxc-tor-relay`) are **not** installed via `auto-install.sh` — the
 interactive menu deliberately excludes them. Don't try to select one there;
 `nixos-install` would bind-mount `/` onto `/mnt` (LXC containers have no raw
 disk to partition) and then refuse to touch the filesystem it's currently
@@ -134,12 +134,14 @@ Flake outputs:
 nixosConfigurations.installer   # ISO/netboot installer image
 
 packages.x86_64-linux.iso       # installer ISO/netboot image
-packages.x86_64-linux.pxe   # netboot-ipxe + netboot-initrd + netboot-kernel, bundled
+packages.x86_64-linux.pxe       # auto-installer netboot bundle (kernel + initrd + ipxe script)
+packages.x86_64-linux.pxe-minimal  # vanilla NixOS minimal netboot bundle (no installer wiring)
 ```
 
 ```sh
 nix build .#iso
 nix build .#pxe
+nix build .#pxe-minimal
 ```
 
 There's no `nixosConfigurations.proxmox-lxc` (installer-boots-as-an-LXC-

flake.lock

@@ -173,11 +173,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1784350909,
+        "lastModified": 1785119570,
-        "narHash": "sha256-ZWyzLbS1yKUTeFJLmdVuWNnHttL333/ldJbEE+KzCrM=",
+        "narHash": "sha256-Rgs2xKnGLFWQscxUaXX07oyZeuMDOHEbqDOsgliLFGM=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "4ce190229c73d44536caa7072f6308fb2d8feeb3",
+        "rev": "d4fd24667c8cbef124bb70a20380cab75ec8474d",
         "type": "github"
       },
       "original": {
@@ -259,11 +259,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1784432872,
+        "lastModified": 1785104993,
-        "narHash": "sha256-n3gKTBIV4ZA5VQpUakffBe3KGu4+mhPoA34rrqS0GkA=",
+        "narHash": "sha256-eKbrvPoAOFutbYMdbB3r5EQVmFxKv24iKqHPPUXA0gM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "fd1462031fdee08f65fd0b4c6b64e22239a77870",
+        "rev": "8623c4c20aa4ca2f5fb81510d2944066c3fb0d96",
         "type": "github"
       },
       "original": {

hosts/docker/host.nix

@@ -14,6 +14,11 @@
   };
   boot.zfs.forceImportRoot = false;
 
+  # Only advertise the LAN interface to IPA DNS. Without this, SSSD registers
+  # every Docker bridge (172.x.x.x) as an A record for docker.sweet.home —
+  # the default dyndns.interface = "*" catches them all.
+  security.ipa.dyndns.interface = vars.lxcLanInterface; # eth0
+
   # Preserved from the pre-refactor `docker` target — stateVersion must never
   # be bumped on an already-installed machine.
   system.stateVersion = "25.05";

hosts/ha-server-1/host.nix

@@ -24,7 +24,7 @@
   };
 
   # Set KEY after pairing this host with the beszel hub; the token is sops-managed.
-  services.beszel.agent.environment.KEY = "";
+  # services.beszel.agent.environment.KEY = "";
 
   system.stateVersion = "26.05";
 }

hosts/ha-server-2/host.nix

@@ -24,7 +24,7 @@
   };
 
   # Set KEY after pairing this host with the beszel hub; the token is sops-managed.
-  services.beszel.agent.environment.KEY = "";
+  # services.beszel.agent.environment.KEY = "";
 
   system.stateVersion = "26.05";
 }

hosts/nix-cache/host.nix

@@ -6,15 +6,10 @@
       name = "nix-cache";
       sopsFile = ../../secrets/nix-cache.yaml;
     })
-    (import ../../modules/ipa/client.nix {
-      keytabSopsFile = ../../secrets/nix-cache.keytab;
-      caCertFile = ../../certs/ipa-ca.crt;
-    })
   ];
 
   networking = {
     hostName = vars.nixCacheHost;
-    domain = vars.homeDomain;
     useDHCP = false;
     interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{
       address = vars.nixCacheIp;
@@ -25,7 +20,6 @@
   };
 
   services.beszel.agent.environment = {
-    #DOCKER_HOST = "tcp://docker-socket-proxy:2375";
     KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
   };
 

hosts/server/host.nix

@@ -21,7 +21,6 @@
   };
 
   services.beszel.agent.environment = {
-    #DOCKER_HOST = "tcp://docker-socket-proxy:2375";
     KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
     EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
     LOG_LEVEL = "debug";

hosts/tailscale-router/host.nix

@@ -6,11 +6,6 @@
       name = "tailscale-router";
       sopsFile = ../../secrets/tailscale-router.yaml;
     })
-    (import ../../modules/ipa/client.nix {
-      keytabSopsFile = ../../secrets/tailscale-router.keytab;
-      caCertFile     = ../../certs/ipa-ca.crt;
-    })
-
   ];
 
   networking = {

modules/build-types/gui.nix

@@ -1,6 +1,17 @@
 { config, pkgs, lib, inputs, vars, ... }:
 
 {
+  imports = [
+    ../docker/enable-service.nix
+  ];
+
+  nixpkgs.overlays = [
+    (final: prev: {
+      docker = prev.docker_29;
+      docker_cli = prev.docker_29;
+    })
+  ];
+
   environment.systemPackages = with pkgs; [
     inputs.nixos-conf-editor.packages.${pkgs.stdenv.hostPlatform.system}.nixos-conf-editor
     nodejs
@@ -18,7 +29,7 @@
   ];
 
   boot.loader.grub.useOSProber = true;
-programs.direnv.enable = true;
+  programs.direnv.enable = true;
   services = {
     xserver = {
       enable = true;
@@ -70,4 +81,70 @@ programs.direnv.enable = true;
   programs.firefox.enable = true;
 
   nixpkgs.config.allowUnfree = true;
+
+  # GUI-specific Home Manager additions for the IPA primary user, extending
+  # the baseline in modules/ipa/client.nix with desktop apps and services
+  # that only make sense on a graphical workstation.
+  home-manager.users.${vars.ipaUser} = { pkgs, ... }: {
+    home = {
+      packages = with pkgs; [
+        git
+        vim
+        nextcloud-client
+        chromium
+        claude-code
+        fish
+        sops
+      ];
+      sessionVariables = {
+        EDITOR = "vim";
+        SOPS_AGE_KEY_FILE = "/home/${vars.ipaUser}/.config/sops/age/keys.txt";
+      };
+      file = {
+        ".local/share/applications/proxmox-chromium-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox (Chromium)
+          Exec=chromium --app=https://pve.${vars.homeDomain}:${toString vars.ports.pveWeb} --window-size=1920,1080 --window-position=0,0
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=Hypervisor;
+          StartupWMClass=PVE
+        '';
+        ".local/share/applications/pbs-chromium-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox Backup Server (Chromium)
+          Exec=chromium --app=https://${vars.pbsIp}:${toString vars.ports.pbsWeb} --window-size=1920,1080 --window-position=0,0
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=backup;
+        '';
+        ".local/share/applications/proxmox-firefox-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox (Firefox)
+          Exec=firefox --new-instance https://pve.${vars.homeDomain}:${toString vars.ports.pveWeb} --profile ProxmoxWebApp --window-size=1920,1080 --class ProxmoxWebApp
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=Hypervisor;
+          StartupWMClass=PVE
+        '';
+        ".local/share/applications/pbs-firefox-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox Backup Server (Firefox)
+          Exec=firefox --new-window https://${vars.pbsIp}:${toString vars.ports.pbsWeb} --profile PbsWebApp --window-size=1920,1080 --class PbsWebApp
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=backup;
+          StartupWMClass=PBS
+        '';
+      };
+    };
+    services.nextcloud-client = {
+      enable = true;
+      startInBackground = true;
+    };
+  };
 }

modules/build-types/ha-server.nix

@@ -14,27 +14,32 @@
 #   the Beszel hub) is not set yet — add it to hosts/ha-server-{1,2}/host.nix
 #   under services.beszel.agent.environment.KEY once the hub accepts the
 #   new agents, following the pattern in hosts/server/host.nix.
-{ lib, vars, ... }:
+{ lib, pkgs, vars, ... }:
+
+let
+  # Generates /etc/exports lines for all nfsShares data entries. Shared
+  # pattern with modules/build-types/server.nix — both export the same
+  # set of shares, differing only in the storage root they serve from.
+  mkNfsExports = storageRoot:
+    lib.concatMapStrings
+      (share: "  ${storageRoot}/${share.subpath} ${vars.lanCidr}${vars.nfsShares.options}\n")
+      (lib.filter builtins.isAttrs (lib.attrValues vars.nfsShares));
+in
 {
   imports = [
     ../ha/pacemaker-stack.nix
     ../ha/iscsi-target.nix
     ../ha/cluster-config.nix
-    ../beszel/enable-agent.nix
+    # ../beszel/enable-agent.nix
   ];
 
+  # xfsprogs: mkfs.xfs/xfs_info needed by cluster-init.sh.
+  # openiscsi: iscsiadm needed by acceptance-tests.sh T4 (iSCSI discovery check).
+  environment.systemPackages = [ pkgs.xfsprogs pkgs.openiscsi ];
+
   services.nfs.server = {
     enable = true;
-    exports = ''
+    exports = mkNfsExports vars.haStorageRoot;
-      ${vars.haStorageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.haStorageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.haStorageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.haStorageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.haStorageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.haStorageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.haStorageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.haStorageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-    '';
   };
 
   # Pacemaker controls nfs-server — prevent systemd from starting it at boot

modules/build-types/server.nix

@@ -13,6 +13,15 @@ let
     lib.concatMap (share: ancestors share.subpath)
       (lib.filter builtins.isAttrs (lib.attrValues vars.nfsShares))
   );
+
+  # Generates /etc/exports lines for all nfsShares data entries (every
+  # attrset value — excludes the bare `options` string). Both server and
+  # ha-server export the same share set from different storage roots, so
+  # this helper is the single source of truth for the export line format.
+  mkNfsExports = storageRoot:
+    lib.concatMapStrings
+      (share: "  ${storageRoot}/${share.subpath} ${vars.lanCidr}${vars.nfsShares.options}\n")
+      (lib.filter builtins.isAttrs (lib.attrValues vars.nfsShares));
 in
 {
   imports = [
@@ -95,16 +104,7 @@ in
 
   services.nfs.server = {
     enable = true;
-    exports = ''
+    exports = mkNfsExports vars.storageRoot;
-      ${vars.storageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.storageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.storageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.storageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.storageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.storageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-      ${vars.storageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
-    '';
   };
 
   # mountd (20048) is needed for showmount/NFSv3 mount protocol — without it

modules/common/aliases.nix

@@ -1,50 +1,7 @@
-{ config, pkgs, lib, vars, ... }:
+_:
 
-let
-  # Flake attribute names are now <platform>-<buildtype> (e.g. proxmox-docker)
-  # and no longer match networking.hostName, since a host's hostname stays
-  # fixed while the platform backing it can change. Each nixosConfiguration
-  # stamps its own active target name into /etc/flake-target at build time.
-  mySwitchCmd = ''
-    sudo nixos-rebuild switch \
-      --no-write-lock-file \
-      --refresh \
-      --flake git+https://${vars.lanDomain}/beatzaplenty/nixos.git#$(cat /etc/flake-target)
-  '';
-  myTestCmd = ''
-    sudo nixos-rebuild test \
-      --no-write-lock-file \
-      --refresh \
-      --flake git+https://${vars.lanDomain}/beatzaplenty/nixos.git#$(cat /etc/flake-target)
-  '';
-
-  # lxc-* hosts pre-seed their SSH host key at build time (see
-  # modules/platforms/lxc.nix) so sops-nix's .sops.yaml recipient matches on
-  # first boot -- without it, secrets permanently fail to decrypt (see that
-  # file's comment for the confirmed failure). That requires --impure plus
-  # NIXOS_HOST_KEYS_DIR pointing at the repo's host-keys/ dir, same pattern
-  # docs/auto-installer.md uses for the installer ISO. A function, not a
-  # shellAlias, since the target name has to interpolate into the middle of
-  # the flake attribute path, not just append after it. Must be run from the
-  # repo root, same as every other host-keys/ command in this repo.
-  buildImageFn = ''
-    buildImage() {
-      if [ -z "$1" ]; then
-        echo "usage: buildImage <flake-target>  (e.g. lxc-docker)" >&2
-        return 1
-      fi
-      NIXOS_HOST_KEYS_DIR="$(pwd)/host-keys" nix build --impure \
-        ".#nixosConfigurations.$1.config.system.build.tarball"
-    }
-  '';
-in
 {
-  programs.bash = {
+  # Switch-nix, Test-nix, and buildImage are defined system-wide in
-    enable = true;
+  # modules/common/configuration.nix so all users (including IPA accounts)
-    shellAliases = {
+  # get them. Add any Home-Manager-only per-user shell config here.
-      "Switch-nix" = mySwitchCmd;
-      "Test-nix" = myTestCmd;
-    };
-    initExtra = buildImageFn;
-  };
 }

modules/common/configuration.nix

@@ -1,17 +1,45 @@
 { config, lib, pkgs, vars, ... }:
 
+let
+  switchCmd = ''
+    sudo nixos-rebuild switch \
+      --no-write-lock-file \
+      --refresh \
+      --flake git+https://${vars.lanDomain}/beatzaplenty/nixos.git#$(cat /etc/flake-target)
+  '';
+  testCmd = ''
+    sudo nixos-rebuild test \
+      --no-write-lock-file \
+      --refresh \
+      --flake git+https://${vars.lanDomain}/beatzaplenty/nixos.git#$(cat /etc/flake-target)
+  '';
+  buildImageFn = ''
+    buildImage() {
+      if [ -z "$1" ]; then
+        echo "usage: buildImage <flake-target>  (e.g. lxc-docker)" >&2
+        return 1
+      fi
+      NIXOS_HOST_KEYS_DIR="$(pwd)/host-keys" nix build --impure \
+        ".#nixosConfigurations.$1.config.system.build.tarball"
+    }
+  '';
+in
 {
-  imports =
+  imports = [
-    [
-      # Include the results of the hardware scan.
-      #      ./hardware-configuration.nix
     ./set-locale.nix
+    ../ipa/client.nix
   ];
-  # Use the GRUB 2 boot loader.
-  # boot.loader.grub.enable = true;
-  #boot.loader.grub.device = "/dev/sda"; # or "nodev" for efi only
 
-  networking.networkmanager.enable = true; # Easiest to use and most distros use this by default.
+  # System-wide shell config so all users (including IPA accounts) get the
+  # same management aliases as the local nixos user's Home Manager provides.
+  programs.bash = {
+    shellAliases = {
+      "Switch-nix" = switchCmd;
+      "Test-nix" = testCmd;
+    };
+    interactiveShellInit = buildImageFn;
+  };
+  networking.networkmanager.enable = true;
 
   # Recommended over the true default (bypasses ZFS's own import safeguards)
   # per the option's own docs; matches hosts/docker/host.nix and

modules/common/preserve-ssh-host-key.nix

@@ -0,0 +1,35 @@
+# Shared activation-script logic to preserve the SSH host key across
+# nixos-rebuild on platforms that embed the key via environment.etc (lxc and
+# proxmox).  When NIXOS_HOST_KEYS_DIR is not set the key is absent from
+# environment.etc, and NixOS's etc activation removes any /etc file not in
+# the new generation — which would destroy the live key and break sops-nix
+# decryption permanently.  These scripts save the key to /run before etc
+# removes it, then restore it afterward.
+#
+# Explicit deps enforce the correct ordering: without them the topological
+# sort places preserveSshHostKey after etc (confirmed live on lxc-tor-relay:
+# position 7 vs etc's position 5), so the key is gone before it can be saved.
+_: {
+  system.activationScripts = {
+    preserveSshHostKey = ''
+      if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
+        cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
+        cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
+      fi
+    '';
+
+    restoreSshHostKey = {
+      deps = [ "etc" ];
+      text = ''
+        if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
+          install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
+          install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
+        fi
+        rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
+      '';
+    };
+
+    etc = { deps = [ "preserveSshHostKey" ]; };
+    setupSecrets = { deps = [ "restoreSshHostKey" ]; };
+  };
+}

modules/docker/enable-service.nix

@@ -1,23 +1,45 @@
-{ pkgs, ... }:
+{ lib, pkgs, vars, ... }:
 
+let
+  gid = toString vars.dockerAccessGid;
+in
 {
-  #  virtualisation.docker.enable = true;
   virtualisation.docker = {
     enable = true;
     package = pkgs.docker;
-    #  listenOptions = [
-    #  "unix:///var/run/docker.sock"
-    #  "tcp://0.0.0.0:2375"
-    #];
-
-    #  daemon.settings = {
-    #    metrics-addr = "0.0.0.0:9323";
-    #    experimental = true;
-    #  };
   };
-
+  # Pin the docker group GID to match the IPA "docker-access" group so that
+  # IPA group membership alone grants access to the Docker socket. Any user
+  # whose supplementary groups (resolved by SSSD from IPA) include GID
+  # vars.dockerAccessGid will pass the socket group-permission check without
+  # any per-host users.groups.docker.members entry.
+  users.groups.docker.gid = lib.mkForce vars.dockerAccessGid;
+  users.users.${vars.primaryUser}.extraGroups = [ "docker" ];
   environment.systemPackages = with pkgs; [
     docker-compose
     docker-buildx
   ];
+
+  # NixOS's group activation uses plain `groupmod` without --non-unique.
+  # When SSSD is active it exposes the IPA "docker-access" group at
+  # vars.dockerAccessGid via NSS, so groupmod sees that GID as already in
+  # use and silently skips the change (warning: "not applying GID change").
+  # This script runs after the normal "groups" step and applies the change
+  # with --non-unique (which lets the local docker group share the GID with
+  # the SSSD-provided IPA group). If the GID actually changed it also
+  # restarts docker.socket so the socket is recreated with the new GID.
+  system.activationScripts.docker-group-gid = {
+    deps = [ "groups" ];
+    text = ''
+      current=$(grep "^docker:" /etc/group | cut -d: -f3)
+      if [ "$current" != "${gid}" ]; then
+        ${pkgs.shadow}/bin/groupmod --non-unique -g ${gid} docker
+        if ${pkgs.systemd}/bin/systemctl is-active --quiet docker.socket; then
+          ${pkgs.systemd}/bin/systemctl stop docker.service docker.socket
+          rm -f /var/run/docker.sock
+          ${pkgs.systemd}/bin/systemctl start docker.socket docker.service
+        fi
+      fi
+    '';
+  };
 }

modules/ha/cluster-config.nix

@@ -11,14 +11,57 @@
 #   Both host keys must be registered via sync-host-keys.sh first so both nodes can decrypt it.
 #
 # DRBD fencing:
-#   Production setting is resource-only: DRBD waits for the STONITH fence
+#   resource-only with crm-fence-peer.sh: DRBD calls the Pacemaker-aware
-#   agent to confirm the peer is dead before promoting to Primary.  This
+#   crm-fence-peer.sh handler before promoting.  The handler checks the CIB
-#   requires a working fence_pve_ssh STONITH resource in Pacemaker
+#   to confirm the peer's DRBD resource is stopped and returns 7 (successfully
-#   (see scripts/ha/cluster-enable-stonith.sh).  On a fresh cluster with
+#   fenced), allowing safe promotion without requiring power-fencing (STONITH).
-#   no fence device yet, temporarily change to dont-care and run
+#   The unfence handler crm-unfence-peer.sh clears the outdate flag when the
-#   cluster-enable-stonith.sh once the fence key is deployed.
+#   peer reconnects.  This is the correct setting for Pacemaker+DRBD clusters
-{ lib, vars, ... }:
+#   with STONITH disabled; crm-fence-peer.sh replaces the need for a separate
+#   STONITH device during the testing phase.  Switch to resource-and-stonith
+#   once the fence_pve_ssh STONITH resource is active (see
+#   scripts/ha/cluster-enable-stonith.sh).
+#
+#   PATH wrapper: when the DRBD kernel module invokes the fence-peer handler
+#   via the UMH (User Mode Helper) mechanism it provides a minimal PATH that
+#   omits /run/current-system/sw/bin.  crm-fence-peer.sh calls cibadmin,
+#   crm_mon etc.; if those aren't found a pipeline in the script breaks with
+#   SIGPIPE.  A process killed by signal has WEXITSTATUS() == 0, so the kernel
+#   sees exit code 0 and logs "fence-peer helper broken, returned 0", looping
+#   forever.  The writeShellScript wrappers below prepend the NixOS sw path
+#   before exec-ing the real handler, giving it a working Pacemaker toolchain.
+{ lib, pkgs, vars, ... }:
+let
+  fencePeerWrapper = pkgs.writeShellScript "drbd-fence-peer" ''
+    export PATH="/run/current-system/sw/bin:/run/current-system/sw/sbin:$PATH"
+    exec /run/current-system/sw/lib/drbd/crm-fence-peer.sh "$@"
+  '';
+  unfencePeerWrapper = pkgs.writeShellScript "drbd-unfence-peer" ''
+    export PATH="/run/current-system/sw/bin:/run/current-system/sw/sbin:$PATH"
+    exec /run/current-system/sw/lib/drbd/crm-unfence-peer.sh "$@"
+  '';
+in
 {
+  # Root SSH access — same key set as nixos user so all admin keys can reach root.
+  users.users.root.openssh.authorizedKeys.keys = [
+    vars.adminSshKey
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
+  ];
+
+  # Passwordless sudo for wheel — operator SSHes as nixos and uses sudo for
+  # cluster management commands (drbdadm, crm*, pcs, etc.)
+  security.sudo.wheelNeedsPassword = lib.mkForce false;
+
+  # DRBD lock-file directory (drbd-utils checks for it; missing → harmless but noisy warnings).
+  systemd.tmpfiles.rules = [ "d /var/lib/drbd 0750 root root -" ];
+
+  # Prevent drbd.service from auto-starting at boot / nixos-rebuild switch.
+  # Pacemaker's OCF drbd agent calls drbdadm up/down directly when managing
+  # the resource.  If drbd.service also runs drbdadm up all while DRBD is
+  # already Primary under Pacemaker, apply-al fails with "device busy" (exit 20).
+  systemd.services.drbd.wantedBy = lib.mkForce [];
+
   services.drbd = {
     enable = true;
     config = ''
@@ -37,12 +80,16 @@
         disk {
           fencing resource-only;
         }
+        handlers {
+          fence-peer   "${fencePeerWrapper}";
+          unfence-peer "${unfencePeerWrapper}";
+        }
       }
 
       resource ha-data {
         volume 0 {
           device    /dev/drbd0;
-          disk      /dev/sdb;
+          disk      ${vars.haServerDrbdDisk};
           meta-disk internal;
         }
 

modules/ha/iscsi-target.nix

@@ -25,7 +25,7 @@
 
 let
   python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]);
-  targetctl = "${pkgs.targetcli-fb}/bin/targetctl";
+  targetctl = "${python3}/bin/targetctl";
 
   targetctlStop = pkgs.writeScript "targetctl-stop" ''
     #!${python3}/bin/python3

modules/ipa/client.nix

@@ -1,70 +1,164 @@
 # Fully declarative FreeIPA domain membership.
 #
-# Configures security.ipa (SSSD, Kerberos, PAM, NSSwitch) and places a
+# Imported by modules/common/configuration.nix — no per-host wiring needed.
-# pre-provisioned host keytab via sops-nix so no imperative ipa-client-install
+# Enables itself automatically on any host that has a sops-encrypted keytab
-# step is needed after deployment.
+# at secrets/<hostname>.keytab; is a no-op for all other hosts.
 #
-# Usage (in a host.nix imports list):
+# To enroll a new host:
-#   (import ../../modules/ipa/client.nix {
+#   0. scripts/secrets/sync-host-keys.sh <flake-target>
-#     keytabSopsFile = ../../secrets/nix-cache.keytab;
+#   1. scripts/ipa/create-nixos-ipa-host-account.sh [--ip <addr>] <hostname>
-#     caCertFile     = ../../certs/ipa-ca.crt;
+#      (adds .sops.yaml rule, runs ipa host-add, encrypts keytab in one step)
-#   })
+#   2. git add secrets/<hostname>.keytab .sops.yaml && git commit
+#   3. Deploy — no further steps required.
 #
-# One-time operator setup per host (do this BEFORE deploying):
+# Manual fallback (if the script isn't usable):
-#
+#   a. On the FreeIPA server: ipa host-add <fqdn> [--ip-address=<ip>] --force
-#   1. Fetch the IPA CA certificate (public — safe to commit):
+#   b. On the FreeIPA server: ipa-getkeytab -s <ipa-server> -p host/<fqdn> -k /tmp/<host>.keytab
-#        curl -o certs/ipa-ca.crt http://<ipa-server>/ipa/config/ca.crt
+#   c. From the repo root (path must match for sops creation rule to apply):
-#      Replace the placeholder at certs/ipa-ca.crt and commit it.
-#
-#   2. On the FreeIPA server, add the host and generate a keytab:
-#        ipa host-add <fqdn> --ip-address=<ip>
-#        ipa-getkeytab -s <ipa-server> -p host/<fqdn> -k /tmp/<host>.keytab
-#
-#   3. sops-encrypt the keytab as a binary secret from your admin machine
-#      (must run from repo root; sops matches creation rules against the file
-#      path, so copy to secrets/ first and encrypt in-place):
 #        cp /tmp/<host>.keytab secrets/<host>.keytab
 #        sops -e --input-type binary -i secrets/<host>.keytab
-#      Add secrets/<host>.keytab to .sops.yaml with the host's age key as a
+#   d. Commit secrets/<host>.keytab and the updated .sops.yaml, then deploy.
-#      recipient (see the nix-cache.keytab entry for the pattern), then run:
-#        scripts/secrets/sync-host-keys.sh <target>   # if not done yet
-#        sops updatekeys secrets/<host>.keytab
-#      Commit the encrypted file.
 #
-#   4. nixos-rebuild switch (or create-proxmox-resource.sh) — no further
+# vars dependencies: homeDomain, ipaServer, domainControllerIp, ipaUser
-#      manual enrollment steps required.
-#
-# vars dependencies: homeDomain, ipaServer
 
-{ keytabSopsFile, caCertFile }:
 { config, lib, pkgs, vars, ... }:
 
 let
+  keytabPath = ../../secrets + "/${config.networking.hostName}.keytab";
+  enabled = builtins.pathExists keytabPath;
+
   realm = lib.strings.toUpper vars.homeDomain;
   fqdn = "${config.networking.hostName}.${vars.homeDomain}";
   # "sweet.home" -> "dc=sweet,dc=home"
   basedn = lib.strings.concatMapStringsSep "," (c: "dc=${c}") (lib.strings.splitString "." vars.homeDomain);
   # security.ipa.certificate expects a derivation (package), not a raw path.
-  caCertPkg = pkgs.writeText "ipa-ca.crt" (builtins.readFile caCertFile);
+  caCertPkg = pkgs.writeText "ipa-ca.crt" (builtins.readFile ../../certs/ipa-ca.crt);
 in
-{
+lib.mkIf enabled {
-  security.ipa = {
+  networking.domain = lib.mkDefault vars.homeDomain;
+  networking.nameservers = lib.mkDefault [ vars.domainControllerIp ];
+
+  security = {
+    ipa = {
       enable = true;
       domain = vars.homeDomain;
-    realm = realm;
+      inherit realm;
       server = vars.ipaServer;
       certificate = caCertPkg;
-    basedn = basedn;
+      inherit basedn;
       ipaHostname = fqdn;
       offlinePasswords = true;
       cacheCredentials = true;
     };
 
+    # Create the home directory on first login if it doesn't exist yet.
+    # IPA users have no pre-created home on the host; without this sshd
+    # opens a session to a non-existent directory and resets the connection.
+    # lightdm also needs this so the GUI login path can create the home dir
+    # if it was not pre-seeded by the tmpfiles rule above (e.g. on first boot
+    # before SSSD has resolved the user).
+    pam.services = {
+      sshd.makeHomeDir = true;
+      lightdm.makeHomeDir = true;
+
+      # pam_unix returns PAM_AUTHINFO_UNAVAIL without prompting when the local
+      # stub has "!" in shadow (account locked), so PAM_AUTHTOK is never set
+      # and pam_sss's use_first_pass fails with "No authentication token".
+      # Changing to try_first_pass makes pam_sss prompt independently when no
+      # prior module has set the token, restoring IPA password login via
+      # LightDM and su.
+      login.rules.auth.sss.settings = lib.mkForce { try_first_pass = true; };
+      su.rules.auth.sss.settings = lib.mkForce { try_first_pass = true; };
+    };
+
+    # HM with useUserPackages = true (flake.nix) sets users.users.${ipaUser}.packages,
+    # which forces the stub into /etc/passwd. pam_sss.so with the "localusers" flag
+    # (added by NixOS when SSSD is enabled) then skips SSSD for any user it finds in
+    # local /etc/passwd — including this stub — falling through to pam_unix, which has
+    # no password for the stub → sudo auth always fails.
+    #
+    # Fix: NOPASSWD for the IPA user. The IPA user already authenticated to reach a
+    # shell (SSH public key from IPA or Kerberos), so re-prompting via a broken PAM
+    # path is security theater on a single-admin homelab.
+    sudo.extraRules = [{
+      users = [ vars.ipaUser ];
+      commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
+    }];
+  };
+
+  systemd = {
+    # Fetch SSH public keys from IPA so users can log in with the key stored
+    # in their IPA profile rather than needing ~/.ssh/authorized_keys on every
+    # host. sss_ssh_authorizedkeys queries SSSD (which queries IPA LDAP).
+    #
+    # /nix/store is 1775 (group-writable by nixbld). OpenSSH 10.0+ rejects
+    # AuthorizedKeysCommand binaries whose path contains any group-writable
+    # component, silently skipping the command. Copy to /usr/local/bin (all
+    # components root-owned, 755) so the path passes sshd's safety check.
+    tmpfiles.rules = [
+      "d /usr/local 0755 root root - -"
+      "d /usr/local/bin 0755 root root - -"
+      "C+ /usr/local/bin/sss_ssh_authorizedkeys 0555 root root - ${pkgs.sssd}/bin/sss_ssh_authorizedkeys"
+      # Pre-create the IPA user's home dir so Home Manager activation succeeds
+      # even before their first login. On a fresh system SSSD may not have
+      # resolved the user yet — tmpfiles warns and skips in that case (non-fatal),
+      # and pam_mkhomedir covers the first-login path as a fallback.
+      "d /home/${vars.ipaUser} 0700 ${vars.ipaUser} ${vars.ipaUser} - -"
+    ];
+
+    # security.ipa enables Kerberos (security.krb5) which causes systemd to
+    # start auth-rpcgss-module.service and rpc-gssd.service for Kerberos NFS
+    # authentication. LXC containers can't load the auth_rpcgss kernel module
+    # and don't have /var/lib/nfs/rpc_pipefs, so both services fail.
+    #
+    # The NixOS IPA module already adds a drop-in for auth-rpcgss-module.service
+    # with ConditionPathExists=/etc/krb5.keytab. We use lib.mkForce to win the
+    # text conflict and add ConditionVirtualization=!container alongside it so
+    # the service is skipped (not failed) in containers that do have a keytab.
+    # Same fix for rpc-gssd.service which also fails in containers.
+    units = lib.mkIf config.boot.isContainer {
+      "auth-rpcgss-module.service" = {
+        overrideStrategy = "asDropinIfExists";
+        text = lib.mkForce ''
+          [Unit]
+          ConditionPathExists=
+          ConditionPathExists=/etc/krb5.keytab
+          ConditionVirtualization=!container
+        '';
+      };
+      # rpc-gssd also has ConditionPathExists from the NixOS IPA module (and an
+      # X-Restart-Triggers store path from systemd.nix). Use mkForce to win;
+      # omit X-Restart-Triggers since this service is skipped in containers anyway.
+      "rpc-gssd.service" = {
+        overrideStrategy = "asDropinIfExists";
+        text = lib.mkForce ''
+          [Unit]
+          ConditionPathExists=
+          ConditionPathExists=/etc/krb5.keytab
+          ConditionVirtualization=!container
+        '';
+      };
+    };
+
+    # home-manager-<user>.service fails on first enrollment because /home/wayne
+    # doesn't exist until the user's first login (pam_mkhomedir creates it then).
+    # ConditionPathExists makes systemd skip the service (exit 0, condition not
+    # met) instead of failing. After first login the dir exists and subsequent
+    # rebuilds activate HM normally.
+    services."home-manager-${vars.ipaUser}".unitConfig.ConditionPathExists =
+      "/home/${vars.ipaUser}";
+  };
+
+  services.openssh.extraConfig = ''
+    AuthorizedKeysCommand /usr/local/bin/sss_ssh_authorizedkeys %u
+    AuthorizedKeysCommandUser nobody
+  '';
+
   # Host keytab: pre-provisioned on the IPA server, sops-encrypted binary.
   # Placed at /etc/krb5.keytab before SSSD starts so the host authenticates
   # to IPA without running ipa-client-install.
   sops.secrets."ipa-host-keytab" = {
-    sopsFile = keytabSopsFile;
+    sopsFile = keytabPath;
     format = "binary";
     path = "/etc/krb5.keytab";
     owner = "root";
@@ -72,4 +166,45 @@ in
     mode = "0600";
     restartUnits = [ "sssd.service" ];
   };
+
+  # NixOS requires isNormalUser/isSystemUser + group on any entry in
+  # users.users. HM with useUserPackages = true (set in flake.nix) adds a stub
+  # entry for each HM user so it can install packages to
+  # /etc/profiles/per-user/<name>/. This definition satisfies those assertions.
+  # With security.ipa setting "passwd: sss files" in nsswitch, SSSD's IPA entry
+  # takes priority for NSS lookups — this local stub is only a fallback when
+  # SSSD is unreachable (at which point auth fails anyway).
+  users.users.${vars.ipaUser} = {
+    isNormalUser = true;
+    group = "users";
+    extraGroups = [ "wheel" ];
+    createHome = false;
+    # "!" is not a password hash — it is the standard "account locked" marker.
+    # It cannot authenticate anyone locally. It exists solely so NixOS generates
+    # a shadow entry for this stub user; without one pam_unix returns
+    # PAM_AUTHINFO_UNAVAIL before prompting, which means PAM_AUTHTOK is never
+    # set and the subsequent pam_sss use_first_pass call has nothing to work
+    # with — blocking LightDM and su logins even when IPA/SSSD auth succeeds.
+    hashedPassword = "!";
+  };
+
+  # Home Manager config for the IPA primary user, applied on every enrolled
+  # host. Manages what IPA doesn't: dotfiles, user-scoped packages, session
+  # variables. Switch-nix/Test-nix/buildImage are system-wide (configuration.nix)
+  # so they don't need to be repeated here.
+  #
+  # homeDirectory uses mkForce because HM's NixOS integration module sets it to
+  # "/var/empty" for users not found in config.users.users at eval time (SSSD
+  # users aren't visible there).
+  home-manager.users.${vars.ipaUser} = { pkgs, ... }: {
+    home = {
+      username = vars.ipaUser;
+      homeDirectory = lib.mkForce "/home/${vars.ipaUser}";
+      stateVersion = "26.05";
+      packages = with pkgs; [ tmux sshfs ];
+      sessionVariables.EDITOR = lib.mkDefault "nano";
+    };
+    programs.home-manager.enable = true;
+    programs.bash.enable = true;
+  };
 }

modules/platforms/lxc.nix

@@ -52,6 +52,7 @@ in
   # LXC container does).
   imports = [
     (modulesPath + "/virtualisation/proxmox-lxc.nix")
+    ../common/preserve-ssh-host-key.nix
   ];
 
   proxmoxLXC = {
@@ -105,49 +106,6 @@ in
     };
   };
 
-  # NixOS's etc activation removes any /etc file that was in the previous
-  # generation's environment.etc but is absent from the current one — even
-  # real (non-symlink) copies.  On every routine nixos-rebuild switch/test that
-  # lacks NIXOS_HOST_KEYS_DIR the key is absent from environment.etc, so it
-  # gets removed as "obsolete".  sops-nix derives its age decryption key from
-  # /etc/ssh/ssh_host_ed25519_key; deletion cascades into every sops secret
-  # failing with "Error getting data key: 0 successful groups required, got 0".
-  #
-  # Fix: activation scripts that bracket the etc step, with explicit deps
-  # to enforce the correct ordering. Without deps the topological sort places
-  # preserveSshHostKey AFTER etc (confirmed live on a deployed lxc-tor-relay:
-  # position 7 vs etc's position 5) -- the key is already gone by the time it
-  # tries to save it. The etc/setupSecrets entries ADD to existing deps
-  # (types.listOf concatenates across module definitions).
-  system.activationScripts = {
-    # Saves the live key to /run before etc can delete it.
-    preserveSshHostKey = ''
-      if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
-        cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
-        cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
-      fi
-    '';
-
-    # Reinstalls the key after etc runs if it was removed as "obsolete".
-    # The resulting file is not registered in environment.etc for either
-    # generation, so subsequent rebuilds leave it alone permanently.
-    restoreSshHostKey = {
-      deps = [ "etc" ];
-      text = ''
-        if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
-          install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
-          install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
-        fi
-        rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
-      '';
-    };
-
-    # Force etc to wait until the key is saved, and sops to wait until the
-    # key is restored. Without these the topological sort breaks the chain.
-    etc = { deps = [ "preserveSshHostKey" ]; };
-    setupSecrets = { deps = [ "restoreSshHostKey" ]; };
-  };
-
   # virtualisation/proxmox-lxc.nix (imported above) registers the Nix
   # store DB via a systemd service (register-nix-paths) -- it never runs
   # an activation script at all. Confirmed live this means neither

modules/platforms/proxmox.nix

@@ -34,6 +34,7 @@ in
     ../hardware-configuration/vm/proxmox.nix
     ../boot/efi.nix
     ../disko/proxmox.nix
+    ../common/preserve-ssh-host-key.nix
   ];
 
   environment.etc = lib.mkIf hasKeyForThisTarget {
@@ -46,36 +47,4 @@ in
       mode = "0644";
     };
   };
-
-  # NixOS's etc activation removes any /etc file that was in the previous
-  # generation's environment.etc but is absent from the current one. Since
-  # the SSH key is only in environment.etc during the --impure build (when
-  # NIXOS_HOST_KEYS_DIR is set), normal rebuilds would remove it as
-  # "obsolete". These scripts mirror lxc.nix's approach: save the live key
-  # before etc runs, restore it after. Without the explicit deps, the
-  # topological sort places preserveSshHostKey after etc (confirmed live on
-  # lxc-tor-relay: position 7 vs etc's position 5), so the key is gone
-  # before it can be saved.
-  system.activationScripts = {
-    preserveSshHostKey = ''
-      if [ -f /etc/ssh/ssh_host_ed25519_key ]; then
-        cp /etc/ssh/ssh_host_ed25519_key /run/sshd-host-key-preserve.tmp
-        cp /etc/ssh/ssh_host_ed25519_key.pub /run/sshd-host-key-preserve.pub.tmp
-      fi
-    '';
-
-    restoreSshHostKey = {
-      deps = [ "etc" ];
-      text = ''
-        if [ ! -f /etc/ssh/ssh_host_ed25519_key ] && [ -f /run/sshd-host-key-preserve.tmp ]; then
-          install -m 0600 /run/sshd-host-key-preserve.tmp /etc/ssh/ssh_host_ed25519_key
-          install -m 0644 /run/sshd-host-key-preserve.pub.tmp /etc/ssh/ssh_host_ed25519_key.pub
-        fi
-        rm -f /run/sshd-host-key-preserve.tmp /run/sshd-host-key-preserve.pub.tmp
-      '';
-    };
-
-    etc = { deps = [ "preserveSshHostKey" ]; };
-    setupSecrets = { deps = [ "restoreSshHostKey" ]; };
-  };
 }

scripts/ha/acceptance-tests.sh

@@ -7,49 +7,109 @@
 set -euo pipefail
 
 # ── Configuration ─────────────────────────────────────────────────────────
-NODE1="ha-server-1"
+# All values override-able via environment variables; defaults match variables.nix.
-NODE2="ha-server-2"
+NODE1="${NODE1:-ha-server-1}"
-NODE1_IP="192.168.2.200"   # vars.haServer1Ip
+NODE2="${NODE2:-ha-server-2}"
-NODE2_IP="192.168.2.201"   # vars.haServer2Ip
+NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
-VIP="192.168.2.202"        # vars.haServerVip
+NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
-XFS_MOUNT="/srv/ha-data"   # vars.haStorageRoot
+VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
-ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage"  # vars.haIscsiIqn
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
+ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}"  # vars.haIscsiIqn
 # ──────────────────────────────────────────────────────────────────────────
 
 PASS=0
 FAIL=0
 RESULTS=()
 
-pass() { echo "  PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); }
+# Use PASS=$((PASS+1)) instead of ((PASS++)) — the latter evaluates to 0 when
-fail() { echo "  FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); }
+# PASS=0, which triggers set -e and kills the script after the very first PASS.
+pass() { echo "  PASS: $1"; PASS=$((PASS+1)); RESULTS+=("PASS $1"); }
+fail() { echo "  FAIL: $1"; FAIL=$((FAIL+1)); RESULTS+=("FAIL $1"); }
 
-n1() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; }
+HA_USER="nixos"
-n2() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; }
+n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
 
 echo "════════════════════════════════════════════════════"
 echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
 echo "════════════════════════════════════════════════════"
 
+# ── Pre-flight: DRBD sync must be complete ────────────────────────────────
+# Tests that check disk state, XFS mount, and iSCSI will fail or give false
+# results while the initial full sync is in progress.  Block until done.
+echo ""
+echo "Pre-flight: verifying DRBD sync is complete..."
+DRBD_PREFLIGHT=$(n1 "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
+if ! echo "$DRBD_PREFLIGHT" | grep -q "^UpToDate/UpToDate$"; then
+  echo ""
+  echo "  ERROR: DRBD initial sync not complete."
+  echo "  Current dstate on $NODE1: $DRBD_PREFLIGHT"
+  echo ""
+  echo "  Monitor progress:"
+  echo "    ssh nixos@$NODE1_IP 'sudo watch -n3 cat /proc/drbd'"
+  echo ""
+  echo "  Re-run this script once dstate shows UpToDate/UpToDate."
+  exit 1
+fi
+echo "  dstate: $DRBD_PREFLIGHT — ready."
+
+# ── Detect Active/Standby nodes ────────────────────────────────────────────
+# Use crm_mon to detect which node holds the Promoted (Primary) DRBD resource.
+# Pacemaker is authoritative; DRBD role can briefly read as Secondary while
+# Pacemaker is mid-transition, giving a false Active/Standby swap.
+# Wait up to 90 s for Pacemaker to settle before giving up.
+echo ""
+echo "Detecting Active/Standby nodes (waiting for Pacemaker to settle)..."
+ACTIVE_NODE=""
+for i in $(seq 1 30); do
+  # crm_mon -1 output contains "Promoted: [ <node> ]" for the DRBD master.
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || n2 "crm_mon -1" 2>/dev/null || true)
+  # crm_mon 2.x formats Promoted lines as "    * Promoted: [ node ]" — the * bullet
+  # means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
+  ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | grep -E '(Promoted|Masters):' | grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+  [[ -n "$ACTIVE_NODE" ]] && break
+  sleep 3
+done
+
+if [[ -z "$ACTIVE_NODE" ]]; then
+  echo "  WARNING: could not determine Active node from crm_mon after 90 s — defaulting to $NODE1"
+  ACTIVE_NODE="$NODE1"
+fi
+
+if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
+  ACTIVE_IP="$NODE1_IP"
+  STANDBY_NODE="$NODE2"; STANDBY_IP="$NODE2_IP"
+  na() { n1 "$@"; }
+  ns() { n2 "$@"; }
+else
+  ACTIVE_IP="$NODE2_IP"
+  STANDBY_NODE="$NODE1"; STANDBY_IP="$NODE1_IP"
+  na() { n2 "$@"; }
+  ns() { n1 "$@"; }
+fi
+echo "  Active:  $ACTIVE_NODE ($ACTIVE_IP)"
+echo "  Standby: $STANDBY_NODE ($STANDBY_IP)"
+
 # ── T1: Corosync quorum established ──────────────────────────────────────
 echo ""
 echo "[T1] Corosync quorum"
-if n1 "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
+if na "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
   pass "cluster has quorum"
 else
   fail "cluster does not have quorum — check corosync on both nodes"
 fi
 
-# ── T2: DRBD Primary on node1, Secondary on node2 ────────────────────────
+# ── T2: DRBD Primary on Active node, Secondary on Standby ────────────────
 echo ""
 echo "[T2] DRBD roles"
-DRBD_ROLE=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
+DRBD_ROLE=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
 if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then
-  pass "DRBD Primary on $NODE1 ($DRBD_ROLE)"
+  pass "DRBD Primary on $ACTIVE_NODE ($DRBD_ROLE)"
 else
-  fail "unexpected DRBD role on $NODE1: $DRBD_ROLE (expected Primary/Secondary)"
+  fail "unexpected DRBD role on $ACTIVE_NODE: $DRBD_ROLE (expected Primary/Secondary)"
 fi
 
-DRBD_DSTATE=$(n1 "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
+DRBD_DSTATE=$(na "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
 if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then
   pass "DRBD disk state UpToDate ($DRBD_DSTATE)"
 else
@@ -59,44 +119,45 @@ fi
 # ── T3: XFS mounted at haStorageRoot on the Active node ──────────────────
 echo ""
 echo "[T3] XFS mount"
-if n1 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+if na "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
-  pass "XFS mounted at ${XFS_MOUNT} on $NODE1"
+  pass "XFS mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
 else
-  fail "XFS not mounted at ${XFS_MOUNT} on $NODE1"
+  fail "XFS not mounted at ${XFS_MOUNT} on $ACTIVE_NODE"
 fi
 
-if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
-  fail "XFS unexpectedly mounted on $NODE2 (should only be on Active node)"
+  fail "XFS unexpectedly mounted on $STANDBY_NODE (should only be on Active node)"
 else
-  pass "XFS not mounted on $NODE2 (correct — Secondary)"
+  pass "XFS not mounted on $STANDBY_NODE (correct — Standby)"
 fi
 
-# ── T4: iSCSI target visible on both nodes ────────────────────────────────
+# ── T4: iSCSI target visible on Active node ───────────────────────────────
 echo ""
 echo "[T4] iSCSI target"
-IQN_COUNT=$(n1 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
+IQN_COUNT=$(na "bash -c 'ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn || true'" 2>/dev/null || echo "0")
 if [[ "$IQN_COUNT" -ge 1 ]]; then
-  pass "iSCSI IQN active on $NODE1 ($IQN_COUNT target(s))"
+  pass "iSCSI IQN active on $ACTIVE_NODE ($IQN_COUNT target(s))"
 else
-  fail "no iSCSI IQN active on $NODE1"
+  fail "no iSCSI IQN active on $ACTIVE_NODE"
 fi
 
-# iSCSI discovery from node2 via VIP
+# iSCSI port reachable from Standby node via VIP.
-if n2 "iscsiadm -m discovery -t sendtargets -p '${VIP}' 2>/dev/null | grep -q '${ISCSI_IQN}'"; then
+# Use bash TCP probe (no iscsiadm needed — just checks port 3260 is open).
-  pass "iSCSI target discoverable from $NODE2 via VIP ${VIP}"
+if ns "bash -c 'echo >/dev/tcp/${VIP}/3260' 2>/dev/null"; then
+  pass "iSCSI port 3260 reachable from $STANDBY_NODE via VIP ${VIP}"
 else
-  fail "iSCSI target not discoverable from $NODE2 via ${VIP}"
+  fail "iSCSI port 3260 not reachable from $STANDBY_NODE via ${VIP}"
 fi
 
-# ── T5: Failover — standby node1, verify resources move to node2 ──────────
+# ── T5: Failover — standby Active node, verify resources move to Standby ──
 echo ""
-echo "[T5] Failover (standby $NODE1)"
+echo "[T5] Failover (standby $ACTIVE_NODE)"
-MYNODE=$(n1 "crm_node -n" 2>/dev/null || echo "")
+ACTIVE_CRMD_NAME=$(na "crm_node -n" 2>/dev/null || echo "")
-n1 "crm_standby -N '${MYNODE}' -v on" 2>/dev/null || true
+na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v on" 2>/dev/null || true
-echo "  Waiting up to 30 s for resources to move to $NODE2..."
+echo "  Waiting up to 120 s for resources to move to $STANDBY_NODE..."
 MOVED=false
-for i in $(seq 1 30); do
+for i in $(seq 1 120); do
-  if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
+  if ns "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
     MOVED=true
     echo "  Resources moved in ${i}s"
     break
@@ -105,49 +166,52 @@ for i in $(seq 1 30); do
 done
 
 if $MOVED; then
-  pass "XFS mounted on $NODE2 after failover"
+  pass "XFS mounted on $STANDBY_NODE after failover"
-  IQN_ON_N2=$(n2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
+  IQN_ON_STANDBY=$(ns "bash -c 'ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn || true'" 2>/dev/null || echo "0")
-  [[ "$IQN_ON_N2" -ge 1 ]] \
+  [[ "$IQN_ON_STANDBY" -ge 1 ]] \
-    && pass "iSCSI target active on $NODE2 after failover" \
+    && pass "iSCSI target active on $STANDBY_NODE after failover" \
-    || fail "iSCSI target NOT active on $NODE2 after failover"
+    || fail "iSCSI target NOT active on $STANDBY_NODE after failover"
 else
-  fail "XFS did not mount on $NODE2 within 30 s — failover incomplete"
+  fail "XFS did not mount on $STANDBY_NODE within 120 s — failover incomplete"
 fi
 
-# ── T6: Data integrity — file written pre-failover readable post-failover ─
+# ── T6: Data integrity — file written post-failover readable ─────────────
 echo ""
 echo "[T6] Data integrity"
-# Write a test file on node2 (now Active) and verify its content
+# Write a test file on the new Active (former Standby) and verify it.
+# Use `echo | sudo tee` for the write: "echo ... > file" via bash -c has the
+# redirect interpreted by the remote nixos shell (not sudo), so the file open
+# runs as nixos and fails with EACCES on the root-owned XFS mount.  Piping
+# through sudo tee lets tee (running as root) open the file instead.
 TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$"
 TEST_CONTENT="ha-acceptance-test-$(date +%s)"
-n2 "echo '${TEST_CONTENT}' > '${TEST_FILE}'" 2>/dev/null || true
+echo "${TEST_CONTENT}" | ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${STANDBY_IP}" sudo tee "${TEST_FILE}" > /dev/null 2>/dev/null || true
-READBACK=$(n2 "cat '${TEST_FILE}' 2>/dev/null" || echo "")
+READBACK=$(ns cat "${TEST_FILE}" 2>/dev/null || echo "")
 if [[ "$READBACK" == "$TEST_CONTENT" ]]; then
-  pass "test file written and read back correctly on $NODE2"
+  pass "test file written and read back correctly on $STANDBY_NODE"
 else
   fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')"
 fi
-n2 "rm -f '${TEST_FILE}'" 2>/dev/null || true
+ns rm -f "${TEST_FILE}" 2>/dev/null || true
 
-# ── T7: Node rejoin — un-standby node1, verify cluster is healthy ─────────
+# ── T7: Node rejoin — un-standby original Active, verify cluster is healthy ─
 echo ""
 echo "[T7] Node rejoin"
-n1 "crm_standby -N '${MYNODE}' -v off" 2>/dev/null || true
+na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
-n1 "crm_resource --cleanup" 2>/dev/null || true
+na "crm_resource --cleanup" 2>/dev/null || true
 sleep 5
 
-ONLINE_NODES=$(n2 "crm_mon -1 2>/dev/null | grep -c 'Online:'" || echo "0")
+if na "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
-if n1 "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
+  pass "$ACTIVE_NODE rejoined — cluster has quorum"
-  pass "$NODE1 rejoined — cluster has quorum"
 else
-  fail "$NODE1 did not rejoin with quorum"
+  fail "$ACTIVE_NODE did not rejoin with quorum"
 fi
 
-DRBD_ROLE_AFTER=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
+DRBD_ROLE_AFTER=$(na "drbdadm role ha-data" 2>/dev/null || echo "unknown")
 if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then
-  pass "$NODE1 is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
+  pass "$ACTIVE_NODE is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
 else
-  fail "unexpected DRBD role on $NODE1 after rejoin: $DRBD_ROLE_AFTER"
+  fail "unexpected DRBD role on $ACTIVE_NODE after rejoin: $DRBD_ROLE_AFTER"
 fi
 
 # ── Summary ───────────────────────────────────────────────────────────────

scripts/ha/cluster-init.sh

@@ -21,22 +21,32 @@
 set -euo pipefail
 
 # ── Configuration ─────────────────────────────────────────────────────────
-# These must match variables.nix haServer* values and the Proxmox VMID
+# All values override-able via environment variables; defaults match variables.nix.
-# assignments.  Update before running.
+NODE1="${NODE1:-ha-server-1}"
-NODE1="ha-server-1"
+NODE2="${NODE2:-ha-server-2}"
-NODE2="ha-server-2"
+NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
-NODE1_IP="192.168.2.200"   # vars.haServer1Ip
+NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
-NODE2_IP="192.168.2.201"   # vars.haServer2Ip
+VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
-VIP="192.168.2.202"        # vars.haServerVip
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
-XFS_MOUNT="/srv/ha-data"   # vars.haStorageRoot
+ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}"  # vars.haIscsiIqn
-ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage"  # vars.haIscsiIqn
 ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
 ISCSI_LUN_SIZE="10G"
 DRBD_DEVICE="/dev/drbd0"
-VMID_NODE1=""              # FILL IN: Proxmox VMID for ha-server-1
+# DRBD backing disk — by-id path that resolves correctly on both nodes
-VMID_NODE2=""              # FILL IN: Proxmox VMID for ha-server-2
+# regardless of whether the OS-level name is sda or sdb (Proxmox VM disk
-PVE_HOST="pve1.sweet.home"
+# ordering is not guaranteed). Matches haServerDrbdDisk in variables.nix.
-PVE_USER="wayne"
+# Override DRBD_DISK if your hardware uses a different controller/slot path.
+DRBD_DISK="${DRBD_DISK:-/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1}"
+VMID_NODE1="${VMID_NODE1:-}"            # set by deploy.sh; needed for STONITH
+VMID_NODE2="${VMID_NODE2:-}"
+PVE_HOST="${PVE_HOST:-pve1.sweet.home}"
+PVE_USER="${PVE_USER:-wayne}"
+# Inter-node SSH: HA_USER is the user to SSH as on NODE2; HA_KEY is the private
+# key to use.  Default is root-to-root (no key arg).  deploy.sh sets HA_USER=nixos
+# and HA_KEY=/tmp/cluster-init-key so the script works even when root-to-root SSH
+# is not available.
+HA_USER="${HA_USER:-root}"
+HA_KEY="${HA_KEY:-}"
 
 # NFS dataset subdirectories to create under XFS_MOUNT.
 # Must mirror vars.nfsShares subpath values in variables.nix.
@@ -59,6 +69,44 @@ warn() { echo "[cluster-init] WARNING: $*" >&2; }
 [[ $(id -u) -eq 0 ]] || die "must run as root"
 [[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1"
 
+# NixOS may not include xfsprogs in root's PATH even when it's in the store.
+# If mkfs.xfs is missing, search the Nix store for it.
+if ! command -v mkfs.xfs &>/dev/null; then
+  _xfs_bin=$(find /nix/store -maxdepth 3 -name mkfs.xfs 2>/dev/null | head -1 | xargs dirname 2>/dev/null || true)
+  [[ -n "$_xfs_bin" ]] && export PATH="$_xfs_bin:$PATH" \
+    || die "mkfs.xfs not found — add xfsprogs to ha-server.nix environment.systemPackages and rebuild"
+fi
+
+# drbdmeta lives alongside drbdadm but may not be in PATH when run via sudo.
+if ! command -v drbdmeta &>/dev/null; then
+  _drbd_bin=$(dirname "$(command -v drbdadm)" 2>/dev/null || true)
+  [[ -n "$_drbd_bin" ]] && export PATH="$_drbd_bin:$PATH" \
+    || die "drbdmeta not found — is drbd-utils in ha-server environment.systemPackages?"
+fi
+
+# Portable 16-hex-char UUID generator (no openssl required).
+_rand_uuid() {
+  cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' | cut -c1-16 | tr '[:lower:]' '[:upper:]'
+}
+
+# Inter-node SSH/SCP helpers — abstract over root-to-root vs nixos+sudo.
+_SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10"
+[[ -n "$HA_KEY" ]] && _SSH_OPTS="-i $HA_KEY $_SSH_OPTS"
+if [[ "$HA_USER" == "root" ]]; then
+  n2_ssh() { ssh $_SSH_OPTS "root@${NODE2_IP}" "$@"; }
+  n2_scp() { scp $_SSH_OPTS "$1" "root@${NODE2_IP}:$2"; }
+else
+  # Non-root user with passwordless sudo; wrap each command with sudo.
+  n2_ssh() { ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo "$@"; }
+  n2_scp() {
+    # SCP to a tmp path, then sudo-move to the real destination as the remote user.
+    local src="$1" dst="$2"
+    local tmp="/tmp/_cluster_init_scp_$$"
+    scp $_SSH_OPTS "$src" "${HA_USER}@${NODE2_IP}:${tmp}"
+    ssh $_SSH_OPTS "${HA_USER}@${NODE2_IP}" sudo mv "${tmp}" "${dst}"
+  }
+fi
+
 # ── 0. Corosync authkey ───────────────────────────────────────────────────
 AUTHKEY="/etc/corosync/authkey"
 mkdir -p /etc/corosync
@@ -68,15 +116,20 @@ if [[ ! -f "$AUTHKEY" ]]; then
   chmod 0400 "$AUTHKEY"
 fi
 log "Distributing authkey to $NODE2..."
-ssh "root@${NODE2_IP}" "mkdir -p /etc/corosync"
+n2_ssh "mkdir -p /etc/corosync"
-scp -q "$AUTHKEY" "root@${NODE2_IP}:${AUTHKEY}"
+n2_scp "$AUTHKEY" "$AUTHKEY"
-ssh "root@${NODE2_IP}" "chmod 0400 '${AUTHKEY}'"
+n2_ssh "chmod 0400 '${AUTHKEY}'"
 
-log "Restarting corosync on both nodes..."
+log "Restarting corosync and pacemaker on both nodes..."
 systemctl restart corosync
-ssh "root@${NODE2_IP}" "systemctl restart corosync"
+n2_ssh "systemctl restart corosync"
 sleep 3
 
+log "Starting pacemaker on both nodes (may have failed at boot before authkey was placed)..."
+systemctl start pacemaker 2>/dev/null || systemctl restart pacemaker 2>/dev/null || true
+n2_ssh "systemctl start pacemaker 2>/dev/null || systemctl restart pacemaker 2>/dev/null || true"
+sleep 2
+
 # ── 1. Corosync quorum ────────────────────────────────────────────────────
 log "Waiting for corosync quorum..."
 for i in $(seq 1 30); do
@@ -99,36 +152,141 @@ for i in $(seq 1 30); do
 done
 
 # ── 2. DRBD initialisation ────────────────────────────────────────────────
+# Put both nodes in Pacemaker standby first so it stops managed resources
+# cleanly, then enable maintenance-mode so Pacemaker's monitor operations are
+# suspended.  Without maintenance-mode, Pacemaker keeps monitoring: when it
+# sees DRBD Primary on a standby node (that it didn't start), it triggers a
+# stop action — killing the initial sync after ~10 s.  Maintenance-mode
+# disables all start/stop/monitor actions for the duration of the sync; it is
+# cleared after UpToDate/UpToDate is confirmed.
+log "Setting both nodes to Pacemaker standby for DRBD metadata init..."
+crm_standby -N "$NODE1" -v on 2>/dev/null || true
+crm_standby -N "$NODE2" -v on 2>/dev/null || true
+
+# Wait for Pacemaker to actually stop DRBD (if it was managing it).
+log "Waiting for DRBD to stop under Pacemaker control..."
+for i in $(seq 1 30); do
+  n1_role=$(drbdadm role ha-data 2>/dev/null || echo "Unconfigured")
+  n2_role=$(n2_ssh "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "Unconfigured")
+  if [[ "$n1_role" == "Unconfigured" ]] && [[ "$n2_role" == "Unconfigured" ]]; then
+    log "DRBD stopped on both nodes"
+    break
+  fi
+  [[ $i -eq 30 ]] && warn "DRBD still active after 60s standby — forcing down anyway"
+  sleep 2
+done
+
+log "Enabling Pacemaker maintenance-mode (suspends monitor/start/stop during sync)..."
+crm_attribute -t crm_config -n maintenance-mode -v true 2>/dev/null || true
+
+log "Detaching DRBD on $NODE1 (belt-and-suspenders after standby)..."
+drbdadm down ha-data 2>/dev/null || true
+log "Detaching DRBD on $NODE2..."
+n2_ssh "drbdadm down ha-data 2>/dev/null || true"
+sleep 2
+
+# Ensure /etc/drbd.conf on both nodes points to DRBD_DISK (the stable by-id
+# path).  VMs built before this fix may have /dev/sda or /dev/sdb hardcoded.
+# NixOS makes /etc/drbd.conf a symlink into the read-only Nix store, so
+# sed -i on the symlink target would fail — we break the symlink first with
+# cp --remove-destination, creating a regular writable copy.
+# Rebuild+redeploy (--force-rebuild) to make this permanent.
+_PATCH_DRBD=$(mktemp)
+cat > "$_PATCH_DRBD" << 'PATCHEOF'
+#!/bin/bash
+WANT="$1"
+conf=/etc/drbd.conf
+if [[ -L "$conf" ]]; then
+  cp --remove-destination "$(readlink -f "$conf")" "$conf"
+fi
+cur=$(drbdadm sh-ll-dev ha-data 2>/dev/null | head -1 || true)
+if [[ -n "$cur" && "$cur" != "$WANT" ]]; then
+  echo "[cluster-init] WARNING: patching $conf: $cur → $WANT (rebuild to make permanent)"
+  sed -i "s,${cur},${WANT},g" "$conf"
+fi
+PATCHEOF
+chmod +x "$_PATCH_DRBD"
+bash "$_PATCH_DRBD" "$DRBD_DISK"
+n2_scp "$_PATCH_DRBD" "/tmp/patch-drbd-disk.sh"
+n2_ssh "bash /tmp/patch-drbd-disk.sh ${DRBD_DISK}"
+n2_ssh "rm -f /tmp/patch-drbd-disk.sh"
+rm -f "$_PATCH_DRBD"
+
 log "Initialising DRBD metadata on $NODE1..."
-if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then
+# Use drbdmeta --force directly for BOTH create-md and write-dev-uuid.
-  drbdadm create-md ha-data --force
+# drbdadm create-md --force passes --force to drbdmeta create-md but NOT to
+# the write-dev-uuid sub-call it makes internally, so write-dev-uuid fails when
+# the backing disk is still busy and stdin is not a TTY:
+# "stdin not a TTY, not waiting for confirmation" → exit 20.
+# Calling drbdmeta --force directly bypasses the exclusive-open confirmation on
+# both steps without needing a TTY, regardless of whether the device is busy.
+# Skip metadata creation only if DRBD is UP and fully synced (UpToDate/UpToDate).
+# When the resource is down, drbdadm dstate reads metadata and returns just
+# "UpToDate" (no slash) — that must not be treated as "already synced".
+# Mismatched UUIDs from an interrupted sync cause instant WFConnection→StandAlone,
+# so we always recreate metadata unless the sync is genuinely complete.
+if [[ "$(drbdadm dstate ha-data 2>/dev/null)" != "UpToDate/UpToDate" ]]; then
+  UUID1=$(_rand_uuid)
+  drbdmeta --force 0 v08 "${DRBD_DISK}" internal create-md
+  drbdmeta --force 0 v08 "${DRBD_DISK}" internal write-dev-uuid "$UUID1"
 fi
 
 log "Initialising DRBD metadata on $NODE2..."
-ssh "root@${NODE2_IP}" "
+if [[ "$(n2_ssh "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null)" != "UpToDate/UpToDate" ]]; then
-  if ! drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent\|Diskless'; then
+  UUID2=$(n2_ssh "cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' | cut -c1-16 | tr '[:lower:]' '[:upper:]'")
-    drbdadm create-md ha-data --force
+  n2_ssh "drbdmeta --force 0 v08 ${DRBD_DISK} internal create-md"
-  fi
+  n2_ssh "drbdmeta --force 0 v08 ${DRBD_DISK} internal write-dev-uuid ${UUID2}"
-"
+fi
 
 log "Bringing up DRBD on both nodes..."
 drbdadm up ha-data 2>/dev/null || true
-ssh "root@${NODE2_IP}" "drbdadm up ha-data 2>/dev/null" || true
+n2_ssh "drbdadm up ha-data" 2>/dev/null || true
 
 log "Forcing $NODE1 to DRBD Primary for initial sync..."
 drbdadm primary ha-data --force
+# NOTE: Pacemaker standby is intentionally kept ON until after the sync
+# completes.  Clearing it here races with the OCF DRBD agent: Pacemaker
+# sees DRBD in WFConnection/SyncSource and may call drbdadm-down thinking
+# something went wrong, killing the sync.  Standby is cleared below, after
+# UpToDate/UpToDate is confirmed.
 
-log "Waiting for DRBD to finish initial sync (this may take several minutes)..."
+log "Waiting for DRBD initial sync to complete (32 GB may take 10–20 min)..."
-for i in $(seq 1 300); do
+log "  (monitor with: watch -n3 cat /proc/drbd)"
-  state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
+_sync_chars=('|' '/' '-' $'\\')
-  if echo "$state" | grep -q "UpToDate/UpToDate"; then
+_sync_iter=0
-    log "DRBD sync complete: $state"
+while true; do
+  _dstate=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
+  if echo "$_dstate" | grep -q "UpToDate/UpToDate"; then
+    printf "\r%-80s\r" ""
+    log "DRBD initial sync complete (dstate: $_dstate)"
     break
   fi
-  [[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)"
+  # Parse connection state from /proc/drbd (cs:SyncSource, cs:Connected, cs:StandAlone …)
-  sleep 1
+  _cs=$(grep -oE 'cs:[A-Za-z]+' /proc/drbd 2>/dev/null | head -1 | sed 's/cs://' || echo "unknown")
+  # /proc/drbd uses variable whitespace: "sync'ed:  5.2%" (two spaces).
+  _pct=$(grep -oE "sync'ed:[[:space:]]+[0-9.]+" /proc/drbd 2>/dev/null | grep -oE "[0-9.]+" | head -1 || echo "")
+  _eta=$(grep -oE "finish:[[:space:]]+[0-9:]+" /proc/drbd 2>/dev/null | grep -oE "[0-9:]+$" | head -1 || echo "")
+  _spd=$(grep -oE "speed:[[:space:]]+[0-9,]+" /proc/drbd 2>/dev/null | grep -oE "[0-9,]+$" | head -1 || echo "")
+  _sync_iter=$(( _sync_iter + 1 ))
+  _sc="${_sync_chars[$_sync_iter % 4]}"
+  if [[ "$_cs" == "StandAlone" && $_sync_iter -gt 5 ]]; then
+    printf "\r%-80s\r" ""
+    die "DRBD is StandAlone after 15 s — peer connection lost (dstate: $_dstate). " \
+        "Check corosync/network and re-run cluster-init."
+  elif [[ -n "$_pct" ]]; then
+    printf "\r  [%s] syncing: %s%% done — ETA %s @ %s K/s     " \
+      "$_sc" "$_pct" "${_eta:-??:??:??}" "${_spd:-?}"
+  else
+    printf "\r  [%s] cs:%s dstate:%s — waiting for sync to start     " "$_sc" "$_cs" "$_dstate"
+  fi
+  sleep 3
 done
 
+log "Disabling Pacemaker maintenance-mode and clearing standby — handing DRBD back to Pacemaker..."
+crm_attribute -t crm_config -n maintenance-mode -v false 2>/dev/null || true
+crm_standby -N "$NODE1" -v off 2>/dev/null || true
+crm_standby -N "$NODE2" -v off 2>/dev/null || true
+
 # ── 3. XFS filesystem ─────────────────────────────────────────────────────
 log "Creating XFS on ${DRBD_DEVICE}..."
 if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
@@ -137,7 +295,7 @@ fi
 
 log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
 mkdir -p "${XFS_MOUNT}"
-mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
+mountpoint -q "${XFS_MOUNT}" || mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
 
 # ── 4. NFS dataset directories ────────────────────────────────────────────
 log "Creating NFS dataset directories..."
@@ -153,108 +311,133 @@ fi
 
 # ── 6. LIO iSCSI target ───────────────────────────────────────────────────
 log "Configuring LIO iSCSI target via targetcli..."
+# Note: do NOT bind portal to ${VIP} here — the VIP isn't assigned yet (Pacemaker
+# creates it).  The default portal (all IPs, port 3260) is correct; Pacemaker's
+# VIP resource will make the target reachable at the VIP address.
+#
+# Clear any existing LIO state first (idempotent: re-run after a partial failure).
+# Use specific delete commands — clearconfig does not reliably clear kernel state.
+if ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -q "${ISCSI_IQN}"; then
+  log "Clearing existing LIO target ${ISCSI_IQN} before reconfiguration..."
+  targetcli "/iscsi delete ${ISCSI_IQN}" 2>/dev/null || true
+fi
+if ls /sys/kernel/config/target/core/ 2>/dev/null | grep -q "fileio"; then
+  log "Clearing existing LIO backstore ha-lun0 before reconfiguration..."
+  targetcli "/backstores/fileio delete ha-lun0" 2>/dev/null || true
+fi
 targetcli <<EOF
 /backstores/fileio create name=ha-lun0 file_or_dev=${ISCSI_LUN_FILE} size=0 write_back=false
 /iscsi create ${ISCSI_IQN}
 /iscsi/${ISCSI_IQN}/tpg1/luns create /backstores/fileio/ha-lun0
-/iscsi/${ISCSI_IQN}/tpg1/portals create ${VIP}
 /iscsi/${ISCSI_IQN}/tpg1 set attribute authentication=0
 /iscsi/${ISCSI_IQN}/tpg1 set attribute demo_mode_write_protect=0
 saveconfig /etc/target/saveconfig.json
 EOF
 
+log "Tearing down LIO kernel objects — Pacemaker will restore via targetctl on the Active node..."
+# LIO holds the backing file open; clear kernel state now so the XFS unmount succeeds.
+# Use specific delete commands (clearconfig does not reliably clear kernel configfs state).
+targetcli "/iscsi delete ${ISCSI_IQN}" 2>/dev/null || warn "LIO iscsi delete failed — umount may fail"
+targetcli "/backstores/fileio delete ha-lun0" 2>/dev/null || warn "LIO backstore delete failed"
+
 log "Distributing iSCSI saveconfig to $NODE2..."
-scp -q /etc/target/saveconfig.json "root@${NODE2_IP}:/etc/target/saveconfig.json"
+n2_scp /etc/target/saveconfig.json /etc/target/saveconfig.json
+
 
 log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..."
-umount "${XFS_MOUNT}"
+umount "${XFS_MOUNT}" || { sync; umount -l "${XFS_MOUNT}"; }
 
 log "Demoting DRBD to Secondary — Pacemaker manages primary role..."
-drbdadm secondary ha-data
+drbdadm role ha-data 2>/dev/null | grep -q "^Primary" && drbdadm secondary ha-data || true
 
 # ── 7. Pacemaker resources ────────────────────────────────────────────────
 log "Configuring Pacemaker cluster properties..."
 crm_attribute -t crm_config -n stonith-enabled -v false
 crm_attribute -t crm_config -n no-quorum-policy -v ignore
 
-log "Creating DRBD promotable clone resource..."
+log "Creating Pacemaker resources via cibadmin..."
-cibadmin --replace --scope resources --xml-text "
+# Use cibadmin --replace with pacemaker-4.0-compatible XML.
-<resources>
+# Key schema rules for pacemaker-4.0:
-  <clone id=\"ms-drbd0\" globally-unique=\"false\">
+#   - globally-unique must be in <meta_attributes>, not a direct <clone> attribute
-    <meta_attributes id=\"ms-drbd0-meta\">
+#   - promoted-max / promoted-node-max (not master-max / master-node-max)
-      <nvpair id=\"ms-drbd0-promotable\" name=\"promotable\" value=\"true\"/>
+#   - constraint with-rsc-role="Promoted" (not "Master")
-      <nvpair id=\"ms-drbd0-master-max\" name=\"master-max\" value=\"1\"/>
+cibadmin --replace --scope resources --xml-text '<resources>
-      <nvpair id=\"ms-drbd0-master-node-max\" name=\"master-node-max\" value=\"1\"/>
+  <clone id="ms-drbd0">
-      <nvpair id=\"ms-drbd0-clone-max\" name=\"clone-max\" value=\"2\"/>
+    <meta_attributes id="ms-drbd0-meta">
-      <nvpair id=\"ms-drbd0-clone-node-max\" name=\"clone-node-max\" value=\"1\"/>
+      <nvpair id="ms-drbd0-globally-unique" name="globally-unique" value="false"/>
-      <nvpair id=\"ms-drbd0-notify\" name=\"notify\" value=\"true\"/>
+      <nvpair id="ms-drbd0-promotable" name="promotable" value="true"/>
-      <nvpair id=\"ms-drbd0-interleave\" name=\"interleave\" value=\"true\"/>
+      <nvpair id="ms-drbd0-promoted-max" name="promoted-max" value="1"/>
+      <nvpair id="ms-drbd0-promoted-node-max" name="promoted-node-max" value="1"/>
+      <nvpair id="ms-drbd0-clone-max" name="clone-max" value="2"/>
+      <nvpair id="ms-drbd0-clone-node-max" name="clone-node-max" value="1"/>
+      <nvpair id="ms-drbd0-notify" name="notify" value="true"/>
+      <nvpair id="ms-drbd0-interleave" name="interleave" value="true"/>
     </meta_attributes>
-    <primitive id=\"drbd0\" class=\"ocf\" type=\"drbd\" provider=\"linbit\">
+    <primitive id="drbd0" class="ocf" type="drbd" provider="linbit">
-      <instance_attributes id=\"drbd0-attrs\">
+      <instance_attributes id="drbd0-attrs">
-        <nvpair id=\"drbd0-resource\" name=\"drbd_resource\" value=\"ha-data\"/>
+        <nvpair id="drbd0-resource" name="drbd_resource" value="ha-data"/>
       </instance_attributes>
       <operations>
-        <op id=\"drbd0-start\" name=\"start\" interval=\"0\" timeout=\"240s\"/>
+        <op id="drbd0-start" name="start" interval="0" timeout="240s"/>
-        <op id=\"drbd0-stop\" name=\"stop\" interval=\"0\" timeout=\"120s\"/>
+        <op id="drbd0-stop" name="stop" interval="0" timeout="120s"/>
-        <op id=\"drbd0-promote\" name=\"promote\" interval=\"0\" timeout=\"90s\"/>
+        <op id="drbd0-promote" name="promote" interval="0" timeout="240s"/>
-        <op id=\"drbd0-demote\" name=\"demote\" interval=\"0\" timeout=\"90s\"/>
+        <op id="drbd0-demote" name="demote" interval="0" timeout="90s"/>
-        <op id=\"drbd0-monitor-master\" name=\"monitor\" interval=\"20s\" timeout=\"20s\" role=\"Promoted\"/>
+        <op id="drbd0-monitor-promoted" name="monitor" interval="20s" timeout="20s" role="Promoted"/>
-        <op id=\"drbd0-monitor-slave\" name=\"monitor\" interval=\"30s\" timeout=\"20s\" role=\"Unpromoted\"/>
+        <op id="drbd0-monitor-unpromoted" name="monitor" interval="30s" timeout="20s" role="Unpromoted"/>
       </operations>
     </primitive>
   </clone>
-  <group id=\"ha-group\">
+  <group id="ha-group">
-    <primitive id=\"xfs-data\" class=\"ocf\" type=\"Filesystem\" provider=\"heartbeat\">
+    <primitive id="xfs-data" class="ocf" type="Filesystem" provider="heartbeat">
-      <instance_attributes id=\"xfs-data-attrs\">
+      <instance_attributes id="xfs-data-attrs">
-        <nvpair id=\"xfs-data-device\" name=\"device\" value=\"${DRBD_DEVICE}\"/>
+        <nvpair id="xfs-data-device" name="device" value="/dev/drbd0"/>
-        <nvpair id=\"xfs-data-directory\" name=\"directory\" value=\"${XFS_MOUNT}\"/>
+        <nvpair id="xfs-data-directory" name="directory" value="/srv/ha-data"/>
-        <nvpair id=\"xfs-data-fstype\" name=\"fstype\" value=\"xfs\"/>
+        <nvpair id="xfs-data-fstype" name="fstype" value="xfs"/>
-        <nvpair id=\"xfs-data-options\" name=\"options\" value=\"defaults\"/>
+        <nvpair id="xfs-data-options" name="options" value="defaults"/>
-        <nvpair id=\"xfs-data-force_unmount\" name=\"force_unmount\" value=\"false\"/>
+        <nvpair id="xfs-data-force_unmount" name="force_unmount" value="true"/>
       </instance_attributes>
       <operations>
-        <op id=\"xfs-data-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
+        <op id="xfs-data-start" name="start" interval="0" timeout="60s"/>
-        <op id=\"xfs-data-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
+        <op id="xfs-data-stop" name="stop" interval="0" timeout="60s"/>
-        <op id=\"xfs-data-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
+        <op id="xfs-data-monitor" name="monitor" interval="20s" timeout="40s"/>
       </operations>
     </primitive>
-    <primitive id=\"iscsi-target\" class=\"systemd\" type=\"targetctl\">
+    <primitive id="iscsi-target" class="systemd" type="targetctl">
       <operations>
-        <op id=\"iscsi-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
+        <op id="iscsi-start" name="start" interval="0" timeout="60s"/>
-        <op id=\"iscsi-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
+        <op id="iscsi-stop" name="stop" interval="0" timeout="60s"/>
-        <op id=\"iscsi-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
+        <op id="iscsi-monitor" name="monitor" interval="20s" timeout="40s"/>
       </operations>
     </primitive>
-    <primitive id=\"nfs-server\" class=\"systemd\" type=\"nfs-server\">
+    <primitive id="nfs-server" class="systemd" type="nfs-server">
       <operations>
-        <op id=\"nfs-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
+        <op id="nfs-start" name="start" interval="0" timeout="60s"/>
-        <op id=\"nfs-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
+        <op id="nfs-stop" name="stop" interval="0" timeout="60s"/>
-        <op id=\"nfs-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"40s\"/>
+        <op id="nfs-monitor" name="monitor" interval="30s" timeout="40s"/>
       </operations>
     </primitive>
-    <primitive id=\"vip\" class=\"ocf\" type=\"IPaddr2\" provider=\"heartbeat\">
+    <primitive id="vip" class="ocf" type="IPaddr2" provider="heartbeat">
-      <instance_attributes id=\"vip-attrs\">
+      <instance_attributes id="vip-attrs">
-        <nvpair id=\"vip-ip\" name=\"ip\" value=\"${VIP}\"/>
+        <nvpair id="vip-ip" name="ip" value="192.168.2.229"/>
-        <nvpair id=\"vip-cidr\" name=\"cidr_netmask\" value=\"24\"/>
+        <nvpair id="vip-cidr" name="cidr_netmask" value="24"/>
       </instance_attributes>
       <operations>
-        <op id=\"vip-start\" name=\"start\" interval=\"0\" timeout=\"20s\"/>
+        <op id="vip-start" name="start" interval="0" timeout="20s"/>
-        <op id=\"vip-stop\" name=\"stop\" interval=\"0\" timeout=\"20s\"/>
+        <op id="vip-stop" name="stop" interval="0" timeout="20s"/>
-        <op id=\"vip-monitor\" name=\"monitor\" interval=\"10s\" timeout=\"20s\"/>
+        <op id="vip-monitor" name="monitor" interval="10s" timeout="20s"/>
       </operations>
     </primitive>
   </group>
-</resources>
+</resources>'
-"
 
-log "Adding ordering and colocation constraints..."
+log "Adding Pacemaker ordering and colocation constraints..."
-cibadmin --create --scope constraints --xml-text "
+cibadmin --replace --scope constraints --xml-text '<constraints>
-<constraints>
+  <rsc_order id="order-drbd-group" first="ms-drbd0" first-action="promote" then="ha-group" then-action="start" kind="Mandatory"/>
-  <rsc_order id=\"order-drbd-group\" first=\"ms-drbd0\" first-action=\"promote\" then=\"ha-group\" then-action=\"start\"/>
+  <rsc_colocation id="coloc-group-with-drbd" score="INFINITY" rsc="ha-group" with-rsc="ms-drbd0" with-rsc-role="Promoted"/>
-  <rsc_colocation id=\"coloc-group-with-drbd\" rsc=\"ha-group\" with-rsc=\"ms-drbd0\" with-rsc-role=\"Master\" score=\"INFINITY\"/>
+</constraints>'
-</constraints>
+
-"
+log "Clearing stale Pacemaker failure history..."
+crm_resource --cleanup 2>/dev/null || true
 
 log "Waiting for resources to start..."
 for i in $(seq 1 60); do

scripts/ha/deploy.sh

@@ -0,0 +1,499 @@
+#!/usr/bin/env bash
+# deploy.sh — Full lifecycle management for the HA file-server cluster.
+#
+# Handles everything from zero (no VMs, no secrets) through a running,
+# tested cluster, and optionally tears it back down.
+#
+# Usage:
+#   scripts/ha/deploy.sh [options]
+#   scripts/ha/deploy.sh --destroy [options]
+#
+# Phases (all run by default; skip any with --skip-*):
+#   1. ensure-bridge   Create storage bridge (vmbr1) on the Proxmox node if absent.
+#   2. sync-keys       Generate SSH host keys and register age keys for both nodes.
+#   3. create-vms      Build disk images and create both VMs via create-proxmox-resource.sh.
+#   4. add-hardware    Attach storage NIC (vmbr1) and DRBD data disk to each VM.
+#   5. boot-wait       Start VMs, wait for SSH on both nodes.
+#   6. cluster-init    Form the cluster: DRBD, corosync, Pacemaker, NFS, VIP.
+#                      Also encrypts the generated corosync authkey into the repo.
+#   7. run-tests       Run acceptance tests (T1–T7).
+#
+# Options:
+#   --node <host>            Proxmox host to deploy on (default: pve1.sweet.home)
+#   --vmid1 <n>              VMID for ha-server-1 (default: 200)
+#   --vmid2 <n>              VMID for ha-server-2 (default: 201)
+#   --storage <pool>         Proxmox storage pool (default: local-zfs)
+#   --storage-bridge <br>    Bridge for HA storage network (default: vmbr1)
+#   --drbd-disk-gb <n>       DRBD data disk size in GB (default: 32)
+#   --memory <MB>            RAM per node (default: 4096)
+#   --cores <n>              vCPUs per node (default: 4)
+#   --skip-ensure-bridge     Skip storage bridge creation/check
+#   --skip-sync-keys         Skip sync-host-keys.sh (clan vars already exist)
+#   --skip-create-vms        Skip VM creation (VMs already exist)
+#   --skip-add-hardware      Skip net1/scsi1 attachment (already attached)
+#   --skip-boot-wait         Skip boot/SSH wait (VMs already running)
+#   --skip-refresh-sops-keys Skip scanning running VMs for fresh SSH host keys
+#   --skip-cluster-init      Skip cluster formation (cluster already configured)
+#   --skip-tests             Skip acceptance tests
+#   --force-rebuild          Pass --force-rebuild to create-proxmox-resource.sh
+#   --destroy                Stop and delete both VMs (skip all other phases)
+#   --dry-run                Print what would run without executing
+#   -h|--help                Show this message
+#
+# Prerequisites:
+#   - SSH access to the Proxmox node as $PROXMOX_SSH_USER (wayne).
+#   - sops age key in the standard location (used by sync-host-keys.sh).
+#   - For --skip-sync-keys: clan vars already in vars/per-machine/proxmox-ha-server-{1,2}/.
+#   - For full tests: secrets/common.yaml decryptable on both nodes (run
+#     `sops updatekeys secrets/common.yaml` after sync-keys).
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# shellcheck source=../env.sh
+source "${REPO_ROOT}/scripts/env.sh"
+
+# ── Defaults ──────────────────────────────────────────────────────────────────
+
+NODE="${PROXMOX_HOST:-$PVE1_HOST}"
+VMID1=200
+VMID2=201
+STORAGE="${PROXMOX_STORAGE:-local-zfs}"
+STORAGE_BRIDGE="vmbr1"
+DRBD_DISK_GB=32
+MEMORY_MB=4096
+CORES=4
+
+SKIP_ENSURE_BRIDGE=false
+SKIP_SYNC_KEYS=false
+SKIP_CREATE_VMS=false
+SKIP_ADD_HARDWARE=false
+SKIP_BOOT_WAIT=false
+SKIP_REFRESH_SOPS_KEYS=false
+SKIP_CLUSTER_INIT=false
+SKIP_TESTS=false
+FORCE_REBUILD=false
+DESTROY=false
+DRY_RUN=false
+
+# ── Variables from repo ───────────────────────────────────────────────────────
+
+NODE1_HOST="ha-server-1"
+NODE2_HOST="ha-server-2"
+NODE1_IP="192.168.2.228"
+NODE2_IP="192.168.2.227"
+STORAGE_IP1="192.168.4.228"
+STORAGE_IP2="192.168.4.227"
+STORAGE_CIDR="192.168.4.0/29"
+SSH_USER="${PROXMOX_SSH_USER:-wayne}"
+
+# ── Argument parsing ──────────────────────────────────────────────────────────
+
+usage() {
+  sed -n '/^# Usage:/,/^[^#]/{ /^#/{ s/^# \?//; p } }' "$0"
+  exit "${1:-0}"
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --node)               NODE="$2";           shift 2 ;;
+    --vmid1)              VMID1="$2";          shift 2 ;;
+    --vmid2)              VMID2="$2";          shift 2 ;;
+    --storage)            STORAGE="$2";        shift 2 ;;
+    --storage-bridge)     STORAGE_BRIDGE="$2"; shift 2 ;;
+    --drbd-disk-gb)       DRBD_DISK_GB="$2";   shift 2 ;;
+    --memory)             MEMORY_MB="$2";      shift 2 ;;
+    --cores)              CORES="$2";          shift 2 ;;
+    --skip-ensure-bridge) SKIP_ENSURE_BRIDGE=true; shift ;;
+    --skip-sync-keys)     SKIP_SYNC_KEYS=true; shift ;;
+    --skip-create-vms)    SKIP_CREATE_VMS=true; shift ;;
+    --skip-add-hardware)  SKIP_ADD_HARDWARE=true; shift ;;
+    --skip-boot-wait)         SKIP_BOOT_WAIT=true; shift ;;
+    --skip-refresh-sops-keys) SKIP_REFRESH_SOPS_KEYS=true; shift ;;
+    --skip-cluster-init)      SKIP_CLUSTER_INIT=true; shift ;;
+    --skip-tests)         SKIP_TESTS=true;     shift ;;
+    --force-rebuild)      FORCE_REBUILD=true;  shift ;;
+    --destroy)            DESTROY=true;        shift ;;
+    --dry-run)            DRY_RUN=true;        shift ;;
+    -h|--help)            usage 0 ;;
+    *) echo "Unknown option: $1" >&2; usage 1 ;;
+  esac
+done
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+log()  { echo "==> $*"; }
+logn() { echo "    $*"; }
+err()  { echo "ERROR: $*" >&2; exit 1; }
+
+run() {
+  if $DRY_RUN; then
+    echo "[dry-run] $*"
+  else
+    "$@"
+  fi
+}
+
+pve() {
+  # Run a command on the Proxmox node via SSH.
+  if $DRY_RUN; then
+    echo "[dry-run] ssh ${SSH_USER}@${NODE} sudo $*"
+  else
+    ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
+  fi
+}
+
+pve_check() {
+  # Run a read-only probe on the Proxmox node — always executes even in dry-run.
+  ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
+}
+
+HA_USER="nixos"
+
+n1() {
+  # Run a command on ha-server-1 via SSH as nixos with sudo.
+  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null
+}
+
+n2() {
+  # Run a command on ha-server-2 via SSH as nixos with sudo.
+  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null
+}
+
+wait_for_ssh() {
+  local ip="$1" label="$2"
+  if $DRY_RUN; then
+    logn "[dry-run] Skipping SSH wait for ${label} (${ip})"
+    return 0
+  fi
+  local deadline=$(( $(date +%s) + 300 ))
+  log "Waiting for SSH on ${label} (${ip}) — up to 5 min..."
+  while [[ $(date +%s) -lt $deadline ]]; do
+    if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=3 \
+        -o BatchMode=yes "${HA_USER}@${ip}" true 2>/dev/null; then
+      logn "${label} is up."
+      return 0
+    fi
+    sleep 5
+  done
+  err "Timed out waiting for SSH on ${label} (${ip})"
+}
+
+# ── Destroy mode ─────────────────────────────────────────────────────────────
+
+if $DESTROY; then
+  log "Destroying HA cluster VMs (${VMID1}=${NODE1_HOST}, ${VMID2}=${NODE2_HOST}) on ${NODE}"
+  for vmid in "$VMID1" "$VMID2"; do
+    STATUS=$(pve "qm status ${vmid} 2>/dev/null" 2>/dev/null || true)
+    if echo "$STATUS" | grep -q "running"; then
+      log "Stopping VMID ${vmid}..."
+      pve "qm stop ${vmid} --skiplock 1"
+      sleep 5
+    fi
+    if $DRY_RUN || pve "qm config ${vmid} >/dev/null 2>&1"; then
+      log "Deleting VMID ${vmid}..."
+      run pve "qm destroy ${vmid} --purge 1"
+    else
+      logn "VMID ${vmid} not found — already gone."
+    fi
+  done
+  log "Done — cluster VMs destroyed."
+  exit 0
+fi
+
+# ── Phase 1: Storage bridge ───────────────────────────────────────────────────
+
+if ! $SKIP_ENSURE_BRIDGE; then
+  log "Phase 1: Ensuring storage bridge ${STORAGE_BRIDGE} on ${NODE}"
+  if pve_check "test -d /sys/class/net/${STORAGE_BRIDGE}" &>/dev/null; then
+    logn "${STORAGE_BRIDGE} already exists — skipping."
+  else
+    logn "Creating isolated internal bridge ${STORAGE_BRIDGE} (no upstream port, ${STORAGE_CIDR})"
+    BRIDGE_CONF="auto ${STORAGE_BRIDGE}
+iface ${STORAGE_BRIDGE} inet manual
+    bridge-ports none
+    bridge-stp off
+    bridge-fd 0"
+    if $DRY_RUN; then
+      echo "[dry-run] Would write /etc/network/interfaces.d/${STORAGE_BRIDGE}.conf and ifup it"
+    else
+      ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
+        "echo '${BRIDGE_CONF}' | sudo tee /etc/network/interfaces.d/${STORAGE_BRIDGE}.conf > /dev/null && sudo ifup ${STORAGE_BRIDGE}"
+      logn "${STORAGE_BRIDGE} created and brought up."
+    fi
+  fi
+fi
+
+# ── Phase 2: Sync host keys ───────────────────────────────────────────────────
+
+if ! $SKIP_SYNC_KEYS; then
+  log "Phase 2: Syncing SSH host keys for both HA targets"
+  for target in proxmox-ha-server-1 proxmox-ha-server-2; do
+    CLAN_DIR="${REPO_ROOT}/vars/per-machine/${target}/openssh"
+    if [[ -d "$CLAN_DIR" ]]; then
+      logn "Clan vars for ${target} already exist — skipping."
+    else
+      logn "Generating host keys for ${target}..."
+      run bash "${REPO_ROOT}/scripts/secrets/sync-host-keys.sh" "$target"
+    fi
+  done
+fi
+
+# ── Phase 2.5: Prepare Proxmox node for building ─────────────────────────────
+if ! $SKIP_CREATE_VMS && ! $DRY_RUN; then
+  CURRENT_BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD)"
+
+  # Fix /nix ownership if it exists but belongs to a different UID.
+  # pve1's IPA-enrolled wayne (UID 50002) can't write to a store created by
+  # another UID — passwordless sudo corrects it once.
+  # Use direct SSH (no sudo) for the writability check so we test wayne's own
+  # access, not root's.
+  local_ssh() { ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "$*"; }
+  if local_ssh "test -d /nix" &>/dev/null && ! local_ssh "test -w /nix" &>/dev/null; then
+    logn "/nix exists but not writable by ${SSH_USER} — fixing ownership with sudo (one-time)..."
+    local_ssh "sudo chown -R ${SSH_USER} /nix"
+    logn "Done."
+  fi
+  unset -f local_ssh
+
+  # Ensure the remote clone is on the correct branch so create-proxmox-resource.sh
+  # builds from the same commits we're deploying.
+  REMOTE_REPO="/home/${SSH_USER}/nixos"
+  if pve_check "test -d ${REMOTE_REPO}/.git" &>/dev/null; then
+    REMOTE_BRANCH=$(ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
+      "cd ${REMOTE_REPO} && git rev-parse --abbrev-ref HEAD 2>/dev/null")
+    if [[ "$REMOTE_BRANCH" != "$CURRENT_BRANCH" ]]; then
+      logn "Remote clone is on '${REMOTE_BRANCH}', switching to '${CURRENT_BRANCH}'..."
+      ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
+        "cd ${REMOTE_REPO} && git fetch origin && git checkout '${CURRENT_BRANCH}' && git pull --ff-only"
+      logn "Done."
+    fi
+  fi
+fi
+
+# ── Phase 3: Create VMs ───────────────────────────────────────────────────────
+
+if ! $SKIP_CREATE_VMS; then
+  log "Phase 3: Building and creating VMs on ${NODE}"
+
+  CREATE="${REPO_ROOT}/scripts/proxmox/create-proxmox-resource.sh"
+
+  for spec in "${VMID1}:ha-server-1:proxmox-ha-server-1" "${VMID2}:ha-server-2:proxmox-ha-server-2"; do
+    IFS=: read -r vmid host_name flake_target <<< "$spec"
+    log "Creating ${flake_target} (VMID ${vmid}) on ${NODE}..."
+    # Always --force-rebuild: create-proxmox-resource.sh only calls
+    # sync_remote_host_keys (which populates host-keys/ for proxmox.nix to
+    # bake the clan-var SSH key into the disko image) when it actually builds.
+    # Reusing a cached image skips that step, so destroy+recreate would reuse
+    # an image with a stale/random key baked in → sops fails on first boot.
+    run bash "$CREATE" \
+      --type vm \
+      --host "$host_name" \
+      --vmid "$vmid" \
+      --node "$NODE" \
+      --storage "$STORAGE" \
+      --memory "$MEMORY_MB" \
+      --cores "$CORES" \
+      --force-rebuild
+  done
+fi
+
+# ── Phase 4: Add storage NIC and DRBD disk ────────────────────────────────────
+
+if ! $SKIP_ADD_HARDWARE; then
+  log "Phase 4: Attaching storage NIC (${STORAGE_BRIDGE}) and DRBD disk (${DRBD_DISK_GB}G) to each VM"
+  for vmid in "$VMID1" "$VMID2"; do
+    log "  VMID ${vmid}: stopping to add hardware..."
+    pve "qm stop ${vmid} --skiplock 1 2>/dev/null; sleep 3" || true
+
+    logn "Adding net1 (${STORAGE_BRIDGE})..."
+    pve "qm set ${vmid} --net1 virtio,bridge=${STORAGE_BRIDGE},firewall=0"
+
+    logn "Adding scsi1 (${STORAGE}:${DRBD_DISK_GB}G for DRBD)..."
+    pve "qm set ${vmid} --scsi1 ${STORAGE}:${DRBD_DISK_GB},format=raw"
+
+    logn "Starting VMID ${vmid}..."
+    pve "qm start ${vmid}"
+  done
+fi
+
+# ── Phase 5: Wait for SSH ─────────────────────────────────────────────────────
+
+if ! $SKIP_BOOT_WAIT; then
+  log "Phase 5: Waiting for both nodes to come up"
+  wait_for_ssh "$NODE1_IP" "$NODE1_HOST"
+  wait_for_ssh "$NODE2_IP" "$NODE2_HOST"
+  logn "Both nodes are SSHable."
+  # Give systemd a few seconds to settle after activation
+  sleep 10
+fi
+
+# ── Phase 5.5: Refresh sops host-key registrations ───────────────────────────
+#
+# Disko builds raw disk images; each new VM boots with a freshly-generated SSH
+# host key rather than the one pre-seeded in clan vars.  This phase scans the
+# actual running VMs, and if their ed25519 host keys differ from what clan vars
+# record: updates the clan var pub-key files, rewrites the .sops.yaml age-key
+# anchors, and re-encrypts all affected sops files so the nodes can decrypt
+# secrets on the next nixos-rebuild.  Safe no-op when keys haven't changed.
+
+if ! $SKIP_REFRESH_SOPS_KEYS; then
+  if $DRY_RUN; then
+    logn "[dry-run] Would scan VM host keys and refresh .sops.yaml / secrets if needed"
+  else
+    log "Phase 5.5: Refreshing sops host-key registrations (disko key drift fix)"
+    SOPS_UPDATED=false
+
+    for spec in \
+      "${NODE1_IP}:proxmox-ha-server-1:${NODE1_HOST}" \
+      "${NODE2_IP}:proxmox-ha-server-2:${NODE2_HOST}"; do
+      IFS=: read -r node_ip flake_target host_name <<< "$spec"
+      CLAN_PUB="${REPO_ROOT}/vars/per-machine/${flake_target}/openssh/ssh_host_ed25519_key.pub/value"
+
+      logn "Scanning ed25519 host key from ${host_name} (${node_ip})..."
+      RAW=$(ssh-keyscan -t ed25519 "${node_ip}" 2>/dev/null | grep -v "^#") || true
+      if [[ -z "$RAW" ]]; then
+        logn "WARNING: no ed25519 key returned by ssh-keyscan for ${node_ip} — skipping"
+        continue
+      fi
+      # ssh-keyscan returns:  <ip>  ssh-ed25519  <b64key>
+      SCANNED_TYPE=$(awk '{print $2}' <<< "$RAW")
+      SCANNED_KEY=$(awk  '{print $3}' <<< "$RAW")
+      SCANNED_PUBKEY="${SCANNED_TYPE} ${SCANNED_KEY} ${host_name}"
+
+      CURRENT=$(tr -d '\n' < "$CLAN_PUB" 2>/dev/null || true)
+      if [[ "$SCANNED_PUBKEY" == "$CURRENT" ]]; then
+        logn "${host_name}: clan var matches running key — no update needed"
+        continue
+      fi
+
+      logn "${host_name}: key drift detected — updating clan var"
+      logn "  old: ${CURRENT}"
+      logn "  new: ${SCANNED_PUBKEY}"
+      echo "$SCANNED_PUBKEY" > "$CLAN_PUB"
+      SOPS_UPDATED=true
+
+      # Rewrite the .sops.yaml anchor for this host with the new age key.
+      ANCHOR="${flake_target}"   # e.g. proxmox-ha-server-1
+      NEW_AGE=$(echo "$SCANNED_PUBKEY" | \
+        nix run --quiet --no-warn-dirty nixpkgs#ssh-to-age 2>/dev/null)
+      if [[ -z "$NEW_AGE" ]]; then
+        err "ssh-to-age produced no output for ${host_name} — check nixpkgs#ssh-to-age"
+      fi
+      logn "  new age key: ${NEW_AGE}"
+      sed -i "/&${ANCHOR} /s| age[a-z0-9]*$| ${NEW_AGE}|" "${REPO_ROOT}/.sops.yaml"
+    done
+
+    if $SOPS_UPDATED; then
+      logn "Running sops updatekeys on affected secrets..."
+      SOPS="nix run --quiet --no-warn-dirty nixpkgs#sops --"
+      (cd "${REPO_ROOT}" && \
+        $SOPS updatekeys -y secrets/common.yaml && \
+        $SOPS updatekeys -y secrets/ha-server-1.yaml && \
+        $SOPS updatekeys -y secrets/ha-server-2.yaml && \
+        $SOPS updatekeys -y secrets/ha-server-1.keytab && \
+        $SOPS updatekeys -y secrets/ha-server-2.keytab)
+      # Note: ha-corosync-authkey is re-generated and re-encrypted by cluster-init below.
+
+      logn "Committing refreshed host keys and re-encrypted secrets..."
+      (cd "${REPO_ROOT}" && \
+        git add \
+          vars/per-machine/proxmox-ha-server-1/openssh/ssh_host_ed25519_key.pub/value \
+          vars/per-machine/proxmox-ha-server-2/openssh/ssh_host_ed25519_key.pub/value \
+          .sops.yaml \
+          secrets/common.yaml \
+          secrets/ha-server-1.yaml \
+          secrets/ha-server-2.yaml \
+          secrets/ha-server-1.keytab \
+          secrets/ha-server-2.keytab && \
+        git commit -m "secrets(ha): refresh sops host-key registrations for new VM instances" || true)
+      logn "Sops keys refreshed and committed."
+    fi
+  fi
+fi
+
+# ── Phase 6: Cluster init ─────────────────────────────────────────────────────
+
+if ! $SKIP_CLUSTER_INIT; then
+  log "Phase 6: Initialising HA cluster"
+
+  CLUSTER_INIT="${REPO_ROOT}/scripts/ha/cluster-init.sh"
+  [[ -x "$CLUSTER_INIT" ]] || chmod +x "$CLUSTER_INIT"
+
+  if $DRY_RUN; then
+    logn "[dry-run] Would generate temp key, authorise on ${NODE2_HOST}, scp cluster-init.sh to ${NODE1_HOST}, and run it as root via sudo"
+  else
+    # Generate a temp keypair so cluster-init.sh can SSH node1→node2 as ${HA_USER}.
+    # Root on node1 has no keys; a temp key bridging node1→node2 nixos solves this.
+    TEMP_KEY="${REPO_ROOT}/.tmp-cluster-init-key"
+    TEMP_KEY_PUB="${TEMP_KEY}.pub"
+    rm -f "$TEMP_KEY" "$TEMP_KEY_PUB"
+    ssh-keygen -t ed25519 -f "$TEMP_KEY" -N "" -C "cluster-init-temp-$(date +%s)" -q
+    TEMP_PUBKEY=$(cat "$TEMP_KEY_PUB")
+
+    logn "Authorising temp key on ${NODE2_HOST} for ${HA_USER}..."
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE2_IP}" \
+      "mkdir -p ~/.ssh && chmod 700 ~/.ssh && echo '${TEMP_PUBKEY}' >> ~/.ssh/authorized_keys"
+
+    logn "Placing temp key on ${NODE1_HOST} for root..."
+    scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
+      "$TEMP_KEY" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init-key"
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo mkdir -p /root/.ssh && sudo cp /tmp/cluster-init-key /root/.ssh/cluster-init-key && \
+       sudo chmod 600 /root/.ssh/cluster-init-key && rm -f /tmp/cluster-init-key"
+
+    logn "Uploading cluster-init.sh to ${NODE1_HOST}..."
+    scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
+      "$CLUSTER_INIT" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init.sh"
+
+    logn "Running cluster-init.sh on ${NODE1_HOST}..."
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo env NODE1=${NODE1_HOST} NODE2=${NODE2_HOST} \
+       NODE1_IP=${NODE1_IP} NODE2_IP=${NODE2_IP} \
+       VIP=192.168.2.229 XFS_MOUNT=/srv/ha-data \
+       ISCSI_IQN=iqn.2026-01.home.sweet:ha-storage \
+       VMID_NODE1=${VMID1} VMID_NODE2=${VMID2} \
+       HA_USER=${HA_USER} HA_KEY=/root/.ssh/cluster-init-key \
+       bash /tmp/cluster-init.sh"
+
+    logn "Cleaning up temp key from both nodes..."
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE2_IP}" \
+      "sed -i '/cluster-init-temp/d' ~/.ssh/authorized_keys" 2>/dev/null || true
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo rm -f /root/.ssh/cluster-init-key" 2>/dev/null || true
+    rm -f "$TEMP_KEY" "$TEMP_KEY_PUB"
+
+    # Encrypt the corosync authkey generated by cluster-init and commit it.
+    log "  Encrypting corosync authkey into secrets/ha-corosync-authkey..."
+    AUTHKEY_TMP="${REPO_ROOT}/secrets/ha-corosync-authkey.tmp"
+    ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \
+      "sudo cat /etc/corosync/authkey" > "$AUTHKEY_TMP"
+    if [[ ! -s "$AUTHKEY_TMP" ]]; then
+      err "corosync authkey on node1 is empty — cluster-init may have failed."
+    fi
+    mv "$AUTHKEY_TMP" "${REPO_ROOT}/secrets/ha-corosync-authkey"
+    (cd "${REPO_ROOT}" && nix run nixpkgs#sops -- -e --input-type binary -i secrets/ha-corosync-authkey)
+    logn "Authkey encrypted. Committing..."
+    (cd "${REPO_ROOT}" && git add secrets/ha-corosync-authkey && \
+      git commit -m "secrets(ha): encrypt corosync authkey generated by cluster-init")
+    logn "Committed."
+  fi
+fi
+
+# ── Phase 7: Acceptance tests ─────────────────────────────────────────────────
+
+if ! $SKIP_TESTS; then
+  log "Phase 7: Running acceptance tests (T1–T7)"
+  if $DRY_RUN; then
+    logn "[dry-run] Would run acceptance-tests.sh against ${NODE1_HOST}/${NODE2_HOST}"
+  else
+    NODE1="$NODE1_HOST" NODE2="$NODE2_HOST" \
+    NODE1_IP="$NODE1_IP" NODE2_IP="$NODE2_IP" \
+    VIP="192.168.2.229" \
+    bash "${REPO_ROOT}/scripts/ha/acceptance-tests.sh"
+  fi
+fi
+
+log "Deploy complete."

scripts/ha/failover.sh

@@ -0,0 +1,256 @@
+#!/usr/bin/env bash
+# failover.sh — graceful HA cluster failover
+#
+# Detects which node is active and moves all resources to the other node by
+# putting the active node into Pacemaker standby.  Waits for the XFS mount to
+# appear on the target before returning.
+#
+# Usage:
+#   scripts/ha/failover.sh [--to node1|node2] [--force] [--timeout <s>] [--dry-run]
+#
+#   --to node1|node2  target node (default: the node that is NOT currently active)
+#   --force           skip the interactive confirmation prompt
+#   --timeout <s>     seconds to wait for resources to move (default: 120)
+#   --dry-run         show what would be done without changing anything
+set -euo pipefail
+
+# ── Configuration ─────────────────────────────────────────────────────────
+NODE1="${NODE1:-ha-server-1}"
+NODE2="${NODE2:-ha-server-2}"
+NODE1_IP="${NODE1_IP:-192.168.2.228}"
+NODE2_IP="${NODE2_IP:-192.168.2.227}"
+VIP="${VIP:-192.168.2.229}"
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"
+HA_USER="${HA_USER:-nixos}"
+# ──────────────────────────────────────────────────────────────────────────
+
+n1() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
+
+# ── Argument parsing ───────────────────────────────────────────────────────
+TARGET_NODE=""
+FORCE=false
+DRY_RUN=false
+TIMEOUT=120
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --to)
+      shift
+      case "${1:-}" in
+        node1|ha-server-1) TARGET_NODE="$NODE1" ;;
+        node2|ha-server-2) TARGET_NODE="$NODE2" ;;
+        *) echo "ERROR: --to must be node1, node2, ha-server-1, or ha-server-2"; exit 1 ;;
+      esac
+      ;;
+    --force)   FORCE=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --timeout) shift; TIMEOUT="${1:?--timeout requires a value}" ;;
+    *) echo "Unknown argument: $1"; echo "Usage: $0 [--to node1|node2] [--force] [--timeout <s>] [--dry-run]"; exit 1 ;;
+  esac
+  shift
+done
+
+DRY_PREFIX=""
+$DRY_RUN && DRY_PREFIX="[dry-run] "
+
+echo "════════════════════════════════════════════════════"
+echo " HA Cluster Failover — $(date '+%Y-%m-%d %H:%M:%S')"
+$DRY_RUN && echo " MODE: dry-run — no changes will be made"
+echo "════════════════════════════════════════════════════"
+
+# ── Detect active node ─────────────────────────────────────────────────────
+echo ""
+echo "Detecting active node..."
+
+CRM_OUT=""
+if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
+fi
+if [[ -z "$CRM_OUT" ]]; then
+  if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then
+    CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
+  fi
+fi
+
+# crm_mon 2.x formats Promoted lines as "    * Promoted: [ node ]" — the * bullet
+# means ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
+ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
+  grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+
+if [[ -z "$ACTIVE_NODE" ]]; then
+  echo ""
+  echo "ERROR: could not determine active node from crm_mon."
+  echo "  Is Pacemaker still settling?  Try running scripts/ha/health.sh first."
+  echo "  If Pacemaker is down on both nodes, manual recovery is required."
+  exit 1
+fi
+
+if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
+  ACTIVE_IP="$NODE1_IP"
+  STANDBY_NODE="$NODE2"
+  STANDBY_IP="$NODE2_IP"
+  na() { n1 "$@"; }
+  ns() { n2 "$@"; }
+else
+  ACTIVE_IP="$NODE2_IP"
+  STANDBY_NODE="$NODE1"
+  STANDBY_IP="$NODE1_IP"
+  na() { n2 "$@"; }
+  ns() { n1 "$@"; }
+fi
+
+echo "  Active:  $ACTIVE_NODE ($ACTIVE_IP)"
+echo "  Standby: $STANDBY_NODE ($STANDBY_IP)"
+
+# ── Validate target ────────────────────────────────────────────────────────
+if [[ -n "$TARGET_NODE" ]]; then
+  if [[ "$TARGET_NODE" == "$ACTIVE_NODE" ]]; then
+    echo ""
+    echo "ERROR: $TARGET_NODE is already the active node — nothing to do."
+    exit 1
+  fi
+  echo "  Target:  $TARGET_NODE (as requested)"
+else
+  echo "  Target:  $STANDBY_NODE (auto — the other node)"
+fi
+
+# ── Pre-checks ─────────────────────────────────────────────────────────────
+echo ""
+echo "Pre-checks..."
+
+DRBD_DSTATE=$(na "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
+if ! echo "$DRBD_DSTATE" | grep -q "UpToDate/UpToDate"; then
+  echo ""
+  echo "  WARNING: DRBD dstate is '$DRBD_DSTATE' (not UpToDate/UpToDate)."
+  echo "  Failing over with a partially-synced disk risks split-brain."
+  if ! $FORCE; then
+    echo "  Use --force to proceed anyway (not recommended)."
+    exit 1
+  fi
+  echo "  --force specified — proceeding despite non-ideal DRBD state."
+else
+  echo "  DRBD dstate: $DRBD_DSTATE — OK"
+fi
+
+QUORUM_OK=$(na "corosync-quorumtool -s 2>/dev/null | grep -c 'Quorate:.*Yes'" 2>/dev/null || echo "0")
+if [[ "$QUORUM_OK" -lt 1 ]]; then
+  echo "  ERROR: cluster does not have quorum — failover would be unsafe."
+  exit 1
+fi
+echo "  Quorum: OK"
+
+# ── Confirm ────────────────────────────────────────────────────────────────
+if ! $FORCE && ! $DRY_RUN; then
+  echo ""
+  echo "  This will move all resources from $ACTIVE_NODE → $STANDBY_NODE."
+  echo "  VIP and services will be unreachable for ~10–30 seconds."
+  printf "  Proceed? [y/N] "
+  read -r ANSWER
+  [[ "${ANSWER,,}" == "y" || "${ANSWER,,}" == "yes" ]] || { echo "Aborted."; exit 0; }
+fi
+
+# ── Capture active node's crm_node name ───────────────────────────────────
+# crm_node -n returns the node name as registered in Pacemaker (may differ
+# from hostname if Pacemaker was configured with explicit node names).
+ACTIVE_CRMD_NAME=$(na "crm_node -n 2>/dev/null" 2>/dev/null || echo "$ACTIVE_NODE")
+
+# ── Perform failover ───────────────────────────────────────────────────────
+echo ""
+echo "${DRY_PREFIX}Putting $ACTIVE_NODE into standby (resources will migrate to $STANDBY_NODE)..."
+if ! $DRY_RUN; then
+  na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v on" 2>/dev/null || true
+fi
+
+# ── Wait for resources to move ─────────────────────────────────────────────
+echo "${DRY_PREFIX}Waiting up to ${TIMEOUT}s for XFS to mount on $STANDBY_NODE..."
+MOVED=false
+SPIN_CHARS=('|' '/' '-' '\')
+SPIN_I=0
+
+if $DRY_RUN; then
+  echo "  [dry-run] would wait for mountpoint $XFS_MOUNT on $STANDBY_NODE"
+  MOVED=true
+else
+  for i in $(seq 1 "$TIMEOUT"); do
+    if ns "mountpoint -q '${XFS_MOUNT}' 2>/dev/null" 2>/dev/null; then
+      printf "\r%-80s\r" ""
+      echo "  Resources moved in ${i}s"
+      MOVED=true
+      break
+    fi
+    SPIN_I=$(( SPIN_I + 1 ))
+    SC="${SPIN_CHARS[$((SPIN_I % 4))]}"
+    printf "\r  [%s] waiting... (%ds)     " "$SC" "$i"
+    sleep 1
+  done
+fi
+
+if ! $MOVED; then
+  echo ""
+  echo "ERROR: XFS did not mount on $STANDBY_NODE within ${TIMEOUT}s."
+  echo ""
+  echo "  Current resource state:"
+  na "crm_mon -1 2>/dev/null" 2>/dev/null | grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED' | sed 's/^/  /' || true
+  echo ""
+  echo "  Clearing standby to restore $ACTIVE_NODE (undo the failover attempt)..."
+  na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
+  na "crm_resource --cleanup" 2>/dev/null || true
+  exit 1
+fi
+
+# ── Clear failure history ──────────────────────────────────────────────────
+echo "${DRY_PREFIX}Clearing Pacemaker failure history..."
+if ! $DRY_RUN; then
+  ns "crm_resource --cleanup 2>/dev/null" 2>/dev/null || true
+fi
+
+# ── Re-enable original active node as standby ─────────────────────────────
+echo "${DRY_PREFIX}Re-enabling $ACTIVE_NODE (now standby — will not claim resources)..."
+if ! $DRY_RUN; then
+  na "crm_standby -N '${ACTIVE_CRMD_NAME}' -v off" 2>/dev/null || true
+fi
+
+# ── Wait briefly for DRBD resync to begin ─────────────────────────────────
+if ! $DRY_RUN; then
+  sleep 5
+fi
+
+# ── Final state ────────────────────────────────────────────────────────────
+echo ""
+echo "Failover complete.  Final state:"
+echo ""
+
+CRM_OUT_AFTER=""
+if ! $DRY_RUN; then
+  CRM_OUT_AFTER=$(ns "crm_mon -1" 2>/dev/null || na "crm_mon -1" 2>/dev/null || true)
+else
+  CRM_OUT_AFTER="$CRM_OUT"
+fi
+
+NEW_ACTIVE=$(echo "$CRM_OUT_AFTER" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
+  grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+
+if [[ -n "$NEW_ACTIVE" ]]; then
+  if [[ "$NEW_ACTIVE" == "$ACTIVE_NODE" ]]; then
+    echo "  WARNING: $ACTIVE_NODE is still showing as active in crm_mon."
+    echo "  Pacemaker may still be settling — check again in a few seconds."
+  else
+    echo "  Active:  $NEW_ACTIVE"
+    echo "  Standby: $ACTIVE_NODE"
+  fi
+fi
+
+echo ""
+RESOURCES_AFTER=$(echo "$CRM_OUT_AFTER" | awk '/Full List of Resources/,0' | tail -n +2 || true)
+[[ -z "$RESOURCES_AFTER" ]] && RESOURCES_AFTER=$(echo "$CRM_OUT_AFTER" | \
+  grep -E 'Started|Stopped|Promoted|Unpromoted|FAILED|Master|Slave' || true)
+[[ -n "$RESOURCES_AFTER" ]] && echo "$RESOURCES_AFTER" | sed 's/^/  /'
+
+echo ""
+echo "  (DRBD resync of $ACTIVE_NODE may take a moment; monitor with:"
+echo "   ssh nixos@${ACTIVE_IP} 'sudo watch -n3 cat /proc/drbd')"
+echo ""
+echo "════════════════════════════════════════════════════"

scripts/ha/health.sh

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+# health.sh — HA cluster health snapshot (read-only, non-destructive)
+#
+# Prints a compact status panel across both nodes: SSH reachability, quorum,
+# DRBD state, Pacemaker resources, and service ports via the VIP.
+# Run from any host with SSH access to the HA nodes.
+set -euo pipefail
+
+# ── Configuration ─────────────────────────────────────────────────────────
+NODE1="${NODE1:-ha-server-1}"
+NODE2="${NODE2:-ha-server-2}"
+NODE1_IP="${NODE1_IP:-192.168.2.228}"   # vars.haServer1Ip
+NODE2_IP="${NODE2_IP:-192.168.2.227}"   # vars.haServer2Ip
+VIP="${VIP:-192.168.2.229}"             # vars.haServerVip
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"  # vars.haStorageRoot
+HA_USER="${HA_USER:-nixos}"
+# ──────────────────────────────────────────────────────────────────────────
+
+REACHABLE_1=false
+REACHABLE_2=false
+
+n1()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
+
+probe_node() {
+  local ip=$1
+  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HA_USER}@${ip}" true 2>/dev/null && echo "ONLINE" || echo "OFFLINE"
+}
+
+section() { echo ""; echo "── $* ──"; }
+
+echo "════════════════════════════════════════════════════"
+echo " HA Cluster Health — $(date '+%Y-%m-%d %H:%M:%S')"
+echo "════════════════════════════════════════════════════"
+
+# ── Node reachability ──────────────────────────────────────────────────────
+section "Nodes"
+N1_STATUS=$(probe_node "$NODE1_IP")
+N2_STATUS=$(probe_node "$NODE2_IP")
+[[ "$N1_STATUS" == "ONLINE" ]] && REACHABLE_1=true
+[[ "$N2_STATUS" == "ONLINE" ]] && REACHABLE_2=true
+
+if ! $REACHABLE_1 && ! $REACHABLE_2; then
+  echo "  ERROR: both nodes unreachable — cannot continue."
+  exit 1
+fi
+
+# ── Detect active node ─────────────────────────────────────────────────────
+# crm_mon 2.x formats the Promoted line as "    * Promoted: [ node ]" — the
+# bullet * means ^\s*(Promoted|Masters): never matches.  Filter out Unpromoted
+# first, then match anywhere on the line.
+ACTIVE_NODE=""
+CRM_OUT=""
+if $REACHABLE_1; then
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
+elif $REACHABLE_2; then
+  CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
+fi
+ACTIVE_NODE=$(echo "${CRM_OUT:-}" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
+  grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+
+STANDBY_NODE=""
+if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
+  STANDBY_NODE="$NODE2"
+elif [[ "$ACTIVE_NODE" == "$NODE2" ]]; then
+  STANDBY_NODE="$NODE1"
+fi
+
+n1_tag=""; n2_tag=""
+[[ "$ACTIVE_NODE" == "$NODE1" ]] && n1_tag=" [ACTIVE]"  || n1_tag=" [STANDBY]"
+[[ "$ACTIVE_NODE" == "$NODE2" ]] && n2_tag=" [ACTIVE]"  || n2_tag=" [STANDBY]"
+[[ -z "$ACTIVE_NODE" ]] && { n1_tag=""; n2_tag=""; }
+
+printf "  %-14s  [%s]%s\n" "$NODE1" "$N1_STATUS" "$n1_tag"
+printf "  %-14s  [%s]%s\n" "$NODE2" "$N2_STATUS" "$n2_tag"
+
+if [[ -z "$ACTIVE_NODE" ]]; then
+  echo ""
+  echo "  WARNING: could not determine active node from crm_mon."
+  echo "  Pacemaker may still be settling, or both nodes may be in standby."
+fi
+
+# ── Quorum ─────────────────────────────────────────────────────────────────
+section "Quorum"
+if $REACHABLE_1; then
+  QUORUM=$(n1 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
+elif $REACHABLE_2; then
+  QUORUM=$(n2 "corosync-quorumtool -s 2>/dev/null" 2>/dev/null || echo "")
+fi
+
+if [[ -z "${QUORUM:-}" ]]; then
+  echo "  corosync-quorumtool: unavailable"
+else
+  QUORATE=$(echo "$QUORUM" | grep "Quorate:" | awk '{print $2}' || echo "?")
+  VOTES=$(echo "$QUORUM" | grep "Total votes:" | awk '{print $3}' || echo "?")
+  NEEDED=$(echo "$QUORUM" | grep "Quorum votes:" | awk '{print $3}' || echo "?")
+  echo "  Quorate: $QUORATE   Votes: $VOTES / Expected: $NEEDED"
+fi
+
+# ── DRBD ───────────────────────────────────────────────────────────────────
+section "DRBD (ha-data)"
+
+drbd_info_from() {
+  local node=$1 run=$2
+  local role dstate cs pct
+  role=$($run "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
+  dstate=$($run "drbdadm dstate ha-data 2>/dev/null" 2>/dev/null || echo "unknown")
+  cs=$($run "grep -oE 'cs:[A-Za-z]+' /proc/drbd 2>/dev/null | head -1 | sed 's/cs://'" 2>/dev/null || echo "unknown")
+  pct=$($run "grep -oE \"sync'ed:[[:space:]]+[0-9.]+\" /proc/drbd 2>/dev/null | grep -oE '[0-9.]+$' | head -1" 2>/dev/null || echo "")
+  printf "  %-14s  role: %-22s  dstate: %-25s  cs: %s" \
+    "$node" "${role:-unknown}" "${dstate:-unknown}" "${cs:-unknown}"
+  [[ -n "$pct" ]] && printf "  syncing: %s%%" "$pct"
+  echo ""
+}
+
+$REACHABLE_1 && drbd_info_from "$NODE1" n1 || echo "  $NODE1  [OFFLINE]"
+$REACHABLE_2 && drbd_info_from "$NODE2" n2 || echo "  $NODE2  [OFFLINE]"
+
+# ── Pacemaker (full crm_mon output) ───────────────────────────────────────
+section "Pacemaker"
+if [[ -n "${CRM_OUT:-}" ]]; then
+  echo "$CRM_OUT" | sed 's/^/  /'
+else
+  echo "  crm_mon returned no output — trying again without suppression:"
+  if $REACHABLE_1; then
+    n1 "crm_mon -1" || true
+  elif $REACHABLE_2; then
+    n2 "crm_mon -1" || true
+  fi
+fi
+
+# ── XFS mount ─────────────────────────────────────────────────────────────
+section "XFS Mount ($XFS_MOUNT)"
+check_mount() {
+  local node=$1 run=$2
+  local status
+  if $run "mountpoint -q '$XFS_MOUNT' 2>/dev/null" 2>/dev/null; then
+    local usage
+    usage=$($run "df -h '$XFS_MOUNT' 2>/dev/null | tail -1 | awk '{print \$3\"/\"\$2\" used (\"\$5\")\";}'" 2>/dev/null || echo "")
+    status="mounted"
+    [[ -n "$usage" ]] && status="mounted  $usage"
+  else
+    status="not mounted"
+  fi
+  printf "  %-14s  %s\n" "$node" "$status"
+}
+
+$REACHABLE_1 && check_mount "$NODE1" n1 || echo "  $NODE1  [OFFLINE]"
+$REACHABLE_2 && check_mount "$NODE2" n2 || echo "  $NODE2  [OFFLINE]"
+
+# ── Service ports via VIP ──────────────────────────────────────────────────
+section "Services via VIP ($VIP)"
+
+check_port() {
+  local name=$1 port=$2
+  if bash -c "echo >/dev/tcp/${VIP}/${port}" 2>/dev/null; then
+    printf "  %-10s  port %-5s  OK\n" "$name" "$port"
+  else
+    printf "  %-10s  port %-5s  UNREACHABLE\n" "$name" "$port"
+  fi
+}
+
+if ping -c1 -W2 "$VIP" >/dev/null 2>&1; then
+  echo "  Ping       OK"
+else
+  echo "  Ping       UNREACHABLE"
+fi
+check_port "NFS"    2049
+check_port "iSCSI"  3260
+
+echo ""
+echo "════════════════════════════════════════════════════"
+if [[ -n "$ACTIVE_NODE" ]]; then
+  echo " Active: $ACTIVE_NODE   Standby: $STANDBY_NODE"
+else
+  echo " Active: unknown (Pacemaker not settled)"
+fi
+echo "════════════════════════════════════════════════════"
+echo ""

scripts/ha/resize-data-disk.sh

@@ -0,0 +1,276 @@
+#!/usr/bin/env bash
+# resize-data-disk.sh — online resize of the HA cluster data disk
+#
+# Three-phase process (all online-safe, no downtime required):
+#   1. Proxmox: grow scsi1 on both HA VMs   (qm resize)
+#   2. Guest:   rescan block device on both nodes so the kernel sees the new size
+#   3. DRBD + XFS: drbdadm resize, then xfs_growfs — active node only
+#
+# Usage:
+#   scripts/ha/resize-data-disk.sh --size +20G [--force] [--dry-run]
+#
+#   --size +NNg    amount to grow scsi1 by, e.g. +20G, +50G (required)
+#                  XFS and DRBD cannot shrink; only positive deltas accepted
+#   --force        skip the interactive confirmation prompt
+#   --dry-run      show what would be done without changing anything
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=../env.sh
+source "${SCRIPT_DIR}/../env.sh"
+
+# ── Configuration ─────────────────────────────────────────────────────────
+NODE1="${NODE1:-ha-server-1}"
+NODE2="${NODE2:-ha-server-2}"
+NODE1_IP="${NODE1_IP:-192.168.2.228}"
+NODE2_IP="${NODE2_IP:-192.168.2.227}"
+XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}"
+DRBD_RESOURCE="${DRBD_RESOURCE:-ha-data}"
+DATA_DISK_SLOT="${DATA_DISK_SLOT:-scsi1}"   # Proxmox disk name (scsi1 = data disk)
+HA_USER="${HA_USER:-nixos}"
+PVE_HOST="${PVE_HOST:-${PVE1_HOST}}"
+PVE_SSH_USER="${PVE_SSH_USER:-${PROXMOX_SSH_USER}}"
+# By-id symlink for the data disk; basename resolves to the raw block device.
+# matches variables.nix's haServerDrbdDisk.
+DATA_DISK_BYID="${DATA_DISK_BYID:-scsi-0QEMU_QEMU_HARDDISK_drive-${DATA_DISK_SLOT}}"
+# ──────────────────────────────────────────────────────────────────────────
+
+pve() { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=10 \
+          "${PVE_SSH_USER}@${PVE_HOST}" "$@"; }
+n1()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
+          "${HA_USER}@${NODE1_IP}" sudo "$@" 2>/dev/null; }
+n2()  { ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
+          "${HA_USER}@${NODE2_IP}" sudo "$@" 2>/dev/null; }
+
+# ── Argument parsing ───────────────────────────────────────────────────────
+SIZE=""
+FORCE=false
+DRY_RUN=false
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --size)    shift; SIZE="${1:?--size requires a value (e.g. +20G)}" ;;
+    --force)   FORCE=true ;;
+    --dry-run) DRY_RUN=true ;;
+    *) echo "Unknown argument: $1"
+       echo "Usage: $0 --size +NNg [--force] [--dry-run]"
+       exit 1 ;;
+  esac
+  shift
+done
+
+if [[ -z "$SIZE" ]]; then
+  echo "ERROR: --size is required (e.g. --size +20G)"
+  echo "Usage: $0 --size +NNg [--force] [--dry-run]"
+  exit 1
+fi
+
+# Only positive deltas — qm resize, DRBD, and XFS all refuse to shrink.
+if [[ ! "$SIZE" =~ ^\+[0-9]+(G|M|T)$ ]]; then
+  echo "ERROR: --size must be a positive delta like +20G, +50G, +500M, +2T"
+  echo "       (qm resize, drbdadm resize, and xfs_growfs can only grow, not shrink)"
+  exit 1
+fi
+
+DRY_PREFIX=""
+$DRY_RUN && DRY_PREFIX="[dry-run] "
+
+echo "════════════════════════════════════════════════════"
+echo " HA Data Disk Resize — $(date '+%Y-%m-%d %H:%M:%S')"
+echo " Size delta: $SIZE  •  Proxmox: $PVE_HOST"
+$DRY_RUN && echo " MODE: dry-run — no changes will be made"
+echo "════════════════════════════════════════════════════"
+
+# ── Detect active node ─────────────────────────────────────────────────────
+echo ""
+echo "Detecting active node..."
+
+CRM_OUT=""
+if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
+     "${HA_USER}@${NODE1_IP}" true 2>/dev/null; then
+  CRM_OUT=$(n1 "crm_mon -1" 2>/dev/null || true)
+fi
+if [[ -z "$CRM_OUT" ]]; then
+  if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
+       "${HA_USER}@${NODE2_IP}" true 2>/dev/null; then
+    CRM_OUT=$(n2 "crm_mon -1" 2>/dev/null || true)
+  fi
+fi
+
+# crm_mon 2.x formats Promoted lines as "    * Promoted: [ node ]" (bullet *),
+# so ^\s*(Promoted|Masters): never matches; filter Unpromoted first instead.
+ACTIVE_NODE=$(echo "$CRM_OUT" | grep -v 'Unpromoted\|Unmanaged' | \
+  grep -E '(Promoted|Masters):' | \
+  grep -oE '\b(ha-server-[0-9]+)\b' | head -1 || true)
+
+if [[ -z "$ACTIVE_NODE" ]]; then
+  echo "ERROR: could not determine active node from crm_mon."
+  echo "  Is Pacemaker still settling?  Try running scripts/ha/health.sh first."
+  exit 1
+fi
+
+if [[ "$ACTIVE_NODE" == "$NODE1" ]]; then
+  ACTIVE_IP="$NODE1_IP"
+  na() { n1 "$@"; }
+else
+  ACTIVE_IP="$NODE2_IP"
+  na() { n2 "$@"; }
+fi
+
+echo "  Active node: $ACTIVE_NODE ($ACTIVE_IP)"
+
+# ── Pre-check DRBD state ───────────────────────────────────────────────────
+echo ""
+echo "Pre-checks..."
+
+DRBD_DSTATE=$(na "drbdadm dstate ${DRBD_RESOURCE} 2>/dev/null" 2>/dev/null || echo "unknown")
+if ! echo "$DRBD_DSTATE" | grep -q "UpToDate/UpToDate"; then
+  echo "  WARNING: DRBD dstate is '$DRBD_DSTATE' (expected UpToDate/UpToDate)."
+  echo "  Resizing with a partially-synced disk may cause issues."
+  if ! $FORCE; then
+    echo "  Use --force to proceed anyway."
+    exit 1
+  fi
+  echo "  --force specified — proceeding despite non-ideal DRBD state."
+else
+  echo "  DRBD dstate: $DRBD_DSTATE — OK"
+fi
+
+# ── Find VMIDs on Proxmox ─────────────────────────────────────────────────
+echo ""
+echo "Looking up VM IDs on ${PVE_HOST}..."
+
+QM_LIST=$(pve "qm list 2>/dev/null" || true)
+VMID1=$(echo "$QM_LIST" | awk -v name="$NODE1" '$0 ~ name {print $1}' | head -1)
+VMID2=$(echo "$QM_LIST" | awk -v name="$NODE2" '$0 ~ name {print $1}' | head -1)
+
+if [[ -z "$VMID1" ]]; then
+  echo "  ERROR: could not find VMID for $NODE1 on $PVE_HOST"
+  echo "  qm list output:"
+  echo "$QM_LIST" | sed 's/^/    /'
+  exit 1
+fi
+if [[ -z "$VMID2" ]]; then
+  echo "  ERROR: could not find VMID for $NODE2 on $PVE_HOST"
+  echo "  qm list output:"
+  echo "$QM_LIST" | sed 's/^/    /'
+  exit 1
+fi
+
+echo "  $NODE1: VMID $VMID1"
+echo "  $NODE2: VMID $VMID2"
+
+# ── Confirm ────────────────────────────────────────────────────────────────
+if ! $FORCE && ! $DRY_RUN; then
+  echo ""
+  echo "  Plan:"
+  echo "    Phase 1 — qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}  (on $PVE_HOST)"
+  echo "              qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}  (on $PVE_HOST)"
+  echo "    Phase 2 — block device rescan on $NODE1 and $NODE2"
+  echo "    Phase 3 — drbdadm resize + xfs_growfs on $ACTIVE_NODE"
+  echo "  No downtime required (all operations are online-safe)."
+  printf "  Proceed? [y/N] "
+  read -r ANSWER
+  [[ "${ANSWER,,}" == "y" || "${ANSWER,,}" == "yes" ]] || { echo "Aborted."; exit 0; }
+fi
+
+# ═══════════════════════════════════════════════════════════════
+# Phase 1 — Resize both VM data disks in Proxmox
+# ═══════════════════════════════════════════════════════════════
+echo ""
+echo "── Phase 1 — Proxmox disk resize (${DATA_DISK_SLOT} ${SIZE} on both VMs) ──"
+
+echo "  ${DRY_PREFIX}qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}  ($NODE1 on ${PVE_HOST})"
+if ! $DRY_RUN; then
+  pve "qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}"
+fi
+
+echo "  ${DRY_PREFIX}qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}  ($NODE2 on ${PVE_HOST})"
+if ! $DRY_RUN; then
+  pve "qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}"
+fi
+
+echo "  Phase 1 done."
+
+# ═══════════════════════════════════════════════════════════════
+# Phase 2 — Rescan block device on both guest nodes
+# ═══════════════════════════════════════════════════════════════
+echo ""
+echo "── Phase 2 — Block device rescan (both nodes) ──"
+
+rescan_node() {
+  local node_name=$1 run_fn=$2
+
+  # Resolve block device name from the stable by-id symlink on the guest.
+  local blk_dev=""
+  if ! $DRY_RUN; then
+    blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
+  fi
+  if [[ -z "$blk_dev" ]]; then
+    # Fall back: scsi1 → index 1 → sdb, scsi2 → sdc, etc.
+    local slot_idx
+    slot_idx=$(echo "$DATA_DISK_SLOT" | grep -oE '[0-9]+$' || echo "1")
+    blk_dev=$(printf "sd%s" "$(echo "abcdefghij" | cut -c$((slot_idx + 1)))")
+    echo "  WARNING: could not resolve /dev/disk/by-id/${DATA_DISK_BYID} on $node_name;" \
+         "falling back to /dev/${blk_dev}" >&2
+  fi
+
+  echo "  ${DRY_PREFIX}Rescanning /dev/${blk_dev} on ${node_name}..."
+  if ! $DRY_RUN; then
+    $run_fn "bash -c 'echo 1 > /sys/block/${blk_dev}/device/rescan'" 2>/dev/null
+    local new_size
+    new_size=$($run_fn "lsblk -nd -o SIZE /dev/${blk_dev} 2>/dev/null" 2>/dev/null || echo "unknown")
+    echo "    /dev/${blk_dev} on $node_name now reports: $new_size"
+  fi
+}
+
+rescan_node "$NODE1" n1
+rescan_node "$NODE2" n2
+
+echo "  Phase 2 done."
+
+# ═══════════════════════════════════════════════════════════════
+# Phase 3 — Grow DRBD metadata, then XFS (active node only)
+# ═══════════════════════════════════════════════════════════════
+echo ""
+echo "── Phase 3 — DRBD resize + XFS grow (on active node: $ACTIVE_NODE) ──"
+
+echo "  ${DRY_PREFIX}drbdadm resize ${DRBD_RESOURCE}"
+if ! $DRY_RUN; then
+  na "drbdadm resize ${DRBD_RESOURCE}"
+fi
+
+echo "  ${DRY_PREFIX}xfs_growfs ${XFS_MOUNT}"
+if ! $DRY_RUN; then
+  na "xfs_growfs ${XFS_MOUNT}"
+fi
+
+echo "  Phase 3 done."
+
+# ── Verify ────────────────────────────────────────────────────────────────
+echo ""
+echo "── Verify ──"
+
+if ! $DRY_RUN; then
+  DF_OUT=$(na "df -h '${XFS_MOUNT}' 2>/dev/null" 2>/dev/null || echo "")
+  if [[ -n "$DF_OUT" ]]; then
+    echo "  ${XFS_MOUNT}:"
+    echo "$DF_OUT" | sed 's/^/    /'
+  fi
+
+  DRBD_DSTATE_AFTER=$(na "drbdadm dstate ${DRBD_RESOURCE} 2>/dev/null" 2>/dev/null || echo "unknown")
+  echo "  DRBD dstate: $DRBD_DSTATE_AFTER"
+  if ! echo "$DRBD_DSTATE_AFTER" | grep -q "UpToDate/UpToDate"; then
+    echo "  NOTE: DRBD is resyncing — normal immediately after resize."
+    echo "        Monitor: ssh nixos@${ACTIVE_IP} 'sudo watch -n3 cat /proc/drbd'"
+  fi
+else
+  echo "  [dry-run] would verify df -h ${XFS_MOUNT} and drbdadm dstate on $ACTIVE_NODE"
+fi
+
+echo ""
+echo "════════════════════════════════════════════════════"
+echo " Resize complete."
+echo " Active node: $ACTIVE_NODE"
+echo "════════════════════════════════════════════════════"
+echo ""

scripts/ipa/create-nixos-ipa-host-account.sh

@@ -17,7 +17,7 @@
 #
 # Options:
 #   --ip <addr>      Register this IP with the IPA host record (optional).
-#   --dc <host>      SSH to this host for ipa-getkeytab.
+#   --dc <host>      SSH to this host to run IPA commands.
 #                    Default: $IPA_SERVER (from env.sh / environment).
 #   --dc-user <u>    SSH user on the domain controller. Default: wayne.
 #   --dry-run        Print what would be done without making any changes.
@@ -35,7 +35,9 @@
 #      scripts/secrets/sync-host-keys.sh <flake-target> first so the host
 #      can decrypt its own keytab on boot. This script adds the .sops.yaml
 #      creation rule for secrets/<hostname>.keytab automatically, but the
-#      host age key anchor (&lxc-<hostname> etc.) must already exist.
+#      host age key anchor (&lxc-<hostname> etc.) must already exist —
+#      otherwise only the admin key can decrypt the keytab and the deployed
+#      host will fail to read it.
 #   4. sops in PATH, or Nix available to run it via `nix run`.
 
 set -euo pipefail
@@ -52,7 +54,7 @@ DC_HOST="${IPA_SERVER}"
 DC_USER="wayne"
 IP_ADDR=""
 DRY_RUN=false
-HOSTNAME=""
+TARGET=""
 
 usage() {
   sed -n '/^# Usage:/,/^[^#]/{ /^#/{ s/^# \?//; p } }' "$0"
@@ -68,22 +70,29 @@ while [[ $# -gt 0 ]]; do
     -h|--help) usage 0 ;;
     -*)  echo "Unknown flag: $1" >&2; usage 1 ;;
     *)
-      if [[ -n "${HOSTNAME}" ]]; then echo "Unexpected argument: $1" >&2; usage 1; fi
+      if [[ -n "${TARGET}" ]]; then echo "Unexpected argument: $1" >&2; usage 1; fi
-      HOSTNAME="$1"; shift
+      TARGET="$1"; shift
       ;;
   esac
 done
 
-if [[ -z "${HOSTNAME}" ]]; then
+if [[ -z "${TARGET}" ]]; then
   echo "Error: hostname required." >&2
   usage 1
 fi
 
-FQDN="${HOSTNAME}.${HOME_DOMAIN}"
+# Reject FQDNs passed by mistake — the script appends HOME_DOMAIN itself.
-REALM="${HOME_DOMAIN^^}"           # uppercase: SWEET.HOME
+# "nixos.sweet.home" → FQDN would become "nixos.sweet.home.sweet.home".
-KEYTAB_SECRET="${REPO_ROOT}/secrets/${HOSTNAME}.keytab"
+if [[ "${TARGET}" == *"."* ]]; then
+  echo "Error: <hostname> must be the short name (e.g. 'nixos'), not a FQDN." >&2
+  echo "       The FQDN is derived automatically as ${TARGET}.${HOME_DOMAIN}." >&2
+  exit 1
+fi
+
+FQDN="${TARGET}.${HOME_DOMAIN}"
+KEYTAB_SECRET="${REPO_ROOT}/secrets/${TARGET}.keytab"
 # Temp path on the domain controller — use a name that won't collide.
-DC_TMP="/tmp/nixos-keytab-${HOSTNAME}-$$.keytab"
+DC_TMP="/tmp/nixos-keytab-${TARGET}-$$.keytab"
 
 # --- Helpers ---
 
@@ -112,8 +121,8 @@ dc_run() {
 if command -v sops &>/dev/null; then
   SOPS_CMD=(sops)
 else
-  log "sops not in PATH — will use 'nix run github:NixOS/nixpkgs/nixos-25.11#sops'"
+  log "sops not in PATH — will use 'nix run nixpkgs#sops'"
-  SOPS_CMD=(nix run "github:NixOS/nixpkgs/nixos-25.11#sops" --)
+  SOPS_CMD=(nix run "nixpkgs#sops" --)
 fi
 
 # --- Preflight checks ---
@@ -131,13 +140,13 @@ cd "${REPO_ROOT}"
 # also exist at that point or sops will refuse with "no matching creation
 # rules found."
 
-log "Checking .sops.yaml for creation rule: secrets/${HOSTNAME}.keytab"
+log "Checking .sops.yaml for creation rule: secrets/${TARGET}.keytab"
 
 RULE_EXISTS=false
 # Match "path_regex: secrets/<hostname>...keytab" — using .*keytab rather
 # than \.keytab because the file stores the regex verbatim (\.keytab = two
 # chars: backslash + dot), which a BRE \.  (= escaped literal dot) won't span.
-if grep -q "path_regex: secrets/${HOSTNAME}.*keytab" .sops.yaml 2>/dev/null; then
+if grep -q "path_regex: secrets/${TARGET}.*keytab" .sops.yaml 2>/dev/null; then
   RULE_EXISTS=true
   logn "Rule already exists — skipping addition."
 fi
@@ -149,12 +158,20 @@ if ! $RULE_EXISTS; then
   # that have been registered get added as recipients.
   RECIPIENTS=("*admin")
   for platform in lxc proxmox linode; do
-    anchor="${platform}-${HOSTNAME}"
+    anchor="${platform}-${TARGET}"
     if grep -q "^  - &${anchor} " .sops.yaml; then
       RECIPIENTS+=("*${anchor}")
     fi
   done
 
+  if [[ ${#RECIPIENTS[@]} -eq 1 ]]; then
+    echo "Warning: no platform age keys found for '${TARGET}' in .sops.yaml." >&2
+    echo "         Run scripts/secrets/sync-host-keys.sh <flake-target> first," >&2
+    echo "         otherwise only the admin key can decrypt the keytab and the" >&2
+    echo "         deployed host won't be able to read it at boot." >&2
+    echo "         Continuing with admin-only encryption..." >&2
+  fi
+
   # Build the indented recipient list for the YAML block.
   RECIPIENT_YAML=""
   for r in "${RECIPIENTS[@]}"; do
@@ -163,9 +180,9 @@ if ! $RULE_EXISTS; then
   RECIPIENT_YAML="${RECIPIENT_YAML%$'\n'}"  # strip trailing newline
 
   NEW_RULE="
-  # Host keytab for ${HOSTNAME} FreeIPA enrollment (binary sops file).
+  # Host keytab for ${TARGET} FreeIPA enrollment (binary sops file).
   # Generated by scripts/ipa/create-nixos-ipa-host-account.sh.
-  - path_regex: secrets/${HOSTNAME}\\.keytab\$
+  - path_regex: secrets/${TARGET}\\.keytab\$
     key_groups:
       - age:
 ${RECIPIENT_YAML}"
@@ -206,23 +223,37 @@ IP_FLAG=""
 [[ -n "${IP_ADDR}" ]] && IP_FLAG="--ip-address=${IP_ADDR}"
 
 # --force: create the host record even if DNS doesn't resolve it yet.
-# Pipe through grep to suppress the "already exists" warning without
+if $DRY_RUN; then
-# hiding real errors (ipa exits 1 for real errors, 0 for already-exists).
+  echo "[dry-run] ssh ${DC_USER}@${DC_HOST} sudo ipa host-add '${FQDN}' ${IP_FLAG} --force"
-HOST_ADD_CMD="sudo ipa host-add '${FQDN}' ${IP_FLAG} --force 2>&1 | \
+else
-  tee /dev/stderr | grep -q 'already exists' && echo '(host already registered)' || true"
+  HOST_ADD_OUT=$(ssh "${DC_USER}@${DC_HOST}" "sudo ipa host-add '${FQDN}' ${IP_FLAG} --force 2>&1") \
-dc_run "bash -c \"${HOST_ADD_CMD}\""
+    && HOST_ADD_RC=0 || HOST_ADD_RC=$?
+  if [[ $HOST_ADD_RC -eq 0 ]]; then
+    echo "${HOST_ADD_OUT}"
+  elif echo "${HOST_ADD_OUT}" | grep -q "already exists"; then
+    logn "(host already registered)"
+  else
+    echo "Error: ipa host-add failed (exit ${HOST_ADD_RC}):" >&2
+    echo "${HOST_ADD_OUT}" >&2
+    exit 1
+  fi
+fi
 
 # --- Step 3: Fetch the keytab from the domain controller ---
 
 log "Fetching keytab for host/${FQDN}"
 
-dc_run "sudo ipa-getkeytab -s '${DC_HOST}' -p 'host/${FQDN}' -k '${DC_TMP}'"
+# Remove the plaintext keytab if the script aborts before encryption completes.
+# The trap is cleared at the end of step 4 once sops has encrypted it in-place.
+trap 'rm -f "${KEYTAB_SECRET}"' EXIT
+
+dc_run "sudo ipa-getkeytab -s '${IPA_SERVER}' -p 'host/${FQDN}' -k '${DC_TMP}'"
 
 if $DRY_RUN; then
-  echo "[dry-run] Would stream ${DC_USER}@${DC_HOST}:${DC_TMP} → secrets/${HOSTNAME}.keytab"
+  echo "[dry-run] Would stream ${DC_USER}@${DC_HOST}:${DC_TMP} → secrets/${TARGET}.keytab"
 else
-  logn "Streaming keytab from ${DC_HOST}:${DC_TMP} → secrets/${HOSTNAME}.keytab"
+  logn "Streaming keytab from ${DC_HOST}:${DC_TMP} → secrets/${TARGET}.keytab"
-  # scp can't read a root-owned temp file as wayne; pipe through sudo cat instead.
+  # scp can't read a root-owned temp file as ${DC_USER}; pipe through sudo cat instead.
   ssh "${DC_USER}@${DC_HOST}" "sudo cat '${DC_TMP}'" > "${KEYTAB_SECRET}"
 
   logn "Removing temp file on ${DC_HOST}"
@@ -235,24 +266,38 @@ fi
 # sops matches the creation rule by path. Using -i (in-place) rather than
 # stdout redirect keeps the path intact through the encrypt call.
 
-log "Encrypting secrets/${HOSTNAME}.keytab in-place with sops"
+log "Encrypting secrets/${TARGET}.keytab in-place with sops"
 run "${SOPS_CMD[@]}" -e --input-type binary -i "${KEYTAB_SECRET}"
 
+# Encryption succeeded — the file is now sops-encrypted; cancel the cleanup trap.
+trap - EXIT
+
 # --- Done ---
 
 if ! $DRY_RUN; then
   echo ""
-  echo "Done. secrets/${HOSTNAME}.keytab is sops-encrypted and ready."
+  echo "Done. secrets/${TARGET}.keytab is sops-encrypted and ready."
   echo ""
   echo "Next steps:"
-  echo "  1. Verify: grep '\"data\": \"ENC' secrets/${HOSTNAME}.keytab"
+  echo "  1. Verify: grep '\"data\": \"ENC' secrets/${TARGET}.keytab"
   echo "  2. Stage and commit:"
-  echo "       git add secrets/${HOSTNAME}.keytab .sops.yaml"
+  echo "       git add secrets/${TARGET}.keytab .sops.yaml"
-  echo "       git commit -m 'secrets: add IPA keytab for ${HOSTNAME}'"
+  echo "       git commit -m 'secrets: add IPA keytab for ${TARGET}'"
-  echo "  3. Add the module to hosts/${HOSTNAME}/host.nix:"
+  echo "  3. Add to hosts/${TARGET}/host.nix (networking block and imports):"
+  echo ""
+  echo "       networking = {"
+  echo "         hostName = \"${TARGET}\";"
+  echo "         domain   = vars.homeDomain;          # required for Kerberos FQDN"
+  echo "         nameservers = [ vars.domainControllerIp ];  # IPA DNS"
+  echo "         ..."
+  echo "       };"
+  echo ""
+  echo "       imports = ["
   echo "         (import ../../modules/ipa/client.nix {"
-  echo "         keytabSopsFile = ../../secrets/${HOSTNAME}.keytab;"
+  echo "           keytabSopsFile = ../../secrets/${TARGET}.keytab;"
   echo "           caCertFile     = ../../certs/ipa-ca.crt;"
   echo "         })"
+  echo "       ];"
+  echo ""
   echo "  4. Deploy: nixos-rebuild switch (or create-proxmox-resource.sh)"
 fi

secrets/common.yaml

@@ -5,211 +5,229 @@ sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrMGd3ZVNlNXdmOUZMUzdQ
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiU2IxZ056SUs3SVZiUTlQ
-            TW1acEs0NFA2Q01rM2dkc1h0NHkzQmhiWFZZCjMyK202VWdlaGhsZW04MnVwUVdO
+            NXZEYmg3VzJzeWJPRnZXcUtOMGQ1MElBU0dVCm5xSURZZ0sxTjVxQnhpVVNEU092
-            alA0Q2FETThsYkhSS0hKdHBaS3VaY28KLS0tIG1Gdk8yalREOUtIZTUyY2p1UHlJ
+            bDBHbFAzZ3hzejc5NmM2U0FQTkRNSTQKLS0tIC9OOURPRlcvQXIxMVZXR1JjUWRX
-            eG5iQnJsaTJBY3Y1dkw1c0VEaDQwdDQKfV04fLy32Lp2ZQ2VnvQ0h/Vsf+qdaJiv
+            bE5ja3FoWXFIZ2ZLTklqWHhMRHBTRWcK2Z4sEcdyvx1rdBIGrbNw7bYvEPXdobvF
-            DnLXGZ9hE5yzpKWkQIRgqYGBkF8PkH0YC4OIaVkA53wrtjqS4ZHR9Q==
+            h0aNihn817DvtVr8eeucj22iYCS40etElQNcO5i68t5wab4/d3rUbQ==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjWFNRY2FiK3VkSm1RdHBn
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrN0lNNGtYbE8rRTB0U2hL
-            cWl1ZStLcTRFZWY5VVI5N0FhODZvR1ltM0FvCnBHZUtTUm9QeHNlbVBoZEx1V3Fa
+            eUNOZUNFRGk1RzE3R1VjV0Q3ckszcDNHWGxJCkhFN25JbDVSaWJIZ09uNzczbkM1
-            Nk9iMmJKVnhocEpERi9leE1ySUtNMFkKLS0tIDRRYkxnbU90S2RyMHdJNzRJNXBi
+            ZWZ1Y2wrTHZ6VzlnVWRoT2JOdDZHM0UKLS0tIHQ2ZE9BaFRCeDNqbldRQkpLclI4
-            QjRmZFhVakVic2tYODZHcWtJRmNQTDQK7G8eSJInt11P0DiL9uzNQ/ZHHLVNIYPe
+            dXZqOXNaR2Q2QWJtRlZUa0hLaTh2YlkKeCh3+yIt0sjh2Ig8m/YQifhMDHnTpuZE
-            bvlhuGkEuQ/+j5sVSKOfSI2Y7CvM7TpE3APyKBcLG3ajYg6F/Ev3SA==
+            zHBE74SUyWFmqLgtd1b8Rx5gn9vBcS/3SMv3yKWispg4DZ3bCOeNFA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age19m0m7vdfg86yqy8l5mmle5jdd0unrn3f55t232w8h5ey42cqw34sfpt32n
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1S3dOcFdwV2NoMW1oMnY1
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5c2luYmpYcVVxRHROQlpk
-            VGdxeXJVR1lsbzNVZHlGb0NGOVo0SndiakVjCmg0QlNnZDV5RlFja2hCbVRXV1VF
+            cTM4cm9uNEtVS21pK2pDRVRySmxYS3NwdGg0CnQvS3g4Q2tSdlpVeWZxbzArcTM1
-            S1ZtbC9KU0U1ZW9zeVoyR3hxNW1XTFUKLS0tIFUzWTJhTzM4QnpWV3h1OFU0N3BK
+            YWRMcDAxcWgySFRqRC9XSElsTVorVEEKLS0tIFd6YjdrY2ZXQVBaZ01wMTIvV2sv
-            RnF1N2k0S0lIVitoNDJLUmZqdHRzZVkKUfNg24p8zxb3749v/A1BOKCNw75AUKpf
+            ZEk0TG0ybENlTlhpZnovMXh0OGd5V1kKEbjrmKP6Su16KRDQ4PMqi0tHRS6+PfuT
-            RUmFCw5DDWF2aNM0mZqcjjVmJ/FRKV2HXwwUGsHPKSOTnKfOUlPNKA==
+            0v5DEyi8EDtIYSMzzdF7Jk2C0h5XuRux45Z6E6JhKW5c788dGQHz/A==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1rrxqea6q6pn39sw8y5te63h2py8jgjl9v0jyper86w3ggtn67upqg3ah39
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzazl5MUpUNEVNMTdLWXBp
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1eW5kd1FrSGJaSDkwTzdY
-            c1hkaVhjcTZXTUNsWS96QVVWb0RVQlZ4VlNvCnE3TFRySU5jTFk2WjBONUQyQUhl
+            SnJhR2hqRkNTTGtSTWsvYmZpSlVWV0xvYkRZCmtqR1FQOTRQR3dGYU8zNFpMdkUr
-            U1lNTDFTRmZhMUFyZmpVY2xpaUVxRW8KLS0tIE9MQ1M2U2ZXQmRCVll2UGRWL1RG
+            SWN2KzhXaE1rYWZYNHJsOVVvRDY3R00KLS0tIG1aUldSWHlHeHkrcjF2TENvazR3
-            K2tnU0NOKzU5dkpiekF0Vk1VM0ZZZDgKK13aFypGAqrKWPOr3UwtXI1EoXf1+UzS
+            YXJLUXV3VVNSWEdseFlZQUJPQUF6OFkKOPKH52muUha+MV2leWDiqpuTaIj0ZKzV
-            rBqcwnX6WPxSKUwWoins4Aojek4QhbhY4R5ei6rRS0KEQeryGxy8bg==
+            keEdyqkIck6KKfIjphNKTW5ujiuKjGQbRliquBknfCRGMnRlMNfmkA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1adur9g330gua4l6ndk8cqjg35qc8yxwgme6wrl2hpylcc7vxm38q05ejuy
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCZUpVT1p4TGZyNC9qMm5G
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmTDExNFpCZHZmWnpwYis4
-            UzNHV08rWURjY2lqS3FTWFdoc0FYYTFjb0NBCk1mZ2JzaXk5RmExNE9xWGZ5K0pv
+            bFJGZXV5UksyS01MT0d3cXE2Wno2Zzh6OFFnCm9hT3MyZ0hTbnkzdXk3NmN1TUVF
-            UEtqMkltV0dIWll5eVBUVVRNOUNDWUUKLS0tIGpKNVJudUM1UGNvaGl1UDBOeFA1
+            OGtCZnlJRmZJaWJWaE1pMGcrTzJzZm8KLS0tIFE1L3ZxeUl6WVA2UXN2b0lIcTBl
-            RjUyRlZ6a0Y4SXNsL21zSURVRk9KTFEKU1L6BQ6ZlYQQtqx3uF/uM5CQ1ercmvRT
+            YmtXZnZaMml6ejY1Y3lRUFFtTEVMS0UKZbcuunA9GRqctCfC0Fh2foKt7n3l8KkE
-            TL3r2/Y07gE7CjRn3pR9z0co8KndGzxV6YR+ubyWptwBS8KQh5stkw==
+            FNjjxUzL7OQjNG9M1aAtRMMO5KXL7YKXRUJWOPcpiLeLUNvDLeLXvA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age17e89ty6p0fw24daanen57wg8uald9s025t3wwxsw269svwpmgvrshfvfvt
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsaWxxWS9xQXViY3VnUEx2
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQYlp5U0FvVkdIKzBGZVRl
-            V09FbTI0WGtNbW0yclhOSGZDbG5NUTNaTkFFCitkcjJ3OE9BSnN4bjFWcE9nYVBk
+            UEtOOG5kVHVLQ0ptVTVpdG9Lb1Y3bzlOODNzCmNYN0Y5ZGNzTGFGYmxJcWFxaExn
-            ZzMyVHlJQ2wwdU5JOXdCQm9oNkhNd2MKLS0tIE9tRzFYS05vSkUwWFRkaTdtc0k0
+            NkkrT2RLSlZTdmFiNks3RVZXQlBEYnMKLS0tIDdCVlBxUG8vYU05d3c2MXhjQjhM
-            blVoMWV0QklBVkluT0Z4NHYyS1F0blUKO+Uc0of/V77ZUZOsxTzeH8/LmmAOQt+J
+            TTVJUGxlWUxWalBRYzFKQUQzd1R6dHcKPz1ciNICph7qnqSH5WYmOjUCNHsIAmTX
-            x/COHxnLCnZ4eWI6q1a0Qn5Br15OJYTxUI2QTV4goTnXBNUDo9wdpQ==
+            lRnbFCZdD/ZIFRK8OYAyCGfQ42UYSTf16kH2jDdTFUy7Fk2+O7lNIA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1hrx8qj02fj2ea6d4g9vqhyj9hl7fppkjqfdx2l37py3h6pdkr95s8n8rvs
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsLzFnZkdVdGQ2dHRwcnRt
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiemVqWW9LdlV0RjRuRHpF
-            Ui8rc1NoQzNKSHVzTk0vYU1vMlRxcjBKZ2hNCmhLYXVGSis4RU9HNGVxZkpvUkd5
+            U2MwN3RDeHU5SWhuQ2l4YStRVVE2eU9XSUc4ClNGMkNXQVdkWFkwcys0aFVhZnZv
-            L0xvalhDYTQ4N21OcHRheTlkaUxvTkEKLS0tIFVkRGxtLzhQT0paV2U3ZnNScVdn
+            Y3J6RmxwakdaZVo5VWZhb0lVTUdTYzQKLS0tIEQ2NW0yb2lGQ0NLZm1PSWNFVkQ1
-            alZnaVppeGI3OUVscGpONkk3YTRXd3MK61na8x5qX7+dyMHasDz2dj7yeaUlX8me
+            a0Y1NlYwZ0dnbHhsQVB5eS9kZVdqbmsKgFLUpZMqGywO9Ext5WRwco2bpenVNrzL
-            N4/SIk1JDBhv9G7mdKLbKhSF1UJrSY7TJqJqx8/dqEc0uG3vptA1ew==
+            XrGCqhn/7TW9DrJlPnFe8r4fl4DFP6GWByn7axDl58Qy1OzeBZwnNg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e7l8dusgmgfzd2cxrrzwepzjxt69hzqj4epee0cs27u6yg4kxcuqm34ncx
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4ZEtvOUhMU1FRWWpJQjF4
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLRzlzWEcwTUx6M1JNYWgr
-            NWphNXp5M0dLZXhkZndhT1Y3L09maytHazJVCm1MeEtMWXg1Zjg3bFVnZEorci9J
+            ZWxnVXhFWkhQVjBrSndGTU9oY2MrOTQxTGhFCjhUdERPc3hzQlkzMjBEa3AvaWVX
-            bkNZQU9Ta1dDTFFHaGFWQVBpK3pYRDQKLS0tIEhPVGliRDR3ZTF2aEl3ZnJEYWtR
+            T0RJWXFFZHdmUVBObGhpQktCQ2NPbWcKLS0tIDN2YkE4bEVtbjJlcndWVmlrQ3ZT
-            anh0SEpnVW8xdXNkZEZQSjcxU1BHMFEKVRJUA71fi1QawB2TnuTWMYhzQR18u4M2
+            dlVvRmt4RklOdUJyZ25ia0ZoYjU4ZHcK5fL1yvRcty6+EMzfSkwwJM3qvrr6h46d
-            s1V4j4TwYyyKZFoNvt8kOUayjC499c5OBUufYs6G2ciC6gK2A9E0EQ==
+            d7Zs+EElXrsGDQvL0fevB/zDcICAtudYnVv4nP3Jf8GQnL19zWlwbg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1jcx3yajjhghn8qh8za3yeu8nxykzlg3p4nrv03vnfvzl0mzayg2qmg940e
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTV2IzSjBEMjUwZjB0dCtj
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhZElRNXNLK0tVUjQ0ektQ
-            aTdZVlEvYkVRbTNjUC9ZRVdNZVhPQTJEd1ZZClc3NDJiR1BVYkZkdVVxMVZGc0VN
+            Tzc5NDJReldiYVFFZnRLS3l5dWw3UVQ0WWhJCm1DYVloZGpYeVJtNUNRcEJvVUZE
-            dWdSSXBFR2xxR0xrV2thRmUwSS96S0UKLS0tIHRRTXVlUi9UYnFRRlhsU21HZVY4
+            Q0xMcmR0WWdVVTBaWko0dGI5V1hqaUkKLS0tIFBBeXdNY1pFai9QT2t3NWFLV2sw
-            SDFYd0NwVEtVZXNsWUI1a1ZZU2xNRGMKuQUhOq2FRD+PGn5OkdODZItbxCzRKjne
+            cWpRU2RISSs1UXh3Wkl1dWI3VEk1bXMKP6v3Lm0XZkp3Gn3iJkB0K0Y0E5BeilqR
-            E60UOYtHjanuGjJ1svuR9cYsLZz7lLOwItklecYaQYpMRZEwzzBGCQ==
+            plmPh1rKbT/wkefFwpa8e2Wwmhx80YK+bjn633MH7mLQDf6AiK83Xw==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1sweerhrga9yf8x6sv0apz4ed4g48rnlcq34rpv20t0rcelwgpgeqwvndzz
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrSjJoUVUyd1JqRm1ZQzZx
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMVk8yT1FtMVoxYjF2YVBs
-            SFJ2cWNUY0E4b3FndVA5Y0hIOEZnZUlVWlhBCm1vY1luOXZBelRUTmF5Y1NMeDBn
+            UTFVbmhzblBCRno2V3YvaERRN0xWcHd2NVE4CmVyK3p1SWtodk5iWjdkQjJRc0dM
-            cE1BTDErc041UjJCWTBQbnk0Wk80dkEKLS0tIDVSZzd1UktvZGdyanFUMkVORUtl
+            Vm8zZHczYlFFSG91QThmUGNaWHBoN0UKLS0tIDZMMXoxbUM0TUVaNFUzRExLc2VM
-            eVB5TnJkMlp6dUpXSTlxRlplZ2NxUlEK0AYOxIbswjM0SUASDfmZ7PqcEU844fgI
+            OVFrNGZNL256cjg0Q3ZBQW9xUFcxSUEKv4HqaTcTzNYRJCmNEeUQLFfEoFXyOBEC
-            ycFWVSEPodwUZ6UFoYXhHlJzHFcgpLvwUd1PMktLHe1qrZ7GOQJIMA==
+            yZrg8D+ROBn98484bxNeoyOjQdcVn7OYiCv1y6utnyeA9sdMUB0ong==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1f7usptjx9rv4rxauasve200gxtdt9jkqhhdqstlf20wvlm7u75rsjfw50m
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPakhQcE92ZU0zYmk2QS9D
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1Rkl0UXduS3k3WVU1RzdG
-            TG0xc1JPZXZCZ0tZOXA3MGNLVnBlZGVtRFRzCmIvMHhQKzFVWCtpMTQrQUhGVGJp
+            N2IybE1TZ1kwVmM4Y2lvUS9DVG0zSkdpZGhRCnFkZnJVS2FBZm9Xa0V4SjRwaGdR
-            RU1jbldYckw3TXI2SlNpZVBIZHRsWWcKLS0tIGtJTUtJejFxem5jajFQUDFTQWU1
+            TWpaaTh4VEk4cUFrb2pURERzbWtSR0UKLS0tIEsybHdvUWg3UlUvL0lhdHIrUE0y
-            VnlxYmVlNG04ay9ETi9FRmVYQXVoRkUK9oFNolI7jRjo9RUs1g4ghrx7aYV4U/ce
+            R0pMdnI5QlpIcWVWdGRYQ3JCaElLWTgK5cTfvehj4BPU22tYiR5piZdouUdeh5Gr
-            ZTc2tFh57+7aKgrDi+2W3jwhfkjvBsThk//p5mLlqEEgw2lwlnhvPA==
+            x/xSbFWqYVauDB6rpzZvhGMI4KCJh2RmDdt7FrlCb/YCB3QoSgvl8w==
             -----END AGE ENCRYPTED FILE-----
           recipient: age17jqc66x9yeshfgd9v78mj483r4zzarqdtuxtrkxe4x5mw679gphshd94th
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoVnh6dFAwTkY4cHJpaEs4
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwL0NNbGlGS3lTZjEydGJm
-            QnZPeXZHK0tXYWZPNytmYXVsdGRWQVI3RlhRClFVb2I5OVZzZFNrRXFaa0JTUkRJ
+            MWZRK0Z5YktuNVNudk84MndLUUdXR0RlRFF3ClB4MWRUTFBWYUdwbityeUU2SlR4
-            OC9GQ1V5K0JhWlhkUjU1WStCa1lPV1kKLS0tIEhzdnBBZkRnK0NtV1FuTkVsNlgv
+            azZMb012YXQzWHUxZnZzeVp3MmdzcU0KLS0tIFJ2SUNyMTBEY1NWejdVQ3hyUytQ
-            QzVEcEVkQm5NL0Z5dUU1U0ZFaTJITnMKaWE9vlrOpQstr6FGP5ObdilsCYk4kYAj
+            VnlQWHRDYWZFaU1ON2RQTXBwNFNGYUEKwnRUWCdsiStia1jLHJ6UNP/kgEt4Aw8U
-            /phboR+Ym7QDTyUF9LZXJCU54YJp6vEWkRnlJFqC75UW/v/lgBhBMQ==
+            pA8PKKu6TZVPI8pw7svtI/uvJuW+8ARuNcyb1FdYmX+sOHnOEqHyjg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1px0h5l9zp2dww0m8fncrc82kfdmzplsfv2ltat7sna28xpg09pqqcl3s2k
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkUHAzWk1KblJxVVZZV3FM
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxM0FyaFlTR2FzMkhGTzZx
-            WndzRGRtbkw2azRVWnBuZmhKWElqRFk0RENzCk5DMHlMVWpwbXEwVUhkaFZUbkp2
+            d1d3dWNGOG1ESllETEVxRGJlYWkrTUNIdkYwCmI4andLMUUvNVU3R0N2WCsvWWhV
-            QXNlZFV4SjBEdmR6UEw0N1JOUnhNKzAKLS0tIHo5RkNDUk1ESWRHQmV6bzkvSTlP
+            UmlkcHdMSFA3bFp0ZG5OckE3YmJrejAKLS0tIDhEVEJYOFc5VWVHcGcvMm5yanBl
-            dk1GQ0Y3V0dTRlByb2xUOERVOTVwbVEKY4sAHyAhvGSYJzPuufWUIQD2xZcSt/nX
+            eGxaNEFxdEN4NWh3cnlGeElBUTJmUzAKvCSJF+LGf1JqEdkzT/Wk6hEzferseyWO
-            t2ZFXu891/QdEzyUXCIzdwAV+Y/LjvroIlCp5Hkbrk0s7N+ghqsB1A==
+            vAYR8Z+u6GDwQi7JdFz9iZKD21LAs0IMzd7cZqXKIXVedyXuaIhPtg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1ufg390ydrmma849t9xfkxxl5xvdkk6mngnlzhmy7mvuaje8sgcmsmnq6l7
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBYVkwYXpYSjdmM1FpbDl3
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGZ2M0cG5nMU5nYVliRzlx
-            QWpycXoyL1AyOEpZUmtpbjl3MFAwTkJoOWpVCmVDd0FBUWxaQmZCU2VmNkZGMk9o
+            TVpDcHpwclhXNkxuc2VKanBvcnRxRW04UXh3CmNJeEl3K0FrWWVHQWRwK3Q4Tzk2
-            TUdLNGtac2N4REg2eVF1eVh0WnNaTE0KLS0tIDJvcFErSjRiWmhPMmpadjROOHdt
+            MEZQbFV1QURLZEhCdEhaMjBRUzllaVUKLS0tIFhSUTJ4SXJwRHhVbkZFc2Rwb3Fh
-            NXp6Y1JpdHFlSlRoa3JTaEt3emdnalUKjoFfZAiKMPF3noX+K0+vc3+p/XUHnhic
+            QzAxTnpWSHZaWlF3NUlFMlBHUkRUa2sK3Kim0mp2adSvruI6Necd3UpZHow0EFmT
-            k888KdUwcZYl2/dAIc8UDSggbMnncJAJgoezoCHLkj97GNNAD7E+gQ==
+            JFOIpIK+3Pc06gsiqrDtDAx2Qtc10Ja4ILTutPlp4va6zzmzrPABiw==
             -----END AGE ENCRYPTED FILE-----
           recipient: age16j42pdc5dr6wnj7xayhkqdj2rny9u68fcqejs50hqq42scssh4gsnrrnlt
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqYnQyYzF5cktZNG5PcThy
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwTUZ1RzJKcVF5bnJGamlU
-            SEszVEwrUkQ1VVRsM3pSTlRQaHVLN0VuSHhrCmN1Z3pwNlFsbDN2UGI0KzYyallM
+            bWJvcU1sTTJuME94YjhMZWRWRWR0aGJ3UUNvCjh6U1BkaEg0TDlCeTNyUlNTRWV6
-            SHJ5eklQeEIxSlhiYW5PUlpJcG5KNjQKLS0tIEk0QkpMdlBlRjVYMmJaMzJUbDNm
+            Nnd0NjE3NmplSWFjVWpSRy9qWjBUbnMKLS0tIGdMZUFzbGdMejZoMlFBVUJXSE13
-            K25pZldwd3JoZi9vdURoa3Myb2RQNG8K6N6bO2YKooPfpKihgsYqilfz/yAYCLZD
+            TW4rUHFjZk5oVkw0UGtpaG5mOHRRbHcKFX4ZyNQ0TUeqwXEaz1qmAXQRIOctDkvz
-            XJ/THgT4URX2VNvSspvBtN8luOiJUVcchp5WtL2m9jARL5txEcDorA==
+            TIwxJseAjS3qVxCslnxzgaqmCh8DzCfjrRxLUS/HVTnwOaE1YAh0vg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1nruncs4l0ufk7yuc4des8p99c0alfndl0lhsws8tycl5pplfp56s30af5f
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvY0VpeW1nazVvRWJFaGNU
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTWStVRDhlblZDMDJ3ejIr
-            RGVzVjRmWWV4dnJzRHdsaW9ENVZYck5lWGtVClNtd3ppelowRjZpRFFSMC9EK09n
+            MkwyeHRwcnpWQVBmOXY0a3NCU1JnTjkvVGtVClVvVUpLTVhUNGNNLzY4U0o2SDRD
-            b2hqNnkzejdrTnhYNGNKblpteDNLRWcKLS0tIE02bWVjazRWNEVKbURITGlQODlR
+            bGZteUU1Uzlzc0NjTW56OVNCcnVpUUEKLS0tIC8wN1V2Nkc0cVU3OHNYa1NBVlhq
-            cTJJVnBVdGIrdzBoSXExelNrVk1XcEUKc77o2EX7PCm/HjUo5GsUiQdm488WB2mg
+            YTV5QUpsQWppNHRpT1dMb2JDQnpoR3MKCoBngnNg5qpktg5j4CcuTUKiooOVI8IN
-            wHd/qDbQhF1W75RrVTuIKgtEtrRjZqpmr8toe+aHJizPofcrToUfzw==
+            04plE+1lra8q5ZK8q4r1/K8hBM23poyfq+XushO1GZ17dPtWs9wITg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1k7d2du5mejsmv5rzavm4xwgpthqvcfsehduquv28nzs53zppa3kqngfxq2
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSMmZnd1pXRWh5NDVSN2xq
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUU2ZSazhZNWFZVXk0UXdx
-            MVhTN3N1OFpkcUh0a0tjYzJnWUlINGNVd3k0CmZkTnBac0l1dllxazdLY2l0Rzli
+            Mjh3R1NLMThlaHUvWVFMZEQ4Z0NYOW83dWgwCkVnd2dhdzQrZkxaL1NRbDJodERQ
-            bE9sNTBVSkJNaWF1T3c0WktoOHl0NU0KLS0tIHEyeWZTUjdQeUN6U2t0d3JwNTBX
+            SkM5S0kxcVBtY3laNWp5OXpXZHhqZ0UKLS0tIHVOT2VUZU5DZjJjVW1kcU5qS3FD
-            ZE1Za0tXb0gwc1FSakVYdU9OTHkyd28KkwmlzSYP8XofB0VGag+S18+S2TyQjLrM
+            YXNhRkpaREMzRDB5aWgrdVNvUk1qcWsKQnPqE+5xSBGeDMmLfKB0tEBt2eylBOxT
-            qaXtbBtLzJGNDhe9FhAKTPFcjTLWbohlG69vxcImyCyCns+QQ+gvug==
+            oi26mC+EoO5T2pCIpbD+DODAAlWKLattqCYHneUO9wRk9DnhuEXRBg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age16kqfmvz4e23hmdlqresnyw69ej604s320mmd49h4hm3fhqchtgyqrws0k2
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2Q2RMWmZVOEwrOC9VcUxp
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZUlFlenlOVDk1NmxQVGUx
-            VVE4R2NqYlZWZlhKSTBHRks4bDNoaUliMWlJCm5FbWdZS05GY0VLc0sxY2x1U21V
+            UHhVK3ZtMHdzaXdtWmE1TXhOekpsTmUwSlJ3CmdncjMxOUw0alBQZkhZZjA0YzJZ
-            eExwK29GVVBqYlRPZ0l5RWVXRFhRNlEKLS0tIDl6dVZJQndwVStFVEJnRHRyMW1W
+            bFNVOHUySXRuYW8wVjdHMDN4ZThuQmsKLS0tIGdvUUdHcXlpWlZkL0xBcVpMN1E1
-            NnFqc1F0SGJqT0xmREpaN21EdnlJK3MKRPE5rfFpVnH5wAOkuB5pNMlMd3omcpku
+            NGZCQURYQVY5ZzdDODNNY0o3SG04eDAKiYVuKsJLaSC7/bQRB51tX282/yqpAjWT
-            do2hFZwyI7t80jxF4+g3J7EolOx8AGjpc9Ba7Gj6IMDjye728q5N+g==
+            XNnqPQBQ7H3xuK0tDVdCAUMruOSlqAe9YRZwmiou1irpg9vZYwgTPA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1arhf2q45zw6wf2uevju4savp575x3m2tfvved5zzq3ay92ynua9s3cm92c
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhSG94VE96TVlTNkEyclFE
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnUUlPS3ZPS1lSVmxwV0lq
-            SmdCSkJpeWlVWDFIMDhwUEkxL1RUTDJ5UG5FCjhKRDB6VGtwTVozdUVzbUxGL3BW
+            WWE2QXltb1BzU3p3cFRuMzVieGJta05nWno4Cnk1RG1CUXZjM3dNQzlJK1ZGaHpZ
-            SnR3cmpSN2RxNnl4QmNvT2lkYmtoVFkKLS0tIHd2V2h2Wk5xOXlISzhjVzBsVkhz
+            b3drMTB2L3hidkpoTXdMQ05vOUtTb28KLS0tIHhyUllpQnlzL093MCs4YWx4VlVn
-            VVpRenVnSVpHUWJqV0JHNXNWWXJOdW8Kv7PJSTDbwFOAcl7pynALaJiTXU/87bSF
+            S3Q3QnpVNEx6ek54RGx5bmlqUk5hbVEKHVUvfk/xKtlOk2o73LodweIivSB4Oii/
-            F3HQllYOwOoibGzBCe18H2N+VxyNxoQL9OWe0TvOIR6bgHFIIF0/Dg==
+            6dwnE8wa+Qx/OU0vDJVelKOUE6JiLwckX/0u2YIUOybkx49n+tYs/Q==
             -----END AGE ENCRYPTED FILE-----
           recipient: age19mn8zrxl8zpps9yvrh4euquvygpp4fp8queg7xc6qhtnl4ng8c9qx02qwn
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsMEZPUExRVHFFb3pSVHNO
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3MHlrTTdxYit0MTkxa293
-            cGJFN1ZzTDFVNEdneVpMZ253ekFJNjVtYkNnCnRGQjU2Q3dsRGRFV25LQ3pCbTJE
+            eFFIcmRLMUpkZjd3eWdsOXYySEljNDB2NUdNCnAyanVkS0dmenZSa2dpSWZTSkZw
-            OTROaFBiT01xb200S1pUK0NYaTQ3R2sKLS0tIEdiQlZTbi9Vcm0zc2t6bHplZktF
+            Tm5LMkRoWEk3L2ExVktnZDZMM09hZ0EKLS0tIGVGcHFuMjh4OTUxdHNJQSsrR1pq
-            ekRySENXcjBuR2psdHZSSUJrR0xUdjgKvBsmnC+cbq5TUDFjXCyImIoPKvh8wsjE
+            WU1jVG5UN2hMbW9iQ3JmbHlwc2VveFUK4JC32glesv/9nMWBR0lFeX7MPgIt9sIo
-            7Shk7Act8Jayrhx0lXBDRmfpHRrB4L16rDSmqO0DTE48VhT3TiFyug==
+            9HsAkOMTxPTI0NZW8q79+G5hq8qG+QvQhwO6wQtwz/KEHPXntKDX+Q==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1jlltcv5jcnm40z5k0q6hv053k2rqpqvemtuecdwn527uw8uqz4es3x7m68
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxNDlya3RmYzNhU2p6RkVw
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRbEZGNmtlUjJVZ2dlVGZC
-            cHJVWWY0Yi93cW1uZi9FWkVONmtpRWh5SzFzCmtWYTRIY3BkTWU5R2Jsa0ZJK3kz
+            MGVEQXV1SFRpMjFqWks1Y2F6c29TY25oL1MwCjVFRk90Q2dJRXhUMmIrRVdHVFRD
-            SnZvZ3YwaGtoMVZ3V2laTlBBK3UyTmMKLS0tIFdpZWtqeGlacjlLbmFySHlSUUlj
+            TWgvQS91VWdFWERJWEYrL3VnRXFraUkKLS0tIExUU01VRWlENlpBSGt3V1Izc1dS
-            RmRqQWVHK0FUT3VDbFhLbXQ5WDhLeEEKcDkgV34lUFJRIHRoLB8F2IOvGAM93sM+
+            RFBhSTVKTGV2KzFidDQ3b3lmUHZ3d0UKHWvWO/Om6ahwBkONreGeENqdFG9UI6JB
-            AkmaM4+WRcGeYWQKMG2x6cYCUKFaT1lDXuWZ9kI8Fd7b9gTSnQMs6w==
+            fRA3gRHy98EorEiia1him6lGY+ewYkTtVNo6iEt0abncW5x5xMQywQ==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1ug787sgt6st6k82fgkrug2lzltw4qsukrrqqs3w27ewwqj8rg4hsxcmylz
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjeFM4ajhHeVd4bzlqZmJ1
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2dFFpeDU2Y2dtQU12cytZ
-            MUxYMUpZVjdjR0NGTzVteVB6WFBQeFI0ZkZrCk55TEt0Zjdwbk51RnhYclNzam1H
+            WUJjKzZ5S2RiZmtvYVJScStYVHI5d2FvcmpzCm9COGZmRllyaElqb2NsczhKRjhO
-            cExKZXQxWFVLa1pDNFpkcGZzcnl6a0kKLS0tIFJ4ZEdJc3JVaEc5RU1aZk1uYm1l
+            ZGhwZCtoaFcrWmNZbGVHMUNLMFQ3Ym8KLS0tIHJQR1JOblZlY0cydnhNUnBCT1Fh
-            d3RHS3hHSkRKRXFnN21FQmh0TlNtNmcKdc2G/1dhTJen6iT9kUWZM5OzCmDVprgx
+            aHJiZVpMU3Y3NXd0ajc0MGMxSGp4MmcKky1oaS8yFydszcvZE2pAL90ZUKoN9MnJ
-            WN1Bl3JzYhLsNKn794887bVAICVqbXqkdpEZztNIS5n/Rw6geKsNvQ==
+            1TdqatDSMtmo3PWkdFeHbWxPjTizQGIcKEcsv6GmO/5KCAJDCHDNMg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1529taqdwr6t0w7cvzmty0d5y5593wffl0krt48j6uc4u39k56g2qf6ywtp
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYa0gxNnBwNjNNTTgxKzgv
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrd3QwWE4yaVlIakdiYkNX
-            YkQwTjZxb25tQ053Ny9tSW4wNGVYYnlIaFZNCkVvOFNPOFkwQzhYZGxjb0FOZzJ5
+            bFVmR01yNjdWa0d6V29tcWh4M0hWRmQwR3pBCkFRV2gzSVdPQmQ5dXZYc0F1dW1r
-            ZjNXam1ZTWZrS1M4cGhhcHZaT1NjblkKLS0tIEdvL1dDTHpWcWF0S3ZqNkxrQW52
+            OFZNTkRwSzRFR0tlUXE2UzFxeU1Ld0kKLS0tIEZ6RlZkMFFiZytPZ05DSEI1Ni9X
-            VlYwa29sZVloOS9qajJWQWFzY2FKRmsKy074SLdttogXsWycaFX8xso4ek7Cbjph
+            WEtpUlMycVhkMERsWDRGQ2lrNHdQcDAKRqVAOzQY4ZM1uH7LyZSqX1T6+1PwNhBY
-            MMEhZd/svmnSiYM81nmeaze7qXEUcsZXuSmZCYATTBEGtx/Srll8aA==
+            5M0RxSuI43E8YKC9axcppvKFJMQI1GAGvgfQxg2J0CEpnmPSVsO5DA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1zhfyuzlq40reuqlr34gf77852nhs3t6mqfzrqmas8z6sxk7tcfhsungrm0
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvb0xNMloyRW5tTUUvZGph
+            YWxIc0ZhSWI5RHNmaEIvdjZJNHFDM2xBMkNzCldlM2RUeFZCNE9tMVp0MFlWM2sr
+            Sm44MjkrWlNkY0U0eUlyNlE2ODl5SFkKLS0tIEdISElQa2VBZlduajhsV0ZaU3dZ
+            ci9COWE4bldUMS9lSVB1eEk4TG1sVjQKYu9zby36DP+41dTV8sRdtDil3Az40pf0
+            vIhqhCTxQymnJHEwEukiEzM/w3Z85R/W/MbOKyzEZmxzG0a5O5svhg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCZlFtOHI0NnlMeWdjZEsr
+            UnQ0UFJXY1pHTm4yLyszY0dBaldtUkIxdzJvCkNQbk9mN0wzK3BtN2pEa3JSWWFk
+            RXNoemFFbDZ1SlUxOEZMUm9mY1RaZW8KLS0tIEk3bCtyaUVGM3MwNUwxS3QyTjA4
+            YVpTTFlOdzljbmI2cnZUNG1nd1NrVFkKxARm3bpBL685Wky4Y06eA18VvUZRkAl3
+            BBYPJMn3lfDgiQPvfXINfhJI6O+bWyjt0WnfodCGFM6EJLARXiTaqA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r
     lastmodified: "2026-07-23T21:15:41Z"
     mac: ENC[AES256_GCM,data:qFhnPra6IE3wyKQ4WKweON0S0YtD5I0adGZVfA0m6BVilN6bX5oC/1j5NK2oHrsz920hSl0SOF8LrpqOrUyGjSRkPsN4kq8qr9bJcrX4URiktP0oRden5LLt6hf+ZRP7WmRXFqixPkPHJnZIoAvkNnTFce7cDq5NEAHkKUEKG7k=,iv:nyblUDGeu3TUfFivYylOn3C/HITj99qiPI2+mh8AGh4=,tag:FrtRzSCylC4wlIoqZdfx7w==,type:str]
     unencrypted_suffix: _unencrypted

secrets/docker.keytab

@@ -0,0 +1,26 @@
+{
+	"data": "ENC[AES256_GCM,data://9IEHIVCfHjyMNao5sCu2zNlZ/CaW+JyxVpGpD/vab2qvURunCUY7eMfSOyvOx/2WPXnWWlkoVJPCR1ec/yUg09EaSMfxrvqlu4UJI3Sxvu9xNuDszMwMMD/sCulJDiMWFNLp8qaYt7UGzexp4+GlGiqWDxk3ZEu/iLmSApzBrpciTF0lfehT4qblDovo9QXG2KDWFhCt2SwEKmHJ61Yl3pVAQnPLyTWaNhwWD/mYL76mIiDVKq7DJlvi9MBxzi3aYh/ttuHgRCvnCL0C9oUvOyT2cljwra0LXuOrum7FhPIXXboA7WTEbTHJm1LB5yLfxDrnWCrGxQzFQAwNixVHIcjuvjjvA/LifTvbj5FYM8x7an+DwXPfPhruFrDew1paAWsEGNNYXSmAlju2QLI/OwLHFFLuDyLnBKQ6RLtZbCEAOUKf2h4iZj7nH86eb/aGU=,iv:rj76+MJBCpiYyx2Ogut5UxJj3Gn+bygvu2mtuY5hFLA=,tag:yQmTvWVu8ZLug/7fo6RnwA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqcURBS2tIMGl1aWxLMHFQ\nS3ZGVTE0cnYzTmpnQ0daL1F3QnlpemRCMmtjClJiZzIrc2syN29sRXVoaXlPRUNF\nakw4RndmbEduTFg5ZVRKUTVwRWpGRjgKLS0tIHZ5ZFlyODlGWTJoNGpXR3RhY0ZH\nOGpPdDZQVmt6UWZXbHkxQTBoeW1JencKbsfH1V1lUj8mmHyLNj36VaRDgaBojcDU\ndoQWmSEXxjticJqdadbVKb3UABpvzAZxASCy81sa3wH0gT+7zLaNyQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrbFdaMmRjcHgyV3Y0eWZL\neG1uMEtpQVUxQmNTU0Z3aC95Zi9uQlgvMEhRCmJiVlRTR2NocGlnbVU1UmlLVVA2\neERiMXpiQlVPNVhRU09XVGxrY0Ixa2sKLS0tIFRuOHdoelJxOXNRRHBJNnlhSE9j\nUUZHZmQzOVFwRmhFV3pvdnpRMTMraGcKRHBuSUpbHaEzH2tuSBE5MsLJDCuH3vUx\nO0jnDldCWkCw7Wvr/tQAkaDI8axZcYVDUkCEk+xqAdxDozLCPhEO6g==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age17e89ty6p0fw24daanen57wg8uald9s025t3wwxsw269svwpmgvrshfvfvt"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZMW5taXh2QkJkT0JjaVpq\nQlBjVFpFYUhXTHhOT3prbFcvUFB0YnRGUzMwCjhHT01lY3dPWDE2dDZWZnJza1hN\nS1AvZWIvdjZoYmRjWlliK1hiOEdrT00KLS0tIDljWWY0NlQveS9VR2hOSUNyTUtH\nK2gvbEVibmZSdktPemdEQ2p5U0Y4d2MKKitoTi2vbxJ41IoWMlj4vO91Ahpj0hHP\nrKCPyx7ws/IUMmKGyvDLpZ3pYqHD9jl5pLfB05Hh4Emhv4lA1qtwwQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age17jqc66x9yeshfgd9v78mj483r4zzarqdtuxtrkxe4x5mw679gphshd94th"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3K2dVV1BHbDk2Z3FhNmdV\nZEN2Um94K0pma1k4b1V5ZUJxQ1hWU1grZjJNCi9iRVVqOVpEMmtBUi80NThlYldY\nTUtZOVErT2VSWk43NndpVzBlL1lQaTgKLS0tIEpCM3Q4NjM3N1lpUE56N04rTXN0\nZkZ6TkV1TU9OV3dpc25RNWRpVnprM00KItUzKBdShakOfX8Sr+k906nsvYPl8QLb\nge//1GA+ukGsaS9rcChOY89vFdm61JDmj1jXSJ0CN4wLMW9/eblZRw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1arhf2q45zw6wf2uevju4savp575x3m2tfvved5zzq3ay92ynua9s3cm92c"
+			}
+		],
+		"lastmodified": "2026-07-28T01:44:28Z",
+		"mac": "ENC[AES256_GCM,data:efFXIqbauOURR7lrVpK7kqRIPgYjgWfenBYoX7mUrQ/thFc+ApcAx58Fi1zg4biwIs7NarJAgDqAxZi+yKb2Sll8H/wlsEjWDU4iQlLJdIQw7wey9eDW8pgBLd+6E7FXrgn1Vh49dS5GXlC6clAYk1lCiB4NvfzPDy5Ktpl+Chs=,iv:+zCqj5I1MLJfRRiIrqgocYB49PZnlV45PUTH4gYlfrQ=,tag:WxY1sF4kFOTEzbSFcfJFbQ==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/ha-corosync-authkey

@@ -1 +1,22 @@
-STUB: run cluster-init.sh to generate, then: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey
+{
+	"data": "ENC[AES256_GCM,data:JwjmYg0qzoer+8/jv7KOfK/BxmpObHOn04GNsUFQd6hSFBWbQlRbODfluL54hnWQ4gsS9MHgwWh8nrLciBhye05kRQCqADB7Crm0CZYubNb65WzM/devh14jmerc3MYIp1M3VmDisML3x5IqhJkFMSqKM/VaNAC3f0otcVR8TTgZw/fJyvPlfqpPTjikGmo5xg++Yk7Cy8A1Dj70kYk10+EQjQ78jf4k/agLoaS+YvMmKXFk4EljtHg8Fe5H6Rn9nfMSSKGTZeAH3Riix2e+fIH+nWEZiPHC0UlPjJrB3SeBqqGRSUUJshVKHFIdKI1Tq2Ea+GDWrAyIRI0qO8e7WQ==,iv:mr39NxbZuYUDHDw0e+fJDshbt9R1hLgVqkUxUVX/+l0=,tag:u31xZRPfvH5j3Sw5U8/nqg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtVW5ENXlkUW83elc3ZnhE\nOEZ4MWIvQWZ4aXhuUmZGYXZNQnhJemsvOWlVCjIwNmFjS3pJOWEwRnhBOGNRamlM\ncXZid1RNYVBXOFZpMlpnU3ZnVE1scnMKLS0tIGFhWFZFd2JPTFNrV1VHSDVKVHRK\nbmhoeFh6YlUySU1ZRVhHbjZnM1IxMFkKo7aHkz2pEeV64m+OEkBZ2V1e+PUzoChu\nUc/Mnh37dNXSSJtg2KnocHdyzDGi1yQbA72xKTxx6QjYJWrAC/jurg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBReWRLb09reXUxS0YyMUVV\nQlR2bEdYY08rK0RoR2pIelpLVEtYL0pqNUVNCklRaDNoSjBWT20xbk1UdmNqWHNt\nR3JoSlVwODVMUVBzMERUNktHMVViZm8KLS0tIHc0SXRIS1IreDF1VCthb0FBR0Na\ncGFRWTROUHF3clFtTmY1azEwNUpPNzQKsgCND/BZcMTgBTAcnHunQcT6LG1jNrtO\n+W7Yx7bFtFBajWnRYiNpUZPibQJlv5SE9os47WDH1gs86xftgV+uyA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGbllNTlhxNVVkd01GSUNi\nR0lxQWZoRXI5NWxoTExRUnBSb2FPR1BONnpzCkxCN3pNOXVrTW5GR2J0WFhZUE9r\ncm1QTThINDVVVlZEalJiS1RXcHJRS28KLS0tIHV6aGh5QjB0M1VPWDVMT1k3cTBh\nNTNGY25NV0tWSUl3UHlJSExFQVdpVDQK3ARj8xhFRYU6oqYxNQ5+Ryza86ALNUwX\n7yK/8ATnquYC8/ZIYUbgTPxzsocmXX9lVT0+ktIALqjtG8PgNTlOow==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r"
+			}
+		],
+		"lastmodified": "2026-07-28T16:04:51Z",
+		"mac": "ENC[AES256_GCM,data:9X0dkAEGJiug16LDy/8//QOkHITTzt4zwTKiIU13tRgSRXRpt/7b27+3eDauscXDMamg5wbYo1YF0+VwLl21Ip4icnnwrZBHd6HWleP30HTgef8rmo21SstshnclZz6RH8SEGVDO/vjrMDChaZ3A2WD+nbcomw47DpK3cG3kBIk=,iv:G7CScSD5Z1Z9WMEMydyeGK/RD4W43xA0PlcpmTCxLOc=,tag:DZmby/xkxpinoJztZZRWwA==,type:str]",
+		"version": "3.13.3"
+	}
+}

secrets/ha-server-1.keytab

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:VpxnzrdX5FY2gfDcZpqg1f2wqL6mAawcrdE5A9A3fQOKr4BWXmCT/hjQEtt8qG+yVOTApdG+vF8SKDj1AKZ25KsZ253WS7W5C4j8oiT/xHo0CtvVani4fZqN60lB/9381h7aUZ+lcBMPaQcNn35zoQqYthuCGrC9PH1qQgD2VVHr0v8J6kceXKp0KTJldtDsQxky6ROC9Zyc44wwkrFPgPss4/yGaRWWqiojDvK5fK8238hmR8HyfD6Tf5KhgbqCsSGS2g6yt1muUhmegaZwtpi/KpoqBmf1XFaObuHGRPKvblq4Fa0x/sSjpPogBZyNuvRKIwRhla0FkKOH1bX4gC9bzOlgiUUfVcRJT6Nr9jocYuhIv9wvIT4TujGbgHyorYaDSQLFPJoJEUa3fgpGq+AC6p1DJJE3b/wqUSngNbdOtAPE/imaEuprOfnbS7cRx+Ti/Yu+ZYZuAlDtLkSVdXi1hQ4fLQ==,iv:rq/BzePXa/w/Gqewz8JfM/NdU+x4ShiH1OuQT7wRyCQ=,tag:IcO3ddoj4M57CkndHEmNkQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRdWFEUlRJMlZId2Yvc29F\nNnkwREkwQkllY25vZnRjV3Nob3BvNE1Zd21NCm56UVg5YUFGbEc2ak5MakxDbCs2\nTFVsRFhjVlRwMXdUbGpMKzQ1N2hBUUEKLS0tIEhmdm9hZEJqUEhRN0dsSk80bDY2\nY1JpajNOVFpEejhibjdKZlNHVjE1bGcKycyDfEBZ1WYc4EfAK2y5x/nKRqq7mnb4\nDlpR1Som4bSt7+B+OCZa48mC0Zv05HbP8PnoZdCe83swGNljvRONcQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0SGFVbXYycTE3amR3ZVhi\nVFJHVVhWNWVQQlJKZUJvZ2p4ditqdnYrWnd3ClZaaUVLUXFKaWlDdDQ4QW5SQUZY\nbG05c0RGNnZUenI1QTBlemVjT0szRncKLS0tIHBWTmNpd2FjWHc3MmR3cWt4SCtH\nWS8xZnVON1lGZVc4YzZFRmRBamdSVlUKdlQTlwiExTUVsiY05MXFG2/IQt1bZwKU\nlFDnuy6YCQzawBLQZuSUAp7WSGedupexlZKQKNUzPf+dis26XEYZoA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6"
+			}
+		],
+		"lastmodified": "2026-07-28T11:53:06Z",
+		"mac": "ENC[AES256_GCM,data:PMsPSffQkxRmqT6HwFezIoT4WKJ9WDlAcflWynsFLtzbqVTREN0YGOJgMwLd0yBR2sbGBWogG5xAuKjO16aiWUVpCFPRvP97um351679ZoXIlrOL+GlFn4NKwoLH9/eCp8BteB9mv5huhEEN631lqCr+jtYO6yS0ULk6QS4+7/c=,iv:oYWV4J1JUuGryvqntVXkj/HOzRDRr9E2C1RKNvr2bT4=,tag:e4p+I40y1LrgdlK9tLRicA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.13.3"
+	}
+}

secrets/ha-server-1.yaml

@@ -1,6 +1,25 @@
-# STUB — not yet encrypted with sops.
+beszel-token: ENC[AES256_GCM,data:tT2a1I6AhVgXWfc=,iv:1iXoOMAakHXpI2lYiipxcVSzXODUdpqYmfrpNKZwljg=,tag:Vuxxz7pyIsZYbkg2x6GvgQ==,type:str]
-# Bootstrap:
+sops:
-#   bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1
+    age:
-#   sops updatekeys secrets/common.yaml  (allows ha-server-1 to decrypt shared secrets)
+        - enc: |
-#   sops secrets/ha-server-1.yaml        (create with: beszel-token)
+            -----BEGIN AGE ENCRYPTED FILE-----
-beszel-token: REPLACE
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLT0oxSDR6WHJGT0VTY211
+            d3hYcU42SzRIQzhvdUxWSnZhRitoNGFlclVBClFEZ0xXVnU2OFV3TmRXZHJyOVp4
+            aDZjQ0g0RlV5bnlzU0Y4ZnYxWm4zK1UKLS0tIDJYZ1F6OG8vdUxnS2NuaG9SL1gv
+            bURjNjRWQXIvQlNyVVZieGpCblV6bmsKQrCQQz7b4+w2WB3E56lEZoZa8/Yx8mn5
+            vWtJa6nvqlE4miJdRpbjQWUeFHghr0A6FsYDg5yf+Mcpf2otTw8psQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5cVFzS3hFVkZpcGRWcUc3
+            YnRqVWUrSU1mUGRaaWhZTFQyRGRLMm9BSWdFCnhzYVhtWExGMFlRdU40K05TV0li
+            L09BK1h6SFlWSGRtQ2F1QW9iM3lDVWMKLS0tIGNQSE9aN0xHNytlOHBzM3MvMVpB
+            VEdyUFc0WWgzVEswRG1XSyt4dThTWGMKRkR+FifFDDKzrMrlJgZj/lbJDjfHsvjI
+            BGfJj1x0whWiozg/SZ6BsYYYlVkcCBwVTmPYx2iQHk4jLqHcMF36dw==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6
+    lastmodified: "2026-07-28T06:44:32Z"
+    mac: ENC[AES256_GCM,data:rtDzbONqaNpbEKo1btGiHArAJVCVsStHhgvM8MTgkRMUNGU8p9W4tiuQLmobDpFh81kj/mDnkx4cCpScuiMhQojz9H74l7P9o9b1Wxy0fSiCtGoQhaq6JoIVJoKtEZoHfMStdIio5EBW9DBB5abvjPLTaC0afgHo5pNn3A9qr6g=,iv:SqtJhel/5qW0V7IV6V8dCRWqPnsK6QCzbC15/BKeq1k=,tag:vF7RJJccm0q7BSyNGYnplQ==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.3

secrets/ha-server-2.keytab

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:Ea5AGwloFMyRgmQhJaDNTW19mpDvx0+dSc3ibENEXEniEIBV4Wv135mAwE6jQPbN8wF6f5ki3B5mDXHOMuhDQEqUXg6jcs4uEm2nwz7wAKYXEX4T8p0gUj7FPISiSQrw8O3jwzt4qF7AVcPsNhMtZLgrx9ssMtVmGct5do+E61Y7dFe0XbpJLynfzeiMKYvaBngTbHwQjv1mn9AMpX98apsp3tyzg8avLDR+WXRcMusds2mdov0jCoKuWPLG/srhMwjKOp1Z1Tu+aZCr9lHTT76mytpGIWGSbrP5l3UTp3oPLmy87Kwi/FJStZYdYikMnvmSo5qovePiYsLKVD74dS+d3oFGARyW4TILd9OQdXEjzdDRak5HcPv2/HhR638kvukVMdEdH0hpeJcqcV0h+BoLyIHe1e037iF1ZkCjuXE+A8LtlzRR0wqP202uzBIyg21ca4X5bRiHkkAOX4P3riaEcMgKfw==,iv:IXq4sHHsjnK6maVd3RRVZzhnxx1hOnxfsvYX8IKLn+4=,tag:jRZinAVLH8nGVf3gE4aSKA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByOTkrVVd0TXlBRzdkTktO\nalU3WlVOb2ltc1k5RG5sRE9EN2M2WUlnQVRnClFGZFYvSHdpVlhtNXExSXZMZnU3\ncnl6aHRzU3k2V3R3dU9RVDA3VWp3VkkKLS0tIHcwREM1dDREK0h0UFp0bU5JTW9Q\ncWdyanNMVjY0SnZUWUl3Y2R2SXNyc1UKI1FPIE66to19oQK2TwM5B4snGOVMLFfx\nl6JMEilDVSgy2RR/tSiovmo4NwOsv0hmmF1t34547rbZb+dB5KrMEQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnbWp2UWphSnFibnA4N0ZV\nRHlNUnBNYUNzMzNZakFVc3JXT1d0R3Yyakg0CkowMFlOcEhPcHE5dXd4ZUd3cDAx\nbjQ2T1NqejNRMnFvbTdNRGFGdW1pM2sKLS0tIDhGb1JBQ0RFQ3NxbmpkdXg2WVF6\nS25uZW1hMVo4Q1JGdGNQWWRyWE84b0EK0FmFZB6sN7uZynhzYU932x461zSZIUWU\n8oyDEKNoUuPqpWK8RktgjWlKHtB0oLXC+SLAOJEhtnhWF+GSqloGfw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r"
+			}
+		],
+		"lastmodified": "2026-07-28T11:53:32Z",
+		"mac": "ENC[AES256_GCM,data:EhWEEIamG62xHI6SqO4tzdK6gcEXxUU/UGcF25m+X11kUVXzHd6yLcfiM2DvqcwGLmwmOOf3SpFLABlQUzTka4QRzTMFIAJi7x1rzQV4J0YIGITHk9rgvy0V50TI0loONBb2du+vDt8IlTBNvF9WMbNCki+fHPAjIemyoRBdubo=,iv:wfsx0H4fD3L7WeGzz57SzZtajBr/xTRDljSpz22DD7Y=,tag:naDyGH0WTKkqgd954Ya8jA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.13.3"
+	}
+}

secrets/ha-server-2.yaml

@@ -1,6 +1,25 @@
-# STUB — not yet encrypted with sops.
+beszel-token: ENC[AES256_GCM,data:0wooMaPiytDTGug=,iv:/WATBCEakrIvujt4oAeprw18W1n/WoK2zjpQdlVJl4Y=,tag:lhJa9hJgFqNFSB1Ogj5CtQ==,type:str]
-# Bootstrap:
+sops:
-#   bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2
+    age:
-#   sops updatekeys secrets/common.yaml  (allows ha-server-2 to decrypt shared secrets)
+        - enc: |
-#   sops secrets/ha-server-2.yaml        (create with: beszel-token)
+            -----BEGIN AGE ENCRYPTED FILE-----
-beszel-token: REPLACE
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlMWJCTUNSVVA0Rk84VWl5
+            emJ5ck0rMTFRMmljUmVOTlpUT1dlM2VFekZnCmRZbWcvL214YjUzakNDYkovQm5q
+            cVJQZnJvVGpIMkpqUnZEenkxZHNDSkkKLS0tIE5tUFl4a0FtZFNNTXFudlFNbjcv
+            a1dEbnl1NTR2T1lWN2wweVZ2ZnErZEEKbn9bLqIsMwSGKBjR97kMfFDuh1l7Hu5o
+            zdLjnAHzCEAsTW6V6wPB1rvi++FO04lqq1RfEiWXJkB2KWSRf+jdTw==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDYUhMZWZCVS9MbnlscVBQ
+            aENENGFLTjJ3MVg5SGJCeXZsNVZ2ZFM4aVZZClYyRUhYTFZENEkvQjJqNGp6b3Q2
+            SWthQ01EMDdQYkVBQ0ZRdmRIMTFnUE0KLS0tIEJHV2YzRmw1MDVsVWhHRW9LMnVH
+            cU9qdUQzcjlNWFdQZmVydHdpamdQQ2MKndSN6wQbwyGqnyS819tHJRqDLRryXonC
+            CPuBYrTwxosRPwgh+nyJto4lj1uZqCmFVMXGjn87em8ykY5arp9o8Q==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r
+    lastmodified: "2026-07-28T06:45:50Z"
+    mac: ENC[AES256_GCM,data:f50JYTzMJiEjB9+fNIs734CXrkN1IRgJKcoJ0GHoI5p75cfLfPxuzUifty9kQ/4DbM7LbWa5w+9occoV/yNOa86BKpPtU0LmlzXf904/SgOCUKhyHqhyO5mp8w71GoyLxx3ld8CbqfBu8t5l4gwbQamftsmQRxnUdU/Oa/CzdZQ=,iv:EewXBjeihw89h8ypH5u/G4YfoKZzkHqRWgSSIXU3/a0=,tag:lEWMvAkebMexDGGpE+wqkg==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.3

secrets/nixos.keytab

@@ -0,0 +1,30 @@
+{
+	"data": "ENC[AES256_GCM,data:apiijrtrqd77CTizITg0R35BfCi8PBnufpxIyC+hLYqwoBzP//3z/yjFyHPLG98m/c/qywoi3Kn+zsaTT7MjP++9OMhhX94YKlSHV1/cHB76OkwsNc+ClqWxl6vpaFX29Qvh3gFX9c/NR3xvYQutYwrIrQ9NR+t/M52IMC8hvtR1LQy0ak3VIuXJlSnG2r4kF2Ym1iP7phjuq39Gd245Axzw8OB7yGvOjNxSdTPxW/qL0fMlzNcMrjr9hw15WlqnZfWPOsB1+gZjHXpGfPD5BCbAAMoTRJd75vhKKXP/ERhIffewuuH2x/QHfSFvXVB3QyhBQMxd2b8QEEE5cjvcExOST3tkj6QARkzoUpRT7AE3jhl3XZ0uA2qu9SwyrSvbr0tBRKxCdK0g2E2/hqwcK/Tck5GB1eKb4aN+UkqxOblNDH+B1RfDoyNAuN+KEg==,iv:0p+ScrKpP4kQvO52gBAlwAis6oAzZ0EHFnU74hYPrn4=,tag:ON7qOjztF52xsJWAou7ogg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlYXB3cVAzb2xEZ2pGa1RJ\nbS9ZVTc4Ums5eUZJUjEvd1g1aGVyNUNramowCnptZXFOZVB3MFRFcUtzSXBEZk1B\neEtKcDdLS0h0b1h3VjRjRXRvV3V5V3MKLS0tIHBDemkyUnV6ZXhTeE5VOVVOMlky\nWWMzVGVzZlAxMjZYUGpQUCs5QmxiYkkKcuBshCgWX4TwfVlQ5lHikzvwWdLEXWD1\n/uSiy0J6yMSiu8u6cg2SxeFrlKJ3j47dDlT6WHCxS0PfeEA0bJb3LA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGa2VTc2NWdkFRNUJxelVR\nWFk4RWxoelYzNHo1UFVhU2ZkLzEySlRWN2xNCmNmcmJod2crL3NMRlVsSmpmVkU2\nMjlXMktjc3piUVNhUXlTdnVGTWJkUTQKLS0tIGQrMUxrNDlNTkRCSUtFWkxRdXgw\nRlV4ZmtYSGhPQU84eWtiQXVqTmxUK3cKk5fn72UZPH68t5ZappfAhZJwzpLkfKmT\ny9TbUPIr4Pbrexau6YiH43QIbDQFdwYPfkBjGkd57zCg8AVo1+MBRw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1adur9g330gua4l6ndk8cqjg35qc8yxwgme6wrl2hpylcc7vxm38q05ejuy"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArNWkwQXFWT1RpaDNvbzYy\nQWc1aHhmNHFEUXVsQjZqb0EzM0wrV0dwN1hnCjFsUFJiT3REK05uSGRWTEw2SFE4\nY1FleE1XVjhBbndiMmZxTWNTYmhYeVEKLS0tIEJiZzJvS3BsYzB3cHIxa2k5N1Ro\nenFFZDVaODNnVGdBZTBOYWJwRjQzc1kKlXJgee8wTSN4Beq4P0t9cYbk0BWHCseQ\nyaWpiPT9aZBEGLFmuEd3zKABc8lrilX/ySTmOG49vRg6CPmr7cT0Wg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1rrxqea6q6pn39sw8y5te63h2py8jgjl9v0jyper86w3ggtn67upqg3ah39"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEa0JYenFMNmNzWnVmcXdz\ndG90ZUZ0WWlIU0FCZG9OaWpBM3ZDWnFhZFhNCjhuV1FTOTJ2WVJGa2RuNVV2MjR0\naVNXa3diaWxWUlJtdkNOQXZ2R2NsQkUKLS0tIDcyYXh3N3B2QmNiK3dzemFFMGV1\nNTZpTk5yNGV5YVo3cGswK0NLWFQxQlEKIe0N5OxooWXzt1cUViBmjihmGEe3G6/f\nkz2/IscnG78ZvNgYKjdoG1jlsyje/3zI4C8aWXLq2DnIyxUyAhPgsQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age19mn8zrxl8zpps9yvrh4euquvygpp4fp8queg7xc6qhtnl4ng8c9qx02qwn"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2b0NVVm9QNm9waUUzcXBi\nWnZWU0JETjZRZmprZVRUL1h6ZHB4cXkrdm5rCm9GZ0VnTXB3S1BYSmlGWFJVcDhJ\naUl3RjR0ak9BRmQvVk1GRnQxNmtYM00KLS0tIFlTU1p2OHhWUGlOWngwbE56NEhF\nRW5QSkVVUWZpdDZXWEIxZ1BkbzVwclEK2P25nBgf8255vaKW/+T97aNTecRgNjLu\nedIUiPdXbFATCe3v/YRo6sqzFwIsvM6Bl9yHh/SXo6Ftc7eWZZd8zQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hrx8qj02fj2ea6d4g9vqhyj9hl7fppkjqfdx2l37py3h6pdkr95s8n8rvs"
+			}
+		],
+		"lastmodified": "2026-07-28T01:44:24Z",
+		"mac": "ENC[AES256_GCM,data:J0D8bEs5mHLraLS6TvYuCgfiNU1xKM2Yfb5Y0f/q/4wM4LzXufNzv3+SWDHumTe328U8UnNXLqjNHEKL0bZi0coxpU5hVM+BvPcmqD72vscETzbQ2hnU05sfW+XjfZhcN8/ke0bpLt7nP0crD5hsZv3esV1E2UWvzjEiYtWzFHY=,iv:lcmXYG2H469UKBYDndWKMO+GP0mSGLztenm+kBaUdYI=,tag:ujiXbLM9CUsuoFwQQWI84Q==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/pxe-boot.keytab

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:Q++XWxg9tvY7ugT8+8FWCC5jgOfQ1+LYLsnqN0/WGxTf5XT2VmoXvszcE/ow2BUOBG3qBTXS3OHYFPwmj1GgaBHB8rpdXX6+LveSBE2gmx1VR+NUDTxy+0/DBq411MK9n/J9eIYcybdFIE11biFSAob9EgfxBF5roCDXIPDPyVCSe6LyhNvqYPnGQsfHCbejSewLTcRQEiguP9BX96CpMPIpmaB9fHN25t5RWCgMI7MtacrRxRsyKg44+2FZstXdZrp2Wv9u86BxqdAFqtZE8qpPeGdrdzluZx9jhnw0wZPzHdKg5wS7/UrLCb0UxIQiDxDMDuuBuLGkKXfAhb6SGZU41wWAWYk548iGRUaG+79BO4HhRZNObOFvfsjpMXEfcH/Vbv/wHVY4OpJUPe/ZWK4wpL9YrY4nT0t62HH7MirOy1uhwLE50h2+6dHKj6ur5G7thkSqgzVWXQ==,iv:zaRBwS+gfXLhH30havn6Q2+oPWuLV3qBfbOj00kewlQ=,tag:Vr5iEQ2u+9YNahryhgzwSw==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArTGJnaVhleUtsWnlITE1s\nNmRCaW1QWTFNSG5LRFdmb1lzRlV5NWl3YTBNCmF6eUN6RkJBZzc0MmJJa0dKTW01\ncVh4K1VLR2lURUpKQXpxNGpQNnpSUlUKLS0tIFJONnJFWkNCR3pqalRsUW9POVBj\nQ0pFQ3ltKzBETTVXTW5sV1ppWTFJc1kKzxUboNZO+Nwn2eTWy11VP9w1pRswCHaJ\nE2dYU0oUOClVzc0oSuIJxraG6TPj1N4WGC24gS+UmpkmSuCiOeZBsw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDSmxNR1I3dUFKT0xUVG9h\nWkRIakNVeWRQOEN3blNRVjZlWHF4K2NRa0hBClRiOXlmTTJ4T2JTUEw2c1l0R2N2\nMnRwdDA5bEZlQWJRTm9vUmNKclBSU1EKLS0tIFcyeDFjbTZyVEVDUjN1VzU1VHly\nNWNDMW9rTXY2bHNWYVR0SmtMckovUzQKhTWr6yFVW9am3okCiIswwqR5+/p9OLmB\nWCgPtwoFaBt1RjUXPK4/eS4LlucR2K6V/mNMn4xVsnkIl193U9632g==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age16j42pdc5dr6wnj7xayhkqdj2rny9u68fcqejs50hqq42scssh4gsnrrnlt"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2ME9CbmVGcENRWksyL0pW\nOEJyUGhpbGMwWlBhVXBSeXQ2MW1EWnFuR0E4CnMwU0pjdk1YMzF5ZEhTVFlBaHZq\nam94UWVGbjhZSEx6VHBmem9JRWgwYzAKLS0tIHFJZWRyRjRHNzhXdDJSYWN3bDlR\nYVp3eGJWWkh3Y09ZWElyclZQN1ZSVFkKjR32//EcFAdMjVlNgky5zvVkwXwEN68D\nrkTuHKjiO5aV7yAQGPkdNw0UM0oRGF0u4YF3oOUcZfSvnKgDeoi2Zw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1ug787sgt6st6k82fgkrug2lzltw4qsukrrqqs3w27ewwqj8rg4hsxcmylz"
+			}
+		],
+		"lastmodified": "2026-07-28T00:27:46Z",
+		"mac": "ENC[AES256_GCM,data:AbJIHYcFpeanQsJ3x7RPL9Yjlg5BJgkepKax0fL9L/PpA03Antab93iUNG95Mp6k/duovp8Jm445lbuppDZq1dh9ij/deBa8GbzJ50wwEe9zMc3EwRKScpqZEhRPF7KlJsIjsHJyd8NkcI5ji49XkHb4Ae1//8zG5HpVgy+3b04=,iv:uxQCbMwMIfP5S1dbsvIx3F79YWEguxwox8T0YZvUBdc=,tag:3utjmBGDmPc8q4JjaXvCkA==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/server.keytab

@@ -0,0 +1,26 @@
+{
+	"data": "ENC[AES256_GCM,data:aqlMnoVkGtH9z3fJRweeC0OYf7LGqJU2sWA9Q25dKK6NuNyJd4BvjPtpfeg/WhVtJsaOtcbwVm4WAhVK9FARE8g8j+vmq0f6BAU4s6mx0ZIhl+mP+/hIpt//LOdd+9YezelJxdpzUyZbdAngU99rsTluLRe2XmZ7Fquxd8yH/OHenSDY6dizp9+5jfEi8EU+EmuXvuWMPY59xnlnqYNPfSFxs43/pS402LzJoJ5H+cBPprddkUBVzy4cBQvMnrRFUSjnqp74ovZkfIWFqDWQ5YgSU2PjatBg18oulZ7wNRhQ6OLqj6gsu+xrMjNFwnp7rMlA3X//hIidxTkVcYITycXd8KzuMIaofUpnwoyT34fy6+H35/39iiEyG4LRTrOOKRDzXkY2rhJUxFSZ8GlhNhMd0RlkmLngVYrtjsswJ9meIwoAFLPYt7BC61PJf0TXdtk=,iv:mCA819J9LpAOj8QxFAkrHI9wFJIy8qVxv31D6IwWFnk=,tag:qqMtM0eyHbQEyl6ND/wf7g==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOZVpRYmNMQ1E4ZlFXMDVq\nbW1FcnYzWTBHQmdhSjlsZUtZK3dzNUZjY1FvCkV4VXU4MTcwNVZoZUpVVS9JeHE5\nbGk3UzE3YmpZQ1JISHV0RHJqSG9ZNXcKLS0tIHhBdnIvMjJkeWlVRE9Fb0FhWjNX\nUjBVWDltK2w0akkwOTJaTGNSWWNrRXMKqZRNnHiXvn1QBoSGdABp7vOqNlsEN6Xr\nDp3NByXow6PuRuWvQXHzd+WC+ADkwNaaiT6TUrbZcd/Pl8Ges9kcZg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2VFBxc2ZMUFp0YkJhVFhW\nc0ZNbnJUcGtkT1VLR2YrVGhRRFVTZFlyZXlJCmVnblNBNGxIdHh4Q0IweWR1b0dq\nRWFyOURuWkdkRE1RTnJRMEpXdk9HaUUKLS0tIFhKV3FoWW1zRk1pMlhuWWlhV2E5\nQzJTRHAyc0JtSjd2NHlJODZVbndaVDgKFA4565X/4FqNq/fZDZTg81/55hZi4c7b\nTti2AnyE3OcY/kurXJFHRinVMqURQf1fx9MxqUYRitiCz4qe5zFF+A==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1sweerhrga9yf8x6sv0apz4ed4g48rnlcq34rpv20t0rcelwgpgeqwvndzz"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwRWlKTFNocnBRN01pZ0hm\na3pyRUkxMWM3dE1iTkZZcGRYUFcyb1RDb2h3Ckt4blpGZHBjcnZ4SVE5UnF4cmY4\nWW0rZG5wUkZINVk5a2lmSzN2L3JJY1kKLS0tIExpWGlBRFZJVGpGbFhxRE5ZYjBo\nQVliTlFIZ1U5dE9xbDhHMUtxenpBSm8KY6sIFEfK8p+70IXsC4Jwb9Lm/pd9+V6K\n4JAzGrpA6mAuIwwSNnbdcA5j8FmBhCpK6nLBWmFhGm9Y+MRTaM7Jrw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1nruncs4l0ufk7yuc4des8p99c0alfndl0lhsws8tycl5pplfp56s30af5f"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoV0pIUXlyOFFTUHVTdllu\neG10VENOYWl5Wk0ycnpLUytTbnlrQ25pMkFRCnhyRU5xWW9vQXJJOHdFMGZqMkR1\nSktqT3lOdVVPRUN4YTlveGp0NXpqd0kKLS0tIGRaTktra0ZtSVpzSHBrY3VSSUph\nRVRZOSsxTTNmMmltMlJnVy9oT2VEWU0Krxf49B1BsrWn05fqg+cZ0k0PtfJJNfn0\nUL44RUWXWbK2igQHaIct9DfYe7DEonBJeROuxDYm8g7yNOv15S+P4Q==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1529taqdwr6t0w7cvzmty0d5y5593wffl0krt48j6uc4u39k56g2qf6ywtp"
+			}
+		],
+		"lastmodified": "2026-07-28T01:44:26Z",
+		"mac": "ENC[AES256_GCM,data:eyOSn03dzHSgkshPzLVwc95382eEFaDQarHs9l83dtcsb1Ui9CjkKipl2DVSUb6bdMUH1qKYmXqJhwFnAZbFZFjT4VTKtutNtM+OkhVXfT+fJs+1+u7k+ZUYFL9HuxKA6AWpwX3eJ8vmJDJZaAayJbm4PRzOyJywvKeneQqdUS0=,iv:QRKltR4qzVofo/Elt2Us/lrHlD7BenX605X31x+Ng78=,tag:SZ97CcfgMJeu3yqNk8Y/cA==,type:str]",
+		"version": "3.13.2"
+	}
+}

secrets/tor-relay.keytab

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:QvViw/s1N7eIN3CoD27llEjriuSrohOou4Cv310nigcW8xMnA2SDN2id3H3AoKii1JlJ+qWpKn+gUmt5HOM0UlbeNe3os2RVwiX38O7eN9xFH9F5kA3TFs6Umqq3EoI586PwIVmB2LyxDnTeEEXVd7v5PFkBcfu7u8YIcNF7lpcj+6rOyHMB8uxPhrGep3yiKawFd9c9wWD0hlSSatV5tMHA1qmdK8VmDbCU/iuGwIoMzN1eZwGAXzG6LkCA63bUfdxU6yGuTboD+kN2Wbo+GZB0EACmiZoofl2wqlXuiw6qwTvlXkyauc9O5EG//PUkIECzDwiXcX+qSOM9DIBlZNth4ebhtic/PskyF09etL/gICz5YvV4ph5lyrWHq41KxljSU7QXOkGhzagruuMrYzhZb35wFH4Tie1ee2DXbGhreJr8V3Zse/zTMaD+iM57V8bvNcarTzOFXyKfp7Y=,iv:XBPhj2wT0k/yRCRHU4d+BQA/k00ZHWSKOucnZ5+PGys=,tag:Nh5MOxZIUXAYVZY5SZh/JA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0aUtETXlBMmRlTC9vK3NF\nRW4zS1FrSmo1dDN0OGN4SmV5ODd5MTRVejFJCnpPQkM2MVJ5WFlYRW1NVjduV1hv\nUWUzY0hHV05LV1BROVZVUlM2NldsTlEKLS0tIHFsTzI2SVZzYUtJWTM0MmFiUlVQ\neHEvUXgzc1pxSU1OZFo0cXhSZDdGUVEKpLVfzQEnntluUGsblnkHZJ9Jezu8tFte\nxEoV96GVHxUca6TFWpTLMqdR7NtuQGCkx295W3i1tkp58DP1OzRTUA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4ZW5ybVBvTkxibWJMWGRL\nZ1lnWEkxZ3l6d0FWbmxMY2VsTVlzOHJJM3owClNYd3BlQi9pL1lIdzJmekJUVEVi\ncWhSK3ZkZEMyeTNoZlpTT0NMUXZFWVEKLS0tIHVkVlhmQXdRQktTK3J5dXZ4aTNt\ncUw3WCt5dXJhTkdUbVpmeWoxWkoxNnMKj4XtdwmgFVOiVsIJs2Du7QJ09A9tv/Lf\nkFOq8y4tlZe0nCwRjq43sVz7hdCTdQ0rsaWjBGY90LLkJbOA+f+Wrw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age16kqfmvz4e23hmdlqresnyw69ej604s320mmd49h4hm3fhqchtgyqrws0k2"
+			}
+		],
+		"lastmodified": "2026-07-28T01:44:30Z",
+		"mac": "ENC[AES256_GCM,data:zR2WLWX7NaHA15gi4kX0jDvzUIe9jtz5bMCAggbPW+IXOEedPrddAHZ8OfPErVMfx8O1pJKkAKSzoPTAbEle54FisSLMHXp8fI0297MByJrF9pOsMFpVcDy/L4Q+pBzmB7aS9r7+u7KRVVTZT3QwG1rFWZaDs5dFTP80RhtCbWQ=,iv:ZzPoO+h7ebS+jsSH7tWMx6QK8umpa2/HFQmx9dnJN+Y=,tag:vG9+ifxw4HaBE6YsmOwXcg==,type:str]",
+		"version": "3.13.2"
+	}
+}

variables.nix

@@ -16,7 +16,7 @@
   serverIp = "192.168.2.226"; # server (NFS/ZFS) Proxmox VM LAN IP
   dockerIp = "192.168.2.225"; # docker Proxmox VM LAN IP
   pbsIp = "192.168.2.244"; # Proxmox Backup Server LAN IP (not NixOS-managed)
-  domainControllerIp = "192.168.2.253"; # FreeIPA domain controller / primary DNS (not NixOS-managed)
+  domainControllerIp = "192.168.2.253"; # FreeIPA domain controller — authoritative DNS for sweet.home (not NixOS-managed)
   ipaServer = "domain-controller.sweet.home"; # FreeIPA server hostname (used by security.ipa and Kerberos; must be a resolvable FQDN, not an IP)
 
   # Cross-host references (LAN hostnames/users other hosts reach over the network)
@@ -80,6 +80,17 @@
   # one-line change.
   primaryUser = "nixos";
 
+  # Primary IPA/domain user. Home Manager is configured for this user on every
+  # IPA-enrolled host (see modules/ipa/client.nix) to manage the environment
+  # that IPA itself doesn't cover: dotfiles, user packages, session variables.
+  ipaUser = "wayne";
+
+  # GID of the IPA "docker-access" group (GID 50010 on the IPA server).
+  # The local "docker" group is pinned to this GID on every host that runs
+  # Docker so that IPA group membership alone grants docker socket access -
+  # no per-host users.groups.docker.members entry for the IPA user needed.
+  dockerAccessGid = 50010;
+
   # HA file server cluster
   # LAN IPs (vmbr0 / ens18) — client-facing: iSCSI initiators, NFS, management.
   # Storage IPs (vmbr1 / ens19) — isolated internal bridge, used for DRBD
@@ -97,6 +108,11 @@
   haStoragePrefixLength = 29; # storage subnet prefix length (/29)
   haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node
   haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage";
+  # DRBD backing disk — identified by SCSI controller path so it resolves to the
+  # correct block device regardless of OS-level naming (sda vs sdb can differ
+  # between Proxmox VMs depending on disk-add order). drive-scsi1 is always the
+  # dedicated data disk on all HA nodes; drive-scsi0 is the OS disk.
+  haServerDrbdDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1";
 
   # Storage
   storageRoot = "/tank"; # ZFS pool root on `server`

vars/per-machine/proxmox-ha-server-1/openssh/ssh_host_ed25519_key.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN7IABG7x9ejHSENmHsL/BF9MQadGuBnHZa1eUEFMYnH ha-server-1

vars/per-machine/proxmox-ha-server-1/openssh/ssh_host_ed25519_key/secret

@@ -0,0 +1,14 @@
+{
+	"data": "ENC[AES256_GCM,data:EhYMNPzlMDyby6g3hfvBKisKJnU+YAxZjf326tPdflQ2PcmPTzZ0GjHqJOPrOWyLU6IyvnsLMBV0JF5/UUi2c/H3p2wvoBgsxDMaaMW0YUK2IZFo+PZpWoYJSMrx40sH90W0qL0VBw17JLwPU6CjeUZrn5+sCN9EhQvncD8NoJ6QwZpWKhWLn9/7mxd/PF1uxjEQXzHYFeuUuVpQdKSUfV/BSJIT3pMM/VdH2203jTMtwUbK1/7UtGKtDWdfFbkEvJ6qsV7hnN3EYU16WtdyAiHEZvvZ3gqg6YK2DemojWG7z7VQkQGbGm40yA6/mYGW2dVK5ULzjxALl05G6lo1vzxODbMHhzsfJZVw6ocGx3r8xwaasiX8S3loq8WNUWJ9QgjbxuEX9uGZja6A3nPhcwVGGZExbUoU+syXUk2m+WfIQ0epplaPDyAxrYO5ZPWLdMYoPRxbDx94rx5fMxA3jst++IEL2kQMTieGQYJy3R8veQIe3Jb074zY+Y79EmRxfo8ruR6ZMqORtbZzJOBcd0OObjUVLUqB4rnC,iv:XkCfOKtmZNz+UTZdElLm8L0PpymkDzCM24VkAa9Y/Qo=,tag:hmr9ik0V7mxEKR0DcDB3/w==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqQ3J3VDVwUVBvUjVXSWNz\nU3YrRXZlT2NLRThCNzd3YWNHVTlHeS9xMHpzCi9HVW5kMXZrdjE4bXdML3NhRnI1\nL0NQQkNQUmg2M0NBYzh2cG1vdHA0clkKLS0tIDJZYUwzK0RGNzJWckJJekI0SFg1\nUXE3N05xWThtdjdCZkdJTFJ2YUx6cm8KVANudVL54WBNc9DK9s9h4WQRLMewUqgN\ntu4LdMOmi5oV3LX06lbxhBq79dmsV5uos/qszhJkVGhEZwD1RGBoZA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			}
+		],
+		"lastmodified": "2026-07-28T06:42:29Z",
+		"mac": "ENC[AES256_GCM,data:Xn3/CWhQJHWtd3QHoqpdwLuJdZTp+oxx43k38j4jspQSBPdVDSG6Ifi7Sb2r1g9YTxpjUto8mv7FH+BqI0wSYUkLfjaxI51xs3dq6e6kl5NWOsvWwIOGmBuhfIwusHGgsfBguxk0J6Ev8Irw674CRFc37jozXTVdpOOzFhPTvFs=,iv:/7OFE9NYkw2kYEetzeZRXah0KXuD5OXmfjQMIeqI75Q=,tag:VamuInZqYGYRwlyU8TtL9g==,type:str]",
+		"version": "3.13.3"
+	}
+}

vars/per-machine/proxmox-ha-server-2/openssh/ssh_host_ed25519_key.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINWuFwPKW2jwQEj6c//xb95DbtandPkJ7M0ceyTNd35G ha-server-2

vars/per-machine/proxmox-ha-server-2/openssh/ssh_host_ed25519_key/secret

@@ -0,0 +1,14 @@
+{
+	"data": "ENC[AES256_GCM,data:GEebcQrPPrQxLC6mWXLLtytGkIy6Q2FbcQ/Z+ImGa3gahI/nigdNyR/avIz1DAUsOibpoMVlrk0GQUBMuxiEXulIwUEzeTh7sMI3XTZiugujOFHPyJoFeynlgeRNBbl2EVVjGmEmOsR/cZJGyKaet9aNTHTPDJUhodY5d+D/sE3SgpHoch1DI02/DRnqlnlOYiBoujk++BW71EDHLVyz3n9NsurTDYDK4L0Ch5cMgdlvwIuxGuPKtkhU9z6PtGxPbNlYlNP8U+3jG4XWEEY5hZHBGTVHn+bd5FViGY+sMiigO6yaOmddbUqzJoblKeI6F/rfrL3kXETLgVP7uVm3gw9KPEl5JaTdVqkG4hQ9G9/LvRlyEh619mCp9IRXxabm5SQ4NWFzah1ZwqIKp6GoAtqeWRgZNVqijoF+bnb0YmNFmjwpSrSMZvKsrZb5geyGmqah+NJii0xqVebnufB5p8lW9eytBSNgC/XLjj0olwHMw7CWGmnY6Q7lxF1tNYAD/K3dkbxQYFM3kBnmAdweC8WYc+hDjtk0HJul,iv:H4BonLmf3VoW2S9+IUE+SzPb0qiWHuNuGpVIUgUYScs=,tag:5+bq7vC81DMmOk4jn3u8lg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrQitsaDlqTmlYM2RvR1Ew\nb3NzZTJzR3F2bjM5NHhsa3dCUzZ4TW1pa3pBCkxIeEJtbWdaUFMwMkFyRHpCZ1Bl\nbmM5dGV3b3BDcDEwVlJ4UWV5RFNvRkUKLS0tIFRDd0U3dHBHb0Y3WDhNV3hkd2Ew\nclRDVEZHYjZUSzNVZkFjWGt3SGVQTlUKOULuXiYD9k2uVUmhuC15Kgezrd69rc9P\n7SocPa8kBliffP9IuxxW2S/hPbK8rqEx/sh/Km85ZIi4pG0AUQJ3fA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
+			}
+		],
+		"lastmodified": "2026-07-28T06:45:10Z",
+		"mac": "ENC[AES256_GCM,data:ZnsqhgELqEdNzlTKJx2+xzcZRMNGTme/0a+PVvpbUv5IApimtif0zkTEC1LJsyLPut3MmO3QGprVlb8ihoRdm/IVvLaQonc36f6vN5cxrpQadru/RWEjLbnge1E8VkN/PVvHqRfLyIOPSq6wtGjAPaC/iHW1scBi50pWr0XO628=,iv:FlqAL1It6JuvoBhZotHWdSz17nckBf2rXnwovl6ZJBA=,tag:tNwcaL1sJakXtgejsB/Sbg==,type:str]",
+		"version": "3.13.3"
+	}
+}