Merge pull request 'refactor(beszel): switch to universal token via secrets/common.yaml' (#97) from worktree-docs-beszel-guide into main

Reviewed-on: #97

CLAUDE.md

@@ -21,7 +21,8 @@ machines when deployed.
   `modules/installer/common.nix` (the auto-installer's own root/nixos login —
   a deliberate, documented choice, see `docs/auto-installer.md`, not
   accidental tech debt) and **SSH public keys** in `variables.nix`
-  (`vars.adminSshKey`, `vars.remoteBuilderAuthorizedKeys`, `vars.beszelHubKey`). Don't use the installer's hardcoded hash as a
+  (`vars.adminSshKey`, `vars.remoteBuilderAuthorizedKeys`) plus per-host
+  `KEY` values for beszel-agent auth (see `docs/beszel.md`). Don't use the installer's hardcoded hash as a
   template for a *real* host — every other host uses sops-nix
   (`hashedPasswordFile`, see "Security Notes" in `README.md`). Flag any *new*
   secret-like string you encounter instead of committing it.

docs/beszel.md

@@ -14,16 +14,13 @@ Everything is handled by a single module:
 **`modules/beszel/enable-agent.nix`** — imported by a build type. It:
 - Enables `beszel-agent`
 - Sets `HUB_URL` to `docker.sweet.home:8090`
-- Sets `KEY` from `vars.beszelHubKey` (`variables.nix`) — the hub's SSH
+- Reads the universal `beszel-token` from `secrets/common.yaml` via sops and
-  public key, shared by every agent. Update `beszelHubKey` if the docker
+  passes it to the agent as `TOKEN` in an env file
-  host is ever rebuilt and the hub generates a new keypair.
-- Reads the universal `beszel-token` from `secrets/common.yaml` via sops
-  and passes it to the agent as `TOKEN` in an env file
 - Fixes an upstream bug where the agent couldn't persist its hub-pairing
   fingerprint across restarts (adds a real `StateDirectory`)
 
-A host file needs no beszel configuration at all — just import the module
+The only thing a host file ever needs to add is `KEY` — the hub's public key
-in the build type and add the system in the hub UI.
+for that agent slot, which comes from the beszel hub UI after first pairing.
 
 ---
 
@@ -39,23 +36,30 @@ imports = [
 ];
 ```
 
-That's the only change required. The host file needs nothing.
+That's the only build-type change required.
 
 ---
 
-## Adding a new system to the hub
+## Wiring the host file
 
-1. Rebuild and deploy the host with its build type importing `enable-agent.nix`.
+No `imports` are needed in the host file. Just set `KEY` once you've paired
-2. Open the beszel hub (`http://docker.sweet.home:8090`).
+the agent with the hub:
-3. Go to **Systems → Add system**, enter the host's IP and the default port
+
-   (45876). The agent will connect and the system will appear as active.
+```nix
+services.beszel.agent.environment = {
+  KEY = "ssh-ed25519 AAAA...";
+};
+```
+
+Leave `KEY` commented out until after the first pairing (see "Pairing with
+the hub" below).
 
 ---
 
 ## One-time setup: add the token to `secrets/common.yaml`
 
 The universal token is stored once in the common secrets file, shared by all
-agents. Only needed once, not per-host:
+agents. You only need to do this once, not per-host:
 
 ```sh
 sops secrets/common.yaml
@@ -63,21 +67,26 @@ sops secrets/common.yaml
 
 Add:
 ```yaml
-beszel-token: <token from the beszel hub Settings → Keys>
+beszel-token: <token from the beszel hub UI>
 ```
 
+The token is found in the beszel hub under **Settings → Keys** or in the
+"Add system" flow.
+
 `secrets/common.yaml` is already a sops recipient for every host via their
-SSH host keys, so no additional sops recipient setup is needed.
+SSH host keys, so no additional sops recipient setup is needed for hosts that
+are already provisioned.
 
 ---
 
 ## Optional: monitoring extra filesystems
 
 To report disk usage for a mount beyond the root filesystem, add
-`EXTRA_FILESYSTEMS` in the host file:
+`EXTRA_FILESYSTEMS` alongside `KEY` in the host file:
 
 ```nix
 services.beszel.agent.environment = {
+  KEY = "ssh-ed25519 AAAA...";
   EXTRA_FILESYSTEMS = "/mnt/data";   # colon-separated for multiple paths
 };
 ```
@@ -85,10 +94,7 @@ services.beszel.agent.environment = {
 The `server` host uses this to expose its ZFS data pool:
 
 ```nix
-services.beszel.agent.environment = {
+EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
-  EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
-  LOG_LEVEL = "debug";
-};
 ```
 
 ---
@@ -106,8 +112,44 @@ stats. Hosts without Docker should leave it commented out.
 
 ---
 
-## If the hub key changes
+## Pairing with the hub
 
-If the docker host is ever rebuilt and beszel generates a new SSH keypair,
+1. Deploy the host with its build type importing `enable-agent.nix`. Leave
-update `beszelHubKey` in `variables.nix` and rebuild all beszel-enabled hosts.
+   `KEY` unset (commented out) for now.
-The new key is visible in the beszel hub under **Settings → Keys**.
+2. Open the beszel hub (`http://docker.sweet.home:8090`).
+3. Go to **Systems → Add system**. The new host should appear as an unpaired
+   entry — copy the `KEY` value shown there.
+4. In `hosts/<name>/host.nix`, set:
+   ```nix
+   services.beszel.agent.environment.KEY = "<copied key>";
+   ```
+5. Rebuild and deploy the host. The agent will now pair permanently.
+
+---
+
+## Example: complete host file
+
+Minimal case (`hosts/tor-relay/host.nix` — one filesystem, no Docker, LXC):
+
+```nix
+{ vars, ... }:
+{
+  networking = { ... };
+
+  services.beszel.agent.environment = {
+    KEY = "ssh-ed25519 AAAA...";
+  };
+
+  system.stateVersion = "26.05";
+}
+```
+
+Fuller case (`hosts/server/host.nix` — extra filesystem, debug logging):
+
+```nix
+services.beszel.agent.environment = {
+  KEY = "ssh-ed25519 AAAA...";
+  EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
+  LOG_LEVEL = "debug";
+};
+```

hosts/ha-server-1/host.nix

@@ -16,5 +16,8 @@
     nameservers = [ vars.domainControllerIp ];
   };
 
+  # Set KEY after pairing this host with the beszel hub (see docs/beszel.md).
+  # services.beszel.agent.environment.KEY = "";
+
   system.stateVersion = "26.05";
 }

hosts/ha-server-2/host.nix

@@ -16,5 +16,8 @@
     nameservers = [ vars.domainControllerIp ];
   };
 
+  # Set KEY after pairing this host with the beszel hub (see docs/beszel.md).
+  # services.beszel.agent.environment.KEY = "";
+
   system.stateVersion = "26.05";
 }

hosts/nix-cache/host.nix

@@ -12,6 +12,10 @@
     nameservers = [ vars.domainControllerIp ];
   };
 
+  services.beszel.agent.environment = {
+    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
+  };
+
   # Preserved from the pre-refactor `nix-cache` target — stateVersion must
   # never be bumped on an already-installed machine.
   system.stateVersion = "25.05";

hosts/server/host.nix

@@ -14,6 +14,7 @@
   };
 
   services.beszel.agent.environment = {
+    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
     EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
     LOG_LEVEL = "debug";
   };

hosts/tailscale-router/host.nix

@@ -12,6 +12,10 @@
     nameservers = [ vars.domainControllerIp ];
   };
 
+  services.beszel.agent.environment = {
+    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
+  };
+
   # No networking.hostId: only ZFS-touching hosts (server, docker) need one
   # for pool-import safety, and this host does neither.
 

hosts/tor-relay/host.nix

@@ -15,6 +15,10 @@
   # No networking.hostId: only ZFS-touching hosts need one for pool-import
   # safety, and this host does neither.
 
+  services.beszel.agent.environment = {
+    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
+  };
+
   # A genuinely new host (not a pre-refactor carry-over), so it tracks the
   # flake's current nixpkgs release rather than being pinned to an older one.
   system.stateVersion = "26.05";

modules/beszel/enable-agent.nix

@@ -16,7 +16,6 @@
     environment = {
       #DOCKER_HOST = "tcp://docker-socket-proxy:2375";
       HUB_URL = "http://${vars.dockerHost}.${vars.homeDomain}:${toString vars.ports.beszelHub}";
-      KEY = vars.beszelHubKey;
     };
   };
 

modules/build-types/ha-server.nix

@@ -45,7 +45,4 @@ in
   # Pacemaker controls nfs-server — prevent systemd from starting it at boot
   # on both nodes (only the Active node should be serving NFS).
   systemd.services.nfs-server.wantedBy = lib.mkForce [ ];
-
-  # Same reason as server.nix: exports use standard auth, not Kerberos.
-  systemd.services.rpc-svcgssd.enable = false;
 }

modules/build-types/server.nix

@@ -102,12 +102,6 @@ in
     requires = [ "zfs-mount.service" ];
   };
 
-  # rpc-svcgssd handles Kerberos/GSS-API for NFS. Not needed: exports use
-  # standard auth, not sec=krb5. On IPA-joined hosts the keytab exists (host/
-  # principal only) but has no nfs/ principal, causing spurious failure.
-  # Mask it so nfs-server's Wants= can't pull it in.
-  systemd.services.rpc-svcgssd.enable = false;
-
   services.nfs.server = {
     enable = true;
     exports = mkNfsExports vars.storageRoot;

variables.nix

@@ -40,24 +40,19 @@
   # from scratch.
   nixCacheHostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICuHUxGNH6ei3BZD+EfZs3l4X8uJNcjQiOsM/G4yo4O/ lxc-nix-cache";
 
-  # Beszel hub's SSH public key — used by every agent to authenticate the
-  # hub's incoming connection. Update if the docker host is ever rebuilt and
-  # the hub generates a new keypair.
-  beszelHubKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
-
   # Public keys authorized to SSH in as remoteBuilderUser on the nix-cache
   # host (modules/nix-cache/server.nix) — one per client host that's allowed
   # to use it as a distributed builder.
   remoteBuilderAuthorizedKeys = [
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK+ioWPhHixlgCB9KIQ0QTHTz6A+Oo2F3uKiINLip5rO root@docker"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface"
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJEj26SL/emsVjW2YhRucJVp2kTz8WgcEQgjBEBLRikk root@claude"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII/rLceRhnDobVXQYiPceuhDHHvVjFQ1pc9A6un/eUlA root@server"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHj11bLPpRzH2oslnwFzEvY9cSgfEFtSZbLQaDm4nZMK root@pxe-boot"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH4/Sesm8NYpj73R0cbGhI0Ubvz73vIVWAnbEDTlBTdh root@tor-relay"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIBCFUUtePndW7pqtlawft1QCdHmBVs3O/c8EJO+RcXV root@tailscale-router"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZ9WKKAlP9Z7GQdgaZ1Xgw9C+vja2lqEZO5rJFpVqYN root@ha-server-1"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEGNKlaaMckd8nLWNGz4B2QokXjnnIvM+rEUv+R6h0sp root@ha-server-2"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK+XeMco7OxUpjrjZm54HogMs9QB5xlcKmElASRvrmlW root@nixos"
   ];
 
   # Admin SSH public key, authorized on the primary user of every host and