fix(ha): resolve data disk device via by-id symlink even in dry-run

The by-id lookup is read-only so it's safe to run in dry-run mode.
Previously it was gated behind `if ! $DRY_RUN`, which always triggered
the sdb fallback warning in dry-run — making it look like the device
path wasn't reliable when the symlink actually exists on both servers.

Now the lookup always runs and the script errors out with a clear message
if the by-id symlink is genuinely missing, instead of silently falling
back to a guessed /dev/sd* name.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

CLAUDE.md

@@ -21,8 +21,7 @@ machines when deployed.
   `modules/installer/common.nix` (the auto-installer's own root/nixos login —
   a deliberate, documented choice, see `docs/auto-installer.md`, not
   accidental tech debt) and **SSH public keys** in `variables.nix`
-  (`vars.adminSshKey`, `vars.remoteBuilderAuthorizedKeys`) plus per-host
-  `KEY` values for beszel-agent auth (see `docs/beszel.md`). Don't use the installer's hardcoded hash as a
+  (`vars.adminSshKey`, `vars.remoteBuilderAuthorizedKeys`, `vars.beszelHubKey`). Don't use the installer's hardcoded hash as a
   template for a *real* host — every other host uses sops-nix
   (`hashedPasswordFile`, see "Security Notes" in `README.md`). Flag any *new*
   secret-like string you encounter instead of committing it.

docs/beszel.md

@@ -14,13 +14,16 @@ Everything is handled by a single module:
 **`modules/beszel/enable-agent.nix`** — imported by a build type. It:
 - Enables `beszel-agent`
 - Sets `HUB_URL` to `docker.sweet.home:8090`
-- Reads the universal `beszel-token` from `secrets/common.yaml` via sops and
-  passes it to the agent as `TOKEN` in an env file
+- Sets `KEY` from `vars.beszelHubKey` (`variables.nix`) — the hub's SSH
+  public key, shared by every agent. Update `beszelHubKey` if the docker
+  host is ever rebuilt and the hub generates a new keypair.
+- Reads the universal `beszel-token` from `secrets/common.yaml` via sops
+  and passes it to the agent as `TOKEN` in an env file
 - Fixes an upstream bug where the agent couldn't persist its hub-pairing
   fingerprint across restarts (adds a real `StateDirectory`)
 
-The only thing a host file ever needs to add is `KEY` — the hub's public key
-for that agent slot, which comes from the beszel hub UI after first pairing.
+A host file needs no beszel configuration at all — just import the module
+in the build type and add the system in the hub UI.
 
 ---
 
@@ -36,30 +39,23 @@ imports = [
 ];
 ```
 
-That's the only build-type change required.
+That's the only change required. The host file needs nothing.
 
 ---
 
-## Wiring the host file
+## Adding a new system to the hub
 
-No `imports` are needed in the host file. Just set `KEY` once you've paired
-the agent with the hub:
-
-```nix
-services.beszel.agent.environment = {
-  KEY = "ssh-ed25519 AAAA...";
-};
-```
-
-Leave `KEY` commented out until after the first pairing (see "Pairing with
-the hub" below).
+1. Rebuild and deploy the host with its build type importing `enable-agent.nix`.
+2. Open the beszel hub (`http://docker.sweet.home:8090`).
+3. Go to **Systems → Add system**, enter the host's IP and the default port
+   (45876). The agent will connect and the system will appear as active.
 
 ---
 
 ## One-time setup: add the token to `secrets/common.yaml`
 
 The universal token is stored once in the common secrets file, shared by all
-agents. You only need to do this once, not per-host:
+agents. Only needed once, not per-host:
 
 ```sh
 sops secrets/common.yaml
@@ -67,26 +63,21 @@ sops secrets/common.yaml
 
 Add:
 ```yaml
-beszel-token: <token from the beszel hub UI>
+beszel-token: <token from the beszel hub Settings → Keys>
 ```
 
-The token is found in the beszel hub under **Settings → Keys** or in the
-"Add system" flow.
-
 `secrets/common.yaml` is already a sops recipient for every host via their
-SSH host keys, so no additional sops recipient setup is needed for hosts that
-are already provisioned.
+SSH host keys, so no additional sops recipient setup is needed.
 
 ---
 
 ## Optional: monitoring extra filesystems
 
 To report disk usage for a mount beyond the root filesystem, add
-`EXTRA_FILESYSTEMS` alongside `KEY` in the host file:
+`EXTRA_FILESYSTEMS` in the host file:
 
 ```nix
 services.beszel.agent.environment = {
-  KEY = "ssh-ed25519 AAAA...";
   EXTRA_FILESYSTEMS = "/mnt/data";   # colon-separated for multiple paths
 };
 ```
@@ -94,7 +85,10 @@ services.beszel.agent.environment = {
 The `server` host uses this to expose its ZFS data pool:
 
 ```nix
-EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
+services.beszel.agent.environment = {
+  EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
+  LOG_LEVEL = "debug";
+};
 ```
 
 ---
@@ -112,44 +106,8 @@ stats. Hosts without Docker should leave it commented out.
 
 ---
 
-## Pairing with the hub
+## If the hub key changes
 
-1. Deploy the host with its build type importing `enable-agent.nix`. Leave
-   `KEY` unset (commented out) for now.
-2. Open the beszel hub (`http://docker.sweet.home:8090`).
-3. Go to **Systems → Add system**. The new host should appear as an unpaired
-   entry — copy the `KEY` value shown there.
-4. In `hosts/<name>/host.nix`, set:
-   ```nix
-   services.beszel.agent.environment.KEY = "<copied key>";
-   ```
-5. Rebuild and deploy the host. The agent will now pair permanently.
-
----
-
-## Example: complete host file
-
-Minimal case (`hosts/tor-relay/host.nix` — one filesystem, no Docker, LXC):
-
-```nix
-{ vars, ... }:
-{
-  networking = { ... };
-
-  services.beszel.agent.environment = {
-    KEY = "ssh-ed25519 AAAA...";
-  };
-
-  system.stateVersion = "26.05";
-}
-```
-
-Fuller case (`hosts/server/host.nix` — extra filesystem, debug logging):
-
-```nix
-services.beszel.agent.environment = {
-  KEY = "ssh-ed25519 AAAA...";
-  EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
-  LOG_LEVEL = "debug";
-};
-```
+If the docker host is ever rebuilt and beszel generates a new SSH keypair,
+update `beszelHubKey` in `variables.nix` and rebuild all beszel-enabled hosts.
+The new key is visible in the beszel hub under **Settings → Keys**.

hosts/ha-server-1/host.nix

@@ -16,8 +16,5 @@
     nameservers = [ vars.domainControllerIp ];
   };
 
-  # Set KEY after pairing this host with the beszel hub (see docs/beszel.md).
-  # services.beszel.agent.environment.KEY = "";
-
   system.stateVersion = "26.05";
 }

hosts/ha-server-2/host.nix

@@ -16,8 +16,5 @@
     nameservers = [ vars.domainControllerIp ];
   };
 
-  # Set KEY after pairing this host with the beszel hub (see docs/beszel.md).
-  # services.beszel.agent.environment.KEY = "";
-
   system.stateVersion = "26.05";
 }

hosts/nix-cache/host.nix

@@ -12,10 +12,6 @@
     nameservers = [ vars.domainControllerIp ];
   };
 
-  services.beszel.agent.environment = {
-    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
-  };
-
   # Preserved from the pre-refactor `nix-cache` target — stateVersion must
   # never be bumped on an already-installed machine.
   system.stateVersion = "25.05";

hosts/server/host.nix

@@ -14,7 +14,6 @@
   };
 
   services.beszel.agent.environment = {
-    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
     EXTRA_FILESYSTEMS = "${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath}";
     LOG_LEVEL = "debug";
   };

hosts/tailscale-router/host.nix

@@ -12,10 +12,6 @@
     nameservers = [ vars.domainControllerIp ];
   };
 
-  services.beszel.agent.environment = {
-    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
-  };
-
   # No networking.hostId: only ZFS-touching hosts (server, docker) need one
   # for pool-import safety, and this host does neither.
 

hosts/tor-relay/host.nix

@@ -15,10 +15,6 @@
   # No networking.hostId: only ZFS-touching hosts need one for pool-import
   # safety, and this host does neither.
 
-  services.beszel.agent.environment = {
-    KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
-  };
-
   # A genuinely new host (not a pre-refactor carry-over), so it tracks the
   # flake's current nixpkgs release rather than being pinned to an older one.
   system.stateVersion = "26.05";

modules/beszel/enable-agent.nix

@@ -16,6 +16,7 @@
     environment = {
       #DOCKER_HOST = "tcp://docker-socket-proxy:2375";
       HUB_URL = "http://${vars.dockerHost}.${vars.homeDomain}:${toString vars.ports.beszelHub}";
+      KEY = vars.beszelHubKey;
     };
   };
 

modules/build-types/ha-server.nix

@@ -30,7 +30,7 @@ in
     ../ha/pacemaker-stack.nix
     ../ha/iscsi-target.nix
     ../ha/cluster-config.nix
-    # ../beszel/enable-agent.nix
+    ../beszel/enable-agent.nix
   ];
 
   # xfsprogs: mkfs.xfs/xfs_info needed by cluster-init.sh.
@@ -45,4 +45,7 @@ in
   # Pacemaker controls nfs-server — prevent systemd from starting it at boot
   # on both nodes (only the Active node should be serving NFS).
   systemd.services.nfs-server.wantedBy = lib.mkForce [ ];
+
+  # Same reason as server.nix: exports use standard auth, not Kerberos.
+  systemd.services.rpc-svcgssd.enable = false;
 }

modules/build-types/server.nix

@@ -102,6 +102,12 @@ in
     requires = [ "zfs-mount.service" ];
   };
 
+  # rpc-svcgssd handles Kerberos/GSS-API for NFS. Not needed: exports use
+  # standard auth, not sec=krb5. On IPA-joined hosts the keytab exists (host/
+  # principal only) but has no nfs/ principal, causing spurious failure.
+  # Mask it so nfs-server's Wants= can't pull it in.
+  systemd.services.rpc-svcgssd.enable = false;
+
   services.nfs.server = {
     enable = true;
     exports = mkNfsExports vars.storageRoot;

scripts/gc-hosts.sh

@@ -0,0 +1,227 @@
+#!/usr/bin/env bash
+# gc-hosts.sh — Run nix-collect-garbage -d on all live NixOS hosts.
+#
+# The host list is rebuilt on every run:
+#   1. This workstation (nixos) — always first
+#   2. pve1 — always second (non-NixOS Proxmox node with Nix installed)
+#   3. Every NixOS guest currently running on pve1 (discovered via pct/qm list)
+#
+# nix-cache is excluded: gc-ing the shared binary cache evicts store paths
+# that other hosts depend on for substitution.
+#
+# NixOS hosts: tries "sudo -n nix-collect-garbage -d" first (works when
+# wheelNeedsPassword = false, e.g. the HA cluster).  Falls back to user-level
+# "nix-collect-garbage -d" if sudo needs a password — still collects
+# unreferenced store paths and old nixos-user profile generations, but leaves
+# old system generations in place.
+# pve1: runs "nix-collect-garbage -d" as the login user (no system generations
+# on a non-NixOS host).
+#
+# Usage (from repo root):
+#   bash scripts/gc-hosts.sh [--dry-run]
+
+set -euo pipefail
+cd "$(dirname "$0")/.."
+source scripts/env.sh 2>/dev/null || true
+source scripts/lib/nix-eval.sh 2>/dev/null || true
+
+# ── config ────────────────────────────────────────────────────────────────────
+
+: "${MAX_JOBS:=8}"
+: "${NIXOS_USER:=nixos}"
+: "${PVE1_SSH_USER:=${PROXMOX_SSH_USER:-wayne}}"
+SSH_OPTS=(-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10)
+
+DRY_RUN=0
+for arg in "$@"; do
+  case "$arg" in
+    --dry-run) DRY_RUN=1 ;;
+    *) echo "Unknown option: $arg" >&2; exit 1 ;;
+  esac
+done
+
+# ── build the host list ───────────────────────────────────────────────────────
+
+# ORDERED_HOSTS: names in display/execution order.
+# HOST_TARGET[name]: SSH target string (user@host).
+# HOST_TYPE[name]: "nixos" (try sudo gc, fallback user) | "nix" (user gc only).
+declare -a  ORDERED_HOSTS=()
+declare -A  HOST_TARGET=()
+declare -A  HOST_TYPE=()
+declare -A  _SEEN_HOSTNAMES=()   # dedup tracker
+
+_add_host() {
+  local name="$1" target="$2" type="$3"
+  if [[ -n "${_SEEN_HOSTNAMES[$name]+_}" ]]; then return; fi
+  _SEEN_HOSTNAMES[$name]=1
+  ORDERED_HOSTS+=("$name")
+  HOST_TARGET[$name]="$target"
+  HOST_TYPE[$name]="$type"
+}
+
+# 1. Workstation (hard-wired first)
+_add_host "nixos" "${NIXOS_USER}@nixos" "nixos"
+
+# 2. pve1 (hard-wired second; non-NixOS, no system generations)
+_add_host "pve1" "${PVE1_SSH_USER}@${PVE1_HOST}" "nix"
+
+# 3. Dynamically discover running NixOS guests on pve1
+echo "Discovering running guests on ${PVE1_HOST}..."
+
+# Evaluate the full flake hostname map in one shot.
+hostname_map="{}"
+if ! hostname_map="$(
+  nix eval --json "${NIX_EVAL_FLAGS[@]}" .#nixosConfigurations \
+    --apply 'cfgs: builtins.mapAttrs (_: cfg: cfg.config.networking.hostName) cfgs' \
+    2>/dev/null
+)"; then
+  echo "  warning: flake eval failed — skipping dynamic host discovery" >&2
+fi
+
+# Get names of all currently running guests from pve1.
+if ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" "true" 2>/dev/null; then
+  running_guests="$(
+    ssh "${SSH_OPTS[@]}" "${PVE1_SSH_USER}@${PVE1_HOST}" bash <<'REMOTE'
+      { sudo pct list 2>/dev/null | awk 'NR>1 && $2=="running" { print $NF }';
+        sudo qm  list 2>/dev/null | awk 'NR>1 && $3=="running" { print $2  }'; } | sort -u
+REMOTE
+  )" || running_guests=""
+
+  while IFS= read -r guest; do
+    [[ -z "$guest" ]] && continue
+
+    # Resolve flake target name → NixOS hostname.
+    hostname="$(printf '%s' "$hostname_map" \
+      | jq -r --arg g "$guest" '.[$g] // empty' 2>/dev/null || true)"
+    [[ -z "$hostname" ]] && continue
+
+    # Exclude nix-cache and any target whose hostname is already in our list.
+    case "$hostname" in nix-cache) continue ;; esac
+    if [[ -n "${_SEEN_HOSTNAMES[$hostname]+_}" ]]; then continue; fi
+
+    echo "  + $guest → $hostname"
+    _add_host "$hostname" "${NIXOS_USER}@${hostname}" "nixos"
+  done <<< "$running_guests"
+else
+  echo "  warning: ${PVE1_HOST} unreachable — skipping dynamic host discovery" >&2
+fi
+
+echo ""
+echo "Hosts: ${ORDERED_HOSTS[*]}"
+echo ""
+
+# ── dry-run ───────────────────────────────────────────────────────────────────
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  echo "[dry-run] commands that would run:"
+  for host in "${ORDERED_HOSTS[@]}"; do
+    target="${HOST_TARGET[$host]}"
+    type="${HOST_TYPE[$host]}"
+    if [[ "$type" == "nixos" ]]; then
+      echo "  ssh ${SSH_OPTS[*]} $target 'sudo -n nix-collect-garbage -d'"
+      echo "  # fallback: ssh ... $target 'nix-collect-garbage -d'"
+    else
+      echo "  ssh ${SSH_OPTS[*]} $target '. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d'"
+    fi
+  done
+  exit 0
+fi
+
+# ── gc worker ─────────────────────────────────────────────────────────────────
+
+gc_one() {
+  local host="$1" target="${HOST_TARGET[$1]}" type="${HOST_TYPE[$1]}" logfile="$2"
+
+  if ! ssh "${SSH_OPTS[@]}" "$target" "true" 2>>"$logfile"; then
+    echo "unreachable"; return
+  fi
+
+  if [[ "$type" == "nixos" ]]; then
+    if ssh "${SSH_OPTS[@]}" "$target" "sudo -n nix-collect-garbage -d" \
+        >>"$logfile" 2>>"$logfile"; then
+      echo "ok(sudo)"; return
+    fi
+    echo "[sudo needs password — falling back to user-level gc]" >>"$logfile"
+    if ssh "${SSH_OPTS[@]}" "$target" "nix-collect-garbage -d" \
+        >>"$logfile" 2>>"$logfile"; then
+      echo "ok(user)"; return
+    fi
+  else
+    # Non-NixOS node: BatchMode SSH doesn't source the Nix daemon profile, so
+    # nix-collect-garbage won't be on PATH unless we source it explicitly.
+    local nix_profile='. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 2>/dev/null || true'
+    if ssh "${SSH_OPTS[@]}" "$target" "$nix_profile && nix-collect-garbage -d" \
+        >>"$logfile" 2>>"$logfile"; then
+      echo "ok"; return
+    fi
+  fi
+
+  echo "failed:$?"
+}
+
+# ── parallel execution ────────────────────────────────────────────────────────
+
+echo "Running gc on ${#ORDERED_HOSTS[@]} hosts (up to ${MAX_JOBS} parallel)..."
+echo ""
+
+TMPDIR_GC="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR_GC"' EXIT
+
+declare -A LOGS=()
+job_count=0
+
+for host in "${ORDERED_HOSTS[@]}"; do
+  logfile="${TMPDIR_GC}/${host}.log"
+  resultfile="${TMPDIR_GC}/${host}.result"
+  LOGS[$host]="$logfile"
+  : > "$logfile"
+
+  ( result="$(gc_one "$host" "$logfile")"; echo "$result" > "$resultfile" ) &
+
+  (( job_count++ )) || true
+  if [[ "$job_count" -ge "$MAX_JOBS" ]]; then
+    wait -n 2>/dev/null || wait
+    (( job_count-- )) || true
+  fi
+done
+
+wait
+
+# ── summary ───────────────────────────────────────────────────────────────────
+
+echo "Results:"
+echo "──────────────────────────────"
+
+ok_hosts=()
+warn_hosts=()
+fail_hosts=()
+
+for host in "${ORDERED_HOSTS[@]}"; do
+  result="$(cat "${TMPDIR_GC}/${host}.result" 2>/dev/null || echo "failed:missing")"
+  case "$result" in
+    ok|"ok(sudo)"|"ok(user)")
+      printf "  %-22s  %s\n" "$host" "$result"
+      ok_hosts+=("$host") ;;
+    unreachable)
+      printf "  %-22s  UNREACHABLE\n" "$host"
+      warn_hosts+=("$host") ;;
+    *)
+      printf "  %-22s  FAILED (%s)\n" "$host" "$result"
+      fail_hosts+=("$host") ;;
+  esac
+done
+
+echo ""
+echo "  ${#ok_hosts[@]} succeeded, ${#warn_hosts[@]} unreachable, ${#fail_hosts[@]} failed"
+
+for host in "${warn_hosts[@]+"${warn_hosts[@]}"}" "${fail_hosts[@]+"${fail_hosts[@]}"}"; do
+  logfile="${LOGS[$host]}"
+  if [[ -s "$logfile" ]]; then
+    echo ""
+    echo "── $host ──"
+    cat "$logfile"
+  fi
+done
+
+echo ""
+[[ "${#fail_hosts[@]}" -eq 0 ]]

scripts/ha/resize-data-disk.sh

@@ -30,6 +30,8 @@ DATA_DISK_SLOT="${DATA_DISK_SLOT:-scsi1}"   # Proxmox disk name (scsi1 = data di
 HA_USER="${HA_USER:-nixos}"
 PVE_HOST="${PVE_HOST:-${PVE1_HOST}}"
 PVE_SSH_USER="${PVE_SSH_USER:-${PROXMOX_SSH_USER}}"
+PVE_SUDO=""
+[[ "$PVE_SSH_USER" != "root" ]] && PVE_SUDO="sudo"
 # By-id symlink for the data disk; basename resolves to the raw block device.
 # matches variables.nix's haServerDrbdDisk.
 DATA_DISK_BYID="${DATA_DISK_BYID:-scsi-0QEMU_QEMU_HARDDISK_drive-${DATA_DISK_SLOT}}"
@@ -140,7 +142,7 @@ fi
 echo ""
 echo "Looking up VM IDs on ${PVE_HOST}..."
 
-QM_LIST=$(pve "qm list 2>/dev/null" || true)
+QM_LIST=$(pve "$PVE_SUDO qm list 2>/dev/null" || true)
 VMID1=$(echo "$QM_LIST" | awk -v name="$NODE1" '$0 ~ name {print $1}' | head -1)
 VMID2=$(echo "$QM_LIST" | awk -v name="$NODE2" '$0 ~ name {print $1}' | head -1)
 
@@ -182,12 +184,12 @@ echo "── Phase 1 — Proxmox disk resize (${DATA_DISK_SLOT} ${SIZE} on both
 
 echo "  ${DRY_PREFIX}qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}  ($NODE1 on ${PVE_HOST})"
 if ! $DRY_RUN; then
-  pve "qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}"
+  pve "$PVE_SUDO qm resize $VMID1 ${DATA_DISK_SLOT} ${SIZE}"
 fi
 
 echo "  ${DRY_PREFIX}qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}  ($NODE2 on ${PVE_HOST})"
 if ! $DRY_RUN; then
-  pve "qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}"
+  pve "$PVE_SUDO qm resize $VMID2 ${DATA_DISK_SLOT} ${SIZE}"
 fi
 
 echo "  Phase 1 done."
@@ -202,17 +204,13 @@ rescan_node() {
   local node_name=$1 run_fn=$2
 
   # Resolve block device name from the stable by-id symlink on the guest.
+  # Read-only lookup — safe to run even in dry-run so we show the real device.
   local blk_dev=""
-  if ! $DRY_RUN; then
-    blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
-  fi
+  blk_dev=$($run_fn "bash -c 'basename \$(readlink -f /dev/disk/by-id/${DATA_DISK_BYID})'" 2>/dev/null || true)
   if [[ -z "$blk_dev" ]]; then
-    # Fall back: scsi1 → index 1 → sdb, scsi2 → sdc, etc.
-    local slot_idx
-    slot_idx=$(echo "$DATA_DISK_SLOT" | grep -oE '[0-9]+$' || echo "1")
-    blk_dev=$(printf "sd%s" "$(echo "abcdefghij" | cut -c$((slot_idx + 1)))")
-    echo "  WARNING: could not resolve /dev/disk/by-id/${DATA_DISK_BYID} on $node_name;" \
-         "falling back to /dev/${blk_dev}" >&2
+    echo "  ERROR: /dev/disk/by-id/${DATA_DISK_BYID} not found on $node_name" >&2
+    echo "         Check DATA_DISK_BYID or DATA_DISK_SLOT configuration." >&2
+    exit 1
   fi
 
   echo "  ${DRY_PREFIX}Rescanning /dev/${blk_dev} on ${node_name}..."

variables.nix

@@ -40,6 +40,11 @@
   # from scratch.
   nixCacheHostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICuHUxGNH6ei3BZD+EfZs3l4X8uJNcjQiOsM/G4yo4O/ lxc-nix-cache";
 
+  # Beszel hub's SSH public key — used by every agent to authenticate the
+  # hub's incoming connection. Update if the docker host is ever rebuilt and
+  # the hub generates a new keypair.
+  beszelHubKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
+
   # Public keys authorized to SSH in as remoteBuilderUser on the nix-cache
   # host (modules/nix-cache/server.nix) — one per client host that's allowed
   # to use it as a distributed builder.
@@ -53,6 +58,7 @@
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZ9WKKAlP9Z7GQdgaZ1Xgw9C+vja2lqEZO5rJFpVqYN root@ha-server-1"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEGNKlaaMckd8nLWNGz4B2QokXjnnIvM+rEUv+R6h0sp root@ha-server-2"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK+XeMco7OxUpjrjZm54HogMs9QB5xlcKmElASRvrmlW root@nixos"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMseQwPpmaa6cgV5U8KhUsiVSYARG85zGa9rho0LJWks wayne@pve1"
   ];
 
   # Admin SSH public key, authorized on the primary user of every host and