From 5856d455753d975079ab44cd9f3b4ac856d55fe7 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 27 Jul 2026 11:45:36 +1000 Subject: [PATCH] feat(ha): wire sops secrets and disable NetworkManager for HA servers - cluster-config.nix: add corosync_authkey sops binary secret (/etc/corosync/authkey, mode 0400) and force-disable NetworkManager (common config enables it; HA nodes need stable static IP networking) - hosts/ha-server-{1,2}/host.nix: add host-token.nix import for sops-managed beszel-token; add KEY placeholder for beszel hub pairing - .sops.yaml: add creation rules for secrets/ha-server-{1,2}.yaml and secrets/ha-corosync-authkey (admin-only until sync-host-keys.sh runs) - secrets/ha-server-{1,2}.yaml, secrets/ha-corosync-authkey: stub files so eval passes before real secrets are provisioned Bootstrap order (post-merge): 1. bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1 2. bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2 3. sops updatekeys secrets/common.yaml (grants HA nodes common secrets) 4. sops secrets/ha-server-{1,2}.yaml (set beszel-token values) 5. On node1: corosync-keygen; sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey; git add/commit Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8 --- .sops.yaml | 26 ++++++++++++++++++++++++++ hosts/ha-server-1/host.nix | 16 +++++++++------- hosts/ha-server-2/host.nix | 16 +++++++++------- modules/ha/cluster-config.nix | 27 +++++++++++++++++++-------- secrets/ha-corosync-authkey | 1 + secrets/ha-server-1.yaml | 6 ++++++ secrets/ha-server-2.yaml | 6 ++++++ 7 files changed, 76 insertions(+), 22 deletions(-) create mode 100644 secrets/ha-corosync-authkey create mode 100644 secrets/ha-server-1.yaml create mode 100644 secrets/ha-server-2.yaml diff --git a/.sops.yaml b/.sops.yaml index b37b4e2..a14898b 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -85,6 +85,32 @@ creation_rules: - *lxc-tailscale-router - *proxmox-tailscale-router + # HA file server per-node secrets (beszel-token). + # proxmox-ha-server-1 / proxmox-ha-server-2 keys are added automatically + # by scripts/secrets/sync-host-keys.sh once the hosts are provisioned; + # until then only the admin key can decrypt these files. + - path_regex: secrets/ha-server-1\.yaml$ + key_groups: + - age: + - *admin + # proxmox-ha-server-1 added by sync-host-keys.sh + + - path_regex: secrets/ha-server-2\.yaml$ + key_groups: + - age: + - *admin + # proxmox-ha-server-2 added by sync-host-keys.sh + + # Shared HA cluster corosync authkey (binary sops file). + # Encrypted for both HA nodes so either can decrypt on boot. + # Both host keys added by sync-host-keys.sh; admin key allows initial creation. + - path_regex: secrets/ha-corosync-authkey$ + key_groups: + - age: + - *admin + # proxmox-ha-server-1 added by sync-host-keys.sh + # proxmox-ha-server-2 added by sync-host-keys.sh + # gui-host-specific secrets (currently: wifi-password, see # modules/networking/wifi.nix). Only *lxc-gui has a registered key today # -- proxmox-gui/linode-gui/baremetal-gui haven't been provisioned via diff --git a/hosts/ha-server-1/host.nix b/hosts/ha-server-1/host.nix index 0fd808b..c380008 100644 --- a/hosts/ha-server-1/host.nix +++ b/hosts/ha-server-1/host.nix @@ -1,5 +1,12 @@ { vars, ... }: { + imports = [ + (import ../../modules/beszel/host-token.nix { + name = "ha-server-1"; + sopsFile = ../../secrets/ha-server-1.yaml; + }) + ]; + networking = { hostName = vars.haServer1Host; hostId = "3a4b5c6d"; @@ -12,13 +19,8 @@ nameservers = [ "192.168.2.1" "8.8.8.8" ]; }; - # TODO: after running `bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1` - # add beszel agent pairing and sops-managed corosync authkey: - # imports = [ (import ../../modules/beszel/host-token.nix { - # name = "ha-server-1"; - # sopsFile = ../../secrets/ha-server-1.yaml; - # }) ]; - # services.beszel.agent.environment.KEY = "..."; + # Set KEY after pairing this host with the beszel hub; the token is sops-managed. + services.beszel.agent.environment.KEY = ""; system.stateVersion = "26.05"; } diff --git a/hosts/ha-server-2/host.nix b/hosts/ha-server-2/host.nix index 1f5a8ba..dcb1464 100644 --- a/hosts/ha-server-2/host.nix +++ b/hosts/ha-server-2/host.nix @@ -1,5 +1,12 @@ { vars, ... }: { + imports = [ + (import ../../modules/beszel/host-token.nix { + name = "ha-server-2"; + sopsFile = ../../secrets/ha-server-2.yaml; + }) + ]; + networking = { hostName = vars.haServer2Host; hostId = "7e8f9a0b"; @@ -12,13 +19,8 @@ nameservers = [ "192.168.2.1" "8.8.8.8" ]; }; - # TODO: after running `bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2` - # add beszel agent pairing and sops-managed corosync authkey: - # imports = [ (import ../../modules/beszel/host-token.nix { - # name = "ha-server-2"; - # sopsFile = ../../secrets/ha-server-2.yaml; - # }) ]; - # services.beszel.agent.environment.KEY = "..."; + # Set KEY after pairing this host with the beszel hub; the token is sops-managed. + services.beszel.agent.environment.KEY = ""; system.stateVersion = "26.05"; } diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix index 9074e99..160c154 100644 --- a/modules/ha/cluster-config.nix +++ b/modules/ha/cluster-config.nix @@ -5,13 +5,10 @@ # (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix. # # Corosync authkey: -# /etc/corosync/authkey must be present (mode 0400) for corosync to start. -# It is NOT managed declaratively here — the initial deploy uses -# scripts/ha/cluster-init.sh to generate it via corosync-keygen and -# distribute it to both nodes. -# TODO: once both hosts have their sops keys registered via -# scripts/secrets/sync-host-keys.sh, add a sops secret here so the -# authkey survives nixos-rebuild. +# /etc/corosync/authkey (mode 0400) is managed by sops-nix below. +# Bootstrap: run scripts/ha/cluster-init.sh on node1 to generate the key, +# then encrypt it with: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey +# Both host keys must be registered via sync-host-keys.sh first so both nodes can decrypt it. # # DRBD fencing: # Production setting is resource-only: DRBD waits for the STONITH fence @@ -20,7 +17,7 @@ # (see scripts/ha/cluster-enable-stonith.sh). On a fresh cluster with # no fence device yet, temporarily change to dont-care and run # cluster-enable-stonith.sh once the fence key is deployed. -{ vars, ... }: +{ lib, vars, ... }: { services.drbd = { enable = true; @@ -60,6 +57,20 @@ ''; }; + # /etc/corosync/authkey — sops binary secret, identical on both nodes. + # Decryptable by both ha-server host keys (added by sync-host-keys.sh). + sops.secrets.corosync_authkey = { + sopsFile = ../../secrets/ha-corosync-authkey; + format = "binary"; + path = "/etc/corosync/authkey"; + mode = "0400"; + restartUnits = [ "corosync.service" ]; + }; + + # NixOS common config enables NetworkManager by default; HA cluster nodes + # need stable static IPs with predictable interface names — NM is not suitable. + networking.networkmanager.enable = lib.mkForce false; + # services.corosync.enable is set by modules/ha/pacemaker-stack.nix. services.corosync = { clusterName = "ha-cluster"; diff --git a/secrets/ha-corosync-authkey b/secrets/ha-corosync-authkey new file mode 100644 index 0000000..74db83c --- /dev/null +++ b/secrets/ha-corosync-authkey @@ -0,0 +1 @@ +STUB: run cluster-init.sh to generate, then: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey diff --git a/secrets/ha-server-1.yaml b/secrets/ha-server-1.yaml new file mode 100644 index 0000000..f33120f --- /dev/null +++ b/secrets/ha-server-1.yaml @@ -0,0 +1,6 @@ +# STUB — not yet encrypted with sops. +# Bootstrap: +# bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1 +# sops updatekeys secrets/common.yaml (allows ha-server-1 to decrypt shared secrets) +# sops secrets/ha-server-1.yaml (create with: beszel-token) +beszel-token: REPLACE diff --git a/secrets/ha-server-2.yaml b/secrets/ha-server-2.yaml new file mode 100644 index 0000000..19bbbe9 --- /dev/null +++ b/secrets/ha-server-2.yaml @@ -0,0 +1,6 @@ +# STUB — not yet encrypted with sops. +# Bootstrap: +# bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2 +# sops updatekeys secrets/common.yaml (allows ha-server-2 to decrypt shared secrets) +# sops secrets/ha-server-2.yaml (create with: beszel-token) +beszel-token: REPLACE