Archived
beatzaplentyandClaude Sonnet 4.6
e3498b1087
Adds proxmox-ha-server-1 and proxmox-ha-server-2 as real mkTarget entries
alongside the existing proxmox-server, backed by a new ha-server build type.
New modules
modules/ha/cluster-config.nix — DRBD resource + corosync nodelist sourced
from vars (haServer1Host/Ip, haServer2Host/Ip); resource-only fencing for
production STONITH; HA port firewall rules for DRBD, iSCSI, Corosync, pcsd
modules/build-types/ha-server.nix — imports pacemaker-stack + iscsi-target
+ cluster-config + beszel; NFS exports from vars.haStorageRoot (XFS-over-DRBD
mount); nfs-server.service.wantedBy force-cleared so Pacemaker controls
start/stop on the Active node only
New hosts
hosts/ha-server-{1,2}/host.nix — static IP from vars, unique hostId; sops
secrets (beszel, corosync authkey) are TODOs pending sync-host-keys.sh
variables.nix
haServer1/2Host, haServer1/2Ip, haServerVip, haStorageRoot, haIscsiIqn
ports.haServerDrbd/Iscsi/Corosync{1,2,Crypto}/PacemakerRemoted/Pcsd
scripts/ha/ (migrated + updated from test-lab/ha/)
cluster-init.sh — generates corosync authkey, initialises DRBD/XFS/iSCSI,
creates NFS dataset dirs, configures Pacemaker with DRBD + XFS + iSCSI
+ nfs-server + VIP; STONITH disabled initially (enable separately)
cluster-enable-stonith.sh — enables fence_pve_ssh STONITH after key deploy
fence-pve-ssh.py — Proxmox SSH fence agent (node names updated to ha-server-1/2)
acceptance-tests.sh — T1–T7 production acceptance tests
test-lab/ha/ removed — all Nix config moved to modules/ha/ and
modules/build-types/; scripts moved to scripts/ha/
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
225 lines
10 KiB
Nix
225 lines
10 KiB
Nix
{
|
|
# Network / domains
|
|
lanDomain = "gitea.lan.ddnsgeek.com"; # Gitea/DDNS domain
|
|
homeDomain = "sweet.home"; # base LAN domain for service subdomains (pve., docker.)
|
|
tailnetDomain = "tail13f623.ts.net"; # Tailscale MagicDNS suffix
|
|
lanCidr = "192.168.2.0/24"; # LAN subnet
|
|
pxeServerIp = "192.168.2.247"; # pxe-boot host's LAN IP
|
|
pbsIp = "192.168.2.108"; # Proxmox Backup Server LAN IP
|
|
|
|
# Cross-host references (LAN hostnames/users other hosts reach over the network)
|
|
nixCacheHost = "nix-cache"; # substituter/remote-builder hostname
|
|
nfsServerHost = "server"; # NFS export source hostname
|
|
dockerHost = "docker"; # docker-compose stack host
|
|
|
|
# Raspberry Pi's own Tailscale hostname (not fronted by `server` — it
|
|
# exports its own NFS share directly). Resolved as
|
|
# "${raspberryPiHost}.${tailnetDomain}" in modules/raspi/mount-data.nix.
|
|
raspberryPiHost = "raspberrypi";
|
|
|
|
remoteBuilderUser = "nixremote"; # remote builder SSH user
|
|
|
|
# nix-cache's own SSH host public key (not a secret — the private half
|
|
# never leaves the host). Wired into every client's
|
|
# programs.ssh.knownHosts by modules/nix-cache/remote-builder-client.nix
|
|
# so distributed builds don't hit "Host key verification failed" on a
|
|
# fresh client that has never manually ssh'd to nix-cache before. Update
|
|
# this if nix-cache's host key is ever rotated or the host is rebuilt
|
|
# from scratch.
|
|
nixCacheHostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICuHUxGNH6ei3BZD+EfZs3l4X8uJNcjQiOsM/G4yo4O/ lxc-nix-cache";
|
|
|
|
# Public keys authorized to SSH in as remoteBuilderUser on the nix-cache
|
|
# host (modules/nix-cache/server.nix) — one per client host that's allowed
|
|
# to use it as a distributed builder.
|
|
remoteBuilderAuthorizedKeys = [
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFDEA1S2ikpObREgbP5uVBWMxIOGbY8B+Wx7VTZK1m6t root@server"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPAYIT9ormlmxZ0SziyDQaUntnKI8HK9/s3Qac1ZKjP2 root@docker"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKKKzoEPl/ZW9KBRHBcp6/ThOngGpwMv5EhkTlgC4aDf root@nixos"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIGtOWOCS+ImHc7NehguoyD7PbonGosKMZqc9+QR3v/h root@nixos"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHxXTQxFnArK5HXG7czeoybZebCGfxpUdusJkPn+BCSp root@server"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJEj26SL/emsVjW2YhRucJVp2kTz8WgcEQgjBEBLRikk root@claude"
|
|
];
|
|
|
|
# Admin SSH public key, authorized on the primary user of every host and
|
|
# the installer image's nixos/root users.
|
|
adminSshKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCq/Q5LvIXlZwO2kdeAN5nLGZ59nZB7JHYMEszHxmNtGMzv1lM31jiPNsr0z2EKVZhE7OOfa2IF9rhWYD7JUA9G0yzdZ4WTXFNGVVOJoOVH6vAF3XCxoVilOEwTc7h2Wiy+rzd0B28/3spffzQQWJhY6GRQVa8j+6xAGF60Fcvl1vLosYT9Bn2ZbK4TCWOwAn2jqXIieGpZdn/UNZbGOeKRiCvhktDfMAzuQzN/9jMu/oF4pkPn2X1UrsQdNlvp0Ci8md612MozIpncQJyAF1ADhunr3sMx0isUXiqD29R5DS4TftpekqLNLak+zcxFa8N7DcRNp3DcKfJvyTkwQrR4r+b7lFLYOLHLagSso9CzeW/paAS2q9I5SBm/2DtE1diLLg2jZikYcstsu/G5RgvbzbKqjiaMwTdXC3AMvDxQrs7U5pDRZFzoofG3cpODbTm+uy3m0kP70z0M1K45UbDG0p+itnTu9x40JbQEgefbx38AItNvAIx1A8HO4I1VX28= wayne@stream";
|
|
|
|
# Prestaged wifi SSID for the gui host's NetworkManager profile
|
|
# (modules/networking/wifi.nix). The password is not here -- it's
|
|
# sops-encrypted in secrets/gui.yaml (wifi-password) instead, since this
|
|
# file isn't a secret store.
|
|
wifiSsid = "nbn-fttp-net-5G";
|
|
|
|
# Bare-metal gui host's two disks for a ZFS RAID0 (striped) root pool
|
|
# (modules/disko/baremetal.nix). Only used transiently at disko-format
|
|
# time (partitioning); the resulting fileSystems/zpool import reference
|
|
# by-partlabel/by-id paths afterward regardless, same as
|
|
# modules/disko/proxmox.nix's own plain "/dev/sda".
|
|
guiRootDisk1 = "/dev/sda";
|
|
guiRootDisk2 = "/dev/sdb";
|
|
|
|
# System
|
|
timeZone = "Australia/Brisbane";
|
|
|
|
# Main interactive user on every host. Every module that grants this user
|
|
# a group, a home directory, or tmpfiles ownership should reference
|
|
# vars.primaryUser rather than the literal "nixos", so renaming it is a
|
|
# one-line change.
|
|
primaryUser = "nixos";
|
|
|
|
# HA file server cluster
|
|
# haServer1Ip / haServer2Ip: static LAN IPs for both HA nodes (must be
|
|
# fixed — DRBD and corosync ring addresses are baked into the NixOS config).
|
|
# haServerVip: floating virtual IP managed by Pacemaker's IPaddr2 resource;
|
|
# NFS and iSCSI clients connect here regardless of which node is Active.
|
|
# Set all three to real values in variables.nix before deploying.
|
|
haServer1Host = "ha-server-1";
|
|
haServer2Host = "ha-server-2";
|
|
haServer1Ip = "192.168.2.200"; # TODO: confirm production IP
|
|
haServer2Ip = "192.168.2.201"; # TODO: confirm production IP
|
|
haServerVip = "192.168.2.202"; # TODO: confirm floating VIP
|
|
haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node
|
|
haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage";
|
|
|
|
# Storage
|
|
storageRoot = "/tank"; # ZFS pool root on `server`
|
|
|
|
# NFS datasets exported from `storageRoot` on `nfsServerHost` and mounted
|
|
# by client hosts. `subpath` is relative to `storageRoot` — combined with
|
|
# it to build both the export line in modules/build-types/server.nix and
|
|
# the "<nfsServerHost>:<storageRoot>/<subpath>" device string each client
|
|
# mount uses in modules/docker/mount-data.nix. `mountpoint` is the
|
|
# absolute local path clients mount it at, referenced by that same file's
|
|
# fileSystems attribute name plus every other place that needs to know
|
|
# where the share lives locally (modules/build-types/docker.nix's
|
|
# tmpfiles rules, modules/traefik/rotate-logs.nix's log path). Renaming a
|
|
# dataset or moving where it's mounted only needs changing it here — the
|
|
# export and every client reference follow automatically.
|
|
nfsShares = {
|
|
options = "(rw,sync,no_subtree_check,no_root_squash)";
|
|
dockerConfig = {
|
|
subpath = "docker/config";
|
|
mountpoint = "/mnt/docker/config";
|
|
};
|
|
dockerDatabases = {
|
|
subpath = "docker/databases";
|
|
mountpoint = "/mnt/docker/databases";
|
|
};
|
|
dockerVolumes = {
|
|
subpath = "docker/volumes";
|
|
mountpoint = "/mnt/docker/volumes";
|
|
};
|
|
nextcloudData = {
|
|
subpath = "docker/nextcloud-data";
|
|
mountpoint = "/mnt/nextcloud-data";
|
|
};
|
|
raspiVolumes = {
|
|
subpath = "raspi/volumes";
|
|
mountpoint = "/mnt/raspi-backup";
|
|
};
|
|
proxmoxIsos = {
|
|
subpath = "proxmox/iso";
|
|
mountpoint = "/mnt/iso";
|
|
};
|
|
proxmoxLxcImages = {
|
|
subpath = "proxmox/lxc";
|
|
mountpoint = "/mnt/lxc";
|
|
};
|
|
pxebootImages = {
|
|
subpath = "pxe-boot/images";
|
|
mountpoint = "/mnt/pxe-images";
|
|
};
|
|
};
|
|
|
|
# The Raspberry Pi's own NFS export — not under storageRoot/nfsServerHost,
|
|
# served directly by the Pi itself over Tailscale (see raspberryPiHost
|
|
# above) and mounted at raspiMountpoint by modules/raspi/mount-data.nix.
|
|
raspiNfsPath = "/home/raspi/raspi";
|
|
raspiMountpoint = "/mnt/raspi";
|
|
|
|
# Every literal port referenced from modules/ or hosts/, grouped by the
|
|
# service/host that opens or connects to it — kept as separate entries
|
|
# even where two happen to share a number today (e.g. nixCacheHttp and
|
|
# pxeBootHttp are both 80) so changing one service's port can never
|
|
# silently change an unrelated one.
|
|
ports = {
|
|
# nix-cache's nginx reverse proxy in front of nix-serve
|
|
# (modules/nix-cache/server.nix).
|
|
nixCacheHttp = 80;
|
|
|
|
# pxe-boot's nginx asset server, also used to build pxeBaseUrl
|
|
# (modules/build-types/pxe-boot.nix).
|
|
pxeBootHttp = 80;
|
|
|
|
# pxe-boot's atftpd TFTP server — UDP, not TCP
|
|
# (modules/build-types/pxe-boot.nix).
|
|
pxeBootTftp = 69;
|
|
|
|
# `server`'s NFS exports: portmapper (rpcbind), NFS data, and the
|
|
# mountd RPC service (used by showmount/NFSv3 mount protocol).
|
|
# Mountd listens on a fixed port so the firewall can whitelist it
|
|
# explicitly rather than opening all of rpcbind's dynamic range.
|
|
# All three need both TCP and UDP (modules/build-types/server.nix and
|
|
# modules/build-types/ha-server.nix).
|
|
nfsRpcbind = 111;
|
|
nfsd = 2049;
|
|
nfsMountd = 20048;
|
|
|
|
# HA cluster ports opened on ha-server-1 and ha-server-2
|
|
# (modules/build-types/ha-server.nix / modules/ha/cluster-config.nix).
|
|
haServerDrbd = 7789; # DRBD replication (TCP)
|
|
haServerIscsi = 3260; # iSCSI target (TCP)
|
|
haServerCorosync1 = 5404; # Corosync totem ring (UDP)
|
|
haServerCorosync2 = 5405; # Corosync totem ring (UDP)
|
|
haServerCorosyncCrypto = 5407; # Corosync crypto sync (UDP)
|
|
haServerPacemakerRemoted = 3121; # pacemaker-remoted (TCP)
|
|
haServerPcsd = 2224; # pcsd cluster daemon (TCP)
|
|
|
|
# Opened on the docker host's firewall for the Traefik-fronted
|
|
# container stack (docker-compose config lives in the separate
|
|
# /home/debian/docker repo, not here): 80/443 are Traefik's own
|
|
# HTTP/HTTPS listeners; 8080 is an additional exposed service whose
|
|
# exact backend isn't declared in this repo (modules/build-types/docker.nix).
|
|
dockerHttp = 80;
|
|
dockerHttps = 443;
|
|
dockerExtra = 8080;
|
|
|
|
# Beszel monitoring hub, reachable at
|
|
# http://<dockerHost>.<homeDomain>:<beszelHub> from every agent
|
|
# (modules/beszel/enable-agent.nix, hosts/nixos/home.nix).
|
|
beszelHub = 8090;
|
|
|
|
# Proxmox VE and Proxmox Backup Server web UIs, opened as desktop
|
|
# shortcuts on the gui build type (hosts/nixos/home.nix).
|
|
pveWeb = 8006;
|
|
pbsWeb = 8007;
|
|
|
|
# Tor relay's ORPort — the port other Tor relays connect to for onion
|
|
# routing traffic (modules/tor/enable-relay.nix). Tor's own conventional
|
|
# default; opened via services.tor.openFirewall rather than
|
|
# networking.firewall.allowedTCPPorts directly, but kept here anyway so
|
|
# it's not a bare literal duplicated between the relay's settings and
|
|
# anything else that ever needs to reference it.
|
|
torRelayOrPort = 9001;
|
|
};
|
|
|
|
# .raw disk image size for every proxmox-* host's standalone Disko image
|
|
# build (modules/disko/proxmox.nix, config.system.build.diskoImagesScript
|
|
# — see docs/proxmox-images.md). Root fills whatever's left after the ESP
|
|
# and swap partitions within this total.
|
|
proxmoxImageSize = "50G";
|
|
|
|
# nix-cache's Nix store garbage collection retention
|
|
# (modules/nix-cache/server.nix).
|
|
nixCacheGcMaxAge = "30d";
|
|
|
|
# Traefik access log rotation, watched on the docker host at
|
|
# nfsShares.dockerVolumes.mountpoint (modules/traefik/rotate-logs.nix).
|
|
traefikLogRotate = {
|
|
maxSize = "100M"; # rotate once a log file exceeds this size
|
|
keep = 20; # number of rotated logs to retain before deleting the oldest
|
|
};
|
|
|
|
}
|