Archived
beatzaplentyandClaude Sonnet 4.6
ead4f55805
Check NixOS configurations / eval-hosts (push) Failing after 9m44s
/dev/sda and /dev/sdb are assigned by the OS based on Proxmox disk-add
order, which is not consistent across VMs. Use the SCSI controller path
instead — drive-scsi1 is always the dedicated data disk on all HA nodes
regardless of which sda/sdb it gets assigned to.
variables.nix: replace per-node haServer{1,2}DrbdDisk with a single
haServerDrbdDisk using /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1.
cluster-config.nix: revert to shared volume block (per-node block was
needed for /dev/sd* but the by-id path is identical on both nodes).
cluster-init.sh:
- single DRBD_DISK variable (matching haServerDrbdDisk)
- robust /etc/drbd.conf patch: NixOS manages this file as a symlink to
a read-only Nix store path; cp --remove-destination breaks the symlink
before sed -i so the edit actually takes effect
- scp helper script to NODE2 rather than bash -c over SSH to avoid
quoting complexity
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
257 lines
13 KiB
Nix
257 lines
13 KiB
Nix
{
|
|
# Network / domains
|
|
lanDomain = "gitea.lan.ddnsgeek.com"; # Gitea/DDNS domain
|
|
homeDomain = "sweet.home"; # base LAN domain for service subdomains (pve., docker.)
|
|
tailnetDomain = "tail13f623.ts.net"; # Tailscale MagicDNS suffix
|
|
lanCidr = "192.168.2.0/24"; # LAN subnet
|
|
lanGateway = "192.168.2.254"; # LAN default gateway (router)
|
|
lanPrefixLength = 24; # LAN subnet prefix length (/24 = 255.255.255.0)
|
|
lxcLanInterface = "eth0"; # LAN NIC name in LXC containers (set by Proxmox --net0 name=eth0)
|
|
vmLanInterface = "ens18"; # LAN NIC name in Proxmox VMs (virtio, first NIC)
|
|
vmStorageInterface = "ens19"; # storage NIC name in HA server VMs (virtio, second NIC on vmbr1)
|
|
pxeServerIp = "192.168.2.223"; # pxe-boot LXC container LAN IP
|
|
nixCacheIp = "192.168.2.224"; # nix-cache LXC container LAN IP
|
|
tailscaleRouterIp = "192.168.2.222"; # tailscale-router LXC container LAN IP
|
|
torRelayIp = "192.168.2.221"; # tor-relay LXC container LAN IP
|
|
serverIp = "192.168.2.226"; # server (NFS/ZFS) Proxmox VM LAN IP
|
|
dockerIp = "192.168.2.225"; # docker Proxmox VM LAN IP
|
|
pbsIp = "192.168.2.244"; # Proxmox Backup Server LAN IP (not NixOS-managed)
|
|
domainControllerIp = "192.168.2.253"; # FreeIPA domain controller — authoritative DNS for sweet.home (not NixOS-managed)
|
|
ipaServer = "domain-controller.sweet.home"; # FreeIPA server hostname (used by security.ipa and Kerberos; must be a resolvable FQDN, not an IP)
|
|
|
|
# Cross-host references (LAN hostnames/users other hosts reach over the network)
|
|
nixCacheHost = "nix-cache"; # substituter/remote-builder hostname
|
|
nfsServerHost = "server"; # NFS export source hostname
|
|
dockerHost = "docker"; # docker-compose stack host
|
|
|
|
# Raspberry Pi's own Tailscale hostname (not fronted by `server` — it
|
|
# exports its own NFS share directly). Resolved as
|
|
# "${raspberryPiHost}.${tailnetDomain}" in modules/raspi/mount-data.nix.
|
|
raspberryPiHost = "raspberrypi";
|
|
|
|
remoteBuilderUser = "nixremote"; # remote builder SSH user
|
|
|
|
# nix-cache's own SSH host public key (not a secret — the private half
|
|
# never leaves the host). Wired into every client's
|
|
# programs.ssh.knownHosts by modules/nix-cache/remote-builder-client.nix
|
|
# so distributed builds don't hit "Host key verification failed" on a
|
|
# fresh client that has never manually ssh'd to nix-cache before. Update
|
|
# this if nix-cache's host key is ever rotated or the host is rebuilt
|
|
# from scratch.
|
|
nixCacheHostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICuHUxGNH6ei3BZD+EfZs3l4X8uJNcjQiOsM/G4yo4O/ lxc-nix-cache";
|
|
|
|
# Public keys authorized to SSH in as remoteBuilderUser on the nix-cache
|
|
# host (modules/nix-cache/server.nix) — one per client host that's allowed
|
|
# to use it as a distributed builder.
|
|
remoteBuilderAuthorizedKeys = [
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFDEA1S2ikpObREgbP5uVBWMxIOGbY8B+Wx7VTZK1m6t root@server"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPAYIT9ormlmxZ0SziyDQaUntnKI8HK9/s3Qac1ZKjP2 root@docker"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKKKzoEPl/ZW9KBRHBcp6/ThOngGpwMv5EhkTlgC4aDf root@nixos"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIGtOWOCS+ImHc7NehguoyD7PbonGosKMZqc9+QR3v/h root@nixos"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHxXTQxFnArK5HXG7czeoybZebCGfxpUdusJkPn+BCSp root@server"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMJhrfFayLBG+gWtO6oAvgambw5nWWgztiTFEaaaVRH debian@surface"
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJEj26SL/emsVjW2YhRucJVp2kTz8WgcEQgjBEBLRikk root@claude"
|
|
];
|
|
|
|
# Admin SSH public key, authorized on the primary user of every host and
|
|
# the installer image's nixos/root users.
|
|
adminSshKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCq/Q5LvIXlZwO2kdeAN5nLGZ59nZB7JHYMEszHxmNtGMzv1lM31jiPNsr0z2EKVZhE7OOfa2IF9rhWYD7JUA9G0yzdZ4WTXFNGVVOJoOVH6vAF3XCxoVilOEwTc7h2Wiy+rzd0B28/3spffzQQWJhY6GRQVa8j+6xAGF60Fcvl1vLosYT9Bn2ZbK4TCWOwAn2jqXIieGpZdn/UNZbGOeKRiCvhktDfMAzuQzN/9jMu/oF4pkPn2X1UrsQdNlvp0Ci8md612MozIpncQJyAF1ADhunr3sMx0isUXiqD29R5DS4TftpekqLNLak+zcxFa8N7DcRNp3DcKfJvyTkwQrR4r+b7lFLYOLHLagSso9CzeW/paAS2q9I5SBm/2DtE1diLLg2jZikYcstsu/G5RgvbzbKqjiaMwTdXC3AMvDxQrs7U5pDRZFzoofG3cpODbTm+uy3m0kP70z0M1K45UbDG0p+itnTu9x40JbQEgefbx38AItNvAIx1A8HO4I1VX28= wayne@stream";
|
|
|
|
# Prestaged wifi SSID for the gui host's NetworkManager profile
|
|
# (modules/networking/wifi.nix). The password is not here -- it's
|
|
# sops-encrypted in secrets/gui.yaml (wifi-password) instead, since this
|
|
# file isn't a secret store.
|
|
wifiSsid = "nbn-fttp-net-5G";
|
|
|
|
# Bare-metal gui host's two disks for a ZFS RAID0 (striped) root pool
|
|
# (modules/disko/baremetal.nix). Only used transiently at disko-format
|
|
# time (partitioning); the resulting fileSystems/zpool import reference
|
|
# by-partlabel/by-id paths afterward regardless, same as
|
|
# modules/disko/proxmox.nix's own plain "/dev/sda".
|
|
guiRootDisk1 = "/dev/sda";
|
|
guiRootDisk2 = "/dev/sdb";
|
|
|
|
# System
|
|
timeZone = "Australia/Brisbane";
|
|
|
|
# Main interactive user on every host. Every module that grants this user
|
|
# a group, a home directory, or tmpfiles ownership should reference
|
|
# vars.primaryUser rather than the literal "nixos", so renaming it is a
|
|
# one-line change.
|
|
primaryUser = "nixos";
|
|
|
|
# Primary IPA/domain user. Home Manager is configured for this user on every
|
|
# IPA-enrolled host (see modules/ipa/client.nix) to manage the environment
|
|
# that IPA itself doesn't cover: dotfiles, user packages, session variables.
|
|
ipaUser = "wayne";
|
|
|
|
# GID of the IPA "docker-access" group (GID 50010 on the IPA server).
|
|
# The local "docker" group is pinned to this GID on every host that runs
|
|
# Docker so that IPA group membership alone grants docker socket access -
|
|
# no per-host users.groups.docker.members entry for the IPA user needed.
|
|
dockerAccessGid = 50010;
|
|
|
|
# HA file server cluster
|
|
# LAN IPs (vmbr0 / ens18) — client-facing: iSCSI initiators, NFS, management.
|
|
# Storage IPs (vmbr1 / ens19) — isolated internal bridge, used for DRBD
|
|
# replication and Corosync heartbeat only; never leaves pve1.
|
|
# haServerVip: floating virtual IP managed by Pacemaker's IPaddr2 resource;
|
|
# NFS and iSCSI clients connect here regardless of which node is Active.
|
|
haServer1Host = "ha-server-1";
|
|
haServer2Host = "ha-server-2";
|
|
haServer1Ip = "192.168.2.228"; # LAN IP, node 1
|
|
haServer2Ip = "192.168.2.227"; # LAN IP, node 2
|
|
haServerVip = "192.168.2.229"; # floating VIP (Pacemaker IPaddr2)
|
|
haServer1StorageIp = "192.168.4.228"; # storage-net IP, node 1 (vmbr1 / ens19)
|
|
haServer2StorageIp = "192.168.4.227"; # storage-net IP, node 2 (vmbr1 / ens19)
|
|
haStorageCidr = "192.168.4.0/29"; # storage subnet — internal to pve1 only
|
|
haStoragePrefixLength = 29; # storage subnet prefix length (/29)
|
|
haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node
|
|
haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage";
|
|
# DRBD backing disk — identified by SCSI controller path so it resolves to the
|
|
# correct block device regardless of OS-level naming (sda vs sdb can differ
|
|
# between Proxmox VMs depending on disk-add order). drive-scsi1 is always the
|
|
# dedicated data disk on all HA nodes; drive-scsi0 is the OS disk.
|
|
haServerDrbdDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1";
|
|
|
|
# Storage
|
|
storageRoot = "/tank"; # ZFS pool root on `server`
|
|
|
|
# NFS datasets exported from `storageRoot` on `nfsServerHost` and mounted
|
|
# by client hosts. `subpath` is relative to `storageRoot` — combined with
|
|
# it to build both the export line in modules/build-types/server.nix and
|
|
# the "<nfsServerHost>:<storageRoot>/<subpath>" device string each client
|
|
# mount uses in modules/docker/mount-data.nix. `mountpoint` is the
|
|
# absolute local path clients mount it at, referenced by that same file's
|
|
# fileSystems attribute name plus every other place that needs to know
|
|
# where the share lives locally (modules/build-types/docker.nix's
|
|
# tmpfiles rules, modules/traefik/rotate-logs.nix's log path). Renaming a
|
|
# dataset or moving where it's mounted only needs changing it here — the
|
|
# export and every client reference follow automatically.
|
|
nfsShares = {
|
|
options = "(rw,sync,no_subtree_check,no_root_squash)";
|
|
dockerConfig = {
|
|
subpath = "docker/config";
|
|
mountpoint = "/mnt/docker/config";
|
|
};
|
|
dockerDatabases = {
|
|
subpath = "docker/databases";
|
|
mountpoint = "/mnt/docker/databases";
|
|
};
|
|
dockerVolumes = {
|
|
subpath = "docker/volumes";
|
|
mountpoint = "/mnt/docker/volumes";
|
|
};
|
|
nextcloudData = {
|
|
subpath = "docker/nextcloud-data";
|
|
mountpoint = "/mnt/nextcloud-data";
|
|
};
|
|
raspiVolumes = {
|
|
subpath = "raspi/volumes";
|
|
mountpoint = "/mnt/raspi-backup";
|
|
};
|
|
proxmoxIsos = {
|
|
subpath = "proxmox/iso";
|
|
mountpoint = "/mnt/iso";
|
|
};
|
|
proxmoxLxcImages = {
|
|
subpath = "proxmox/lxc";
|
|
mountpoint = "/mnt/lxc";
|
|
};
|
|
pxebootImages = {
|
|
subpath = "pxe-boot/images";
|
|
mountpoint = "/mnt/pxe-images";
|
|
};
|
|
};
|
|
|
|
# The Raspberry Pi's own NFS export — not under storageRoot/nfsServerHost,
|
|
# served directly by the Pi itself over Tailscale (see raspberryPiHost
|
|
# above) and mounted at raspiMountpoint by modules/raspi/mount-data.nix.
|
|
raspiNfsPath = "/home/raspi/raspi";
|
|
raspiMountpoint = "/mnt/raspi";
|
|
|
|
# Every literal port referenced from modules/ or hosts/, grouped by the
|
|
# service/host that opens or connects to it — kept as separate entries
|
|
# even where two happen to share a number today (e.g. nixCacheHttp and
|
|
# pxeBootHttp are both 80) so changing one service's port can never
|
|
# silently change an unrelated one.
|
|
ports = {
|
|
# nix-cache's nginx reverse proxy in front of nix-serve
|
|
# (modules/nix-cache/server.nix).
|
|
nixCacheHttp = 80;
|
|
|
|
# pxe-boot's nginx asset server, also used to build pxeBaseUrl
|
|
# (modules/build-types/pxe-boot.nix).
|
|
pxeBootHttp = 80;
|
|
|
|
# pxe-boot's atftpd TFTP server — UDP, not TCP
|
|
# (modules/build-types/pxe-boot.nix).
|
|
pxeBootTftp = 69;
|
|
|
|
# `server`'s NFS exports: portmapper (rpcbind), NFS data, and the
|
|
# mountd RPC service (used by showmount/NFSv3 mount protocol).
|
|
# Mountd listens on a fixed port so the firewall can whitelist it
|
|
# explicitly rather than opening all of rpcbind's dynamic range.
|
|
# All three need both TCP and UDP (modules/build-types/server.nix and
|
|
# modules/build-types/ha-server.nix).
|
|
nfsRpcbind = 111;
|
|
nfsd = 2049;
|
|
nfsMountd = 20048;
|
|
|
|
# HA cluster ports opened on ha-server-1 and ha-server-2
|
|
# (modules/build-types/ha-server.nix / modules/ha/cluster-config.nix).
|
|
haServerDrbd = 7789; # DRBD replication (TCP)
|
|
haServerIscsi = 3260; # iSCSI target (TCP)
|
|
haServerCorosync1 = 5404; # Corosync totem ring (UDP)
|
|
haServerCorosync2 = 5405; # Corosync totem ring (UDP)
|
|
haServerCorosyncCrypto = 5407; # Corosync crypto sync (UDP)
|
|
haServerPacemakerRemoted = 3121; # pacemaker-remoted (TCP)
|
|
haServerPcsd = 2224; # pcsd cluster daemon (TCP)
|
|
|
|
# Opened on the docker host's firewall for the Traefik-fronted
|
|
# container stack (docker-compose config lives in the separate
|
|
# /home/debian/docker repo, not here): 80/443 are Traefik's own
|
|
# HTTP/HTTPS listeners; 8080 is an additional exposed service whose
|
|
# exact backend isn't declared in this repo (modules/build-types/docker.nix).
|
|
dockerHttp = 80;
|
|
dockerHttps = 443;
|
|
dockerExtra = 8080;
|
|
|
|
# Beszel monitoring hub, reachable at
|
|
# http://<dockerHost>.<homeDomain>:<beszelHub> from every agent
|
|
# (modules/beszel/enable-agent.nix, hosts/nixos/home.nix).
|
|
beszelHub = 8090;
|
|
|
|
# Proxmox VE and Proxmox Backup Server web UIs, opened as desktop
|
|
# shortcuts on the gui build type (hosts/nixos/home.nix).
|
|
pveWeb = 8006;
|
|
pbsWeb = 8007;
|
|
|
|
# Tor relay's ORPort — the port other Tor relays connect to for onion
|
|
# routing traffic (modules/tor/enable-relay.nix). Tor's own conventional
|
|
# default; opened via services.tor.openFirewall rather than
|
|
# networking.firewall.allowedTCPPorts directly, but kept here anyway so
|
|
# it's not a bare literal duplicated between the relay's settings and
|
|
# anything else that ever needs to reference it.
|
|
torRelayOrPort = 9001;
|
|
};
|
|
|
|
# .raw disk image size for every proxmox-* host's standalone Disko image
|
|
# build (modules/disko/proxmox.nix, config.system.build.diskoImagesScript
|
|
# — see docs/proxmox-images.md). Root fills whatever's left after the ESP
|
|
# and swap partitions within this total.
|
|
proxmoxImageSize = "50G";
|
|
|
|
# nix-cache's Nix store garbage collection retention
|
|
# (modules/nix-cache/server.nix).
|
|
nixCacheGcMaxAge = "30d";
|
|
|
|
# Traefik access log rotation, watched on the docker host at
|
|
# nfsShares.dockerVolumes.mountpoint (modules/traefik/rotate-logs.nix).
|
|
traefikLogRotate = {
|
|
maxSize = "100M"; # rotate once a log file exceeds this size
|
|
keep = 20; # number of rotated logs to retain before deleting the oldest
|
|
};
|
|
|
|
}
|