diff --git a/.sops.yaml b/.sops.yaml index b37b4e2..a14898b 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -85,6 +85,32 @@ creation_rules: - *lxc-tailscale-router - *proxmox-tailscale-router + # HA file server per-node secrets (beszel-token). + # proxmox-ha-server-1 / proxmox-ha-server-2 keys are added automatically + # by scripts/secrets/sync-host-keys.sh once the hosts are provisioned; + # until then only the admin key can decrypt these files. + - path_regex: secrets/ha-server-1\.yaml$ + key_groups: + - age: + - *admin + # proxmox-ha-server-1 added by sync-host-keys.sh + + - path_regex: secrets/ha-server-2\.yaml$ + key_groups: + - age: + - *admin + # proxmox-ha-server-2 added by sync-host-keys.sh + + # Shared HA cluster corosync authkey (binary sops file). + # Encrypted for both HA nodes so either can decrypt on boot. + # Both host keys added by sync-host-keys.sh; admin key allows initial creation. + - path_regex: secrets/ha-corosync-authkey$ + key_groups: + - age: + - *admin + # proxmox-ha-server-1 added by sync-host-keys.sh + # proxmox-ha-server-2 added by sync-host-keys.sh + # gui-host-specific secrets (currently: wifi-password, see # modules/networking/wifi.nix). Only *lxc-gui has a registered key today # -- proxmox-gui/linode-gui/baremetal-gui haven't been provisioned via diff --git a/docs/ip-addressing.md b/docs/ip-addressing.md new file mode 100644 index 0000000..767bfd6 --- /dev/null +++ b/docs/ip-addressing.md @@ -0,0 +1,128 @@ +# IP Addressing Scheme + +## Subnets + +| Subnet | CIDR | Purpose | Routed? | +|---|---|---|---| +| LAN | `192.168.2.0/24` | General LAN — clients and infrastructure | Yes (gateway .254) | +| Storage | `192.168.4.0/29` | HA file server DRBD replication | No — internal `vmbr1` only, no uplink | + +The storage subnet never leaves pve1. `vmbr1` is a Proxmox Linux bridge with no physical port +attached; traffic between the two HA file server VMs stays in-kernel. + +The host octet is consistent across subnets for any host that has multiple interfaces — e.g. +ha-node1 is always `.228` (LAN: `192.168.2.228`, storage: `192.168.4.228`). + +--- + +## LAN — 192.168.2.0/24 + +### Address map + +| Range | Purpose | +|---|---| +| .1–.9 | Reserved, never assign | +| .10–.59 | Client DHCP pool (router-assigned) | +| .60–.219 | Unallocated buffer | +| .220–.229 | Virtual nodes (VMs / LXC containers) | +| .230–.239 | Expansion buffer (reserved, unallocated) | +| .240–.249 | Physical nodes (bare-metal hosts) | +| .250–.253 | Network services | +| .254 | Router / gateway | + +### Network services (.250–.253) + +| IP | Hostname | Role | +|---|---|---| +| `192.168.2.254` | router | Gateway (TP-Link) | +| `192.168.2.253` | domain-controller | FreeIPA — authoritative DNS for `sweet.home`, Kerberos, LDAP | +| `192.168.2.250`–`.252` | — | Reserved for future network services | + +### Physical nodes (.240–.249) + +| IP | Hostname | Role | +|---|---|---| +| `192.168.2.245` | pve1 | Proxmox VE hypervisor | +| `192.168.2.244` | pbs | Proxmox Backup Server | +| `192.168.2.243` | nixos | Bare-metal workstation (`baremetal-gui`) | +| `192.168.2.246`–`.249` | — | Reserved — second Proxmox node and associated services | +| `192.168.2.240`–`.242` | — | Reserved | + +pve1 sits mid-range deliberately so a second Proxmox node can slot in on either side. + +### Virtual nodes (.220–.229) + +All VMs and LXC containers run on pve1. + +| IP | Hostname | Role | Status | +|---|---|---|---| +| `192.168.2.229` | ha-vip | HA file server iSCSI floating VIP (Pacemaker) | Future | +| `192.168.2.228` | ha-node1 | HA file server node 1 (DRBD + XFS + iSCSI) | Future | +| `192.168.2.227` | ha-node2 | HA file server node 2 (DRBD + XFS + iSCSI) | Future | +| `192.168.2.226` | server | Current NFS/ZFS file server — retires when HA is live | Retiring | +| `192.168.2.225` | docker | Docker / Traefik stack | Active | +| `192.168.2.224` | nix-cache | Nix binary cache + remote builder | Active | +| `192.168.2.223` | pxe-boot | PXE / TFTP / HTTP netboot server | Active | +| `192.168.2.222` | tailscale-router | Tailscale exit node / router | Active | +| `192.168.2.221` | tor-relay | Tor relay | Active | +| `192.168.2.220` | pdm | Proxmox Deploy Manager | Active | + +### Client DHCP pool (.10–.59) + +Assigned by the router. DNS option points to `192.168.2.253` (domain-controller). + +Devices in this range: phones, laptops, IoT, Canon printer, any non-infrastructure host. +No static reservations for infrastructure hosts — all infra uses static IP configuration +on the guest itself (not DHCP reservations), so IPs survive VM recreation regardless of +MAC address churn. + +--- + +## Storage network — 192.168.4.0/29 + +Internal to pve1 only. Proxmox bridge `vmbr1`, no physical NIC attached. + +| IP | Hostname | Interface role | +|---|---|---| +| `192.168.4.228` | ha-node1 | DRBD replication NIC | +| `192.168.4.227` | ha-node2 | DRBD replication NIC | +| — | no gateway | Isolated — not routed to LAN or internet | + +--- + +## Migration reference + +Current → target IP for every host being renumbered. + +| Host | Current IP | New IP | Config location | +|---|---|---|---| +| router | `192.168.2.254` | `192.168.2.254` | unchanged | +| domain-controller | `192.168.2.138` | `192.168.2.253` | `/etc/sysconfig/network-scripts/ifcfg-eth0` on guest | +| pve1 | `192.168.2.250` | `192.168.2.245` | `/etc/network/interfaces` on Proxmox host | +| pbs | `192.168.2.108` | `192.168.2.244` | static config on PBS host | +| nixos workstation | `192.168.2.119` | `192.168.2.243` | `networking.interfaces` / NetworkManager on guest | +| ha-node1 | — | `192.168.2.228` | future | +| ha-node2 | — | `192.168.2.227` | future | +| ha-vip | — | `192.168.2.229` | future (Pacemaker resource) | +| server | `192.168.2.252` | `192.168.2.226` | static config on guest | +| docker | `192.168.2.249` | `192.168.2.225` | static config on guest | +| nix-cache | `192.168.2.120` | `192.168.2.224` | static config on guest | +| pxe-boot | `192.168.2.247` | `192.168.2.223` | static config on guest; update `vars.pxeServerIp` in `variables.nix` ✓ | +| tailscale-router | `192.168.2.121` | `192.168.2.222` | static config on guest | +| tor-relay | `192.168.2.107` | `192.168.2.221` | static config on guest | +| pdm | `192.168.2.248` | `192.168.2.220` | static config on guest | + +### Cutover notes + +- **Do domain-controller first** — it becomes the DNS server; everything else depends on it + having its new IP and FreeIPA DNS configured before Pi-hole is retired. +- **pve1 last among physical hosts** — changing the Proxmox management IP drops the web UI + briefly; all guests keep running. +- **Update Pi-hole custom.list / FreeIPA DNS A records** to new IPs before flipping any host, + so name resolution stays valid throughout the migration. +- **variables.nix already updated** for `pxeServerIp` (.247→.223), `pbsIp` (.108→.244), and + new `domainControllerIp` (.253). Rebuild affected hosts after renumbering. +- **Router DHCP**: once domain-controller is at .253 and FreeIPA DNS is serving `sweet.home`, + switch router DHCP on with pool .10–.59 and DNS option pointing to .253; retire Pi-hole CT. +- **Pi-hole's iPXE dnsmasq config** (`99-ipxe-chainload.conf`) moves to the pxe-boot CT as a + dnsmasq proxy-mode config before Pi-hole is decommissioned. diff --git a/docs/network-cutover.md b/docs/network-cutover.md new file mode 100644 index 0000000..631250c --- /dev/null +++ b/docs/network-cutover.md @@ -0,0 +1,366 @@ +# Network Cutover Plan + +Moves the LAN from the current flat/Pi-hole-managed state to the new IP scheme +defined in `docs/ip-addressing.md`. Works in five independent stages — each +stage is safe to pause after and resume later. Rollback steps are given at +every point where something can break. + +**Before starting anything:** confirm you have +- SSH access to `192.168.2.138` (domain-controller, current IP) +- SSH access to `192.168.2.250` (pve1) +- Browser access to Pi-hole admin at `http://192.168.2.253` +- Browser access to router admin at `http://192.168.2.254` +- The FreeIPA `admin` password to hand + +--- + +## Stage 1 — Prepare FreeIPA DNS (zero downtime) + +Everything here is additive. Pi-hole keeps running. Nothing breaks if you stop +mid-stage. + +### 1a. Add NextDNS forwarders + +```bash +ssh wayne@192.168.2.138 +kinit admin # enter FreeIPA admin password when prompted +ipa dnsconfig-mod \ + --forwarder=45.90.28.142 \ + --forwarder=45.90.30.142 \ + --forward-policy=only +``` + +**Verify external resolution works through FreeIPA before continuing:** +```bash +dig @127.0.0.1 google.com +short # must return an IP, not SERVFAIL +``` + +### 1b. Add A records for every host at their CURRENT IPs + +These represent the live state now. You'll update each record to the new IP +when you renumber that host in Stage 5. + +```bash +ipa dnsrecord-add sweet.home pve1 --a-rec 192.168.2.250 +ipa dnsrecord-add sweet.home pbs --a-rec 192.168.2.108 +ipa dnsrecord-add sweet.home nixos --a-rec 192.168.2.119 +ipa dnsrecord-add sweet.home server --a-rec 192.168.2.252 +ipa dnsrecord-add sweet.home docker --a-rec 192.168.2.249 +ipa dnsrecord-add sweet.home nix-cache --a-rec 192.168.2.120 +ipa dnsrecord-add sweet.home pxe-boot --a-rec 192.168.2.247 +ipa dnsrecord-add sweet.home tailscale-router --a-rec 192.168.2.121 +ipa dnsrecord-add sweet.home tor-relay --a-rec 192.168.2.107 +ipa dnsrecord-add sweet.home pdm --a-rec 192.168.2.248 +ipa dnsrecord-add sweet.home router --a-rec 192.168.2.254 +``` + +### 1c. Clean up stale reverse-zone PTR records + +FreeIPA already has PTR records from an earlier import but some are wrong. +Fix them now so reverse DNS is accurate from day one. + +```bash +# Remove stale "win11" entry at .250 (should be pve1) +ipa dnsrecord-del 2.168.192.in-addr.arpa 250 --ptr-rec win11. +ipa dnsrecord-add 2.168.192.in-addr.arpa 250 --ptr-rec pve1.sweet.home. + +# Fix unqualified PTR records (missing .sweet.home. suffix) +ipa dnsrecord-mod 2.168.192.in-addr.arpa 108 --ptr-rec pbs.sweet.home. +ipa dnsrecord-mod 2.168.192.in-addr.arpa 248 --ptr-rec pdm.sweet.home. +ipa dnsrecord-mod 2.168.192.in-addr.arpa 249 --ptr-rec docker.sweet.home. +ipa dnsrecord-mod 2.168.192.in-addr.arpa 252 --ptr-rec server.sweet.home. + +# Add any missing PTR records +ipa dnsrecord-add 2.168.192.in-addr.arpa 119 --ptr-rec nixos.sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa 120 --ptr-rec nix-cache.sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa 121 --ptr-rec tailscale-router.sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa 247 --ptr-rec pxe-boot.sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa 254 --ptr-rec router.sweet.home. +``` + +### 1d. Point domain-controller's own DNS at itself + +```bash +sudo nmcli connection modify "System eth0" ipv4.dns "127.0.0.1" +sudo nmcli connection up "System eth0" +``` + +**Verify:** +```bash +dig pve1.sweet.home +short # must return 192.168.2.250 +dig google.com +short # must return an IP (NextDNS forwarding) +``` + +**Rollback 1d:** `sudo nmcli connection modify "System eth0" ipv4.dns "192.168.2.253" && sudo nmcli connection up "System eth0"` + +--- + +## Stage 2 — Move pxe-boot DHCP options off Pi-hole (zero downtime) + +Pi-hole's dnsmasq currently serves the iPXE boot options via +`99-ipxe-chainload.conf`. Before Pi-hole is retired, that config must move to +the pxe-boot CT running dnsmasq in proxy mode so PXE boot keeps working. + +### 2a. Add dnsmasq proxy config to the pxe-boot NixOS module + +In `modules/build-types/pxe-boot.nix`, add: + +```nix +services.dnsmasq = { + enable = true; + settings = { + # Proxy mode: respond only to PXE DHCP requests, leave normal leases to router + dhcp-range = [ "192.168.2.0,proxy" ]; + # iPXE client detection + dhcp-match = [ + "set:ipxe,175" + "set:efi64,option:client-arch,7" + "set:efi64,option:client-arch,9" + ]; + dhcp-userclass = "set:ipxe,iPXE"; + # Boot file selection + dhcp-boot = [ + "tag:ipxe,tag:efi64,http://${vars.pxeServerIp}/boot.ipxe" + "tag:ipxe,http://${vars.pxeServerIp}/boot.ipxe" + "tag:efi64,ipxe.efi,,${vars.pxeServerIp}" + "undionly.kpxe,,${vars.pxeServerIp}" + ]; + }; +}; +``` + +### 2b. Rebuild and deploy the pxe-boot CT + +```bash +# On pve1 — build the new tarball +nix build .#lxc-pxe-boot.config.system.build.tarball + +# Verify dnsmasq starts correctly in the CT after deploy +ssh nixos@192.168.2.247 systemctl status dnsmasq +``` + +### 2c. Remove the iPXE config from Pi-hole + +In the Pi-hole CT, remove `/etc/dnsmasq.d/99-ipxe-chainload.conf` and +restart the FTL service: + +```bash +ssh wayne@pve1.sweet.home \ + "sudo pct exec 100 -- bash -c 'rm /etc/dnsmasq.d/99-ipxe-chainload.conf && systemctl restart pihole-FTL'" +``` + +**Verify:** PXE boot a test machine — it should still get an iPXE response and +reach the boot menu. + +**Rollback 2c:** restore the file from the Pi-hole config backup at +`/etc/pihole/config_backups/` and restart pihole-FTL. + +--- + +## Stage 3 — DHCP migration: Pi-hole → router (brief maintenance window) + +**Do this in the evening.** Existing DHCP leases stay valid during the +switchover so connected devices don't drop — only new lease requests fail +during the gap, which is under 60 seconds if you follow the steps in order. + +The key: configure the router's DHCP DNS option to point at `.253` (Pi-hole's +current IP). This way, all new leases issued by the router still get the same +DNS server address — clients never need to change their DNS config. When Pi-hole +is retired and the DC takes `.253` in Stage 4, `.253` just starts answering +differently. No client reconfiguration. + +### 3a. Pre-configure router DHCP (do not enable yet) + +Log into `http://192.168.2.254`, find the DHCP settings and fill in — but +leave DHCP **disabled** until step 3b: + +| Setting | Value | +|---|---| +| Start IP | 192.168.2.10 | +| End IP | 192.168.2.59 | +| Subnet mask | 255.255.255.0 | +| Gateway | 192.168.2.254 | +| Primary DNS | 192.168.2.253 | +| Secondary DNS | *(leave blank)* | +| Lease time | 24h | + +Save without enabling. + +### 3b. Switchover (do steps in quick succession) + +1. **Disable Pi-hole DHCP:** Pi-hole admin UI → Settings → DHCP → uncheck + "DHCP server enabled" → Save +2. **Enable router DHCP** immediately after step 1 + +### 3c. Verify router DHCP is working + +On a phone or laptop, disconnect from WiFi and reconnect (or run +`sudo dhclient -r && sudo dhclient` on a Linux host): + +```bash +ip addr show # IP should be in 192.168.2.10–59 range +dig google.com # should resolve (Pi-hole DNS still running at .253) +dig pve1.sweet.home # should resolve via FreeIPA at .138 (relayed via Pi-hole) +``` + +Wait 10–15 minutes for the most active devices to renew their leases. There's +no need to wait for all leases to expire before proceeding. + +**Rollback 3b:** Re-enable Pi-hole DHCP. Disable router DHCP. Done — existing +leases remain valid so most devices are unaffected. + +--- + +## Stage 4 — Move domain-controller from .138 to .253 + +Pi-hole lives at `.253`. The DC must take `.253` the moment Pi-hole stops so +clients that still have `.253` as their DNS server don't notice the change. +Script these commands in advance and run them in rapid succession. + +**Pre-stage: have this SSH command ready before running step 4a:** +```bash +ssh wayne@192.168.2.138 " + sudo nmcli connection modify 'System eth0' \ + ipv4.addresses '192.168.2.253/24' \ + ipv4.gateway '192.168.2.254' \ + ipv4.dns '127.0.0.1' \ + ipv4.method manual && \ + sudo nmcli connection up 'System eth0' +" +``` + +**Also update the Proxmox VM config to match (run from pve1):** +```bash +sudo qm set 108 \ + --ipconfig0 ip=192.168.2.253/24,gw=192.168.2.254 \ + --nameserver 192.168.2.253 +``` + +### 4a. Stop Pi-hole + +```bash +ssh wayne@pve1.sweet.home "sudo pct stop 100" +``` + +### 4b. Immediately: change DC's IP to .253 + +Run the pre-staged SSH command from above. You have ~30 seconds before any +client notices Pi-hole is gone. If SSH to `.138` refuses (the IP is already +changing), open a Proxmox console to VM 108 and run the `nmcli` commands +there. + +### 4c. Update Proxmox VM config + +Run the pre-staged `qm set 108` command from above. + +### 4d. Verify + +```bash +ssh wayne@192.168.2.253 # must connect (new DC IP) +dig @192.168.2.253 pve1.sweet.home +short # must return 192.168.2.250 +dig @192.168.2.253 google.com +short # must return an IP +``` + +From a client device that renewed its DHCP lease in Stage 3: +```bash +cat /etc/resolv.conf # should show 192.168.2.253 +dig pve1.sweet.home # should resolve +``` + +**Rollback 4:** `ssh wayne@pve1.sweet.home "sudo pct start 100"`. Change DC IP +back to .138 via Proxmox console. This restores full Pi-hole DNS/DHCP service. +Leave Pi-hole CT stopped-but-intact for 48 hours before deleting it. + +--- + +## Stage 5 — Host renumbering (one at a time, any order) + +For each host: +1. Update FreeIPA DNS A record and PTR record to the new IP +2. Change the static IP on the host itself +3. Verify SSH to new IP +4. Update `variables.nix` if that host has an IP variable (pxe-boot, pbs — already done in this PR) + +**FreeIPA record update template** (run as admin on domain-controller): +```bash +ipa dnsrecord-mod sweet.home --a-rec +ipa dnsrecord-del 2.168.192.in-addr.arpa --ptr-rec .sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa --ptr-rec .sweet.home. +``` + +### Renumbering order + +| # | Host | Old IP | New IP | How to change IP | +|---|---|---|---|---| +| 1 | nixos workstation | .119 | .243 | NetworkManager on guest; or `nmcli connection modify` | +| 2 | nix-cache | .120 | .224 | `pct set 102 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.224/24,gw=192.168.2.254` then `pct reboot 102` | +| 3 | tailscale-router | .121 | .222 | Static config on guest; check Tailscale ACLs if IP is referenced there | +| 4 | tor-relay | .107 | .221 | `pct set 104 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.221/24,gw=192.168.2.254` then `pct reboot 104` | +| 5 | pdm | .248 | .220 | `pct set 106 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.220/24,gw=192.168.2.254` then `pct reboot 106` | +| 6 | pxe-boot | .247 | .223 | `pct set 103 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.223/24,gw=192.168.2.254` then rebuild NixOS (already updated in variables.nix) | +| 7 | server | .252 | .226 | Static config on guest; NFS clients (docker) lose mounts briefly — they remount automatically | +| 8 | docker | .249 | .225 | Static config on guest; do this after server is at .226 | +| 9 | pbs | .108 | .244 | Static config on PBS host itself; update in `pbsIp` already done in variables.nix | +| 10 | pve1 | .250 | .245 | Edit `/etc/network/interfaces` on the Proxmox host — see below | + +### pve1 renumber (step 10 — do last) + +All guests keep running; only the Proxmox web UI is briefly unreachable. + +```bash +ssh wayne@pve1.sweet.home + +# Edit /etc/network/interfaces: change address from .250 to .245 +sudo nano /etc/network/interfaces +# Change: address 192.168.2.250/24 +# To: address 192.168.2.245/24 + +sudo systemctl restart networking +# SSH will drop here — reconnect to new IP +``` + +```bash +ssh wayne@192.168.2.245 # verify +``` + +Update FreeIPA DNS: +```bash +ipa dnsrecord-mod sweet.home pve1 --a-rec 192.168.2.245 +ipa dnsrecord-del 2.168.192.in-addr.arpa 250 --ptr-rec pve1.sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa 245 --ptr-rec pve1.sweet.home. +``` + +**Rollback any step 5 host:** change the IP back on the guest and update the +FreeIPA record back to the old IP. The old IP is unoccupied so you can +temporarily use either. + +--- + +## Stage 6 — Final cleanup + +Once all hosts are at their new IPs and verified: + +```bash +# Delete the Pi-hole CT (already stopped since Stage 4) +ssh wayne@pve1.sweet.home "sudo pct destroy 100" + +# Remove stale FreeIPA records for retired addresses +ipa dnsrecord-del sweet.home pihole --del-all +ipa dnsrecord-del 2.168.192.in-addr.arpa 253 --ptr-rec pihole.sweet.home. + +# Rebuild any NixOS hosts that reference pbsIp or pxeServerIp to pick up +# the updated variables.nix values (pxe-boot mandatory; others as convenient) +``` + +--- + +## Rollback summary + +| What broke | How to roll back | +|---|---| +| FreeIPA DNS not resolving | Check `systemctl status named` on DC; restart if failed | +| FreeIPA DNS unreachable | `pct start 100` on pve1 (restores Pi-hole) | +| Router DHCP not handing out leases | Re-enable Pi-hole DHCP; disable router DHCP | +| DC unreachable after IP change | Proxmox console on VM 108 → `nmcli connection up "System eth0"` with old IP | +| Host unreachable after renumber | Proxmox console → revert IP; or `pct set --net0 ...` old IP and reboot CT | +| pve1 web UI gone after renumber | SSH to .245 and check `/etc/network/interfaces`; if wrong, fix and restart networking | diff --git a/docs/pxe-boot.md b/docs/pxe-boot.md index 5d1fe18..126b262 100644 --- a/docs/pxe-boot.md +++ b/docs/pxe-boot.md @@ -15,6 +15,7 @@ rescue/inspection use. - TFTP root for first-stage bootloaders: `/srv/pxe/tftp` - iPXE entry script: `/srv/pxe/http/boot.ipxe` - Generated iPXE menu: `/srv/pxe/http/menu.ipxe` +- Debian Minimal iPXE script: `/srv/pxe/http/debian.ipxe` - SystemRescue iPXE script: `/srv/pxe/http/systemrescue.ipxe` - TFTP fallback script: `/srv/pxe/tftp/autoexec.ipxe` - Boot binaries copied from the Nix `ipxe` package: @@ -28,32 +29,51 @@ The host creates these directories with systemd tmpfiles: ```text /srv/pxe /srv/pxe/http -/srv/pxe/http/images +/srv/pxe/http/images -> /mnt/pxe-images (symlink to NFS share) /srv/pxe/http/auto-installer /srv/pxe/http/nixos-minimal +/srv/pxe/http/debian /srv/pxe/http/systemrescue /srv/pxe/http/ubuntu /srv/pxe/http/rescue /srv/pxe/tftp ``` -Mount shared image storage under `/srv/pxe/http`, preferably -`/srv/pxe/http/images` unless a menu entry expects files in a specific -directory such as `/srv/pxe/http/auto-installer`. +`/srv/pxe/http/images` is a symlink to `/mnt/pxe-images`, which is an NFS +mount of `server.sweet.home:/tank/pxe-boot/images` +(`modules/pxe-boot/mount-pxe-images.nix`). Place large images there (ISOs, +disk images) rather than on the pxe-boot host's own root disk. For an LXC +pxe-boot container the mount uses NFSv3+nolock with `nofail` (eager, +non-blocking on server unavailability); for a Proxmox VM it uses NFSv4.2 +with `x-systemd.automount` (lazy, triggered on first access). + +When running as `lxc-pxe-boot`, the Proxmox container must have +`features: nesting=1,mount=nfs` (at minimum) in its Proxmox config. `nesting=1` +is required by systemd 260+ for credential isolation (user namespace creation +and internal move-mounts); without it, AppArmor denies both, and every +systemd service that uses `PrivateUsers`, `PrivateDevices`, or credential +passing fails on boot. `mount=nfs` allows the NFSv3 mount. Both are set +automatically by `scripts/proxmox/create-proxmox-resource.sh` (via +`PROXMOX_DEFAULT_LXC_FEATURES` in `scripts/env.sh` which defaults to +`nesting=1,keyctl=1,mount=nfs;nfs4`). If you ever change these features +manually via `pct set`, be sure to include both — `pct set` replaces the +entire features string, it does not append to it. The HTTP iPXE chain is: ```text undionly.kpxe or ipxe.efi -> autoexec.ipxe from the TFTP root, when iPXE requests it - -> http://192.168.2.247/boot.ipxe - -> http://192.168.2.247/menu.ipxe + -> http://192.168.2.223/boot.ipxe + -> http://192.168.2.223/menu.ipxe ``` The generated menu currently exposes entries for: - NixOS Auto-Installer - NixOS Minimal +- Debian Minimal +- FreeIPA Server (Rocky Linux 9) - SystemRescue environment - iPXE shell - Reboot @@ -84,19 +104,82 @@ directory name (`auto-installer` / `nixos-minimal`), so each one's generated system name (`nixos-system--*`) is self-describing rather than the nixpkgs default of `nixos-system-nixos-*` for both. +The Debian Minimal entry chains `http:///debian.ipxe`, which loads +the Debian bookworm netboot kernel and initrd from `/srv/pxe/http/debian/`. The +`fetch-debian-netboot.service` oneshot downloads these files from +`deb.debian.org` on first boot (idempotent — skips if files are already +present): + +```text +/srv/pxe/http/debian/linux (Debian bookworm netboot kernel) +/srv/pxe/http/debian/initrd.gz (Debian bookworm netboot initrd) +``` + +The service requires outbound internet access on the pxe-boot host. To +re-download (e.g. after a Debian point release), delete the files and restart +the service: + +```bash +rm /srv/pxe/http/debian/linux /srv/pxe/http/debian/initrd.gz +systemctl restart fetch-debian-netboot.service +``` + +To update to a different Debian release, change `debianRelease` in +`modules/build-types/pxe-boot.nix` and redeploy. + +The **FreeIPA Server (Rocky Linux 9)** entry chains +`http:///rocky-freeipa.ipxe`, which boots the Rocky Linux 9 +Anaconda installer with a Kickstart file (`rocky-freeipa.ks`) hosted on the +same server. The `fetch-rocky-pxeboot.service` oneshot downloads the pxeboot +kernel and initrd from the Rocky Linux mirror on first boot (idempotent): + +```text +/srv/pxe/http/rocky/vmlinuz (Rocky Linux 9 Anaconda pxeboot kernel) +/srv/pxe/http/rocky/initrd.img (Rocky Linux 9 Anaconda pxeboot initrd) +``` + +The Kickstart file is generated from the NixOS module and staged at +`/srv/pxe/http/rocky-freeipa.ks`. It performs a fully unattended install: + +1. Installs Rocky Linux 9 with `ipa-server` + `ipa-server-dns` packages +2. Configures static IP `192.168.2.138`, hostname `domain-controller.sweet.home` +3. Creates user `wayne` with the `adminSshKey` from `variables.nix` +4. Generates random IPA passwords and writes them to `/root/ipa-credentials.txt` +5. Creates a `freeipa-first-boot.service` oneshot that runs `ipa-server-install` + on first reboot (~20 minutes) + +After the install completes: +- SSH in as `wayne@domain-controller` using the admin key +- Monitor FreeIPA install progress: `sudo tail -f /root/freeipa-install.log` +- Retrieve credentials: `sudo cat /root/ipa-credentials.txt` (save to password manager) +- Configure Pi-hole: `server=/sweet.home/192.168.2.138` in dnsmasq + +To refresh the pxeboot files (e.g. after a Rocky point release): + +```bash +rm /srv/pxe/http/rocky/vmlinuz /srv/pxe/http/rocky/initrd.img +systemctl restart fetch-rocky-pxeboot.service +``` + +To update to a different Rocky release, change `rockyRelease` in +`modules/build-types/pxe-boot.nix` and redeploy. + The SystemRescue entry expects the source ISO at: ```text /srv/pxe/http/images/systemrescue.iso ``` +Since `/srv/pxe/http/images` is the NFS-backed symlink, place the ISO on the +NFS share at `server.sweet.home:/tank/pxe-boot/images/systemrescue.iso`. + The `stage-systemrescue.service` oneshot extracts that ISO into: ```text /srv/pxe/http/systemrescue ``` -The rescue menu entry then chains `http://192.168.2.247/systemrescue.ipxe`, +The rescue menu entry then chains `http://192.168.2.223/systemrescue.ipxe`, which loads the SystemRescue kernel and initramfs from the extracted tree and uses `archiso_http_srv` to fetch the squashfs payload over HTTP. @@ -113,6 +196,13 @@ After deployment by an operator, basic service checks are: ```bash curl http://pxe-boot/boot.ipxe curl http://pxe-boot/menu.ipxe +curl http://pxe-boot/debian.ipxe +curl -I http://pxe-boot/debian/linux +curl -I http://pxe-boot/debian/initrd.gz +curl http://pxe-boot/rocky-freeipa.ipxe +curl http://pxe-boot/rocky-freeipa.ks +curl -I http://pxe-boot/rocky/vmlinuz +curl -I http://pxe-boot/rocky/initrd.img curl http://pxe-boot/systemrescue.ipxe curl -I http://pxe-boot/systemrescue/sysresccd/boot/x86_64/vmlinuz curl -I http://pxe-boot/systemrescue/sysresccd/boot/x86_64/sysresccd.img diff --git a/flake.nix b/flake.nix index 47ac295..d4b0aa0 100644 --- a/flake.nix +++ b/flake.nix @@ -130,6 +130,9 @@ lxc-tailscale-router = mkTarget { platform = "lxc"; buildType = "tailscale-router"; hostPath = ./hosts/tailscale-router/host.nix; }; lxc-tor-relay = mkTarget { platform = "lxc"; buildType = "tor-relay"; hostPath = ./hosts/tor-relay/host.nix; }; + + proxmox-ha-server-1 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-1/host.nix; }; + proxmox-ha-server-2 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-2/host.nix; }; }; # Auto-install environments (migrated from the former nix-auto-installer diff --git a/hosts/docker/host.nix b/hosts/docker/host.nix index f625e1e..4f9fda4 100644 --- a/hosts/docker/host.nix +++ b/hosts/docker/host.nix @@ -1,8 +1,17 @@ -_: +{ vars, ... }: { - networking.hostName = "docker"; - networking.hostId = "007f0200"; + networking = { + hostName = "docker"; + hostId = "007f0200"; + useDHCP = false; + interfaces.${vars.vmLanInterface}.ipv4.addresses = [{ + address = vars.dockerIp; + prefixLength = vars.lanPrefixLength; + }]; + defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; }; + nameservers = [ vars.domainControllerIp ]; + }; boot.zfs.forceImportRoot = false; # Preserved from the pre-refactor `docker` target — stateVersion must never diff --git a/hosts/ha-server-1/host.nix b/hosts/ha-server-1/host.nix new file mode 100644 index 0000000..6afb486 --- /dev/null +++ b/hosts/ha-server-1/host.nix @@ -0,0 +1,30 @@ +{ vars, ... }: +{ + imports = [ + (import ../../modules/beszel/host-token.nix { + name = "ha-server-1"; + sopsFile = ../../secrets/ha-server-1.yaml; + }) + ]; + + networking = { + hostName = vars.haServer1Host; + hostId = "3a4b5c6d"; + useDHCP = false; + interfaces.${vars.vmLanInterface}.ipv4.addresses = [{ + address = vars.haServer1Ip; + prefixLength = vars.lanPrefixLength; + }]; + interfaces.${vars.vmStorageInterface}.ipv4.addresses = [{ + address = vars.haServer1StorageIp; + prefixLength = vars.haStoragePrefixLength; + }]; + defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; }; + nameservers = [ vars.domainControllerIp ]; + }; + + # Set KEY after pairing this host with the beszel hub; the token is sops-managed. + services.beszel.agent.environment.KEY = ""; + + system.stateVersion = "26.05"; +} diff --git a/hosts/ha-server-2/host.nix b/hosts/ha-server-2/host.nix new file mode 100644 index 0000000..2f798f4 --- /dev/null +++ b/hosts/ha-server-2/host.nix @@ -0,0 +1,30 @@ +{ vars, ... }: +{ + imports = [ + (import ../../modules/beszel/host-token.nix { + name = "ha-server-2"; + sopsFile = ../../secrets/ha-server-2.yaml; + }) + ]; + + networking = { + hostName = vars.haServer2Host; + hostId = "7e8f9a0b"; + useDHCP = false; + interfaces.${vars.vmLanInterface}.ipv4.addresses = [{ + address = vars.haServer2Ip; + prefixLength = vars.lanPrefixLength; + }]; + interfaces.${vars.vmStorageInterface}.ipv4.addresses = [{ + address = vars.haServer2StorageIp; + prefixLength = vars.haStoragePrefixLength; + }]; + defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; }; + nameservers = [ vars.domainControllerIp ]; + }; + + # Set KEY after pairing this host with the beszel hub; the token is sops-managed. + services.beszel.agent.environment.KEY = ""; + + system.stateVersion = "26.05"; +} diff --git a/hosts/nix-cache/host.nix b/hosts/nix-cache/host.nix index 44146eb..89f6c55 100644 --- a/hosts/nix-cache/host.nix +++ b/hosts/nix-cache/host.nix @@ -8,7 +8,16 @@ }) ]; - networking.hostName = vars.nixCacheHost; + networking = { + hostName = vars.nixCacheHost; + useDHCP = false; + interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{ + address = vars.nixCacheIp; + prefixLength = vars.lanPrefixLength; + }]; + defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; }; + nameservers = [ vars.domainControllerIp ]; + }; services.beszel.agent.environment = { #DOCKER_HOST = "tcp://docker-socket-proxy:2375"; diff --git a/hosts/nixos/home.nix b/hosts/nixos/home.nix index 64b8b7f..65702e6 100644 --- a/hosts/nixos/home.nix +++ b/hosts/nixos/home.nix @@ -27,7 +27,7 @@ # Optional: set environment vars sessionVariables = { EDITOR = "vim"; - SOPS_AGE_KEY_FILE = "~/.config/sops/age/keys.txt"; + SOPS_AGE_KEY_FILE = "${config.home.homeDirectory}/.config/sops/age/keys.txt"; }; file = { diff --git a/hosts/pxe-boot/host.nix b/hosts/pxe-boot/host.nix index 99378bb..639ecd2 100644 --- a/hosts/pxe-boot/host.nix +++ b/hosts/pxe-boot/host.nix @@ -1,7 +1,16 @@ -_: +{ vars, ... }: { - networking.hostName = "pxe-boot"; + networking = { + hostName = "pxe-boot"; + useDHCP = false; + interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{ + address = vars.pxeServerIp; + prefixLength = vars.lanPrefixLength; + }]; + defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; }; + nameservers = [ vars.domainControllerIp ]; + }; # Preserved from the pre-refactor `pxe-boot` target — stateVersion must # never be bumped on an already-installed machine. diff --git a/hosts/server/host.nix b/hosts/server/host.nix index ec43972..38544d5 100644 --- a/hosts/server/host.nix +++ b/hosts/server/host.nix @@ -8,8 +8,17 @@ }) ]; - networking.hostName = vars.nfsServerHost; - networking.hostId = "6689f93e"; + networking = { + hostName = vars.nfsServerHost; + hostId = "6689f93e"; + useDHCP = false; + interfaces.${vars.vmLanInterface}.ipv4.addresses = [{ + address = vars.serverIp; + prefixLength = vars.lanPrefixLength; + }]; + defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; }; + nameservers = [ vars.domainControllerIp ]; + }; services.beszel.agent.environment = { #DOCKER_HOST = "tcp://docker-socket-proxy:2375"; diff --git a/hosts/tailscale-router/host.nix b/hosts/tailscale-router/host.nix index d5b09f1..6198bf4 100644 --- a/hosts/tailscale-router/host.nix +++ b/hosts/tailscale-router/host.nix @@ -1,4 +1,4 @@ -_: +{ vars, ... }: { imports = [ @@ -8,7 +8,16 @@ _: }) ]; - networking.hostName = "tailscale-router"; + networking = { + hostName = "tailscale-router"; + useDHCP = false; + interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{ + address = vars.tailscaleRouterIp; + prefixLength = vars.lanPrefixLength; + }]; + defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; }; + nameservers = [ vars.domainControllerIp ]; + }; services.beszel.agent.environment = { KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG"; diff --git a/hosts/tor-relay/host.nix b/hosts/tor-relay/host.nix index bac4552..8cdcad3 100644 --- a/hosts/tor-relay/host.nix +++ b/hosts/tor-relay/host.nix @@ -1,4 +1,4 @@ -{ ... }: +{ vars, ... }: { imports = [ @@ -8,10 +8,19 @@ }) ]; - networking.hostName = "tor-relay"; + networking = { + hostName = "tor-relay"; + useDHCP = false; + interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{ + address = vars.torRelayIp; + prefixLength = vars.lanPrefixLength; + }]; + defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; }; + nameservers = [ vars.domainControllerIp ]; + }; - # No networking.hostId: only ZFS-touching hosts (server, docker) need one - # for pool-import safety, and this host does neither. + # No networking.hostId: only ZFS-touching hosts need one for pool-import + # safety, and this host does neither. services.beszel.agent.environment = { KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG"; diff --git a/modules/build-types/ha-server.nix b/modules/build-types/ha-server.nix new file mode 100644 index 0000000..26026ac --- /dev/null +++ b/modules/build-types/ha-server.nix @@ -0,0 +1,43 @@ +# HA file server build type: DRBD + XFS + LIO iSCSI + NFS, managed by +# Corosync + Pacemaker. Both ha-server-1 and ha-server-2 use this type. +# +# NFS start/stop: +# services.nfs.server.enable = true configures /etc/exports, wires up +# rpcbind, and loads kernel modules — but nfs-server.service.wantedBy is +# force-cleared so systemd does NOT auto-start it at boot. Pacemaker's +# ha-group resource group (configured by scripts/ha/cluster-init.sh) +# starts and stops nfs-server as part of the failover sequence after the +# XFS mount and iSCSI target are brought up on the new Active node. +# +# Beszel agent: +# Enabled here via enable-agent.nix. The agent KEY (used to pair with +# the Beszel hub) is not set yet — add it to hosts/ha-server-{1,2}/host.nix +# under services.beszel.agent.environment.KEY once the hub accepts the +# new agents, following the pattern in hosts/server/host.nix. +{ lib, vars, ... }: +{ + imports = [ + ../ha/pacemaker-stack.nix + ../ha/iscsi-target.nix + ../ha/cluster-config.nix + ../beszel/enable-agent.nix + ]; + + services.nfs.server = { + enable = true; + exports = '' + ${vars.haStorageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.haStorageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ''; + }; + + # Pacemaker controls nfs-server — prevent systemd from starting it at boot + # on both nodes (only the Active node should be serving NFS). + systemd.services.nfs-server.wantedBy = lib.mkForce [ ]; +} diff --git a/modules/build-types/pxe-boot.nix b/modules/build-types/pxe-boot.nix index 4dcd88b..034ddc2 100644 --- a/modules/build-types/pxe-boot.nix +++ b/modules/build-types/pxe-boot.nix @@ -21,6 +21,216 @@ let chain ${pxeBaseUrl}/boot.ipxe ''; + debianRelease = "bookworm"; + debianMirror = "https://deb.debian.org/debian"; + debianNetbootBase = "${debianMirror}/dists/${debianRelease}/main/installer-amd64/current/images/netboot/debian-installer/amd64"; + + rockyRelease = "9"; + rockyArch = "x86_64"; + rockyMirror = "https://dl.rockylinux.org/pub/rocky/${rockyRelease}"; + rockyPxebootBase = "${rockyMirror}/BaseOS/${rockyArch}/os/images/pxeboot"; + + debianIpxe = pkgs.writeText "debian.ipxe" '' + #!ipxe + + set base ${pxeBaseUrl} + + kernel ''${base}/debian/linux + initrd ''${base}/debian/initrd.gz + boot + ''; + + fetchDebianNetboot = pkgs.writeShellScript "fetch-debian-netboot" '' + set -eu + + dir="${httpRoot}/debian" + mirror="${debianNetbootBase}" + + if [ -f "$dir/linux" ] && [ -f "$dir/initrd.gz" ]; then + echo "Debian ${debianRelease} netboot files already present; skipping download." + exit 0 + fi + + echo "Downloading Debian ${debianRelease} netboot kernel and initrd from $mirror ..." + ${pkgs.curl}/bin/curl -fsSL -o "$dir/linux.tmp" "$mirror/linux" + ${pkgs.curl}/bin/curl -fsSL -o "$dir/initrd.gz.tmp" "$mirror/initrd.gz" + mv "$dir/linux.tmp" "$dir/linux" + mv "$dir/initrd.gz.tmp" "$dir/initrd.gz" + echo "Debian ${debianRelease} netboot files staged." + ''; + + # Rocky Linux 9 iPXE script — boots vmlinuz+initrd.img from the staged + # /rocky/ directory and hands Anaconda the hosted Kickstart URL. + # net.ifnames=0 biosdevname=0 ensures the NIC is eth0 in both the + # installer and the installed system (matches the Kickstart NM config). + rockyFreeIpaIpxe = pkgs.writeText "rocky-freeipa.ipxe" '' + #!ipxe + + set base ${pxeBaseUrl} + + kernel ''${base}/rocky/vmlinuz inst.ks=''${base}/rocky-freeipa.ks inst.repo=${rockyMirror}/BaseOS/${rockyArch}/os/ net.ifnames=0 biosdevname=0 ip=dhcp quiet + initrd ''${base}/rocky/initrd.img + boot + ''; + + # Kickstart file for domain-controller.sweet.home. + # Installs Rocky Linux 9, sets a static IP, creates wayne with the + # admin SSH key, then on first reboot runs ipa-server-install via a + # systemd oneshot service. Passwords are generated at %post time, + # written to /root/ipa-credentials.txt (chmod 600), and read back by + # the first-boot script — never hardcoded here or in the repo. + rockyFreeIpaKs = pkgs.writeText "rocky-freeipa.ks" '' + #version=RHEL9 + # Unattended Rocky Linux 9 + FreeIPA install + # Target: domain-controller.${vars.homeDomain} ${vars.domainControllerIp} + + url --url=${rockyMirror}/BaseOS/${rockyArch}/os/ + repo --name=appstream --baseurl=${rockyMirror}/AppStream/${rockyArch}/os/ + + lang en_US.UTF-8 + keyboard us + timezone UTC --utc + + # DHCP during install; static IP configured in %post via NM config file + network --bootproto=dhcp --device=link --activate + network --hostname=domain-controller.sweet.home + + selinux --enforcing + firewall --enabled --service=ssh + + rootpw --lock + user --name=wayne --groups=wheel --shell=/bin/bash + sshkey --username=wayne "${vars.adminSshKey}" + + zerombr + clearpart --all --initlabel --drives=sda + # Keep net.ifnames=0 biosdevname=0 in the installed GRUB so the NIC + # stays eth0 after reboot (matches the NM connection file below). + bootloader --location=mbr --boot-drive=sda --append="net.ifnames=0 biosdevname=0" + + part /boot --fstype=xfs --size=1024 --ondisk=sda + part swap --fstype=swap --size=2048 --ondisk=sda + part / --fstype=xfs --grow --size=1 --ondisk=sda --asprimary + + %packages + @^minimal-environment + ipa-server + ipa-server-dns + %end + + reboot + + %post --log=/root/ks-post.log + set -euo pipefail + + # -- Static IP: write NM connection file directly (NM not running in chroot) -- + mkdir -p /etc/NetworkManager/system-connections + cat > /etc/NetworkManager/system-connections/eth0.nmconnection << 'NMCONN' + [connection] + id=eth0 + type=ethernet + interface-name=eth0 + autoconnect=true + + [ethernet] + + [ipv4] + method=manual + addresses=${vars.domainControllerIp}/${toString vars.lanPrefixLength} + gateway=${vars.lanGateway} + dns=${vars.domainControllerIp}; + dns-search=${vars.homeDomain}; + + [ipv6] + method=auto + NMCONN + chmod 600 /etc/NetworkManager/system-connections/eth0.nmconnection + + # -- /etc/hosts: FQDN must resolve to the real IP (not loopback) for IPA -- + sed -i '/domain-controller/d' /etc/hosts + echo '${vars.domainControllerIp} domain-controller.${vars.homeDomain} domain-controller' >> /etc/hosts + + # -- Generate IPA passwords and store securely -- + DM_PASS=$(openssl rand -base64 24 | tr -dc 'A-Za-z0-9' | head -c 24) + ADMIN_PASS=$(openssl rand -base64 24 | tr -dc 'A-Za-z0-9' | head -c 24) + printf 'Directory Manager: %s\nIPA Admin: %s\n' "$DM_PASS" "$ADMIN_PASS" \ + > /root/ipa-credentials.txt + chmod 600 /root/ipa-credentials.txt + + # -- First-boot script: reads passwords back, runs ipa-server-install -- + cat > /usr/local/sbin/freeipa-first-boot.sh << 'FIRSTBOOT' + #!/bin/bash + set -euo pipefail + exec >> /root/freeipa-install.log 2>&1 + echo "=== FreeIPA first-boot install started at $(date) ===" + + DM_PASS=$(grep '^Directory Manager:' /root/ipa-credentials.txt | awk '{print $NF}') + ADMIN_PASS=$(grep '^IPA Admin:' /root/ipa-credentials.txt | awk '{print $NF}') + + ipa-server-install \ + --realm=SWEET.HOME \ + --domain=sweet.home \ + --hostname=domain-controller.sweet.home \ + --ds-password="$DM_PASS" \ + --admin-password="$ADMIN_PASS" \ + --setup-dns \ + --forwarder=192.168.2.253 \ + --no-dnssec-validation \ + --no-ntp \ + --unattended + + echo "=== FreeIPA install complete at $(date) ===" + echo "Credentials: /root/ipa-credentials.txt (save to password manager)" + echo "CA backup: /root/cacert.p12 (encrypted with Directory Manager password)" + systemctl disable freeipa-first-boot.service + FIRSTBOOT + chmod 700 /usr/local/sbin/freeipa-first-boot.sh + + # -- Systemd oneshot service: runs freeipa-first-boot.sh on first real boot -- + cat > /etc/systemd/system/freeipa-first-boot.service << 'UNIT' + [Unit] + Description=FreeIPA first-boot installation + After=network-online.target + Wants=network-online.target + ConditionPathExists=/root/ipa-credentials.txt + + [Service] + Type=oneshot + ExecStart=/usr/local/sbin/freeipa-first-boot.sh + TimeoutStartSec=1800 + RemainAfterExit=yes + + [Install] + WantedBy=multi-user.target + UNIT + + mkdir -p /etc/systemd/system/multi-user.target.wants + ln -sf /etc/systemd/system/freeipa-first-boot.service \ + /etc/systemd/system/multi-user.target.wants/freeipa-first-boot.service + + echo "Kickstart %post complete. FreeIPA installs on first reboot (~20 min)." + %end + ''; + + fetchRockyPxeboot = pkgs.writeShellScript "fetch-rocky-pxeboot" '' + set -eu + + dir="${httpRoot}/rocky" + base="${rockyPxebootBase}" + + if [ -f "$dir/vmlinuz" ] && [ -f "$dir/initrd.img" ]; then + echo "Rocky Linux ${rockyRelease} pxeboot files already present; skipping download." + exit 0 + fi + + echo "Downloading Rocky Linux ${rockyRelease} pxeboot kernel and initrd from $base ..." + ${pkgs.curl}/bin/curl -fsSL -o "$dir/vmlinuz.tmp" "$base/vmlinuz" + ${pkgs.curl}/bin/curl -fsSL -o "$dir/initrd.img.tmp" "$base/initrd.img" + mv "$dir/vmlinuz.tmp" "$dir/vmlinuz" + mv "$dir/initrd.img.tmp" "$dir/initrd.img" + echo "Rocky Linux ${rockyRelease} pxeboot files staged." + ''; + systemRescueIpxe = pkgs.writeText "systemrescue.ipxe" '' #!ipxe @@ -68,11 +278,13 @@ let set base ${pxeBaseUrl} menu PXE Boot Menu - item auto-installer NixOS Auto-Installer - item nixos-minimal NixOS Minimal - item rescue Rescue Environment - item shell iPXE Shell - item reboot Reboot + item auto-installer NixOS Auto-Installer + item nixos-minimal NixOS Minimal + item debian Debian Minimal + item rocky-freeipa FreeIPA Server (Rocky Linux 9) + item rescue Rescue Environment + item shell iPXE Shell + item reboot Reboot choose target && goto ''${target} @@ -82,6 +294,12 @@ let :nixos-minimal chain ''${base}/nixos-minimal/netboot.ipxe + :debian + chain ''${base}/debian.ipxe + + :rocky-freeipa + chain ''${base}/rocky-freeipa.ipxe + :rescue chain ''${base}/systemrescue.ipxe @@ -95,6 +313,7 @@ in { imports = [ ../pxe-boot/stage-installer-artifacts.nix + ../pxe-boot/mount-pxe-images.nix ]; environment.systemPackages = with pkgs; [ @@ -129,37 +348,101 @@ in openssh.settings.PermitRootLogin = "yes"; }; - systemd.tmpfiles.rules = [ - "d ${pxeRoot} 0755 root root -" - "d ${httpRoot} 0755 root root -" - "d ${httpRoot}/images 0755 root root -" - "d ${httpRoot}/auto-installer 0755 root root -" - "d ${httpRoot}/nixos-minimal 0755 root root -" - "d ${httpRoot}/systemrescue 0755 root root -" - "d ${httpRoot}/ubuntu 0755 root root -" - "d ${httpRoot}/rescue 0755 root root -" - "d ${tftpRoot} 0755 root root -" - "C+ ${httpRoot}/boot.ipxe 0644 root root - ${bootIpxe}" - "C+ ${httpRoot}/menu.ipxe 0644 root root - ${menuIpxe}" - "C+ ${httpRoot}/systemrescue.ipxe 0644 root root - ${systemRescueIpxe}" - "C+ ${tftpRoot}/autoexec.ipxe 0644 root root - ${autoexecIpxe}" - "C+ ${tftpRoot}/ipxe.efi 0644 root root - ${pkgs.ipxe}/ipxe.efi" - "C+ ${tftpRoot}/undionly.kpxe 0644 root root - ${pkgs.ipxe}/undionly.kpxe" - ]; - - systemd.services.stage-systemrescue = { - description = "Stage SystemRescue ISO contents for HTTP PXE boot"; - after = [ - "local-fs.target" - "systemd-tmpfiles-setup.service" + systemd = { + tmpfiles.rules = [ + "d ${pxeRoot} 0755 root root -" + "d ${httpRoot} 0755 root root -" + "L+ ${httpRoot}/images - - - - ${vars.nfsShares.pxebootImages.mountpoint}" + "d ${httpRoot}/auto-installer 0755 root root -" + "d ${httpRoot}/nixos-minimal 0755 root root -" + "d ${httpRoot}/systemrescue 0755 root root -" + "d ${httpRoot}/debian 0755 root root -" + "d ${httpRoot}/ubuntu 0755 root root -" + "d ${httpRoot}/rescue 0755 root root -" + "d ${httpRoot}/rocky 0755 root root -" + "d ${tftpRoot} 0755 root root -" + "C+ ${httpRoot}/boot.ipxe 0644 root root - ${bootIpxe}" + "C+ ${httpRoot}/menu.ipxe 0644 root root - ${menuIpxe}" + "C+ ${httpRoot}/debian.ipxe 0644 root root - ${debianIpxe}" + "C+ ${httpRoot}/rocky-freeipa.ipxe 0644 root root - ${rockyFreeIpaIpxe}" + "C+ ${httpRoot}/rocky-freeipa.ks 0644 root root - ${rockyFreeIpaKs}" + "C+ ${httpRoot}/systemrescue.ipxe 0644 root root - ${systemRescueIpxe}" + "C+ ${tftpRoot}/autoexec.ipxe 0644 root root - ${autoexecIpxe}" + "C+ ${tftpRoot}/ipxe.efi 0644 root root - ${pkgs.ipxe}/ipxe.efi" + "C+ ${tftpRoot}/undionly.kpxe 0644 root root - ${pkgs.ipxe}/undionly.kpxe" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - Type = "oneshot"; - ExecStart = stageSystemRescue; + + services = { + fetch-debian-netboot = { + description = "Download Debian ${debianRelease} netboot kernel and initrd for HTTP PXE boot"; + after = [ + "local-fs.target" + "systemd-tmpfiles-setup.service" + "network-online.target" + ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = fetchDebianNetboot; + RemainAfterExit = true; + }; + }; + + fetch-rocky-pxeboot = { + description = "Download Rocky Linux ${rockyRelease} pxeboot kernel and initrd for HTTP PXE boot"; + after = [ + "local-fs.target" + "systemd-tmpfiles-setup.service" + "network-online.target" + ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = fetchRockyPxeboot; + RemainAfterExit = true; + }; + }; + + stage-systemrescue = { + description = "Stage SystemRescue ISO contents for HTTP PXE boot"; + after = [ + "local-fs.target" + "systemd-tmpfiles-setup.service" + ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = stageSystemRescue; + }; + }; + }; + }; + + services.dnsmasq = { + enable = true; + settings = { + # Disable DNS listener — only proxy DHCP is needed here. + # Without this dnsmasq tries to bind port 53 which systemd-resolved + # already owns, causing startup failure. + port = 0; + dhcp-range = [ "192.168.2.0,proxy" ]; + dhcp-match = [ + "set:ipxe,175" + "set:efi64,option:client-arch,7" + "set:efi64,option:client-arch,9" + ]; + dhcp-userclass = "set:ipxe,iPXE"; + dhcp-boot = [ + "tag:ipxe,tag:efi64,http://${vars.pxeServerIp}/boot.ipxe" + "tag:ipxe,http://${vars.pxeServerIp}/boot.ipxe" + "tag:efi64,ipxe.efi,,${vars.pxeServerIp}" + "undionly.kpxe,,${vars.pxeServerIp}" + ]; }; }; networking.firewall.allowedTCPPorts = [ vars.ports.pxeBootHttp ]; - networking.firewall.allowedUDPPorts = [ vars.ports.pxeBootTftp ]; + networking.firewall.allowedUDPPorts = [ vars.ports.pxeBootTftp 67 ]; } diff --git a/modules/build-types/server.nix b/modules/build-types/server.nix index fdce1af..5455065 100644 --- a/modules/build-types/server.nix +++ b/modules/build-types/server.nix @@ -10,7 +10,8 @@ let in lib.imap1 (i: _: lib.concatStringsSep "/" (lib.take i parts)) parts; poolDatasets = lib.unique ( - lib.concatMap (share: ancestors share.subpath) (lib.attrValues vars.nfsShares) + lib.concatMap (share: ancestors share.subpath) + (lib.filter builtins.isAttrs (lib.attrValues vars.nfsShares)) ); in { @@ -43,14 +44,8 @@ in exit 0 fi - # Pool exists on a device but not yet imported — let the standard - # zfs-import-${poolName}.service handle it normally. - if zpool import -d /dev/disk/by-id -N "${poolName}" 2>/dev/null; then - exit 0 - fi - - # No pool found at all. Create it on the Proxmox data disk (scsi1), - # which appears as /dev/disk/by-id/scsi-*drive-scsi1 inside the VM. + # Locate the data disk first — used for both the fallback import + # attempt and, only if the disk is genuinely blank, pool creation. DATA_DISK="" for candidate in /dev/disk/by-id/scsi-*drive-scsi1; do [[ "$candidate" == *-part* ]] && continue @@ -62,8 +57,31 @@ in exit 1 fi + # Try importing via the by-id symlink directory first (normal path), + # then fall back to scanning the disk directly. The two-step exists + # because of a udev race: systemd-udev-settle.service can clear before + # /dev/disk/by-id/ entries are fully populated, causing the first + # import to fail even when the pool is intact on the disk. + if zpool import -d /dev/disk/by-id -N "${poolName}" 2>/dev/null; then + exit 0 + fi + if zpool import -d "$DATA_DISK" -N "${poolName}" 2>/dev/null; then + exit 0 + fi + + # Both import attempts failed. Before creating a new pool, verify the + # disk is genuinely blank — if ZFS label metadata is present the import + # failed for some other reason and we must not clobber existing data. + if zdb -l "$DATA_DISK" 2>/dev/null | grep -q "name: '${poolName}'"; then + echo "zfs-init-${poolName}: $DATA_DISK has ZFS pool '${poolName}' metadata but import failed — refusing to overwrite existing data. Run 'zpool import -d $DATA_DISK ${poolName}' manually to investigate." >&2 + exit 1 + fi + + # Disk is genuinely blank: create the pool. -f is intentionally + # omitted so that if we somehow reach this point with an existing pool + # on the disk, zpool refuses rather than silently destroying data. echo "zfs-init-${poolName}: creating pool on $DATA_DISK" - zpool create -f "${poolName}" "$DATA_DISK" + zpool create "${poolName}" "$DATA_DISK" ${lib.concatMapStrings (ds: '' zfs create "${poolName}/${ds}" '') poolDatasets} @@ -78,13 +96,20 @@ in services.nfs.server = { enable = true; exports = '' - ${vars.storageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash) - ${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash) - ${vars.storageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash) - ${vars.storageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash) - ${vars.storageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash) + ${vars.storageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.storageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.storageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.storageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.storageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.storageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options} + ${vars.storageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options} ''; }; - networking.firewall.allowedTCPPorts = [ vars.ports.nfsRpcbind vars.ports.nfsd ]; + # mountd (20048) is needed for showmount/NFSv3 mount protocol — without it + # clients can reach portmapper (111) and get the mountd port back, then + # time out trying to connect to it. All three ports need TCP and UDP. + networking.firewall.allowedTCPPorts = [ vars.ports.nfsRpcbind vars.ports.nfsd vars.ports.nfsMountd ]; + networking.firewall.allowedUDPPorts = [ vars.ports.nfsRpcbind vars.ports.nfsd vars.ports.nfsMountd ]; } diff --git a/modules/build-types/tailscale-router.nix b/modules/build-types/tailscale-router.nix index bf3e1b2..557eef3 100644 --- a/modules/build-types/tailscale-router.nix +++ b/modules/build-types/tailscale-router.nix @@ -3,6 +3,7 @@ { imports = [ ../tailscale/subnet-router.nix + ../tailscale/ts-dns-forwarder.nix ../beszel/enable-agent.nix ]; diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix new file mode 100644 index 0000000..5b55776 --- /dev/null +++ b/modules/ha/cluster-config.nix @@ -0,0 +1,107 @@ +# Cluster-wide HA config shared by both ha-server nodes. +# +# Covers everything that is identical on both nodes and references cluster +# topology (node IPs, hostnames, DRBD resource). Per-node identity +# (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix. +# +# Corosync authkey: +# /etc/corosync/authkey (mode 0400) is managed by sops-nix below. +# Bootstrap: run scripts/ha/cluster-init.sh on node1 to generate the key, +# then encrypt it with: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey +# Both host keys must be registered via sync-host-keys.sh first so both nodes can decrypt it. +# +# DRBD fencing: +# Production setting is resource-only: DRBD waits for the STONITH fence +# agent to confirm the peer is dead before promoting to Primary. This +# requires a working fence_pve_ssh STONITH resource in Pacemaker +# (see scripts/ha/cluster-enable-stonith.sh). On a fresh cluster with +# no fence device yet, temporarily change to dont-care and run +# cluster-enable-stonith.sh once the fence key is deployed. +{ lib, vars, ... }: +{ + services.drbd = { + enable = true; + config = '' + global { + usage-count yes; + } + + common { + net { + protocol C; + ping-int 1; + verify-alg sha256; + after-sb-0pri discard-zero-changes; + after-sb-1pri discard-secondary; + } + disk { + fencing resource-only; + } + } + + resource ha-data { + volume 0 { + device /dev/drbd0; + disk /dev/sdb; + meta-disk internal; + } + + on ${vars.haServer1Host} { + address ${vars.haServer1StorageIp}:${toString vars.ports.haServerDrbd}; + } + + on ${vars.haServer2Host} { + address ${vars.haServer2StorageIp}:${toString vars.ports.haServerDrbd}; + } + } + ''; + }; + + # /etc/corosync/authkey — sops binary secret, identical on both nodes. + # Decryptable by both ha-server host keys (added by sync-host-keys.sh). + sops.secrets.corosync_authkey = { + sopsFile = ../../secrets/ha-corosync-authkey; + format = "binary"; + path = "/etc/corosync/authkey"; + mode = "0400"; + restartUnits = [ "corosync.service" ]; + }; + + # NixOS common config enables NetworkManager by default; HA cluster nodes + # need stable static IPs with predictable interface names — NM is not suitable. + networking.networkmanager.enable = lib.mkForce false; + + # services.corosync.enable is set by modules/ha/pacemaker-stack.nix. + services.corosync = { + clusterName = "ha-cluster"; + nodelist = [ + { nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1StorageIp ]; } + { nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2StorageIp ]; } + ]; + }; + + networking.firewall = { + allowedTCPPorts = [ + vars.ports.haServerIscsi + vars.ports.haServerPacemakerRemoted + vars.ports.haServerPcsd + vars.ports.haServerDrbd + vars.ports.nfsRpcbind + vars.ports.nfsd + vars.ports.nfsMountd + ]; + allowedUDPPorts = [ + vars.ports.haServerCorosync1 + vars.ports.haServerCorosync2 + vars.ports.haServerCorosyncCrypto + vars.ports.nfsRpcbind + vars.ports.nfsd + vars.ports.nfsMountd + ]; + extraCommands = '' + iptables -A INPUT -s ${vars.haServer1Ip}/32 -j ACCEPT + iptables -A INPUT -s ${vars.haServer2Ip}/32 -j ACCEPT + iptables -A INPUT -s ${vars.haStorageCidr} -j ACCEPT + ''; + }; +} diff --git a/modules/ha/iscsi-target.nix b/modules/ha/iscsi-target.nix new file mode 100644 index 0000000..c39ec88 --- /dev/null +++ b/modules/ha/iscsi-target.nix @@ -0,0 +1,99 @@ +# LIO iSCSI target service (targetctl) for NixOS HA clusters. +# +# Provides the targetctl.service that saves/restores LIO configuration from +# /etc/target/saveconfig.json. Pacemaker manages this service via its +# systemd resource agent (class="systemd" type="targetctl"). +# +# Why ExecStop is not simply "targetctl save": +# targetctl save writes the LIO config to JSON but does NOT remove the LIO +# target from the kernel's configfs. As a result, any fileio backing store +# that LIO has open (e.g. iscsi-lun.img on an XFS-over-DRBD filesystem) +# stays referenced in the kernel. The subsequent XFS umount from the +# Filesystem OCF resource then returns EBUSY and either hangs for the full +# op-stop timeout or fails outright, blocking the entire failover. +# +# The ExecStop script here additionally tears down the kernel LIO state +# via rtslib_fb after saving, so the backing-store file descriptor is +# released and umount succeeds immediately. +# +# Empty-config guard: +# The save step is skipped when no iSCSI targets are currently active. +# This prevents the secondary node (where LIO was never started) from +# overwriting a valid saveconfig.json with an empty one when Pacemaker +# stops the iscsi-target resource as part of a failover or cleanup. +{ pkgs, ... }: + +let + python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]); + targetctl = "${pkgs.targetcli-fb}/bin/targetctl"; + + targetctlStop = pkgs.writeScript "targetctl-stop" '' + #!${python3}/bin/python3 + import subprocess, sys + import rtslib_fb + + root = rtslib_fb.RTSRoot() + targets = list(root.targets) + if targets: + subprocess.run( + ["${targetctl}", "save", "/etc/target/saveconfig.json"], + capture_output=True, + ) + print(f"saved {len(targets)} iSCSI target(s)") + else: + print("no active LIO targets — saveconfig.json unchanged") + + for target in targets: + try: + for tpg in list(target.tpgs): + tpg.enable = False + target.delete() + except Exception as e: + print(f"warn (target): {e}", file=sys.stderr) + for so in list(root.storage_objects): + try: + so.delete() + except Exception as e: + print(f"warn (backstore): {e}", file=sys.stderr) + print("LIO kernel target cleared") + ''; +in +{ + boot.kernelModules = [ + "target_core_mod" + "iscsi_target_mod" + "target_core_file" + "target_core_pscsi" + "target_core_user" + "configfs" + ]; + + systemd = { + mounts = [{ + where = "/sys/kernel/config"; + what = "configfs"; + type = "configfs"; + wantedBy = [ "multi-user.target" ]; + before = [ "targetctl.service" ]; + }]; + services.targetctl = { + description = "LIO iSCSI target config save/restore"; + wantedBy = [ "multi-user.target" ]; + after = [ "sys-kernel-config.mount" "network.target" ]; + requires = [ "sys-kernel-config.mount" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${targetctl} restore /etc/target/saveconfig.json"; + ExecStop = "${targetctlStop}"; + }; + unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json"; + }; + tmpfiles.rules = [ + "d /etc/target 0750 root root -" + "f /etc/target/saveconfig.json 0640 root root -" + ]; + }; + + environment.systemPackages = [ pkgs.targetcli-fb ]; +} diff --git a/modules/ha/pacemaker-stack.nix b/modules/ha/pacemaker-stack.nix new file mode 100644 index 0000000..8fcab2c --- /dev/null +++ b/modules/ha/pacemaker-stack.nix @@ -0,0 +1,94 @@ +# Pacemaker + Corosync HA stack for NixOS with known-good workarounds. +# +# Issues fixed here (confirmed through live testing on NixOS 25.11): +# +# 1. StateDirectory ownership reset: systemd's StateDirectory=pacemaker +# creates /var/lib/pacemaker owned root:root. pacemaker-based (the CIB +# daemon) runs as the hacluster user and calls pcmk__daemon_can_write, +# which requires the CIB directory to be owned by hacluster or be +# group-writable by haclient. Workaround: remove StateDirectory and let +# ExecStartPre create every required subdirectory with correct ownership. +# +# 2. HA_SBIN_DIR wrong path: ocf-shellfuncs sets HA_SBIN_DIR to the Nix +# store path of the resource-agents derivation's /sbin, which doesn't +# exist. The DRBD OCF agent uses ${HA_SBIN_DIR}/crm_master, so it exits +# 127 without this override. Fix: export HA_SBIN_DIR=/run/current-system/sw/bin. +# +# 3. Broad PATH for OCF agents: the resource executor (pacemaker-execd) runs +# OCF agent scripts as children. NixOS provides no implicit PATH for +# system services; without an explicit PATH the agents can't find ip, ss, +# mount, umount, drbdadm, etc. +# +# 4. FUSER=true: the Filesystem OCF agent calls check_binary $FUSER (default: +# fuser from psmisc), which is not installed. Setting FUSER=true makes +# check_binary succeed (true is always in PATH) and the subsequent +# "$FUSER -km $mountpoint" becomes a no-op. Pair with force_unmount=false +# on each Filesystem resource unless you want lazy unmount behaviour. +{ lib, pkgs, ... }: + +let + ocfBinPath = lib.concatStringsSep ":" [ + "${pkgs.iproute2}/bin" + "${pkgs.iproute2}/sbin" + "${pkgs.iputils}/bin" + "${pkgs.util-linux}/bin" + "${pkgs.util-linux}/sbin" + "${pkgs.gawk}/bin" + "${pkgs.gnugrep}/bin" + "${pkgs.gnused}/bin" + "${pkgs.coreutils}/bin" + "${pkgs.bash}/bin" + "${pkgs.procps}/bin" + "${pkgs.xfsprogs}/bin" + "${pkgs.drbd}/bin" + "${pkgs.python3}/bin" + "/run/current-system/sw/bin" + "/run/current-system/sw/sbin" + "/usr/local/sbin" + "/usr/local/bin" + "/usr/sbin" + "/usr/bin" + "/sbin" + "/bin" + ]; + + # Single pre-start script: schemas symlink + directory ownership. + # Runs before pacemakerd so pacemaker-based finds hacluster-owned dirs. + preStartCmd = "${pkgs.bash}/bin/bash -c '" + + "ln -sfn ${pkgs.pacemaker}/share/pacemaker /var/lib/pacemaker/schemas; " + + "for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores " + + "/var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox " + + "/var/lib/pacemaker/hostcache; do " + + "mkdir -p \"\\$d\" && chown hacluster:pacemaker \"\\$d\" && chmod 2770 \"\\$d\"; " + + "done'"; + + ocfEnv = { + PATH = lib.mkForce ocfBinPath; + OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; + HA_SBIN_DIR = "/run/current-system/sw/bin"; + FUSER = "true"; + }; +in +{ + users.groups.haclient = { }; + + services.corosync.enable = true; + services.pacemaker.enable = true; + + systemd.services = { + pacemaker = { + serviceConfig = { + StateDirectory = lib.mkForce ""; + ExecStartPre = lib.mkBefore [ preStartCmd ]; + }; + environment = ocfEnv; + }; + pacemaker-execd.environment = ocfEnv; + }; + + environment.systemPackages = with pkgs; [ + corosync + pacemaker + ocf-resource-agents + ]; +} diff --git a/modules/platforms/lxc.nix b/modules/platforms/lxc.nix index d9c37a7..0eb0f42 100644 --- a/modules/platforms/lxc.nix +++ b/modules/platforms/lxc.nix @@ -63,23 +63,22 @@ in # back to decide `pct create`'s --unprivileged flag, so the two stay # in sync). # - # lxc-docker is the one exception: the kernel's NFS client doesn't set - # FS_USERNS_MOUNT, so mounting NFS from inside *any* non-init user - # namespace -- which is exactly what an unprivileged container's - # UID-mapped root runs in -- is rejected at the VFS layer with EPERM, - # no matter what Proxmox's own `mount=nfs;nfs4` container feature - # allows at the AppArmor layer (confirmed live: TCP to the NFS server - # succeeds, the server's export table matches the container's IP, and - # `mount.nfs: Operation not permitted` still fires immediately with no - # corresponding denial anywhere in the server's logs -- a kernel-level - # rejection, not a network or export-permission one). Keying off - # hostName rather than something docker-build-type-specific because - # modules/build-types/docker.nix is also composed for linode-docker/ - # proxmox-docker, which don't import proxmox-lxc.nix at all --setting - # this option there would break their eval with "option does not - # exist" regardless of any mkIf guard, since mkIf only makes a value - # conditional, not whether the option needs to exist somewhere. - privileged = config.networking.hostName == "docker"; + # Any lxc-* host with an NFS fileSystem must be privileged: the kernel's + # NFS client doesn't set FS_USERNS_MOUNT, so mounting NFS from inside + # *any* non-init user namespace -- which is exactly what an unprivileged + # container's UID-mapped root runs in -- is rejected at the VFS layer + # with EPERM, no matter what Proxmox's own `mount=nfs;nfs4` container + # feature allows at the AppArmor layer (confirmed live: TCP to the NFS + # server succeeds, the server's export table matches the container's IP, + # and `mount.nfs: Operation not permitted` still fires immediately with + # no corresponding denial anywhere in the server's logs -- a kernel-level + # rejection, not a network or export-permission one). Deriving this from + # fileSystems rather than a per-host override keeps it self-consistent: + # any new lxc-* host that declares an NFS mount automatically gets the + # privilege level it needs without a separate manual flag. + privileged = builtins.any + (fs: fs.fsType == "nfs" || fs.fsType == "nfs4") + (builtins.attrValues config.fileSystems); }; boot.loader = { diff --git a/modules/pxe-boot/mount-pxe-images.nix b/modules/pxe-boot/mount-pxe-images.nix new file mode 100644 index 0000000..b3879c6 --- /dev/null +++ b/modules/pxe-boot/mount-pxe-images.nix @@ -0,0 +1,42 @@ +{ config, lib, vars, ... }: + +let + # Use the same FQDN approach as docker/mount-data.nix — a bare hostname is + # unreliable: systemd-resolved only tries LLMNR for single-label names, and + # a global search domain causes it to skip the interface-scoped LAN DNS. + nfsServer = "${vars.nfsServerHost}.${vars.homeDomain}"; +in +{ + fileSystems.${vars.nfsShares.pxebootImages.mountpoint} = { + device = "${nfsServer}:${vars.storageRoot}/${vars.nfsShares.pxebootImages.subpath}"; + fsType = "nfs"; + options = [ + "_netdev" + "noatime" + ] ++ (if config.boot.isContainer + # NFSv4 requires rpc_pipefs (sunrpc filesystem), which Proxmox LXC + # containers block unless `features: mount=nfs` is set. Use NFSv3+nolock + # instead: no rpc_pipefs dependency at the protocol level, and rpcbind + # on the server handles port resolution without needing client-side + # sunrpc infrastructure. nofail keeps boot clean if server is unreachable. + then [ "nfsvers=3" "proto=tcp" "nolock" "nofail" ] + else [ "nfsvers=4.2" "x-systemd.automount" ]); + }; + + # NixOS pulls var-lib-nfs-rpc_pipefs.mount (the sunrpc filesystem) into + # nfs-client.target for any nfs fileSystems entry. In LXC containers the + # sunrpc mount is blocked by Proxmox's AppArmor profile, causing it to fail + # and the activation to report an error even though our mount uses nofail. + # Add ConditionVirtualization=!container via drop-in so systemd skips the + # unit entirely in containers (skip = inactive, not failed), which keeps + # nfs-client.target green and activation clean. + systemd.units = lib.mkIf config.boot.isContainer { + "var-lib-nfs-rpc_pipefs.mount" = { + overrideStrategy = "asDropin"; + text = '' + [Unit] + ConditionVirtualization=!container + ''; + }; + }; +} diff --git a/modules/tailscale/ts-dns-forwarder.nix b/modules/tailscale/ts-dns-forwarder.nix new file mode 100644 index 0000000..1745177 --- /dev/null +++ b/modules/tailscale/ts-dns-forwarder.nix @@ -0,0 +1,52 @@ +{ vars, ... }: + +{ + # Run dnsmasq on the LAN interface as a forwarding-only resolver for + # *.ts.net (Tailscale MagicDNS names). FreeIPA's bind-dyndb-ldap + # cannot reach 100.100.100.100 (Tailscale's internal resolver) directly + # because the DC is not a Tailscale node. This host IS a Tailscale node + # and can reach 100.100.100.100 via its tailscale0 interface, so it + # acts as an intermediary: FreeIPA has a conditional forward zone for + # ts.net pointing here (vars.tailscaleRouterIp), and this dnsmasq + # instance forwards those queries onward to Tailscale's resolver. + # + # Configure FreeIPA once after deploying this host: + # kinit admin + # ipa dnsforwardzone-add ${vars.tailnetDomain} \ + # --forwarder=${vars.tailscaleRouterIp} \ + # --forward-policy=only + # Note: IPA refuses to shadow ts.net (a real public TLD); use the + # tailnet-specific subdomain (vars.tailnetDomain) instead. + services.dnsmasq = { + enable = true; + settings = { + # Listen only on the LAN interface — not tailscale0 or loopback. + # bind-interfaces prevents dnsmasq from binding to 0.0.0.0 and + # then filtering by interface later; combined with `interface` this + # ensures it genuinely listens only on eth0. + bind-interfaces = true; + interface = [ vars.lxcLanInterface ]; + + # Forward-only: no local /etc/hosts or /etc/resolv.conf reading, + # no negative caching of NXDOMAIN for names this instance doesn't + # serve. All ts.net queries come from FreeIPA's conditional forwarder + # and must be answered by Tailscale's resolver. + no-hosts = true; + no-resolv = true; + + # Tailscale's internal "Quad100" resolver — reachable from any + # Tailscale node via the tailscale0 interface. Scoped to the + # specific tailnet subdomain (vars.tailnetDomain) rather than + # all of ts.net: FreeIPA refuses to shadow ts.net (a real public + # TLD with DNSimple nameservers) so the conditional forward zone + # in FreeIPA must use the tailnet-specific subdomain instead: + # ipa dnsforwardzone-add ${vars.tailnetDomain} \ + # --forwarder=${vars.tailscaleRouterIp} \ + # --forward-policy=only + server = [ "/${vars.tailnetDomain}/100.100.100.100" ]; + }; + }; + + networking.firewall.allowedUDPPorts = [ 53 ]; + networking.firewall.allowedTCPPorts = [ 53 ]; +} diff --git a/scripts/ha/acceptance-tests.sh b/scripts/ha/acceptance-tests.sh new file mode 100644 index 0000000..ab3081c --- /dev/null +++ b/scripts/ha/acceptance-tests.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# acceptance-tests.sh — HA cluster acceptance tests (T1–T7) +# +# Run from a host with SSH access to both HA nodes (or from node1 itself). +# All 7 tests must pass before considering the cluster production-ready. +# Test values below must match variables.nix haServer* values. +set -euo pipefail + +# ── Configuration ───────────────────────────────────────────────────────── +NODE1="ha-server-1" +NODE2="ha-server-2" +NODE1_IP="192.168.2.200" # vars.haServer1Ip +NODE2_IP="192.168.2.201" # vars.haServer2Ip +VIP="192.168.2.202" # vars.haServerVip +XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot +ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn +# ────────────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 +RESULTS=() + +pass() { echo " PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); } +fail() { echo " FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); } + +n1() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; } +n2() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; } + +echo "════════════════════════════════════════════════════" +echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')" +echo "════════════════════════════════════════════════════" + +# ── T1: Corosync quorum established ────────────────────────────────────── +echo "" +echo "[T1] Corosync quorum" +if n1 "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then + pass "cluster has quorum" +else + fail "cluster does not have quorum — check corosync on both nodes" +fi + +# ── T2: DRBD Primary on node1, Secondary on node2 ──────────────────────── +echo "" +echo "[T2] DRBD roles" +DRBD_ROLE=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown") +if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then + pass "DRBD Primary on $NODE1 ($DRBD_ROLE)" +else + fail "unexpected DRBD role on $NODE1: $DRBD_ROLE (expected Primary/Secondary)" +fi + +DRBD_DSTATE=$(n1 "drbdadm dstate ha-data" 2>/dev/null || echo "unknown") +if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then + pass "DRBD disk state UpToDate ($DRBD_DSTATE)" +else + fail "DRBD disk not UpToDate: $DRBD_DSTATE" +fi + +# ── T3: XFS mounted at haStorageRoot on the Active node ────────────────── +echo "" +echo "[T3] XFS mount" +if n1 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + pass "XFS mounted at ${XFS_MOUNT} on $NODE1" +else + fail "XFS not mounted at ${XFS_MOUNT} on $NODE1" +fi + +if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + fail "XFS unexpectedly mounted on $NODE2 (should only be on Active node)" +else + pass "XFS not mounted on $NODE2 (correct — Secondary)" +fi + +# ── T4: iSCSI target visible on both nodes ──────────────────────────────── +echo "" +echo "[T4] iSCSI target" +IQN_COUNT=$(n1 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0") +if [[ "$IQN_COUNT" -ge 1 ]]; then + pass "iSCSI IQN active on $NODE1 ($IQN_COUNT target(s))" +else + fail "no iSCSI IQN active on $NODE1" +fi + +# iSCSI discovery from node2 via VIP +if n2 "iscsiadm -m discovery -t sendtargets -p '${VIP}' 2>/dev/null | grep -q '${ISCSI_IQN}'"; then + pass "iSCSI target discoverable from $NODE2 via VIP ${VIP}" +else + fail "iSCSI target not discoverable from $NODE2 via ${VIP}" +fi + +# ── T5: Failover — standby node1, verify resources move to node2 ────────── +echo "" +echo "[T5] Failover (standby $NODE1)" +MYNODE=$(n1 "crm_node -n" 2>/dev/null || echo "") +n1 "crm_standby -N '${MYNODE}' -v on" 2>/dev/null || true +echo " Waiting up to 30 s for resources to move to $NODE2..." +MOVED=false +for i in $(seq 1 30); do + if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then + MOVED=true + echo " Resources moved in ${i}s" + break + fi + sleep 1 +done + +if $MOVED; then + pass "XFS mounted on $NODE2 after failover" + IQN_ON_N2=$(n2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0") + [[ "$IQN_ON_N2" -ge 1 ]] \ + && pass "iSCSI target active on $NODE2 after failover" \ + || fail "iSCSI target NOT active on $NODE2 after failover" +else + fail "XFS did not mount on $NODE2 within 30 s — failover incomplete" +fi + +# ── T6: Data integrity — file written pre-failover readable post-failover ─ +echo "" +echo "[T6] Data integrity" +# Write a test file on node2 (now Active) and verify its content +TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$" +TEST_CONTENT="ha-acceptance-test-$(date +%s)" +n2 "echo '${TEST_CONTENT}' > '${TEST_FILE}'" 2>/dev/null || true +READBACK=$(n2 "cat '${TEST_FILE}' 2>/dev/null" || echo "") +if [[ "$READBACK" == "$TEST_CONTENT" ]]; then + pass "test file written and read back correctly on $NODE2" +else + fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')" +fi +n2 "rm -f '${TEST_FILE}'" 2>/dev/null || true + +# ── T7: Node rejoin — un-standby node1, verify cluster is healthy ───────── +echo "" +echo "[T7] Node rejoin" +n1 "crm_standby -N '${MYNODE}' -v off" 2>/dev/null || true +n1 "crm_resource --cleanup" 2>/dev/null || true +sleep 5 + +ONLINE_NODES=$(n2 "crm_mon -1 2>/dev/null | grep -c 'Online:'" || echo "0") +if n1 "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then + pass "$NODE1 rejoined — cluster has quorum" +else + fail "$NODE1 did not rejoin with quorum" +fi + +DRBD_ROLE_AFTER=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown") +if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then + pass "$NODE1 is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)" +else + fail "unexpected DRBD role on $NODE1 after rejoin: $DRBD_ROLE_AFTER" +fi + +# ── Summary ─────────────────────────────────────────────────────────────── +echo "" +echo "════════════════════════════════════════════════════" +echo " Results: ${PASS} PASS, ${FAIL} FAIL" +echo "════════════════════════════════════════════════════" +for r in "${RESULTS[@]}"; do echo " $r"; done +echo "" + +if [[ "$FAIL" -eq 0 ]]; then + echo "ALL PASS — cluster is production-ready." + exit 0 +else + echo "SOME TESTS FAILED — investigate before deploying." + exit 1 +fi diff --git a/scripts/ha/cluster-enable-stonith.sh b/scripts/ha/cluster-enable-stonith.sh new file mode 100644 index 0000000..6ef47a5 --- /dev/null +++ b/scripts/ha/cluster-enable-stonith.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# cluster-enable-stonith.sh — enable STONITH fence agent after the fence SSH +# key is deployed to both nodes and authorised on the Proxmox host. +# +# Run from ha-server-1 as root AFTER: +# - /etc/pacemaker/fence_pve_ssh exists on both nodes (chmod +x) +# (copy from scripts/ha/fence-pve-ssh.py) +# - /etc/fence-pve-ssh-key (SSH private key) exists on both nodes +# - The corresponding public key is in authorized_keys on PVE_HOST +# - VMID_NODE1 / VMID_NODE2 filled in below +set -euo pipefail + +# ── Configuration ───────────────────────────────────────────────────────── +NODE1="ha-server-1" +NODE2="ha-server-2" +VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1 +VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2 +PVE_HOST="pve1.sweet.home" +PVE_USER="wayne" +FENCE_KEY="/etc/fence-pve-ssh-key" +FENCE_SCRIPT="/etc/pacemaker/fence_pve_ssh" +# ────────────────────────────────────────────────────────────────────────── + +log() { echo "[stonith-setup] $*"; } +die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; } + +[[ $(id -u) -eq 0 ]] || die "must run as root" +[[ -n "$VMID_NODE1" ]] || die "VMID_NODE1 not set — edit this script" +[[ -n "$VMID_NODE2" ]] || die "VMID_NODE2 not set — edit this script" +[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY" +[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT" + +log "Verifying fence agent can reach ${PVE_HOST}..." +ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \ + -o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" \ + "sudo /usr/sbin/qm list" &>/dev/null \ + || die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo" +log "Fence agent SSH connectivity confirmed" + +log "Creating Pacemaker STONITH resources..." +cibadmin --create --scope resources --xml-text " + + + + + + + + + + + + + + +" 2>/dev/null || true + +cibadmin --create --scope resources --xml-text " + + + + + + + + + + + + + + +" 2>/dev/null || true + +log "Enabling STONITH and restoring quorum policy..." +crm_attribute -t crm_config -n stonith-enabled -v true +crm_attribute -t crm_config -n no-quorum-policy -v stop + +log "DRBD fencing mode must also be updated to resource-only (already the" +log "default in cluster-config.nix; confirm with: cat /etc/drbd.d/ha-data.conf)" + +log "Testing fence agent..." +stonith_admin --list-devices && log "Fence devices listed successfully." \ + || warn "stonith_admin --list-devices failed — check config" + +log "STONITH enabled. Cluster is now fully HA." diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh new file mode 100644 index 0000000..1aac090 --- /dev/null +++ b/scripts/ha/cluster-init.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# cluster-init.sh — one-time HA cluster initialisation script +# +# Run ONCE from ha-server-1 as root AFTER both VMs are booted and have SSH +# access. It: +# 1. Generates and distributes the corosync authkey +# 2. Waits for corosync quorum and pacemaker +# 3. Initialises DRBD metadata, promotes node1 to primary +# 4. Creates XFS on /dev/drbd0 and mounts it +# 5. Creates the directory tree and iSCSI LUN backing file +# 6. Configures LIO iSCSI target (file-backed LUN) +# 7. Configures Pacemaker resources: DRBD → XFS → iSCSI → NFS → VIP +# +# Prerequisites: +# - Both VMs booted with the ha-server config (nixos-rebuild done) +# - SSH key access from node1 to root@NODE2_IP +# - VMID_NODE1 / VMID_NODE2 filled in below (needed for STONITH setup; +# cluster starts without STONITH, which you enable separately via +# scripts/ha/cluster-enable-stonith.sh) +# - Run as root on ha-server-1 +set -euo pipefail + +# ── Configuration ───────────────────────────────────────────────────────── +# These must match variables.nix haServer* values and the Proxmox VMID +# assignments. Update before running. +NODE1="ha-server-1" +NODE2="ha-server-2" +NODE1_IP="192.168.2.200" # vars.haServer1Ip +NODE2_IP="192.168.2.201" # vars.haServer2Ip +VIP="192.168.2.202" # vars.haServerVip +XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot +ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn +ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img" +ISCSI_LUN_SIZE="10G" +DRBD_DEVICE="/dev/drbd0" +VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1 +VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2 +PVE_HOST="pve1.sweet.home" +PVE_USER="wayne" + +# NFS dataset subdirectories to create under XFS_MOUNT. +# Must mirror vars.nfsShares subpath values in variables.nix. +NFS_SUBDIRS=( + "docker/config" + "docker/volumes" + "docker/databases" + "docker/nextcloud-data" + "raspi/volumes" + "proxmox/iso" + "proxmox/lxc" + "pxe-boot/images" +) +# ────────────────────────────────────────────────────────────────────────── + +log() { echo "[cluster-init] $*"; } +die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; } +warn() { echo "[cluster-init] WARNING: $*" >&2; } + +[[ $(id -u) -eq 0 ]] || die "must run as root" +[[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1" + +# ── 0. Corosync authkey ─────────────────────────────────────────────────── +AUTHKEY="/etc/corosync/authkey" +mkdir -p /etc/corosync +if [[ ! -f "$AUTHKEY" ]]; then + log "Generating corosync authkey..." + corosync-keygen -k "$AUTHKEY" + chmod 0400 "$AUTHKEY" +fi +log "Distributing authkey to $NODE2..." +ssh "root@${NODE2_IP}" "mkdir -p /etc/corosync" +scp -q "$AUTHKEY" "root@${NODE2_IP}:${AUTHKEY}" +ssh "root@${NODE2_IP}" "chmod 0400 '${AUTHKEY}'" + +log "Restarting corosync on both nodes..." +systemctl restart corosync +ssh "root@${NODE2_IP}" "systemctl restart corosync" +sleep 3 + +# ── 1. Corosync quorum ──────────────────────────────────────────────────── +log "Waiting for corosync quorum..." +for i in $(seq 1 30); do + if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then + log "Quorum established" + break + fi + [[ $i -eq 30 ]] && die "corosync quorum not established after 60 s" + sleep 2 +done + +log "Waiting for pacemaker..." +for i in $(seq 1 30); do + if crm_mon -1 &>/dev/null; then + log "Pacemaker running" + break + fi + [[ $i -eq 30 ]] && die "pacemaker not running after 60 s" + sleep 2 +done + +# ── 2. DRBD initialisation ──────────────────────────────────────────────── +log "Initialising DRBD metadata on $NODE1..." +if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then + drbdadm create-md ha-data --force +fi + +log "Initialising DRBD metadata on $NODE2..." +ssh "root@${NODE2_IP}" " + if ! drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent\|Diskless'; then + drbdadm create-md ha-data --force + fi +" + +log "Bringing up DRBD on both nodes..." +drbdadm up ha-data 2>/dev/null || true +ssh "root@${NODE2_IP}" "drbdadm up ha-data 2>/dev/null" || true + +log "Forcing $NODE1 to DRBD Primary for initial sync..." +drbdadm primary ha-data --force + +log "Waiting for DRBD to finish initial sync (this may take several minutes)..." +for i in $(seq 1 300); do + state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown") + if echo "$state" | grep -q "UpToDate/UpToDate"; then + log "DRBD sync complete: $state" + break + fi + [[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)" + sleep 1 +done + +# ── 3. XFS filesystem ───────────────────────────────────────────────────── +log "Creating XFS on ${DRBD_DEVICE}..." +if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then + mkfs.xfs -f "${DRBD_DEVICE}" +fi + +log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..." +mkdir -p "${XFS_MOUNT}" +mount "${DRBD_DEVICE}" "${XFS_MOUNT}" + +# ── 4. NFS dataset directories ──────────────────────────────────────────── +log "Creating NFS dataset directories..." +for subdir in "${NFS_SUBDIRS[@]}"; do + mkdir -p "${XFS_MOUNT}/${subdir}" +done + +# ── 5. iSCSI LUN backing file ───────────────────────────────────────────── +log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..." +if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then + fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}" +fi + +# ── 6. LIO iSCSI target ─────────────────────────────────────────────────── +log "Configuring LIO iSCSI target via targetcli..." +targetcli < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +" + +log "Adding ordering and colocation constraints..." +cibadmin --create --scope constraints --xml-text " + + + + +" + +log "Waiting for resources to start..." +for i in $(seq 1 60); do + if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then + log "VIP is up: $(crm_resource -r vip --locate)" + break + fi + [[ $i -eq 60 ]] && { warn "VIP not up after 120 s — check: crm_mon -1"; break; } + sleep 2 +done + +log "" +log "═══════════════════════════════════════════════════════════════" +log " HA cluster initialised." +log "" +log " crm_mon -1 — cluster status" +log " iscsiadm -m discovery -t st -p ${VIP} — verify iSCSI target" +log " showmount -e ${VIP} — verify NFS exports" +log "" +log " To enable STONITH (after deploying fence SSH key):" +log " 1. Fill in VMID_NODE1 / VMID_NODE2 in cluster-enable-stonith.sh" +log " 2. Copy scripts/ha/fence-pve-ssh.py to /etc/pacemaker/fence_pve_ssh" +log " on both nodes (chmod +x)" +log " 3. Generate and distribute the fence SSH key" +log " (see docs or cluster-enable-stonith.sh header)" +log " 4. bash scripts/ha/cluster-enable-stonith.sh" +log "═══════════════════════════════════════════════════════════════" diff --git a/scripts/ha/fence-pve-ssh.py b/scripts/ha/fence-pve-ssh.py new file mode 100644 index 0000000..58c24ed --- /dev/null +++ b/scripts/ha/fence-pve-ssh.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +""" +fence_pve_ssh - Proxmox VE SSH fence agent for Pacemaker. + +Uses SSH to reach the Proxmox host and run 'qm stop/start '. +Deploy to /etc/pacemaker/fence_pve_ssh on both HA nodes (chmod +x). + +Configuration (as pacemaker stonith resource attributes): + pve_host Proxmox host to SSH to (default: pve1.sweet.home) + pve_user SSH user (default: wayne) + key_file SSH private key path (default: /etc/fence-pve-ssh-key) + vmid_node1 VMID for ha-server-1 + vmid_node2 VMID for ha-server-2 + plug Node name to act on (set by pacemaker: ha-server-1 or ha-server-2) + action Action: off|on|reboot|status|list|metadata +""" + +import argparse +import subprocess +import sys +import os + + +METADATA = """ + + Fences a VM on a Proxmox VE host by SSHing to the PVE host and + running qm stop/start. For test use only. + https://proxmox.com + + + + + Fencing action: off|on|reboot|status|list + + + + + Cluster node name to fence + + + + + Proxmox VE host to SSH to + + + + + SSH user on the Proxmox host + + + + + SSH private key file path + + + + + VMID for ha-test-node1 + + + + + VMID for ha-test-node2 + + + + + + + + + + + +""" + + +def parse_args(): + p = argparse.ArgumentParser(add_help=False) + p.add_argument("-a", "--action", default="reboot") + p.add_argument("-n", "--plug") + p.add_argument("--pve-host", default="pve1.sweet.home") + p.add_argument("--pve-user", default="wayne") + p.add_argument("--key-file", default="/etc/fence-pve-ssh-key") + p.add_argument("--vmid-node1") + p.add_argument("--vmid-node2") + # Allow remaining unknown args (pacemaker may pass extra ones) + return p.parse_known_args()[0] + + +def ssh(pve_host, pve_user, key_file, cmd): + result = subprocess.run( + [ + "ssh", + "-i", key_file, + "-o", "StrictHostKeyChecking=no", + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=10", + f"{pve_user}@{pve_host}", + cmd, + ], + capture_output=True, + text=True, + timeout=30, + ) + return result + + +def get_vmid(args): + node = args.plug + if not node: + print("ERROR: --plug not specified", file=sys.stderr) + sys.exit(1) + mapping = { + "ha-server-1": args.vmid_node1, + "ha-server-2": args.vmid_node2, + } + vmid = mapping.get(node) + if not vmid: + print(f"ERROR: unknown node '{node}'", file=sys.stderr) + sys.exit(1) + return vmid + + +def main(): + args = parse_args() + action = args.action.lower() + + if action == "metadata": + print(METADATA) + sys.exit(0) + + if action == "list": + if args.vmid_node1: + print("ha-server-1") + if args.vmid_node2: + print("ha-server-2") + sys.exit(0) + + vmid = get_vmid(args) + + if not os.path.exists(args.key_file): + print(f"ERROR: SSH key not found at {args.key_file}", file=sys.stderr) + sys.exit(1) + + if action in ("off", "reboot"): + print(f"Stopping VM {vmid} ({args.plug}) on {args.pve_host}...") + r = ssh(args.pve_host, args.pve_user, args.key_file, + f"sudo /usr/sbin/qm stop {vmid}") + if r.returncode != 0: + print(f"ERROR stopping VM: {r.stderr}", file=sys.stderr) + sys.exit(1) + print(f"VM {vmid} stopped") + + if action in ("on", "reboot"): + print(f"Starting VM {vmid} ({args.plug}) on {args.pve_host}...") + r = ssh(args.pve_host, args.pve_user, args.key_file, + f"sudo /usr/sbin/qm start {vmid}") + if r.returncode != 0: + print(f"ERROR starting VM: {r.stderr}", file=sys.stderr) + sys.exit(1) + print(f"VM {vmid} started") + + if action == "status": + r = ssh(args.pve_host, args.pve_user, args.key_file, + f"sudo /usr/sbin/qm status {vmid}") + if r.returncode != 0: + print(f"ERROR querying VM status: {r.stderr}", file=sys.stderr) + sys.exit(1) + # qm status returns "status: running" or "status: stopped" + status_line = r.stdout.strip() + print(status_line) + if "stopped" in status_line: + sys.exit(2) # pacemaker interprets exit 2 as "off" + sys.exit(0) # running = exit 0 + + +if __name__ == "__main__": + main() diff --git a/scripts/proxmox/create-proxmox-resource.sh b/scripts/proxmox/create-proxmox-resource.sh index e0b5374..918085f 100755 --- a/scripts/proxmox/create-proxmox-resource.sh +++ b/scripts/proxmox/create-proxmox-resource.sh @@ -867,17 +867,17 @@ if [[ "$type" == "lxc" ]]; then local_swap="${swap:-$memory}" # --unprivileged: read back from modules/platforms/lxc.nix's own # proxmoxLXC.privileged (via flake_target_lxc_privileged) rather than - # hardcoded, since that's no longer the same for every lxc-* target -- - # lxc-docker sets it true so the container's NFS mounts work at all (the - # kernel's NFS client can't mount from inside any unprivileged - # container's user namespace, no matter what AppArmor allows -- see that - # option's own comment). The NixOS config inside the image bakes in - # cgroup/capability/mount expectations matching whichever value it was - # built with, so this must stay in sync with it -- `pct create`'s own - # CLI default for this flag is privileged (unlike the web UI, which - # defaults its checkbox the other way), so leaving it unset would create - # a privileged container running a NixOS config that assumes - # unprivileged for every target except lxc-docker, a real mismatch. + # hardcoded. lxc.nix derives this automatically: any lxc-* host whose + # config.fileSystems has an NFS entry gets privileged=true, because the + # kernel's NFS client (FS_USERNS_MOUNT not set) rejects NFS mounts from + # inside any non-init user namespace -- exactly what an unprivileged + # container's UID-mapped root lives in -- with EPERM at the VFS layer, + # regardless of AppArmor (see lxc.nix's own comment). The NixOS config + # bakes in cgroup/capability/mount expectations matching whichever value + # it was built with, so this must stay in sync -- `pct create`'s CLI + # default is privileged (unlike the web UI, which defaults the other + # way), so leaving it unset would create a privileged container running + # a NixOS config that assumes unprivileged, a real mismatch. privileged_eval="$(flake_target_lxc_privileged "$repo_root" "$flake_target")" unprivileged_flag=1 [[ "$privileged_eval" == "true" ]] && unprivileged_flag=0 diff --git a/scripts/secrets/sync-host-keys.sh b/scripts/secrets/sync-host-keys.sh index 512929e..3a57373 100755 --- a/scripts/secrets/sync-host-keys.sh +++ b/scripts/secrets/sync-host-keys.sh @@ -85,6 +85,10 @@ ensure_admin_decrypt_key() { fi local key_file="$DEFAULT_SOPS_AGE_KEY_FILE" + # Expand a leading ~ that survived variable substitution without tilde + # expansion (happens when SOPS_AGE_KEY_FILE or XDG_CONFIG_HOME is set with + # a literal ~ in the caller's environment). + key_file="${key_file/#~\//$HOME/}" if [[ -s "$key_file" ]]; then echo "Found existing sops age key at ${key_file}." @@ -93,36 +97,23 @@ ensure_admin_decrypt_key() { if [[ "$dry_run" -eq 1 ]]; then echo "[dry-run] No sops age decryption key found (checked \$SOPS_AGE_KEY, \$SOPS_AGE_KEY_FILE, ${key_file})." - echo "[dry-run] Would generate a new one here -- continuing the dry run without one; any" - echo "[dry-run] 'would re-encrypt' output below couldn't actually run for real yet." + echo "[dry-run] Continuing dry run without one -- any 'would re-encrypt' output below" + echo "[dry-run] couldn't actually run for real until a key is present." return fi - echo "No sops age decryption key found (checked \$SOPS_AGE_KEY, \$SOPS_AGE_KEY_FILE, ${key_file})." - echo "Generating a new one at ${key_file}..." - mkdir -p "$(dirname "$key_file")" - nix-shell "${NIX_OPTS[@]}" -p age --run "age-keygen -o '${key_file}'" 2>&1 | grep -v "^Public key:" || true - local new_pub - new_pub="$(age_pubkey_from_identity_file "$key_file")" + cat >&2 < secrets/ha-corosync-authkey diff --git a/secrets/ha-server-1.yaml b/secrets/ha-server-1.yaml new file mode 100644 index 0000000..f33120f --- /dev/null +++ b/secrets/ha-server-1.yaml @@ -0,0 +1,6 @@ +# STUB — not yet encrypted with sops. +# Bootstrap: +# bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1 +# sops updatekeys secrets/common.yaml (allows ha-server-1 to decrypt shared secrets) +# sops secrets/ha-server-1.yaml (create with: beszel-token) +beszel-token: REPLACE diff --git a/secrets/ha-server-2.yaml b/secrets/ha-server-2.yaml new file mode 100644 index 0000000..19bbbe9 --- /dev/null +++ b/secrets/ha-server-2.yaml @@ -0,0 +1,6 @@ +# STUB — not yet encrypted with sops. +# Bootstrap: +# bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2 +# sops updatekeys secrets/common.yaml (allows ha-server-2 to decrypt shared secrets) +# sops secrets/ha-server-2.yaml (create with: beszel-token) +beszel-token: REPLACE diff --git a/variables.nix b/variables.nix index a1f23f3..ec50ec2 100644 --- a/variables.nix +++ b/variables.nix @@ -4,8 +4,19 @@ homeDomain = "sweet.home"; # base LAN domain for service subdomains (pve., docker.) tailnetDomain = "tail13f623.ts.net"; # Tailscale MagicDNS suffix lanCidr = "192.168.2.0/24"; # LAN subnet - pxeServerIp = "192.168.2.247"; # pxe-boot host's LAN IP - pbsIp = "192.168.2.108"; # Proxmox Backup Server LAN IP + lanGateway = "192.168.2.254"; # LAN default gateway (router) + lanPrefixLength = 24; # LAN subnet prefix length (/24 = 255.255.255.0) + lxcLanInterface = "eth0"; # LAN NIC name in LXC containers (set by Proxmox --net0 name=eth0) + vmLanInterface = "ens18"; # LAN NIC name in Proxmox VMs (virtio, first NIC) + vmStorageInterface = "ens19"; # storage NIC name in HA server VMs (virtio, second NIC on vmbr1) + pxeServerIp = "192.168.2.223"; # pxe-boot LXC container LAN IP + nixCacheIp = "192.168.2.224"; # nix-cache LXC container LAN IP + tailscaleRouterIp = "192.168.2.222"; # tailscale-router LXC container LAN IP + torRelayIp = "192.168.2.221"; # tor-relay LXC container LAN IP + serverIp = "192.168.2.226"; # server (NFS/ZFS) Proxmox VM LAN IP + dockerIp = "192.168.2.225"; # docker Proxmox VM LAN IP + pbsIp = "192.168.2.244"; # Proxmox Backup Server LAN IP (not NixOS-managed) + domainControllerIp = "192.168.2.253"; # FreeIPA domain controller / primary DNS (not NixOS-managed) # Cross-host references (LAN hostnames/users other hosts reach over the network) nixCacheHost = "nix-cache"; # substituter/remote-builder hostname @@ -68,6 +79,24 @@ # one-line change. primaryUser = "nixos"; + # HA file server cluster + # LAN IPs (vmbr0 / ens18) — client-facing: iSCSI initiators, NFS, management. + # Storage IPs (vmbr1 / ens19) — isolated internal bridge, used for DRBD + # replication and Corosync heartbeat only; never leaves pve1. + # haServerVip: floating virtual IP managed by Pacemaker's IPaddr2 resource; + # NFS and iSCSI clients connect here regardless of which node is Active. + haServer1Host = "ha-server-1"; + haServer2Host = "ha-server-2"; + haServer1Ip = "192.168.2.228"; # LAN IP, node 1 + haServer2Ip = "192.168.2.227"; # LAN IP, node 2 + haServerVip = "192.168.2.229"; # floating VIP (Pacemaker IPaddr2) + haServer1StorageIp = "192.168.4.228"; # storage-net IP, node 1 (vmbr1 / ens19) + haServer2StorageIp = "192.168.4.227"; # storage-net IP, node 2 (vmbr1 / ens19) + haStorageCidr = "192.168.4.0/29"; # storage subnet — internal to pve1 only + haStoragePrefixLength = 29; # storage subnet prefix length (/29) + haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node + haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage"; + # Storage storageRoot = "/tank"; # ZFS pool root on `server` @@ -83,6 +112,7 @@ # dataset or moving where it's mounted only needs changing it here — the # export and every client reference follow automatically. nfsShares = { + options = "(rw,sync,no_subtree_check,no_root_squash)"; dockerConfig = { subpath = "docker/config"; mountpoint = "/mnt/docker/config"; @@ -103,6 +133,18 @@ subpath = "raspi/volumes"; mountpoint = "/mnt/raspi-backup"; }; + proxmoxIsos = { + subpath = "proxmox/iso"; + mountpoint = "/mnt/iso"; + }; + proxmoxLxcImages = { + subpath = "proxmox/lxc"; + mountpoint = "/mnt/lxc"; + }; + pxebootImages = { + subpath = "pxe-boot/images"; + mountpoint = "/mnt/pxe-images"; + }; }; # The Raspberry Pi's own NFS export — not under storageRoot/nfsServerHost, @@ -129,10 +171,25 @@ # (modules/build-types/pxe-boot.nix). pxeBootTftp = 69; - # `server`'s NFS exports need both the portmapper (rpcbind) and the - # NFS data port itself opened (modules/build-types/server.nix). + # `server`'s NFS exports: portmapper (rpcbind), NFS data, and the + # mountd RPC service (used by showmount/NFSv3 mount protocol). + # Mountd listens on a fixed port so the firewall can whitelist it + # explicitly rather than opening all of rpcbind's dynamic range. + # All three need both TCP and UDP (modules/build-types/server.nix and + # modules/build-types/ha-server.nix). nfsRpcbind = 111; nfsd = 2049; + nfsMountd = 20048; + + # HA cluster ports opened on ha-server-1 and ha-server-2 + # (modules/build-types/ha-server.nix / modules/ha/cluster-config.nix). + haServerDrbd = 7789; # DRBD replication (TCP) + haServerIscsi = 3260; # iSCSI target (TCP) + haServerCorosync1 = 5404; # Corosync totem ring (UDP) + haServerCorosync2 = 5405; # Corosync totem ring (UDP) + haServerCorosyncCrypto = 5407; # Corosync crypto sync (UDP) + haServerPacemakerRemoted = 3121; # pacemaker-remoted (TCP) + haServerPcsd = 2224; # pcsd cluster daemon (TCP) # Opened on the docker host's firewall for the Traefik-fronted # container stack (docker-compose config lives in the separate