From 4444398cac3d21259617b52124ee96c09ad0b3c4 Mon Sep 17 00:00:00 2001 From: beatzaplenty Date: Mon, 27 Jul 2026 12:32:54 +1000 Subject: [PATCH] fix(network): wire correct IPs throughout and add cutover plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit variables.nix: - HA server LAN IPs: .200/.201/.202 → .228/.227/.229 (from ip-addressing.md) - Add haServer1StorageIp (.228), haServer2StorageIp (.227) for 192.168.4.0/29 - Add haStorageCidr for firewall rules ha-server host.nix (both nodes): - Add ens19 interface on storage subnet (/29) - Fix defaultGateway: 192.168.2.1 → 192.168.2.254 - Fix nameservers: 192.168.2.1/8.8.8.8 → domainControllerIp (.253) cluster-config.nix: - DRBD replication addresses: LAN IPs → storage IPs (keep replication off LAN) - Corosync ring_addrs: LAN IPs → storage IPs - Firewall: add haStorageCidr to allowed sources docs/network-cutover.md: step-by-step cutover plan with rollback at every stage Co-Authored-By: Claude Sonnet 4.6 --- docs/network-cutover.md | 366 ++++++++++++++++++++++++++++++++++ hosts/ha-server-1/host.nix | 8 +- hosts/ha-server-2/host.nix | 8 +- modules/ha/cluster-config.nix | 9 +- variables.nix | 15 +- 5 files changed, 392 insertions(+), 14 deletions(-) create mode 100644 docs/network-cutover.md diff --git a/docs/network-cutover.md b/docs/network-cutover.md new file mode 100644 index 0000000..631250c --- /dev/null +++ b/docs/network-cutover.md @@ -0,0 +1,366 @@ +# Network Cutover Plan + +Moves the LAN from the current flat/Pi-hole-managed state to the new IP scheme +defined in `docs/ip-addressing.md`. Works in five independent stages — each +stage is safe to pause after and resume later. Rollback steps are given at +every point where something can break. + +**Before starting anything:** confirm you have +- SSH access to `192.168.2.138` (domain-controller, current IP) +- SSH access to `192.168.2.250` (pve1) +- Browser access to Pi-hole admin at `http://192.168.2.253` +- Browser access to router admin at `http://192.168.2.254` +- The FreeIPA `admin` password to hand + +--- + +## Stage 1 — Prepare FreeIPA DNS (zero downtime) + +Everything here is additive. Pi-hole keeps running. Nothing breaks if you stop +mid-stage. + +### 1a. Add NextDNS forwarders + +```bash +ssh wayne@192.168.2.138 +kinit admin # enter FreeIPA admin password when prompted +ipa dnsconfig-mod \ + --forwarder=45.90.28.142 \ + --forwarder=45.90.30.142 \ + --forward-policy=only +``` + +**Verify external resolution works through FreeIPA before continuing:** +```bash +dig @127.0.0.1 google.com +short # must return an IP, not SERVFAIL +``` + +### 1b. Add A records for every host at their CURRENT IPs + +These represent the live state now. You'll update each record to the new IP +when you renumber that host in Stage 5. + +```bash +ipa dnsrecord-add sweet.home pve1 --a-rec 192.168.2.250 +ipa dnsrecord-add sweet.home pbs --a-rec 192.168.2.108 +ipa dnsrecord-add sweet.home nixos --a-rec 192.168.2.119 +ipa dnsrecord-add sweet.home server --a-rec 192.168.2.252 +ipa dnsrecord-add sweet.home docker --a-rec 192.168.2.249 +ipa dnsrecord-add sweet.home nix-cache --a-rec 192.168.2.120 +ipa dnsrecord-add sweet.home pxe-boot --a-rec 192.168.2.247 +ipa dnsrecord-add sweet.home tailscale-router --a-rec 192.168.2.121 +ipa dnsrecord-add sweet.home tor-relay --a-rec 192.168.2.107 +ipa dnsrecord-add sweet.home pdm --a-rec 192.168.2.248 +ipa dnsrecord-add sweet.home router --a-rec 192.168.2.254 +``` + +### 1c. Clean up stale reverse-zone PTR records + +FreeIPA already has PTR records from an earlier import but some are wrong. +Fix them now so reverse DNS is accurate from day one. + +```bash +# Remove stale "win11" entry at .250 (should be pve1) +ipa dnsrecord-del 2.168.192.in-addr.arpa 250 --ptr-rec win11. +ipa dnsrecord-add 2.168.192.in-addr.arpa 250 --ptr-rec pve1.sweet.home. + +# Fix unqualified PTR records (missing .sweet.home. suffix) +ipa dnsrecord-mod 2.168.192.in-addr.arpa 108 --ptr-rec pbs.sweet.home. +ipa dnsrecord-mod 2.168.192.in-addr.arpa 248 --ptr-rec pdm.sweet.home. +ipa dnsrecord-mod 2.168.192.in-addr.arpa 249 --ptr-rec docker.sweet.home. +ipa dnsrecord-mod 2.168.192.in-addr.arpa 252 --ptr-rec server.sweet.home. + +# Add any missing PTR records +ipa dnsrecord-add 2.168.192.in-addr.arpa 119 --ptr-rec nixos.sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa 120 --ptr-rec nix-cache.sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa 121 --ptr-rec tailscale-router.sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa 247 --ptr-rec pxe-boot.sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa 254 --ptr-rec router.sweet.home. +``` + +### 1d. Point domain-controller's own DNS at itself + +```bash +sudo nmcli connection modify "System eth0" ipv4.dns "127.0.0.1" +sudo nmcli connection up "System eth0" +``` + +**Verify:** +```bash +dig pve1.sweet.home +short # must return 192.168.2.250 +dig google.com +short # must return an IP (NextDNS forwarding) +``` + +**Rollback 1d:** `sudo nmcli connection modify "System eth0" ipv4.dns "192.168.2.253" && sudo nmcli connection up "System eth0"` + +--- + +## Stage 2 — Move pxe-boot DHCP options off Pi-hole (zero downtime) + +Pi-hole's dnsmasq currently serves the iPXE boot options via +`99-ipxe-chainload.conf`. Before Pi-hole is retired, that config must move to +the pxe-boot CT running dnsmasq in proxy mode so PXE boot keeps working. + +### 2a. Add dnsmasq proxy config to the pxe-boot NixOS module + +In `modules/build-types/pxe-boot.nix`, add: + +```nix +services.dnsmasq = { + enable = true; + settings = { + # Proxy mode: respond only to PXE DHCP requests, leave normal leases to router + dhcp-range = [ "192.168.2.0,proxy" ]; + # iPXE client detection + dhcp-match = [ + "set:ipxe,175" + "set:efi64,option:client-arch,7" + "set:efi64,option:client-arch,9" + ]; + dhcp-userclass = "set:ipxe,iPXE"; + # Boot file selection + dhcp-boot = [ + "tag:ipxe,tag:efi64,http://${vars.pxeServerIp}/boot.ipxe" + "tag:ipxe,http://${vars.pxeServerIp}/boot.ipxe" + "tag:efi64,ipxe.efi,,${vars.pxeServerIp}" + "undionly.kpxe,,${vars.pxeServerIp}" + ]; + }; +}; +``` + +### 2b. Rebuild and deploy the pxe-boot CT + +```bash +# On pve1 — build the new tarball +nix build .#lxc-pxe-boot.config.system.build.tarball + +# Verify dnsmasq starts correctly in the CT after deploy +ssh nixos@192.168.2.247 systemctl status dnsmasq +``` + +### 2c. Remove the iPXE config from Pi-hole + +In the Pi-hole CT, remove `/etc/dnsmasq.d/99-ipxe-chainload.conf` and +restart the FTL service: + +```bash +ssh wayne@pve1.sweet.home \ + "sudo pct exec 100 -- bash -c 'rm /etc/dnsmasq.d/99-ipxe-chainload.conf && systemctl restart pihole-FTL'" +``` + +**Verify:** PXE boot a test machine — it should still get an iPXE response and +reach the boot menu. + +**Rollback 2c:** restore the file from the Pi-hole config backup at +`/etc/pihole/config_backups/` and restart pihole-FTL. + +--- + +## Stage 3 — DHCP migration: Pi-hole → router (brief maintenance window) + +**Do this in the evening.** Existing DHCP leases stay valid during the +switchover so connected devices don't drop — only new lease requests fail +during the gap, which is under 60 seconds if you follow the steps in order. + +The key: configure the router's DHCP DNS option to point at `.253` (Pi-hole's +current IP). This way, all new leases issued by the router still get the same +DNS server address — clients never need to change their DNS config. When Pi-hole +is retired and the DC takes `.253` in Stage 4, `.253` just starts answering +differently. No client reconfiguration. + +### 3a. Pre-configure router DHCP (do not enable yet) + +Log into `http://192.168.2.254`, find the DHCP settings and fill in — but +leave DHCP **disabled** until step 3b: + +| Setting | Value | +|---|---| +| Start IP | 192.168.2.10 | +| End IP | 192.168.2.59 | +| Subnet mask | 255.255.255.0 | +| Gateway | 192.168.2.254 | +| Primary DNS | 192.168.2.253 | +| Secondary DNS | *(leave blank)* | +| Lease time | 24h | + +Save without enabling. + +### 3b. Switchover (do steps in quick succession) + +1. **Disable Pi-hole DHCP:** Pi-hole admin UI → Settings → DHCP → uncheck + "DHCP server enabled" → Save +2. **Enable router DHCP** immediately after step 1 + +### 3c. Verify router DHCP is working + +On a phone or laptop, disconnect from WiFi and reconnect (or run +`sudo dhclient -r && sudo dhclient` on a Linux host): + +```bash +ip addr show # IP should be in 192.168.2.10–59 range +dig google.com # should resolve (Pi-hole DNS still running at .253) +dig pve1.sweet.home # should resolve via FreeIPA at .138 (relayed via Pi-hole) +``` + +Wait 10–15 minutes for the most active devices to renew their leases. There's +no need to wait for all leases to expire before proceeding. + +**Rollback 3b:** Re-enable Pi-hole DHCP. Disable router DHCP. Done — existing +leases remain valid so most devices are unaffected. + +--- + +## Stage 4 — Move domain-controller from .138 to .253 + +Pi-hole lives at `.253`. The DC must take `.253` the moment Pi-hole stops so +clients that still have `.253` as their DNS server don't notice the change. +Script these commands in advance and run them in rapid succession. + +**Pre-stage: have this SSH command ready before running step 4a:** +```bash +ssh wayne@192.168.2.138 " + sudo nmcli connection modify 'System eth0' \ + ipv4.addresses '192.168.2.253/24' \ + ipv4.gateway '192.168.2.254' \ + ipv4.dns '127.0.0.1' \ + ipv4.method manual && \ + sudo nmcli connection up 'System eth0' +" +``` + +**Also update the Proxmox VM config to match (run from pve1):** +```bash +sudo qm set 108 \ + --ipconfig0 ip=192.168.2.253/24,gw=192.168.2.254 \ + --nameserver 192.168.2.253 +``` + +### 4a. Stop Pi-hole + +```bash +ssh wayne@pve1.sweet.home "sudo pct stop 100" +``` + +### 4b. Immediately: change DC's IP to .253 + +Run the pre-staged SSH command from above. You have ~30 seconds before any +client notices Pi-hole is gone. If SSH to `.138` refuses (the IP is already +changing), open a Proxmox console to VM 108 and run the `nmcli` commands +there. + +### 4c. Update Proxmox VM config + +Run the pre-staged `qm set 108` command from above. + +### 4d. Verify + +```bash +ssh wayne@192.168.2.253 # must connect (new DC IP) +dig @192.168.2.253 pve1.sweet.home +short # must return 192.168.2.250 +dig @192.168.2.253 google.com +short # must return an IP +``` + +From a client device that renewed its DHCP lease in Stage 3: +```bash +cat /etc/resolv.conf # should show 192.168.2.253 +dig pve1.sweet.home # should resolve +``` + +**Rollback 4:** `ssh wayne@pve1.sweet.home "sudo pct start 100"`. Change DC IP +back to .138 via Proxmox console. This restores full Pi-hole DNS/DHCP service. +Leave Pi-hole CT stopped-but-intact for 48 hours before deleting it. + +--- + +## Stage 5 — Host renumbering (one at a time, any order) + +For each host: +1. Update FreeIPA DNS A record and PTR record to the new IP +2. Change the static IP on the host itself +3. Verify SSH to new IP +4. Update `variables.nix` if that host has an IP variable (pxe-boot, pbs — already done in this PR) + +**FreeIPA record update template** (run as admin on domain-controller): +```bash +ipa dnsrecord-mod sweet.home --a-rec +ipa dnsrecord-del 2.168.192.in-addr.arpa --ptr-rec .sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa --ptr-rec .sweet.home. +``` + +### Renumbering order + +| # | Host | Old IP | New IP | How to change IP | +|---|---|---|---|---| +| 1 | nixos workstation | .119 | .243 | NetworkManager on guest; or `nmcli connection modify` | +| 2 | nix-cache | .120 | .224 | `pct set 102 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.224/24,gw=192.168.2.254` then `pct reboot 102` | +| 3 | tailscale-router | .121 | .222 | Static config on guest; check Tailscale ACLs if IP is referenced there | +| 4 | tor-relay | .107 | .221 | `pct set 104 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.221/24,gw=192.168.2.254` then `pct reboot 104` | +| 5 | pdm | .248 | .220 | `pct set 106 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.220/24,gw=192.168.2.254` then `pct reboot 106` | +| 6 | pxe-boot | .247 | .223 | `pct set 103 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.223/24,gw=192.168.2.254` then rebuild NixOS (already updated in variables.nix) | +| 7 | server | .252 | .226 | Static config on guest; NFS clients (docker) lose mounts briefly — they remount automatically | +| 8 | docker | .249 | .225 | Static config on guest; do this after server is at .226 | +| 9 | pbs | .108 | .244 | Static config on PBS host itself; update in `pbsIp` already done in variables.nix | +| 10 | pve1 | .250 | .245 | Edit `/etc/network/interfaces` on the Proxmox host — see below | + +### pve1 renumber (step 10 — do last) + +All guests keep running; only the Proxmox web UI is briefly unreachable. + +```bash +ssh wayne@pve1.sweet.home + +# Edit /etc/network/interfaces: change address from .250 to .245 +sudo nano /etc/network/interfaces +# Change: address 192.168.2.250/24 +# To: address 192.168.2.245/24 + +sudo systemctl restart networking +# SSH will drop here — reconnect to new IP +``` + +```bash +ssh wayne@192.168.2.245 # verify +``` + +Update FreeIPA DNS: +```bash +ipa dnsrecord-mod sweet.home pve1 --a-rec 192.168.2.245 +ipa dnsrecord-del 2.168.192.in-addr.arpa 250 --ptr-rec pve1.sweet.home. +ipa dnsrecord-add 2.168.192.in-addr.arpa 245 --ptr-rec pve1.sweet.home. +``` + +**Rollback any step 5 host:** change the IP back on the guest and update the +FreeIPA record back to the old IP. The old IP is unoccupied so you can +temporarily use either. + +--- + +## Stage 6 — Final cleanup + +Once all hosts are at their new IPs and verified: + +```bash +# Delete the Pi-hole CT (already stopped since Stage 4) +ssh wayne@pve1.sweet.home "sudo pct destroy 100" + +# Remove stale FreeIPA records for retired addresses +ipa dnsrecord-del sweet.home pihole --del-all +ipa dnsrecord-del 2.168.192.in-addr.arpa 253 --ptr-rec pihole.sweet.home. + +# Rebuild any NixOS hosts that reference pbsIp or pxeServerIp to pick up +# the updated variables.nix values (pxe-boot mandatory; others as convenient) +``` + +--- + +## Rollback summary + +| What broke | How to roll back | +|---|---| +| FreeIPA DNS not resolving | Check `systemctl status named` on DC; restart if failed | +| FreeIPA DNS unreachable | `pct start 100` on pve1 (restores Pi-hole) | +| Router DHCP not handing out leases | Re-enable Pi-hole DHCP; disable router DHCP | +| DC unreachable after IP change | Proxmox console on VM 108 → `nmcli connection up "System eth0"` with old IP | +| Host unreachable after renumber | Proxmox console → revert IP; or `pct set --net0 ...` old IP and reboot CT | +| pve1 web UI gone after renumber | SSH to .245 and check `/etc/network/interfaces`; if wrong, fix and restart networking | diff --git a/hosts/ha-server-1/host.nix b/hosts/ha-server-1/host.nix index c380008..455fb30 100644 --- a/hosts/ha-server-1/host.nix +++ b/hosts/ha-server-1/host.nix @@ -15,8 +15,12 @@ address = vars.haServer1Ip; prefixLength = 24; }]; - defaultGateway = "192.168.2.1"; - nameservers = [ "192.168.2.1" "8.8.8.8" ]; + interfaces.ens19.ipv4.addresses = [{ + address = vars.haServer1StorageIp; + prefixLength = 29; + }]; + defaultGateway = "192.168.2.254"; + nameservers = [ vars.domainControllerIp ]; }; # Set KEY after pairing this host with the beszel hub; the token is sops-managed. diff --git a/hosts/ha-server-2/host.nix b/hosts/ha-server-2/host.nix index dcb1464..2034fef 100644 --- a/hosts/ha-server-2/host.nix +++ b/hosts/ha-server-2/host.nix @@ -15,8 +15,12 @@ address = vars.haServer2Ip; prefixLength = 24; }]; - defaultGateway = "192.168.2.1"; - nameservers = [ "192.168.2.1" "8.8.8.8" ]; + interfaces.ens19.ipv4.addresses = [{ + address = vars.haServer2StorageIp; + prefixLength = 29; + }]; + defaultGateway = "192.168.2.254"; + nameservers = [ vars.domainControllerIp ]; }; # Set KEY after pairing this host with the beszel hub; the token is sops-managed. diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix index 160c154..5b55776 100644 --- a/modules/ha/cluster-config.nix +++ b/modules/ha/cluster-config.nix @@ -47,11 +47,11 @@ } on ${vars.haServer1Host} { - address ${vars.haServer1Ip}:${toString vars.ports.haServerDrbd}; + address ${vars.haServer1StorageIp}:${toString vars.ports.haServerDrbd}; } on ${vars.haServer2Host} { - address ${vars.haServer2Ip}:${toString vars.ports.haServerDrbd}; + address ${vars.haServer2StorageIp}:${toString vars.ports.haServerDrbd}; } } ''; @@ -75,8 +75,8 @@ services.corosync = { clusterName = "ha-cluster"; nodelist = [ - { nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1Ip ]; } - { nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2Ip ]; } + { nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1StorageIp ]; } + { nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2StorageIp ]; } ]; }; @@ -101,6 +101,7 @@ extraCommands = '' iptables -A INPUT -s ${vars.haServer1Ip}/32 -j ACCEPT iptables -A INPUT -s ${vars.haServer2Ip}/32 -j ACCEPT + iptables -A INPUT -s ${vars.haStorageCidr} -j ACCEPT ''; }; } diff --git a/variables.nix b/variables.nix index 70c3772..a16d792 100644 --- a/variables.nix +++ b/variables.nix @@ -70,16 +70,19 @@ primaryUser = "nixos"; # HA file server cluster - # haServer1Ip / haServer2Ip: static LAN IPs for both HA nodes (must be - # fixed — DRBD and corosync ring addresses are baked into the NixOS config). + # LAN IPs (vmbr0 / ens18) — client-facing: iSCSI initiators, NFS, management. + # Storage IPs (vmbr1 / ens19) — isolated internal bridge, used for DRBD + # replication and Corosync heartbeat only; never leaves pve1. # haServerVip: floating virtual IP managed by Pacemaker's IPaddr2 resource; # NFS and iSCSI clients connect here regardless of which node is Active. - # Set all three to real values in variables.nix before deploying. haServer1Host = "ha-server-1"; haServer2Host = "ha-server-2"; - haServer1Ip = "192.168.2.200"; # TODO: confirm production IP - haServer2Ip = "192.168.2.201"; # TODO: confirm production IP - haServerVip = "192.168.2.202"; # TODO: confirm floating VIP + haServer1Ip = "192.168.2.228"; # LAN IP, node 1 + haServer2Ip = "192.168.2.227"; # LAN IP, node 2 + haServerVip = "192.168.2.229"; # floating VIP (Pacemaker IPaddr2) + haServer1StorageIp = "192.168.4.228"; # storage-net IP, node 1 (vmbr1 / ens19) + haServer2StorageIp = "192.168.4.227"; # storage-net IP, node 2 (vmbr1 / ens19) + haStorageCidr = "192.168.4.0/29"; # storage subnet — internal to pve1 only haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage";