Archived
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  beatzaplentyandClaude Sonnet 4.6
|
928646f7d0 | ||
|
beatzaplentyandClaude Sonnet 4.6
|
c90f2cffc0 |
-64
@@ -63,16 +63,6 @@ creation_rules:
|
||||
- *lxc-nix-cache
|
||||
- *proxmox-nix-cache
|
||||
|
||||
# Host keytab for nix-cache FreeIPA enrollment (binary sops file).
|
||||
# Generate with: sops -e --input-type binary /tmp/nix-cache.keytab > secrets/nix-cache.keytab
|
||||
- path_regex: secrets/nix-cache\.keytab$
|
||||
key_groups:
|
||||
- age:
|
||||
- *admin
|
||||
- *linode-nix-cache
|
||||
- *lxc-nix-cache
|
||||
- *proxmox-nix-cache
|
||||
|
||||
- path_regex: secrets/server\.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
@@ -87,56 +77,12 @@ creation_rules:
|
||||
- *admin
|
||||
- *lxc-tor-relay
|
||||
|
||||
- path_regex: secrets/tailscale-router\.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
- *admin
|
||||
- *linode-tailscale-router
|
||||
- *lxc-tailscale-router
|
||||
- *proxmox-tailscale-router
|
||||
|
||||
# HA file server per-node secrets (beszel-token).
|
||||
# proxmox-ha-server-1 / proxmox-ha-server-2 keys are added automatically
|
||||
# by scripts/secrets/sync-host-keys.sh once the hosts are provisioned;
|
||||
# until then only the admin key can decrypt these files.
|
||||
- path_regex: secrets/ha-server-1\.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
- *admin
|
||||
# proxmox-ha-server-1 added by sync-host-keys.sh
|
||||
|
||||
- path_regex: secrets/ha-server-2\.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
- *admin
|
||||
# proxmox-ha-server-2 added by sync-host-keys.sh
|
||||
|
||||
# Shared HA cluster corosync authkey (binary sops file).
|
||||
# Encrypted for both HA nodes so either can decrypt on boot.
|
||||
# Both host keys added by sync-host-keys.sh; admin key allows initial creation.
|
||||
- path_regex: secrets/ha-corosync-authkey$
|
||||
key_groups:
|
||||
- age:
|
||||
- *admin
|
||||
# proxmox-ha-server-1 added by sync-host-keys.sh
|
||||
# proxmox-ha-server-2 added by sync-host-keys.sh
|
||||
|
||||
# gui-host-specific secrets (currently: wifi-password, see
|
||||
# modules/networking/wifi.nix). Only *lxc-gui has a registered key today
|
||||
# -- proxmox-gui/linode-gui/baremetal-gui haven't been provisioned via
|
||||
# scripts/secrets/sync-host-keys.sh yet, so whichever variant is actually
|
||||
# deployed next needs its recipient added here (and `sops updatekeys` rerun)
|
||||
# before it can decrypt this.
|
||||
# Host keytab for tailscale-router FreeIPA enrollment (binary sops file).
|
||||
# Generated by scripts/ipa/create-nixos-ipa-host-account.sh.
|
||||
- path_regex: secrets/tailscale-router\.keytab$
|
||||
key_groups:
|
||||
- age:
|
||||
- *admin
|
||||
- *lxc-tailscale-router
|
||||
- *proxmox-tailscale-router
|
||||
- *linode-tailscale-router
|
||||
|
||||
- path_regex: secrets/gui\.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
@@ -145,13 +91,3 @@ creation_rules:
|
||||
- *baremetal-gui
|
||||
- *linode-gui
|
||||
- *proxmox-gui
|
||||
|
||||
# Host keytab for tailscale-router FreeIPA enrollment (binary sops file).
|
||||
# Generated by scripts/ipa/create-nixos-ipa-host-account.sh.
|
||||
- path_regex: secrets/tailscale-router\.keytab$
|
||||
key_groups:
|
||||
- age:
|
||||
- *admin
|
||||
- *lxc-tailscale-router
|
||||
- *proxmox-tailscale-router
|
||||
- *linode-tailscale-router
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIESDCCArCgAwIBAgIBATANBgkqhkiG9w0BAQsFADA1MRMwEQYDVQQKDApTV0VF
|
||||
VC5IT01FMR4wHAYDVQQDDBVDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwHhcNMjYwNzI2
|
||||
MjExMzQxWhcNNDYwNzI2MjExMzQxWjA1MRMwEQYDVQQKDApTV0VFVC5IT01FMR4w
|
||||
HAYDVQQDDBVDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggGiMA0GCSqGSIb3DQEBAQUA
|
||||
A4IBjwAwggGKAoIBgQCzljYktbHdMGVJ6Wq0XQJuHLN6dkCSOgtoIzQtriPQkkNI
|
||||
uo28LwobaiQQ8sX4kGRH/BTKnH8QlId/jug4Uc+sDHnABYu++AiOhPbBX8gCpRQ0
|
||||
hebBjZiktHSBUEJR31siWOVdBoKBDJEoxehx7XUXvcxIJcaRN+LHYjO86nJN55HB
|
||||
VwFU2JcYDk98c+144dFJxXdr++MjWe4Z/oVVU8JHIOtNtKhVhvij6oOSWxcYoJO/
|
||||
S80LRj1vx/o6o/3G6bYug7PjY7JjZk/Oj61whijZkcsoO1MXSYI6UywJZGflv+ZB
|
||||
7HyufdYAsK3WhE8O2FX3/kq64Ol83HNtoR8Dt68rTg1xpW6K45jS6iDPKueYGkb0
|
||||
oSx7e++90VAW2PDhj6QQ3JJ4O5VQwrrecekJzUrAean0FOEbmgyi4PsEp1Vk6LDQ
|
||||
SsIn1x0euyxVivQMlzNX2XrZL3urn1BNPqAdntXQMkR0Wl8sbUiJPe0kxG52CGXs
|
||||
6yfNEXbPmVGcC0TBdGECAwEAAaNjMGEwHQYDVR0OBBYEFLh5QbI1UWMH0WR4z8bG
|
||||
lhrOX3X5MB8GA1UdIwQYMBaAFLh5QbI1UWMH0WR4z8bGlhrOX3X5MA8GA1UdEwEB
|
||||
/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgHGMA0GCSqGSIb3DQEBCwUAA4IBgQCVodVN
|
||||
owwo53OQe02QhtEbIur2PL7zIfvhvCTRD4J8gwpbMIqT7JQK0tV6Mvsg2L8yTb2O
|
||||
KjrWeLKHGWaZZlhGSPTbkMFdb/Ls8M9FSnkc2bwcdWW3Z1lOiCjBYYqwLCG6JhvB
|
||||
5SXVwWNJwXeasL2m7oFTSwhsqPpARJ2t25u2N35o+tqIoCjijKwkmEOT66N9EAbu
|
||||
2VQjtYZWPkBtP4YCe0Ey6u4oy7sy8ThNAjOylZok+J4JW7QEFjK4Q/emhA4aQq5H
|
||||
gg9qgMuG+5oi6D1g2Wy+fMTRBaukJtLYZbBpQMQhMYWg44uPp/2bbNPTID/nV1KB
|
||||
GcPyHaskcVxPdYWxAPMwk3AeJXWyOq7atAPTF5sbk0kQQf2m+vyOqcli5CxRMUgV
|
||||
rcyi9l6+dZW4U+38Q0ET5M3OuxNI4hA7kVY2cfTakXWNqh97+TIHnstblDhAxECK
|
||||
6ZLMJQYUy7LqJTX84H27CBWLexEMjXwdr5HCV88Fj6mAK0fRufnIw5FeneA=
|
||||
-----END CERTIFICATE-----
|
||||
@@ -1,128 +0,0 @@
|
||||
# IP Addressing Scheme
|
||||
|
||||
## Subnets
|
||||
|
||||
| Subnet | CIDR | Purpose | Routed? |
|
||||
|---|---|---|---|
|
||||
| LAN | `192.168.2.0/24` | General LAN — clients and infrastructure | Yes (gateway .254) |
|
||||
| Storage | `192.168.4.0/29` | HA file server DRBD replication | No — internal `vmbr1` only, no uplink |
|
||||
|
||||
The storage subnet never leaves pve1. `vmbr1` is a Proxmox Linux bridge with no physical port
|
||||
attached; traffic between the two HA file server VMs stays in-kernel.
|
||||
|
||||
The host octet is consistent across subnets for any host that has multiple interfaces — e.g.
|
||||
ha-node1 is always `.228` (LAN: `192.168.2.228`, storage: `192.168.4.228`).
|
||||
|
||||
---
|
||||
|
||||
## LAN — 192.168.2.0/24
|
||||
|
||||
### Address map
|
||||
|
||||
| Range | Purpose |
|
||||
|---|---|
|
||||
| .1–.9 | Reserved, never assign |
|
||||
| .10–.59 | Client DHCP pool (router-assigned) |
|
||||
| .60–.219 | Unallocated buffer |
|
||||
| .220–.229 | Virtual nodes (VMs / LXC containers) |
|
||||
| .230–.239 | Expansion buffer (reserved, unallocated) |
|
||||
| .240–.249 | Physical nodes (bare-metal hosts) |
|
||||
| .250–.253 | Network services |
|
||||
| .254 | Router / gateway |
|
||||
|
||||
### Network services (.250–.253)
|
||||
|
||||
| IP | Hostname | Role |
|
||||
|---|---|---|
|
||||
| `192.168.2.254` | router | Gateway (TP-Link) |
|
||||
| `192.168.2.253` | domain-controller | FreeIPA — authoritative DNS for `sweet.home`, Kerberos, LDAP |
|
||||
| `192.168.2.250`–`.252` | — | Reserved for future network services |
|
||||
|
||||
### Physical nodes (.240–.249)
|
||||
|
||||
| IP | Hostname | Role |
|
||||
|---|---|---|
|
||||
| `192.168.2.245` | pve1 | Proxmox VE hypervisor |
|
||||
| `192.168.2.244` | pbs | Proxmox Backup Server |
|
||||
| `192.168.2.243` | nixos | Bare-metal workstation (`baremetal-gui`) |
|
||||
| `192.168.2.246`–`.249` | — | Reserved — second Proxmox node and associated services |
|
||||
| `192.168.2.240`–`.242` | — | Reserved |
|
||||
|
||||
pve1 sits mid-range deliberately so a second Proxmox node can slot in on either side.
|
||||
|
||||
### Virtual nodes (.220–.229)
|
||||
|
||||
All VMs and LXC containers run on pve1.
|
||||
|
||||
| IP | Hostname | Role | Status |
|
||||
|---|---|---|---|
|
||||
| `192.168.2.229` | ha-vip | HA file server iSCSI floating VIP (Pacemaker) | Future |
|
||||
| `192.168.2.228` | ha-node1 | HA file server node 1 (DRBD + XFS + iSCSI) | Future |
|
||||
| `192.168.2.227` | ha-node2 | HA file server node 2 (DRBD + XFS + iSCSI) | Future |
|
||||
| `192.168.2.226` | server | Current NFS/ZFS file server — retires when HA is live | Retiring |
|
||||
| `192.168.2.225` | docker | Docker / Traefik stack | Active |
|
||||
| `192.168.2.224` | nix-cache | Nix binary cache + remote builder | Active |
|
||||
| `192.168.2.223` | pxe-boot | PXE / TFTP / HTTP netboot server | Active |
|
||||
| `192.168.2.222` | tailscale-router | Tailscale exit node / router | Active |
|
||||
| `192.168.2.221` | tor-relay | Tor relay | Active |
|
||||
| `192.168.2.220` | pdm | Proxmox Deploy Manager | Active |
|
||||
|
||||
### Client DHCP pool (.10–.59)
|
||||
|
||||
Assigned by the router. DNS option points to `192.168.2.253` (domain-controller).
|
||||
|
||||
Devices in this range: phones, laptops, IoT, Canon printer, any non-infrastructure host.
|
||||
No static reservations for infrastructure hosts — all infra uses static IP configuration
|
||||
on the guest itself (not DHCP reservations), so IPs survive VM recreation regardless of
|
||||
MAC address churn.
|
||||
|
||||
---
|
||||
|
||||
## Storage network — 192.168.4.0/29
|
||||
|
||||
Internal to pve1 only. Proxmox bridge `vmbr1`, no physical NIC attached.
|
||||
|
||||
| IP | Hostname | Interface role |
|
||||
|---|---|---|
|
||||
| `192.168.4.228` | ha-node1 | DRBD replication NIC |
|
||||
| `192.168.4.227` | ha-node2 | DRBD replication NIC |
|
||||
| — | no gateway | Isolated — not routed to LAN or internet |
|
||||
|
||||
---
|
||||
|
||||
## Migration reference
|
||||
|
||||
Current → target IP for every host being renumbered.
|
||||
|
||||
| Host | Current IP | New IP | Config location |
|
||||
|---|---|---|---|
|
||||
| router | `192.168.2.254` | `192.168.2.254` | unchanged |
|
||||
| domain-controller | `192.168.2.138` | `192.168.2.253` | `/etc/sysconfig/network-scripts/ifcfg-eth0` on guest |
|
||||
| pve1 | `192.168.2.250` | `192.168.2.245` | `/etc/network/interfaces` on Proxmox host |
|
||||
| pbs | `192.168.2.108` | `192.168.2.244` | static config on PBS host |
|
||||
| nixos workstation | `192.168.2.119` | `192.168.2.243` | `networking.interfaces` / NetworkManager on guest |
|
||||
| ha-node1 | — | `192.168.2.228` | future |
|
||||
| ha-node2 | — | `192.168.2.227` | future |
|
||||
| ha-vip | — | `192.168.2.229` | future (Pacemaker resource) |
|
||||
| server | `192.168.2.252` | `192.168.2.226` | static config on guest |
|
||||
| docker | `192.168.2.249` | `192.168.2.225` | static config on guest |
|
||||
| nix-cache | `192.168.2.120` | `192.168.2.224` | static config on guest |
|
||||
| pxe-boot | `192.168.2.247` | `192.168.2.223` | static config on guest; update `vars.pxeServerIp` in `variables.nix` ✓ |
|
||||
| tailscale-router | `192.168.2.121` | `192.168.2.222` | static config on guest |
|
||||
| tor-relay | `192.168.2.107` | `192.168.2.221` | static config on guest |
|
||||
| pdm | `192.168.2.248` | `192.168.2.220` | static config on guest |
|
||||
|
||||
### Cutover notes
|
||||
|
||||
- **Do domain-controller first** — it becomes the DNS server; everything else depends on it
|
||||
having its new IP and FreeIPA DNS configured before Pi-hole is retired.
|
||||
- **pve1 last among physical hosts** — changing the Proxmox management IP drops the web UI
|
||||
briefly; all guests keep running.
|
||||
- **Update Pi-hole custom.list / FreeIPA DNS A records** to new IPs before flipping any host,
|
||||
so name resolution stays valid throughout the migration.
|
||||
- **variables.nix already updated** for `pxeServerIp` (.247→.223), `pbsIp` (.108→.244), and
|
||||
new `domainControllerIp` (.253). Rebuild affected hosts after renumbering.
|
||||
- **Router DHCP**: once domain-controller is at .253 and FreeIPA DNS is serving `sweet.home`,
|
||||
switch router DHCP on with pool .10–.59 and DNS option pointing to .253; retire Pi-hole CT.
|
||||
- **Pi-hole's iPXE dnsmasq config** (`99-ipxe-chainload.conf`) moves to the pxe-boot CT as a
|
||||
dnsmasq proxy-mode config before Pi-hole is decommissioned.
|
||||
@@ -1,366 +0,0 @@
|
||||
# Network Cutover Plan
|
||||
|
||||
Moves the LAN from the current flat/Pi-hole-managed state to the new IP scheme
|
||||
defined in `docs/ip-addressing.md`. Works in five independent stages — each
|
||||
stage is safe to pause after and resume later. Rollback steps are given at
|
||||
every point where something can break.
|
||||
|
||||
**Before starting anything:** confirm you have
|
||||
- SSH access to `192.168.2.138` (domain-controller, current IP)
|
||||
- SSH access to `192.168.2.250` (pve1)
|
||||
- Browser access to Pi-hole admin at `http://192.168.2.253`
|
||||
- Browser access to router admin at `http://192.168.2.254`
|
||||
- The FreeIPA `admin` password to hand
|
||||
|
||||
---
|
||||
|
||||
## Stage 1 — Prepare FreeIPA DNS (zero downtime)
|
||||
|
||||
Everything here is additive. Pi-hole keeps running. Nothing breaks if you stop
|
||||
mid-stage.
|
||||
|
||||
### 1a. Add NextDNS forwarders
|
||||
|
||||
```bash
|
||||
ssh wayne@192.168.2.138
|
||||
kinit admin # enter FreeIPA admin password when prompted
|
||||
ipa dnsconfig-mod \
|
||||
--forwarder=45.90.28.142 \
|
||||
--forwarder=45.90.30.142 \
|
||||
--forward-policy=only
|
||||
```
|
||||
|
||||
**Verify external resolution works through FreeIPA before continuing:**
|
||||
```bash
|
||||
dig @127.0.0.1 google.com +short # must return an IP, not SERVFAIL
|
||||
```
|
||||
|
||||
### 1b. Add A records for every host at their CURRENT IPs
|
||||
|
||||
These represent the live state now. You'll update each record to the new IP
|
||||
when you renumber that host in Stage 5.
|
||||
|
||||
```bash
|
||||
ipa dnsrecord-add sweet.home pve1 --a-rec 192.168.2.250
|
||||
ipa dnsrecord-add sweet.home pbs --a-rec 192.168.2.108
|
||||
ipa dnsrecord-add sweet.home nixos --a-rec 192.168.2.119
|
||||
ipa dnsrecord-add sweet.home server --a-rec 192.168.2.252
|
||||
ipa dnsrecord-add sweet.home docker --a-rec 192.168.2.249
|
||||
ipa dnsrecord-add sweet.home nix-cache --a-rec 192.168.2.120
|
||||
ipa dnsrecord-add sweet.home pxe-boot --a-rec 192.168.2.247
|
||||
ipa dnsrecord-add sweet.home tailscale-router --a-rec 192.168.2.121
|
||||
ipa dnsrecord-add sweet.home tor-relay --a-rec 192.168.2.107
|
||||
ipa dnsrecord-add sweet.home pdm --a-rec 192.168.2.248
|
||||
ipa dnsrecord-add sweet.home router --a-rec 192.168.2.254
|
||||
```
|
||||
|
||||
### 1c. Clean up stale reverse-zone PTR records
|
||||
|
||||
FreeIPA already has PTR records from an earlier import but some are wrong.
|
||||
Fix them now so reverse DNS is accurate from day one.
|
||||
|
||||
```bash
|
||||
# Remove stale "win11" entry at .250 (should be pve1)
|
||||
ipa dnsrecord-del 2.168.192.in-addr.arpa 250 --ptr-rec win11.
|
||||
ipa dnsrecord-add 2.168.192.in-addr.arpa 250 --ptr-rec pve1.sweet.home.
|
||||
|
||||
# Fix unqualified PTR records (missing .sweet.home. suffix)
|
||||
ipa dnsrecord-mod 2.168.192.in-addr.arpa 108 --ptr-rec pbs.sweet.home.
|
||||
ipa dnsrecord-mod 2.168.192.in-addr.arpa 248 --ptr-rec pdm.sweet.home.
|
||||
ipa dnsrecord-mod 2.168.192.in-addr.arpa 249 --ptr-rec docker.sweet.home.
|
||||
ipa dnsrecord-mod 2.168.192.in-addr.arpa 252 --ptr-rec server.sweet.home.
|
||||
|
||||
# Add any missing PTR records
|
||||
ipa dnsrecord-add 2.168.192.in-addr.arpa 119 --ptr-rec nixos.sweet.home.
|
||||
ipa dnsrecord-add 2.168.192.in-addr.arpa 120 --ptr-rec nix-cache.sweet.home.
|
||||
ipa dnsrecord-add 2.168.192.in-addr.arpa 121 --ptr-rec tailscale-router.sweet.home.
|
||||
ipa dnsrecord-add 2.168.192.in-addr.arpa 247 --ptr-rec pxe-boot.sweet.home.
|
||||
ipa dnsrecord-add 2.168.192.in-addr.arpa 254 --ptr-rec router.sweet.home.
|
||||
```
|
||||
|
||||
### 1d. Point domain-controller's own DNS at itself
|
||||
|
||||
```bash
|
||||
sudo nmcli connection modify "System eth0" ipv4.dns "127.0.0.1"
|
||||
sudo nmcli connection up "System eth0"
|
||||
```
|
||||
|
||||
**Verify:**
|
||||
```bash
|
||||
dig pve1.sweet.home +short # must return 192.168.2.250
|
||||
dig google.com +short # must return an IP (NextDNS forwarding)
|
||||
```
|
||||
|
||||
**Rollback 1d:** `sudo nmcli connection modify "System eth0" ipv4.dns "192.168.2.253" && sudo nmcli connection up "System eth0"`
|
||||
|
||||
---
|
||||
|
||||
## Stage 2 — Move pxe-boot DHCP options off Pi-hole (zero downtime)
|
||||
|
||||
Pi-hole's dnsmasq currently serves the iPXE boot options via
|
||||
`99-ipxe-chainload.conf`. Before Pi-hole is retired, that config must move to
|
||||
the pxe-boot CT running dnsmasq in proxy mode so PXE boot keeps working.
|
||||
|
||||
### 2a. Add dnsmasq proxy config to the pxe-boot NixOS module
|
||||
|
||||
In `modules/build-types/pxe-boot.nix`, add:
|
||||
|
||||
```nix
|
||||
services.dnsmasq = {
|
||||
enable = true;
|
||||
settings = {
|
||||
# Proxy mode: respond only to PXE DHCP requests, leave normal leases to router
|
||||
dhcp-range = [ "192.168.2.0,proxy" ];
|
||||
# iPXE client detection
|
||||
dhcp-match = [
|
||||
"set:ipxe,175"
|
||||
"set:efi64,option:client-arch,7"
|
||||
"set:efi64,option:client-arch,9"
|
||||
];
|
||||
dhcp-userclass = "set:ipxe,iPXE";
|
||||
# Boot file selection
|
||||
dhcp-boot = [
|
||||
"tag:ipxe,tag:efi64,http://${vars.pxeServerIp}/boot.ipxe"
|
||||
"tag:ipxe,http://${vars.pxeServerIp}/boot.ipxe"
|
||||
"tag:efi64,ipxe.efi,,${vars.pxeServerIp}"
|
||||
"undionly.kpxe,,${vars.pxeServerIp}"
|
||||
];
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
### 2b. Rebuild and deploy the pxe-boot CT
|
||||
|
||||
```bash
|
||||
# On pve1 — build the new tarball
|
||||
nix build .#lxc-pxe-boot.config.system.build.tarball
|
||||
|
||||
# Verify dnsmasq starts correctly in the CT after deploy
|
||||
ssh nixos@192.168.2.247 systemctl status dnsmasq
|
||||
```
|
||||
|
||||
### 2c. Remove the iPXE config from Pi-hole
|
||||
|
||||
In the Pi-hole CT, remove `/etc/dnsmasq.d/99-ipxe-chainload.conf` and
|
||||
restart the FTL service:
|
||||
|
||||
```bash
|
||||
ssh wayne@pve1.sweet.home \
|
||||
"sudo pct exec 100 -- bash -c 'rm /etc/dnsmasq.d/99-ipxe-chainload.conf && systemctl restart pihole-FTL'"
|
||||
```
|
||||
|
||||
**Verify:** PXE boot a test machine — it should still get an iPXE response and
|
||||
reach the boot menu.
|
||||
|
||||
**Rollback 2c:** restore the file from the Pi-hole config backup at
|
||||
`/etc/pihole/config_backups/` and restart pihole-FTL.
|
||||
|
||||
---
|
||||
|
||||
## Stage 3 — DHCP migration: Pi-hole → router (brief maintenance window)
|
||||
|
||||
**Do this in the evening.** Existing DHCP leases stay valid during the
|
||||
switchover so connected devices don't drop — only new lease requests fail
|
||||
during the gap, which is under 60 seconds if you follow the steps in order.
|
||||
|
||||
The key: configure the router's DHCP DNS option to point at `.253` (Pi-hole's
|
||||
current IP). This way, all new leases issued by the router still get the same
|
||||
DNS server address — clients never need to change their DNS config. When Pi-hole
|
||||
is retired and the DC takes `.253` in Stage 4, `.253` just starts answering
|
||||
differently. No client reconfiguration.
|
||||
|
||||
### 3a. Pre-configure router DHCP (do not enable yet)
|
||||
|
||||
Log into `http://192.168.2.254`, find the DHCP settings and fill in — but
|
||||
leave DHCP **disabled** until step 3b:
|
||||
|
||||
| Setting | Value |
|
||||
|---|---|
|
||||
| Start IP | 192.168.2.10 |
|
||||
| End IP | 192.168.2.59 |
|
||||
| Subnet mask | 255.255.255.0 |
|
||||
| Gateway | 192.168.2.254 |
|
||||
| Primary DNS | 192.168.2.253 |
|
||||
| Secondary DNS | *(leave blank)* |
|
||||
| Lease time | 24h |
|
||||
|
||||
Save without enabling.
|
||||
|
||||
### 3b. Switchover (do steps in quick succession)
|
||||
|
||||
1. **Disable Pi-hole DHCP:** Pi-hole admin UI → Settings → DHCP → uncheck
|
||||
"DHCP server enabled" → Save
|
||||
2. **Enable router DHCP** immediately after step 1
|
||||
|
||||
### 3c. Verify router DHCP is working
|
||||
|
||||
On a phone or laptop, disconnect from WiFi and reconnect (or run
|
||||
`sudo dhclient -r && sudo dhclient` on a Linux host):
|
||||
|
||||
```bash
|
||||
ip addr show # IP should be in 192.168.2.10–59 range
|
||||
dig google.com # should resolve (Pi-hole DNS still running at .253)
|
||||
dig pve1.sweet.home # should resolve via FreeIPA at .138 (relayed via Pi-hole)
|
||||
```
|
||||
|
||||
Wait 10–15 minutes for the most active devices to renew their leases. There's
|
||||
no need to wait for all leases to expire before proceeding.
|
||||
|
||||
**Rollback 3b:** Re-enable Pi-hole DHCP. Disable router DHCP. Done — existing
|
||||
leases remain valid so most devices are unaffected.
|
||||
|
||||
---
|
||||
|
||||
## Stage 4 — Move domain-controller from .138 to .253
|
||||
|
||||
Pi-hole lives at `.253`. The DC must take `.253` the moment Pi-hole stops so
|
||||
clients that still have `.253` as their DNS server don't notice the change.
|
||||
Script these commands in advance and run them in rapid succession.
|
||||
|
||||
**Pre-stage: have this SSH command ready before running step 4a:**
|
||||
```bash
|
||||
ssh wayne@192.168.2.138 "
|
||||
sudo nmcli connection modify 'System eth0' \
|
||||
ipv4.addresses '192.168.2.253/24' \
|
||||
ipv4.gateway '192.168.2.254' \
|
||||
ipv4.dns '127.0.0.1' \
|
||||
ipv4.method manual && \
|
||||
sudo nmcli connection up 'System eth0'
|
||||
"
|
||||
```
|
||||
|
||||
**Also update the Proxmox VM config to match (run from pve1):**
|
||||
```bash
|
||||
sudo qm set 108 \
|
||||
--ipconfig0 ip=192.168.2.253/24,gw=192.168.2.254 \
|
||||
--nameserver 192.168.2.253
|
||||
```
|
||||
|
||||
### 4a. Stop Pi-hole
|
||||
|
||||
```bash
|
||||
ssh wayne@pve1.sweet.home "sudo pct stop 100"
|
||||
```
|
||||
|
||||
### 4b. Immediately: change DC's IP to .253
|
||||
|
||||
Run the pre-staged SSH command from above. You have ~30 seconds before any
|
||||
client notices Pi-hole is gone. If SSH to `.138` refuses (the IP is already
|
||||
changing), open a Proxmox console to VM 108 and run the `nmcli` commands
|
||||
there.
|
||||
|
||||
### 4c. Update Proxmox VM config
|
||||
|
||||
Run the pre-staged `qm set 108` command from above.
|
||||
|
||||
### 4d. Verify
|
||||
|
||||
```bash
|
||||
ssh wayne@192.168.2.253 # must connect (new DC IP)
|
||||
dig @192.168.2.253 pve1.sweet.home +short # must return 192.168.2.250
|
||||
dig @192.168.2.253 google.com +short # must return an IP
|
||||
```
|
||||
|
||||
From a client device that renewed its DHCP lease in Stage 3:
|
||||
```bash
|
||||
cat /etc/resolv.conf # should show 192.168.2.253
|
||||
dig pve1.sweet.home # should resolve
|
||||
```
|
||||
|
||||
**Rollback 4:** `ssh wayne@pve1.sweet.home "sudo pct start 100"`. Change DC IP
|
||||
back to .138 via Proxmox console. This restores full Pi-hole DNS/DHCP service.
|
||||
Leave Pi-hole CT stopped-but-intact for 48 hours before deleting it.
|
||||
|
||||
---
|
||||
|
||||
## Stage 5 — Host renumbering (one at a time, any order)
|
||||
|
||||
For each host:
|
||||
1. Update FreeIPA DNS A record and PTR record to the new IP
|
||||
2. Change the static IP on the host itself
|
||||
3. Verify SSH to new IP
|
||||
4. Update `variables.nix` if that host has an IP variable (pxe-boot, pbs — already done in this PR)
|
||||
|
||||
**FreeIPA record update template** (run as admin on domain-controller):
|
||||
```bash
|
||||
ipa dnsrecord-mod sweet.home <hostname> --a-rec <new-ip>
|
||||
ipa dnsrecord-del 2.168.192.in-addr.arpa <old-last-octet> --ptr-rec <hostname>.sweet.home.
|
||||
ipa dnsrecord-add 2.168.192.in-addr.arpa <new-last-octet> --ptr-rec <hostname>.sweet.home.
|
||||
```
|
||||
|
||||
### Renumbering order
|
||||
|
||||
| # | Host | Old IP | New IP | How to change IP |
|
||||
|---|---|---|---|---|
|
||||
| 1 | nixos workstation | .119 | .243 | NetworkManager on guest; or `nmcli connection modify` |
|
||||
| 2 | nix-cache | .120 | .224 | `pct set 102 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.224/24,gw=192.168.2.254` then `pct reboot 102` |
|
||||
| 3 | tailscale-router | .121 | .222 | Static config on guest; check Tailscale ACLs if IP is referenced there |
|
||||
| 4 | tor-relay | .107 | .221 | `pct set 104 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.221/24,gw=192.168.2.254` then `pct reboot 104` |
|
||||
| 5 | pdm | .248 | .220 | `pct set 106 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.220/24,gw=192.168.2.254` then `pct reboot 106` |
|
||||
| 6 | pxe-boot | .247 | .223 | `pct set 103 --net0 name=eth0,bridge=vmbr0,ip=192.168.2.223/24,gw=192.168.2.254` then rebuild NixOS (already updated in variables.nix) |
|
||||
| 7 | server | .252 | .226 | Static config on guest; NFS clients (docker) lose mounts briefly — they remount automatically |
|
||||
| 8 | docker | .249 | .225 | Static config on guest; do this after server is at .226 |
|
||||
| 9 | pbs | .108 | .244 | Static config on PBS host itself; update in `pbsIp` already done in variables.nix |
|
||||
| 10 | pve1 | .250 | .245 | Edit `/etc/network/interfaces` on the Proxmox host — see below |
|
||||
|
||||
### pve1 renumber (step 10 — do last)
|
||||
|
||||
All guests keep running; only the Proxmox web UI is briefly unreachable.
|
||||
|
||||
```bash
|
||||
ssh wayne@pve1.sweet.home
|
||||
|
||||
# Edit /etc/network/interfaces: change address from .250 to .245
|
||||
sudo nano /etc/network/interfaces
|
||||
# Change: address 192.168.2.250/24
|
||||
# To: address 192.168.2.245/24
|
||||
|
||||
sudo systemctl restart networking
|
||||
# SSH will drop here — reconnect to new IP
|
||||
```
|
||||
|
||||
```bash
|
||||
ssh wayne@192.168.2.245 # verify
|
||||
```
|
||||
|
||||
Update FreeIPA DNS:
|
||||
```bash
|
||||
ipa dnsrecord-mod sweet.home pve1 --a-rec 192.168.2.245
|
||||
ipa dnsrecord-del 2.168.192.in-addr.arpa 250 --ptr-rec pve1.sweet.home.
|
||||
ipa dnsrecord-add 2.168.192.in-addr.arpa 245 --ptr-rec pve1.sweet.home.
|
||||
```
|
||||
|
||||
**Rollback any step 5 host:** change the IP back on the guest and update the
|
||||
FreeIPA record back to the old IP. The old IP is unoccupied so you can
|
||||
temporarily use either.
|
||||
|
||||
---
|
||||
|
||||
## Stage 6 — Final cleanup
|
||||
|
||||
Once all hosts are at their new IPs and verified:
|
||||
|
||||
```bash
|
||||
# Delete the Pi-hole CT (already stopped since Stage 4)
|
||||
ssh wayne@pve1.sweet.home "sudo pct destroy 100"
|
||||
|
||||
# Remove stale FreeIPA records for retired addresses
|
||||
ipa dnsrecord-del sweet.home pihole --del-all
|
||||
ipa dnsrecord-del 2.168.192.in-addr.arpa 253 --ptr-rec pihole.sweet.home.
|
||||
|
||||
# Rebuild any NixOS hosts that reference pbsIp or pxeServerIp to pick up
|
||||
# the updated variables.nix values (pxe-boot mandatory; others as convenient)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Rollback summary
|
||||
|
||||
| What broke | How to roll back |
|
||||
|---|---|
|
||||
| FreeIPA DNS not resolving | Check `systemctl status named` on DC; restart if failed |
|
||||
| FreeIPA DNS unreachable | `pct start 100` on pve1 (restores Pi-hole) |
|
||||
| Router DHCP not handing out leases | Re-enable Pi-hole DHCP; disable router DHCP |
|
||||
| DC unreachable after IP change | Proxmox console on VM 108 → `nmcli connection up "System eth0"` with old IP |
|
||||
| Host unreachable after renumber | Proxmox console → revert IP; or `pct set <id> --net0 ...` old IP and reboot CT |
|
||||
| pve1 web UI gone after renumber | SSH to .245 and check `/etc/network/interfaces`; if wrong, fix and restart networking |
|
||||
+7
-97
@@ -15,7 +15,6 @@ rescue/inspection use.
|
||||
- TFTP root for first-stage bootloaders: `/srv/pxe/tftp`
|
||||
- iPXE entry script: `/srv/pxe/http/boot.ipxe`
|
||||
- Generated iPXE menu: `/srv/pxe/http/menu.ipxe`
|
||||
- Debian Minimal iPXE script: `/srv/pxe/http/debian.ipxe`
|
||||
- SystemRescue iPXE script: `/srv/pxe/http/systemrescue.ipxe`
|
||||
- TFTP fallback script: `/srv/pxe/tftp/autoexec.ipxe`
|
||||
- Boot binaries copied from the Nix `ipxe` package:
|
||||
@@ -29,51 +28,32 @@ The host creates these directories with systemd tmpfiles:
|
||||
```text
|
||||
/srv/pxe
|
||||
/srv/pxe/http
|
||||
/srv/pxe/http/images -> /mnt/pxe-images (symlink to NFS share)
|
||||
/srv/pxe/http/images
|
||||
/srv/pxe/http/auto-installer
|
||||
/srv/pxe/http/nixos-minimal
|
||||
/srv/pxe/http/debian
|
||||
/srv/pxe/http/systemrescue
|
||||
/srv/pxe/http/ubuntu
|
||||
/srv/pxe/http/rescue
|
||||
/srv/pxe/tftp
|
||||
```
|
||||
|
||||
`/srv/pxe/http/images` is a symlink to `/mnt/pxe-images`, which is an NFS
|
||||
mount of `server.sweet.home:/tank/pxe-boot/images`
|
||||
(`modules/pxe-boot/mount-pxe-images.nix`). Place large images there (ISOs,
|
||||
disk images) rather than on the pxe-boot host's own root disk. For an LXC
|
||||
pxe-boot container the mount uses NFSv3+nolock with `nofail` (eager,
|
||||
non-blocking on server unavailability); for a Proxmox VM it uses NFSv4.2
|
||||
with `x-systemd.automount` (lazy, triggered on first access).
|
||||
|
||||
When running as `lxc-pxe-boot`, the Proxmox container must have
|
||||
`features: nesting=1,mount=nfs` (at minimum) in its Proxmox config. `nesting=1`
|
||||
is required by systemd 260+ for credential isolation (user namespace creation
|
||||
and internal move-mounts); without it, AppArmor denies both, and every
|
||||
systemd service that uses `PrivateUsers`, `PrivateDevices`, or credential
|
||||
passing fails on boot. `mount=nfs` allows the NFSv3 mount. Both are set
|
||||
automatically by `scripts/proxmox/create-proxmox-resource.sh` (via
|
||||
`PROXMOX_DEFAULT_LXC_FEATURES` in `scripts/env.sh` which defaults to
|
||||
`nesting=1,keyctl=1,mount=nfs;nfs4`). If you ever change these features
|
||||
manually via `pct set`, be sure to include both — `pct set` replaces the
|
||||
entire features string, it does not append to it.
|
||||
Mount shared image storage under `/srv/pxe/http`, preferably
|
||||
`/srv/pxe/http/images` unless a menu entry expects files in a specific
|
||||
directory such as `/srv/pxe/http/auto-installer`.
|
||||
|
||||
The HTTP iPXE chain is:
|
||||
|
||||
```text
|
||||
undionly.kpxe or ipxe.efi
|
||||
-> autoexec.ipxe from the TFTP root, when iPXE requests it
|
||||
-> http://192.168.2.223/boot.ipxe
|
||||
-> http://192.168.2.223/menu.ipxe
|
||||
-> http://192.168.2.247/boot.ipxe
|
||||
-> http://192.168.2.247/menu.ipxe
|
||||
```
|
||||
|
||||
The generated menu currently exposes entries for:
|
||||
|
||||
- NixOS Auto-Installer
|
||||
- NixOS Minimal
|
||||
- Debian Minimal
|
||||
- FreeIPA Server (Rocky Linux 9)
|
||||
- SystemRescue environment
|
||||
- iPXE shell
|
||||
- Reboot
|
||||
@@ -104,82 +84,19 @@ directory name (`auto-installer` / `nixos-minimal`), so each one's
|
||||
generated system name (`nixos-system-<name>-*`) is self-describing rather
|
||||
than the nixpkgs default of `nixos-system-nixos-*` for both.
|
||||
|
||||
The Debian Minimal entry chains `http://<pxeServerIp>/debian.ipxe`, which loads
|
||||
the Debian bookworm netboot kernel and initrd from `/srv/pxe/http/debian/`. The
|
||||
`fetch-debian-netboot.service` oneshot downloads these files from
|
||||
`deb.debian.org` on first boot (idempotent — skips if files are already
|
||||
present):
|
||||
|
||||
```text
|
||||
/srv/pxe/http/debian/linux (Debian bookworm netboot kernel)
|
||||
/srv/pxe/http/debian/initrd.gz (Debian bookworm netboot initrd)
|
||||
```
|
||||
|
||||
The service requires outbound internet access on the pxe-boot host. To
|
||||
re-download (e.g. after a Debian point release), delete the files and restart
|
||||
the service:
|
||||
|
||||
```bash
|
||||
rm /srv/pxe/http/debian/linux /srv/pxe/http/debian/initrd.gz
|
||||
systemctl restart fetch-debian-netboot.service
|
||||
```
|
||||
|
||||
To update to a different Debian release, change `debianRelease` in
|
||||
`modules/build-types/pxe-boot.nix` and redeploy.
|
||||
|
||||
The **FreeIPA Server (Rocky Linux 9)** entry chains
|
||||
`http://<pxeServerIp>/rocky-freeipa.ipxe`, which boots the Rocky Linux 9
|
||||
Anaconda installer with a Kickstart file (`rocky-freeipa.ks`) hosted on the
|
||||
same server. The `fetch-rocky-pxeboot.service` oneshot downloads the pxeboot
|
||||
kernel and initrd from the Rocky Linux mirror on first boot (idempotent):
|
||||
|
||||
```text
|
||||
/srv/pxe/http/rocky/vmlinuz (Rocky Linux 9 Anaconda pxeboot kernel)
|
||||
/srv/pxe/http/rocky/initrd.img (Rocky Linux 9 Anaconda pxeboot initrd)
|
||||
```
|
||||
|
||||
The Kickstart file is generated from the NixOS module and staged at
|
||||
`/srv/pxe/http/rocky-freeipa.ks`. It performs a fully unattended install:
|
||||
|
||||
1. Installs Rocky Linux 9 with `ipa-server` + `ipa-server-dns` packages
|
||||
2. Configures static IP `192.168.2.138`, hostname `domain-controller.sweet.home`
|
||||
3. Creates user `wayne` with the `adminSshKey` from `variables.nix`
|
||||
4. Generates random IPA passwords and writes them to `/root/ipa-credentials.txt`
|
||||
5. Creates a `freeipa-first-boot.service` oneshot that runs `ipa-server-install`
|
||||
on first reboot (~20 minutes)
|
||||
|
||||
After the install completes:
|
||||
- SSH in as `wayne@domain-controller` using the admin key
|
||||
- Monitor FreeIPA install progress: `sudo tail -f /root/freeipa-install.log`
|
||||
- Retrieve credentials: `sudo cat /root/ipa-credentials.txt` (save to password manager)
|
||||
- Configure Pi-hole: `server=/sweet.home/192.168.2.138` in dnsmasq
|
||||
|
||||
To refresh the pxeboot files (e.g. after a Rocky point release):
|
||||
|
||||
```bash
|
||||
rm /srv/pxe/http/rocky/vmlinuz /srv/pxe/http/rocky/initrd.img
|
||||
systemctl restart fetch-rocky-pxeboot.service
|
||||
```
|
||||
|
||||
To update to a different Rocky release, change `rockyRelease` in
|
||||
`modules/build-types/pxe-boot.nix` and redeploy.
|
||||
|
||||
The SystemRescue entry expects the source ISO at:
|
||||
|
||||
```text
|
||||
/srv/pxe/http/images/systemrescue.iso
|
||||
```
|
||||
|
||||
Since `/srv/pxe/http/images` is the NFS-backed symlink, place the ISO on the
|
||||
NFS share at `server.sweet.home:/tank/pxe-boot/images/systemrescue.iso`.
|
||||
|
||||
The `stage-systemrescue.service` oneshot extracts that ISO into:
|
||||
|
||||
```text
|
||||
/srv/pxe/http/systemrescue
|
||||
```
|
||||
|
||||
The rescue menu entry then chains `http://192.168.2.223/systemrescue.ipxe`,
|
||||
The rescue menu entry then chains `http://192.168.2.247/systemrescue.ipxe`,
|
||||
which loads the SystemRescue kernel and initramfs from the extracted tree and
|
||||
uses `archiso_http_srv` to fetch the squashfs payload over HTTP.
|
||||
|
||||
@@ -196,13 +113,6 @@ After deployment by an operator, basic service checks are:
|
||||
```bash
|
||||
curl http://pxe-boot/boot.ipxe
|
||||
curl http://pxe-boot/menu.ipxe
|
||||
curl http://pxe-boot/debian.ipxe
|
||||
curl -I http://pxe-boot/debian/linux
|
||||
curl -I http://pxe-boot/debian/initrd.gz
|
||||
curl http://pxe-boot/rocky-freeipa.ipxe
|
||||
curl http://pxe-boot/rocky-freeipa.ks
|
||||
curl -I http://pxe-boot/rocky/vmlinuz
|
||||
curl -I http://pxe-boot/rocky/initrd.img
|
||||
curl http://pxe-boot/systemrescue.ipxe
|
||||
curl -I http://pxe-boot/systemrescue/sysresccd/boot/x86_64/vmlinuz
|
||||
curl -I http://pxe-boot/systemrescue/sysresccd/boot/x86_64/sysresccd.img
|
||||
|
||||
Generated
+6
-6
@@ -173,11 +173,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1785119570,
|
||||
"narHash": "sha256-Rgs2xKnGLFWQscxUaXX07oyZeuMDOHEbqDOsgliLFGM=",
|
||||
"lastModified": 1784350909,
|
||||
"narHash": "sha256-ZWyzLbS1yKUTeFJLmdVuWNnHttL333/ldJbEE+KzCrM=",
|
||||
"owner": "nix-community",
|
||||
"repo": "home-manager",
|
||||
"rev": "d4fd24667c8cbef124bb70a20380cab75ec8474d",
|
||||
"rev": "4ce190229c73d44536caa7072f6308fb2d8feeb3",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -259,11 +259,11 @@
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1785104993,
|
||||
"narHash": "sha256-eKbrvPoAOFutbYMdbB3r5EQVmFxKv24iKqHPPUXA0gM=",
|
||||
"lastModified": 1784432872,
|
||||
"narHash": "sha256-n3gKTBIV4ZA5VQpUakffBe3KGu4+mhPoA34rrqS0GkA=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "8623c4c20aa4ca2f5fb81510d2944066c3fb0d96",
|
||||
"rev": "fd1462031fdee08f65fd0b4c6b64e22239a77870",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -130,9 +130,6 @@
|
||||
lxc-tailscale-router = mkTarget { platform = "lxc"; buildType = "tailscale-router"; hostPath = ./hosts/tailscale-router/host.nix; };
|
||||
|
||||
lxc-tor-relay = mkTarget { platform = "lxc"; buildType = "tor-relay"; hostPath = ./hosts/tor-relay/host.nix; };
|
||||
|
||||
proxmox-ha-server-1 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-1/host.nix; };
|
||||
proxmox-ha-server-2 = mkTarget { platform = "proxmox"; buildType = "ha-server"; hostPath = ./hosts/ha-server-2/host.nix; };
|
||||
};
|
||||
|
||||
# Auto-install environments (migrated from the former nix-auto-installer
|
||||
|
||||
+3
-12
@@ -1,17 +1,8 @@
|
||||
{ vars, ... }:
|
||||
_:
|
||||
|
||||
{
|
||||
networking = {
|
||||
hostName = "docker";
|
||||
hostId = "007f0200";
|
||||
useDHCP = false;
|
||||
interfaces.${vars.vmLanInterface}.ipv4.addresses = [{
|
||||
address = vars.dockerIp;
|
||||
prefixLength = vars.lanPrefixLength;
|
||||
}];
|
||||
defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; };
|
||||
nameservers = [ vars.domainControllerIp ];
|
||||
};
|
||||
networking.hostName = "docker";
|
||||
networking.hostId = "007f0200";
|
||||
boot.zfs.forceImportRoot = false;
|
||||
|
||||
# Preserved from the pre-refactor `docker` target — stateVersion must never
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{ vars, ... }:
|
||||
{
|
||||
imports = [
|
||||
(import ../../modules/beszel/host-token.nix {
|
||||
name = "ha-server-1";
|
||||
sopsFile = ../../secrets/ha-server-1.yaml;
|
||||
})
|
||||
];
|
||||
|
||||
networking = {
|
||||
hostName = vars.haServer1Host;
|
||||
hostId = "3a4b5c6d";
|
||||
useDHCP = false;
|
||||
interfaces.${vars.vmLanInterface}.ipv4.addresses = [{
|
||||
address = vars.haServer1Ip;
|
||||
prefixLength = vars.lanPrefixLength;
|
||||
}];
|
||||
interfaces.${vars.vmStorageInterface}.ipv4.addresses = [{
|
||||
address = vars.haServer1StorageIp;
|
||||
prefixLength = vars.haStoragePrefixLength;
|
||||
}];
|
||||
defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; };
|
||||
nameservers = [ vars.domainControllerIp ];
|
||||
};
|
||||
|
||||
# Set KEY after pairing this host with the beszel hub; the token is sops-managed.
|
||||
services.beszel.agent.environment.KEY = "";
|
||||
|
||||
system.stateVersion = "26.05";
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
{ vars, ... }:
|
||||
{
|
||||
imports = [
|
||||
(import ../../modules/beszel/host-token.nix {
|
||||
name = "ha-server-2";
|
||||
sopsFile = ../../secrets/ha-server-2.yaml;
|
||||
})
|
||||
];
|
||||
|
||||
networking = {
|
||||
hostName = vars.haServer2Host;
|
||||
hostId = "7e8f9a0b";
|
||||
useDHCP = false;
|
||||
interfaces.${vars.vmLanInterface}.ipv4.addresses = [{
|
||||
address = vars.haServer2Ip;
|
||||
prefixLength = vars.lanPrefixLength;
|
||||
}];
|
||||
interfaces.${vars.vmStorageInterface}.ipv4.addresses = [{
|
||||
address = vars.haServer2StorageIp;
|
||||
prefixLength = vars.haStoragePrefixLength;
|
||||
}];
|
||||
defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; };
|
||||
nameservers = [ vars.domainControllerIp ];
|
||||
};
|
||||
|
||||
# Set KEY after pairing this host with the beszel hub; the token is sops-managed.
|
||||
services.beszel.agent.environment.KEY = "";
|
||||
|
||||
system.stateVersion = "26.05";
|
||||
}
|
||||
@@ -6,23 +6,9 @@
|
||||
name = "nix-cache";
|
||||
sopsFile = ../../secrets/nix-cache.yaml;
|
||||
})
|
||||
(import ../../modules/ipa/client.nix {
|
||||
keytabSopsFile = ../../secrets/nix-cache.keytab;
|
||||
caCertFile = ../../certs/ipa-ca.crt;
|
||||
})
|
||||
];
|
||||
|
||||
networking = {
|
||||
hostName = vars.nixCacheHost;
|
||||
domain = vars.homeDomain;
|
||||
useDHCP = false;
|
||||
interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{
|
||||
address = vars.nixCacheIp;
|
||||
prefixLength = vars.lanPrefixLength;
|
||||
}];
|
||||
defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; };
|
||||
nameservers = [ vars.domainControllerIp ];
|
||||
};
|
||||
networking.hostName = vars.nixCacheHost;
|
||||
|
||||
services.beszel.agent.environment = {
|
||||
#DOCKER_HOST = "tcp://docker-socket-proxy:2375";
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
# Optional: set environment vars
|
||||
sessionVariables = {
|
||||
EDITOR = "vim";
|
||||
SOPS_AGE_KEY_FILE = "${config.home.homeDirectory}/.config/sops/age/keys.txt";
|
||||
SOPS_AGE_KEY_FILE = "~/.config/sops/age/keys.txt";
|
||||
};
|
||||
|
||||
file = {
|
||||
|
||||
+2
-11
@@ -1,16 +1,7 @@
|
||||
{ vars, ... }:
|
||||
_:
|
||||
|
||||
{
|
||||
networking = {
|
||||
hostName = "pxe-boot";
|
||||
useDHCP = false;
|
||||
interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{
|
||||
address = vars.pxeServerIp;
|
||||
prefixLength = vars.lanPrefixLength;
|
||||
}];
|
||||
defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; };
|
||||
nameservers = [ vars.domainControllerIp ];
|
||||
};
|
||||
networking.hostName = "pxe-boot";
|
||||
|
||||
# Preserved from the pre-refactor `pxe-boot` target — stateVersion must
|
||||
# never be bumped on an already-installed machine.
|
||||
|
||||
+2
-11
@@ -8,17 +8,8 @@
|
||||
})
|
||||
];
|
||||
|
||||
networking = {
|
||||
hostName = vars.nfsServerHost;
|
||||
hostId = "6689f93e";
|
||||
useDHCP = false;
|
||||
interfaces.${vars.vmLanInterface}.ipv4.addresses = [{
|
||||
address = vars.serverIp;
|
||||
prefixLength = vars.lanPrefixLength;
|
||||
}];
|
||||
defaultGateway = { address = vars.lanGateway; interface = vars.vmLanInterface; };
|
||||
nameservers = [ vars.domainControllerIp ];
|
||||
};
|
||||
networking.hostName = vars.nfsServerHost;
|
||||
networking.hostId = "6689f93e";
|
||||
|
||||
services.beszel.agent.environment = {
|
||||
#DOCKER_HOST = "tcp://docker-socket-proxy:2375";
|
||||
|
||||
@@ -1,32 +1,7 @@
|
||||
{ vars, ... }:
|
||||
_:
|
||||
|
||||
{
|
||||
imports = [
|
||||
(import ../../modules/beszel/host-token.nix {
|
||||
name = "tailscale-router";
|
||||
sopsFile = ../../secrets/tailscale-router.yaml;
|
||||
})
|
||||
(import ../../modules/ipa/client.nix {
|
||||
keytabSopsFile = ../../secrets/tailscale-router.keytab;
|
||||
caCertFile = ../../certs/ipa-ca.crt;
|
||||
})
|
||||
];
|
||||
|
||||
networking = {
|
||||
hostName = "tailscale-router";
|
||||
domain = vars.homeDomain;
|
||||
useDHCP = false;
|
||||
interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{
|
||||
address = vars.tailscaleRouterIp;
|
||||
prefixLength = vars.lanPrefixLength;
|
||||
}];
|
||||
defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; };
|
||||
nameservers = [ vars.domainControllerIp ];
|
||||
};
|
||||
|
||||
services.beszel.agent.environment = {
|
||||
KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
|
||||
};
|
||||
networking.hostName = "tailscale-router";
|
||||
|
||||
# No networking.hostId: only ZFS-touching hosts (server, docker) need one
|
||||
# for pool-import safety, and this host does neither.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{ vars, ... }:
|
||||
{ ... }:
|
||||
|
||||
{
|
||||
imports = [
|
||||
@@ -8,19 +8,10 @@
|
||||
})
|
||||
];
|
||||
|
||||
networking = {
|
||||
hostName = "tor-relay";
|
||||
useDHCP = false;
|
||||
interfaces.${vars.lxcLanInterface}.ipv4.addresses = [{
|
||||
address = vars.torRelayIp;
|
||||
prefixLength = vars.lanPrefixLength;
|
||||
}];
|
||||
defaultGateway = { address = vars.lanGateway; interface = vars.lxcLanInterface; };
|
||||
nameservers = [ vars.domainControllerIp ];
|
||||
};
|
||||
networking.hostName = "tor-relay";
|
||||
|
||||
# No networking.hostId: only ZFS-touching hosts need one for pool-import
|
||||
# safety, and this host does neither.
|
||||
# No networking.hostId: only ZFS-touching hosts (server, docker) need one
|
||||
# for pool-import safety, and this host does neither.
|
||||
|
||||
services.beszel.agent.environment = {
|
||||
KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG";
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
../docker/enable-service.nix
|
||||
../docker/nextcloud-cron-job.nix
|
||||
../docker/docker-health-to-gotify.nix
|
||||
../tailscale/enable-service.nix
|
||||
../traefik/rotate-logs.nix
|
||||
../raspi/mount-data.nix
|
||||
../services/enable-rpcbind.nix
|
||||
|
||||
@@ -1,17 +1,6 @@
|
||||
{ config, pkgs, lib, inputs, vars, ... }:
|
||||
|
||||
{
|
||||
imports = [
|
||||
../docker/enable-service.nix
|
||||
];
|
||||
|
||||
nixpkgs.overlays = [
|
||||
(final: prev: {
|
||||
docker = prev.docker_29;
|
||||
docker_cli = prev.docker_29;
|
||||
})
|
||||
];
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
inputs.nixos-conf-editor.packages.${pkgs.stdenv.hostPlatform.system}.nixos-conf-editor
|
||||
nodejs
|
||||
@@ -29,7 +18,7 @@
|
||||
];
|
||||
|
||||
boot.loader.grub.useOSProber = true;
|
||||
programs.direnv.enable = true;
|
||||
programs.direnv.enable = true;
|
||||
services = {
|
||||
xserver = {
|
||||
enable = true;
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
# HA file server build type: DRBD + XFS + LIO iSCSI + NFS, managed by
|
||||
# Corosync + Pacemaker. Both ha-server-1 and ha-server-2 use this type.
|
||||
#
|
||||
# NFS start/stop:
|
||||
# services.nfs.server.enable = true configures /etc/exports, wires up
|
||||
# rpcbind, and loads kernel modules — but nfs-server.service.wantedBy is
|
||||
# force-cleared so systemd does NOT auto-start it at boot. Pacemaker's
|
||||
# ha-group resource group (configured by scripts/ha/cluster-init.sh)
|
||||
# starts and stops nfs-server as part of the failover sequence after the
|
||||
# XFS mount and iSCSI target are brought up on the new Active node.
|
||||
#
|
||||
# Beszel agent:
|
||||
# Enabled here via enable-agent.nix. The agent KEY (used to pair with
|
||||
# the Beszel hub) is not set yet — add it to hosts/ha-server-{1,2}/host.nix
|
||||
# under services.beszel.agent.environment.KEY once the hub accepts the
|
||||
# new agents, following the pattern in hosts/server/host.nix.
|
||||
{ lib, vars, ... }:
|
||||
{
|
||||
imports = [
|
||||
../ha/pacemaker-stack.nix
|
||||
../ha/iscsi-target.nix
|
||||
../ha/cluster-config.nix
|
||||
../beszel/enable-agent.nix
|
||||
];
|
||||
|
||||
services.nfs.server = {
|
||||
enable = true;
|
||||
exports = ''
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
'';
|
||||
};
|
||||
|
||||
# Pacemaker controls nfs-server — prevent systemd from starting it at boot
|
||||
# on both nodes (only the Active node should be serving NFS).
|
||||
systemd.services.nfs-server.wantedBy = lib.mkForce [ ];
|
||||
}
|
||||
@@ -21,216 +21,6 @@ let
|
||||
chain ${pxeBaseUrl}/boot.ipxe
|
||||
'';
|
||||
|
||||
debianRelease = "bookworm";
|
||||
debianMirror = "https://deb.debian.org/debian";
|
||||
debianNetbootBase = "${debianMirror}/dists/${debianRelease}/main/installer-amd64/current/images/netboot/debian-installer/amd64";
|
||||
|
||||
rockyRelease = "9";
|
||||
rockyArch = "x86_64";
|
||||
rockyMirror = "https://dl.rockylinux.org/pub/rocky/${rockyRelease}";
|
||||
rockyPxebootBase = "${rockyMirror}/BaseOS/${rockyArch}/os/images/pxeboot";
|
||||
|
||||
debianIpxe = pkgs.writeText "debian.ipxe" ''
|
||||
#!ipxe
|
||||
|
||||
set base ${pxeBaseUrl}
|
||||
|
||||
kernel ''${base}/debian/linux
|
||||
initrd ''${base}/debian/initrd.gz
|
||||
boot
|
||||
'';
|
||||
|
||||
fetchDebianNetboot = pkgs.writeShellScript "fetch-debian-netboot" ''
|
||||
set -eu
|
||||
|
||||
dir="${httpRoot}/debian"
|
||||
mirror="${debianNetbootBase}"
|
||||
|
||||
if [ -f "$dir/linux" ] && [ -f "$dir/initrd.gz" ]; then
|
||||
echo "Debian ${debianRelease} netboot files already present; skipping download."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Downloading Debian ${debianRelease} netboot kernel and initrd from $mirror ..."
|
||||
${pkgs.curl}/bin/curl -fsSL -o "$dir/linux.tmp" "$mirror/linux"
|
||||
${pkgs.curl}/bin/curl -fsSL -o "$dir/initrd.gz.tmp" "$mirror/initrd.gz"
|
||||
mv "$dir/linux.tmp" "$dir/linux"
|
||||
mv "$dir/initrd.gz.tmp" "$dir/initrd.gz"
|
||||
echo "Debian ${debianRelease} netboot files staged."
|
||||
'';
|
||||
|
||||
# Rocky Linux 9 iPXE script — boots vmlinuz+initrd.img from the staged
|
||||
# /rocky/ directory and hands Anaconda the hosted Kickstart URL.
|
||||
# net.ifnames=0 biosdevname=0 ensures the NIC is eth0 in both the
|
||||
# installer and the installed system (matches the Kickstart NM config).
|
||||
rockyFreeIpaIpxe = pkgs.writeText "rocky-freeipa.ipxe" ''
|
||||
#!ipxe
|
||||
|
||||
set base ${pxeBaseUrl}
|
||||
|
||||
kernel ''${base}/rocky/vmlinuz inst.ks=''${base}/rocky-freeipa.ks inst.repo=${rockyMirror}/BaseOS/${rockyArch}/os/ net.ifnames=0 biosdevname=0 ip=dhcp quiet
|
||||
initrd ''${base}/rocky/initrd.img
|
||||
boot
|
||||
'';
|
||||
|
||||
# Kickstart file for domain-controller.sweet.home.
|
||||
# Installs Rocky Linux 9, sets a static IP, creates wayne with the
|
||||
# admin SSH key, then on first reboot runs ipa-server-install via a
|
||||
# systemd oneshot service. Passwords are generated at %post time,
|
||||
# written to /root/ipa-credentials.txt (chmod 600), and read back by
|
||||
# the first-boot script — never hardcoded here or in the repo.
|
||||
rockyFreeIpaKs = pkgs.writeText "rocky-freeipa.ks" ''
|
||||
#version=RHEL9
|
||||
# Unattended Rocky Linux 9 + FreeIPA install
|
||||
# Target: domain-controller.${vars.homeDomain} ${vars.domainControllerIp}
|
||||
|
||||
url --url=${rockyMirror}/BaseOS/${rockyArch}/os/
|
||||
repo --name=appstream --baseurl=${rockyMirror}/AppStream/${rockyArch}/os/
|
||||
|
||||
lang en_US.UTF-8
|
||||
keyboard us
|
||||
timezone UTC --utc
|
||||
|
||||
# DHCP during install; static IP configured in %post via NM config file
|
||||
network --bootproto=dhcp --device=link --activate
|
||||
network --hostname=domain-controller.sweet.home
|
||||
|
||||
selinux --enforcing
|
||||
firewall --enabled --service=ssh
|
||||
|
||||
rootpw --lock
|
||||
user --name=wayne --groups=wheel --shell=/bin/bash
|
||||
sshkey --username=wayne "${vars.adminSshKey}"
|
||||
|
||||
zerombr
|
||||
clearpart --all --initlabel --drives=sda
|
||||
# Keep net.ifnames=0 biosdevname=0 in the installed GRUB so the NIC
|
||||
# stays eth0 after reboot (matches the NM connection file below).
|
||||
bootloader --location=mbr --boot-drive=sda --append="net.ifnames=0 biosdevname=0"
|
||||
|
||||
part /boot --fstype=xfs --size=1024 --ondisk=sda
|
||||
part swap --fstype=swap --size=2048 --ondisk=sda
|
||||
part / --fstype=xfs --grow --size=1 --ondisk=sda --asprimary
|
||||
|
||||
%packages
|
||||
@^minimal-environment
|
||||
ipa-server
|
||||
ipa-server-dns
|
||||
%end
|
||||
|
||||
reboot
|
||||
|
||||
%post --log=/root/ks-post.log
|
||||
set -euo pipefail
|
||||
|
||||
# -- Static IP: write NM connection file directly (NM not running in chroot) --
|
||||
mkdir -p /etc/NetworkManager/system-connections
|
||||
cat > /etc/NetworkManager/system-connections/eth0.nmconnection << 'NMCONN'
|
||||
[connection]
|
||||
id=eth0
|
||||
type=ethernet
|
||||
interface-name=eth0
|
||||
autoconnect=true
|
||||
|
||||
[ethernet]
|
||||
|
||||
[ipv4]
|
||||
method=manual
|
||||
addresses=${vars.domainControllerIp}/${toString vars.lanPrefixLength}
|
||||
gateway=${vars.lanGateway}
|
||||
dns=${vars.domainControllerIp};
|
||||
dns-search=${vars.homeDomain};
|
||||
|
||||
[ipv6]
|
||||
method=auto
|
||||
NMCONN
|
||||
chmod 600 /etc/NetworkManager/system-connections/eth0.nmconnection
|
||||
|
||||
# -- /etc/hosts: FQDN must resolve to the real IP (not loopback) for IPA --
|
||||
sed -i '/domain-controller/d' /etc/hosts
|
||||
echo '${vars.domainControllerIp} domain-controller.${vars.homeDomain} domain-controller' >> /etc/hosts
|
||||
|
||||
# -- Generate IPA passwords and store securely --
|
||||
DM_PASS=$(openssl rand -base64 24 | tr -dc 'A-Za-z0-9' | head -c 24)
|
||||
ADMIN_PASS=$(openssl rand -base64 24 | tr -dc 'A-Za-z0-9' | head -c 24)
|
||||
printf 'Directory Manager: %s\nIPA Admin: %s\n' "$DM_PASS" "$ADMIN_PASS" \
|
||||
> /root/ipa-credentials.txt
|
||||
chmod 600 /root/ipa-credentials.txt
|
||||
|
||||
# -- First-boot script: reads passwords back, runs ipa-server-install --
|
||||
cat > /usr/local/sbin/freeipa-first-boot.sh << 'FIRSTBOOT'
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
exec >> /root/freeipa-install.log 2>&1
|
||||
echo "=== FreeIPA first-boot install started at $(date) ==="
|
||||
|
||||
DM_PASS=$(grep '^Directory Manager:' /root/ipa-credentials.txt | awk '{print $NF}')
|
||||
ADMIN_PASS=$(grep '^IPA Admin:' /root/ipa-credentials.txt | awk '{print $NF}')
|
||||
|
||||
ipa-server-install \
|
||||
--realm=SWEET.HOME \
|
||||
--domain=sweet.home \
|
||||
--hostname=domain-controller.sweet.home \
|
||||
--ds-password="$DM_PASS" \
|
||||
--admin-password="$ADMIN_PASS" \
|
||||
--setup-dns \
|
||||
--forwarder=192.168.2.253 \
|
||||
--no-dnssec-validation \
|
||||
--no-ntp \
|
||||
--unattended
|
||||
|
||||
echo "=== FreeIPA install complete at $(date) ==="
|
||||
echo "Credentials: /root/ipa-credentials.txt (save to password manager)"
|
||||
echo "CA backup: /root/cacert.p12 (encrypted with Directory Manager password)"
|
||||
systemctl disable freeipa-first-boot.service
|
||||
FIRSTBOOT
|
||||
chmod 700 /usr/local/sbin/freeipa-first-boot.sh
|
||||
|
||||
# -- Systemd oneshot service: runs freeipa-first-boot.sh on first real boot --
|
||||
cat > /etc/systemd/system/freeipa-first-boot.service << 'UNIT'
|
||||
[Unit]
|
||||
Description=FreeIPA first-boot installation
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
ConditionPathExists=/root/ipa-credentials.txt
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/freeipa-first-boot.sh
|
||||
TimeoutStartSec=1800
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNIT
|
||||
|
||||
mkdir -p /etc/systemd/system/multi-user.target.wants
|
||||
ln -sf /etc/systemd/system/freeipa-first-boot.service \
|
||||
/etc/systemd/system/multi-user.target.wants/freeipa-first-boot.service
|
||||
|
||||
echo "Kickstart %post complete. FreeIPA installs on first reboot (~20 min)."
|
||||
%end
|
||||
'';
|
||||
|
||||
fetchRockyPxeboot = pkgs.writeShellScript "fetch-rocky-pxeboot" ''
|
||||
set -eu
|
||||
|
||||
dir="${httpRoot}/rocky"
|
||||
base="${rockyPxebootBase}"
|
||||
|
||||
if [ -f "$dir/vmlinuz" ] && [ -f "$dir/initrd.img" ]; then
|
||||
echo "Rocky Linux ${rockyRelease} pxeboot files already present; skipping download."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Downloading Rocky Linux ${rockyRelease} pxeboot kernel and initrd from $base ..."
|
||||
${pkgs.curl}/bin/curl -fsSL -o "$dir/vmlinuz.tmp" "$base/vmlinuz"
|
||||
${pkgs.curl}/bin/curl -fsSL -o "$dir/initrd.img.tmp" "$base/initrd.img"
|
||||
mv "$dir/vmlinuz.tmp" "$dir/vmlinuz"
|
||||
mv "$dir/initrd.img.tmp" "$dir/initrd.img"
|
||||
echo "Rocky Linux ${rockyRelease} pxeboot files staged."
|
||||
'';
|
||||
|
||||
systemRescueIpxe = pkgs.writeText "systemrescue.ipxe" ''
|
||||
#!ipxe
|
||||
|
||||
@@ -278,13 +68,11 @@ let
|
||||
set base ${pxeBaseUrl}
|
||||
|
||||
menu PXE Boot Menu
|
||||
item auto-installer NixOS Auto-Installer
|
||||
item nixos-minimal NixOS Minimal
|
||||
item debian Debian Minimal
|
||||
item rocky-freeipa FreeIPA Server (Rocky Linux 9)
|
||||
item rescue Rescue Environment
|
||||
item shell iPXE Shell
|
||||
item reboot Reboot
|
||||
item auto-installer NixOS Auto-Installer
|
||||
item nixos-minimal NixOS Minimal
|
||||
item rescue Rescue Environment
|
||||
item shell iPXE Shell
|
||||
item reboot Reboot
|
||||
|
||||
choose target && goto ''${target}
|
||||
|
||||
@@ -294,12 +82,6 @@ let
|
||||
:nixos-minimal
|
||||
chain ''${base}/nixos-minimal/netboot.ipxe
|
||||
|
||||
:debian
|
||||
chain ''${base}/debian.ipxe
|
||||
|
||||
:rocky-freeipa
|
||||
chain ''${base}/rocky-freeipa.ipxe
|
||||
|
||||
:rescue
|
||||
chain ''${base}/systemrescue.ipxe
|
||||
|
||||
@@ -313,7 +95,6 @@ in
|
||||
{
|
||||
imports = [
|
||||
../pxe-boot/stage-installer-artifacts.nix
|
||||
../pxe-boot/mount-pxe-images.nix
|
||||
];
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
@@ -348,101 +129,37 @@ in
|
||||
openssh.settings.PermitRootLogin = "yes";
|
||||
};
|
||||
|
||||
systemd = {
|
||||
tmpfiles.rules = [
|
||||
"d ${pxeRoot} 0755 root root -"
|
||||
"d ${httpRoot} 0755 root root -"
|
||||
"L+ ${httpRoot}/images - - - - ${vars.nfsShares.pxebootImages.mountpoint}"
|
||||
"d ${httpRoot}/auto-installer 0755 root root -"
|
||||
"d ${httpRoot}/nixos-minimal 0755 root root -"
|
||||
"d ${httpRoot}/systemrescue 0755 root root -"
|
||||
"d ${httpRoot}/debian 0755 root root -"
|
||||
"d ${httpRoot}/ubuntu 0755 root root -"
|
||||
"d ${httpRoot}/rescue 0755 root root -"
|
||||
"d ${httpRoot}/rocky 0755 root root -"
|
||||
"d ${tftpRoot} 0755 root root -"
|
||||
"C+ ${httpRoot}/boot.ipxe 0644 root root - ${bootIpxe}"
|
||||
"C+ ${httpRoot}/menu.ipxe 0644 root root - ${menuIpxe}"
|
||||
"C+ ${httpRoot}/debian.ipxe 0644 root root - ${debianIpxe}"
|
||||
"C+ ${httpRoot}/rocky-freeipa.ipxe 0644 root root - ${rockyFreeIpaIpxe}"
|
||||
"C+ ${httpRoot}/rocky-freeipa.ks 0644 root root - ${rockyFreeIpaKs}"
|
||||
"C+ ${httpRoot}/systemrescue.ipxe 0644 root root - ${systemRescueIpxe}"
|
||||
"C+ ${tftpRoot}/autoexec.ipxe 0644 root root - ${autoexecIpxe}"
|
||||
"C+ ${tftpRoot}/ipxe.efi 0644 root root - ${pkgs.ipxe}/ipxe.efi"
|
||||
"C+ ${tftpRoot}/undionly.kpxe 0644 root root - ${pkgs.ipxe}/undionly.kpxe"
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${pxeRoot} 0755 root root -"
|
||||
"d ${httpRoot} 0755 root root -"
|
||||
"d ${httpRoot}/images 0755 root root -"
|
||||
"d ${httpRoot}/auto-installer 0755 root root -"
|
||||
"d ${httpRoot}/nixos-minimal 0755 root root -"
|
||||
"d ${httpRoot}/systemrescue 0755 root root -"
|
||||
"d ${httpRoot}/ubuntu 0755 root root -"
|
||||
"d ${httpRoot}/rescue 0755 root root -"
|
||||
"d ${tftpRoot} 0755 root root -"
|
||||
"C+ ${httpRoot}/boot.ipxe 0644 root root - ${bootIpxe}"
|
||||
"C+ ${httpRoot}/menu.ipxe 0644 root root - ${menuIpxe}"
|
||||
"C+ ${httpRoot}/systemrescue.ipxe 0644 root root - ${systemRescueIpxe}"
|
||||
"C+ ${tftpRoot}/autoexec.ipxe 0644 root root - ${autoexecIpxe}"
|
||||
"C+ ${tftpRoot}/ipxe.efi 0644 root root - ${pkgs.ipxe}/ipxe.efi"
|
||||
"C+ ${tftpRoot}/undionly.kpxe 0644 root root - ${pkgs.ipxe}/undionly.kpxe"
|
||||
];
|
||||
|
||||
systemd.services.stage-systemrescue = {
|
||||
description = "Stage SystemRescue ISO contents for HTTP PXE boot";
|
||||
after = [
|
||||
"local-fs.target"
|
||||
"systemd-tmpfiles-setup.service"
|
||||
];
|
||||
|
||||
services = {
|
||||
fetch-debian-netboot = {
|
||||
description = "Download Debian ${debianRelease} netboot kernel and initrd for HTTP PXE boot";
|
||||
after = [
|
||||
"local-fs.target"
|
||||
"systemd-tmpfiles-setup.service"
|
||||
"network-online.target"
|
||||
];
|
||||
wants = [ "network-online.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = fetchDebianNetboot;
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
};
|
||||
|
||||
fetch-rocky-pxeboot = {
|
||||
description = "Download Rocky Linux ${rockyRelease} pxeboot kernel and initrd for HTTP PXE boot";
|
||||
after = [
|
||||
"local-fs.target"
|
||||
"systemd-tmpfiles-setup.service"
|
||||
"network-online.target"
|
||||
];
|
||||
wants = [ "network-online.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = fetchRockyPxeboot;
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
};
|
||||
|
||||
stage-systemrescue = {
|
||||
description = "Stage SystemRescue ISO contents for HTTP PXE boot";
|
||||
after = [
|
||||
"local-fs.target"
|
||||
"systemd-tmpfiles-setup.service"
|
||||
];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = stageSystemRescue;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.dnsmasq = {
|
||||
enable = true;
|
||||
settings = {
|
||||
# Disable DNS listener — only proxy DHCP is needed here.
|
||||
# Without this dnsmasq tries to bind port 53 which systemd-resolved
|
||||
# already owns, causing startup failure.
|
||||
port = 0;
|
||||
dhcp-range = [ "192.168.2.0,proxy" ];
|
||||
dhcp-match = [
|
||||
"set:ipxe,175"
|
||||
"set:efi64,option:client-arch,7"
|
||||
"set:efi64,option:client-arch,9"
|
||||
];
|
||||
dhcp-userclass = "set:ipxe,iPXE";
|
||||
dhcp-boot = [
|
||||
"tag:ipxe,tag:efi64,http://${vars.pxeServerIp}/boot.ipxe"
|
||||
"tag:ipxe,http://${vars.pxeServerIp}/boot.ipxe"
|
||||
"tag:efi64,ipxe.efi,,${vars.pxeServerIp}"
|
||||
"undionly.kpxe,,${vars.pxeServerIp}"
|
||||
];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = stageSystemRescue;
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ vars.ports.pxeBootHttp ];
|
||||
networking.firewall.allowedUDPPorts = [ vars.ports.pxeBootTftp 67 ];
|
||||
networking.firewall.allowedUDPPorts = [ vars.ports.pxeBootTftp ];
|
||||
}
|
||||
|
||||
@@ -10,8 +10,7 @@ let
|
||||
in lib.imap1 (i: _: lib.concatStringsSep "/" (lib.take i parts)) parts;
|
||||
|
||||
poolDatasets = lib.unique (
|
||||
lib.concatMap (share: ancestors share.subpath)
|
||||
(lib.filter builtins.isAttrs (lib.attrValues vars.nfsShares))
|
||||
lib.concatMap (share: ancestors share.subpath) (lib.attrValues vars.nfsShares)
|
||||
);
|
||||
in
|
||||
{
|
||||
@@ -44,8 +43,14 @@ in
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Locate the data disk first — used for both the fallback import
|
||||
# attempt and, only if the disk is genuinely blank, pool creation.
|
||||
# Pool exists on a device but not yet imported — let the standard
|
||||
# zfs-import-${poolName}.service handle it normally.
|
||||
if zpool import -d /dev/disk/by-id -N "${poolName}" 2>/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# No pool found at all. Create it on the Proxmox data disk (scsi1),
|
||||
# which appears as /dev/disk/by-id/scsi-*drive-scsi1 inside the VM.
|
||||
DATA_DISK=""
|
||||
for candidate in /dev/disk/by-id/scsi-*drive-scsi1; do
|
||||
[[ "$candidate" == *-part* ]] && continue
|
||||
@@ -57,31 +62,8 @@ in
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Try importing via the by-id symlink directory first (normal path),
|
||||
# then fall back to scanning the disk directly. The two-step exists
|
||||
# because of a udev race: systemd-udev-settle.service can clear before
|
||||
# /dev/disk/by-id/ entries are fully populated, causing the first
|
||||
# import to fail even when the pool is intact on the disk.
|
||||
if zpool import -d /dev/disk/by-id -N "${poolName}" 2>/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
if zpool import -d "$DATA_DISK" -N "${poolName}" 2>/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Both import attempts failed. Before creating a new pool, verify the
|
||||
# disk is genuinely blank — if ZFS label metadata is present the import
|
||||
# failed for some other reason and we must not clobber existing data.
|
||||
if zdb -l "$DATA_DISK" 2>/dev/null | grep -q "name: '${poolName}'"; then
|
||||
echo "zfs-init-${poolName}: $DATA_DISK has ZFS pool '${poolName}' metadata but import failed — refusing to overwrite existing data. Run 'zpool import -d $DATA_DISK ${poolName}' manually to investigate." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Disk is genuinely blank: create the pool. -f is intentionally
|
||||
# omitted so that if we somehow reach this point with an existing pool
|
||||
# on the disk, zpool refuses rather than silently destroying data.
|
||||
echo "zfs-init-${poolName}: creating pool on $DATA_DISK"
|
||||
zpool create "${poolName}" "$DATA_DISK"
|
||||
zpool create -f "${poolName}" "$DATA_DISK"
|
||||
${lib.concatMapStrings (ds: ''
|
||||
zfs create "${poolName}/${ds}"
|
||||
'') poolDatasets}
|
||||
@@ -96,20 +78,13 @@ in
|
||||
services.nfs.server = {
|
||||
enable = true;
|
||||
exports = ''
|
||||
${vars.storageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.storageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.storageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.storageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.storageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.storageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.storageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.storageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash)
|
||||
${vars.storageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash)
|
||||
${vars.storageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash)
|
||||
${vars.storageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash)
|
||||
${vars.storageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}(rw,sync,no_subtree_check,no_root_squash)
|
||||
'';
|
||||
};
|
||||
|
||||
# mountd (20048) is needed for showmount/NFSv3 mount protocol — without it
|
||||
# clients can reach portmapper (111) and get the mountd port back, then
|
||||
# time out trying to connect to it. All three ports need TCP and UDP.
|
||||
networking.firewall.allowedTCPPorts = [ vars.ports.nfsRpcbind vars.ports.nfsd vars.ports.nfsMountd ];
|
||||
networking.firewall.allowedUDPPorts = [ vars.ports.nfsRpcbind vars.ports.nfsd vars.ports.nfsMountd ];
|
||||
networking.firewall.allowedTCPPorts = [ vars.ports.nfsRpcbind vars.ports.nfsd ];
|
||||
}
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
{ vars, ... }:
|
||||
{ ... }:
|
||||
|
||||
{
|
||||
imports = [
|
||||
../tailscale/subnet-router.nix
|
||||
../tailscale/ts-dns-forwarder.nix
|
||||
../beszel/enable-agent.nix
|
||||
];
|
||||
|
||||
# "server", not "both": this build type advertises LAN subnet routes but
|
||||
@@ -14,31 +12,8 @@
|
||||
# own setting) so the intent is clear at the build-type level.
|
||||
services.tailscale.useRoutingFeatures = "server";
|
||||
|
||||
# Advertise the LAN subnet so Tailscale peers can route back to LAN machines.
|
||||
# Must also be approved in the Tailscale admin console (Machines → Edit route settings).
|
||||
services.tailscale.extraUpFlags = [ "--advertise-routes=${vars.lanCidr}" ];
|
||||
|
||||
networking.firewall = {
|
||||
# Forwarded subnet-router traffic arrives on tailscale0 already
|
||||
# tailscale-authenticated -- the firewall's normal per-port allow-list
|
||||
# would otherwise drop it. Standard NixOS/Tailscale subnet-router guidance.
|
||||
trustedInterfaces = [ "tailscale0" ];
|
||||
|
||||
# SNAT LAN traffic going into Tailscale so the remote peer sees it as
|
||||
# coming from this router's Tailscale IP rather than a raw LAN IP.
|
||||
# Without this, Tailscale drops forwarded packets whose source is not a
|
||||
# recognised Tailscale address.
|
||||
#
|
||||
# We target POSTROUTING directly (always-existing built-in chain) rather
|
||||
# than nixos-nat-post: extraCommands runs after the old nixos-nat-post is
|
||||
# deleted but before the new one is created, so -A nixos-nat-post silently
|
||||
# fails. The -C check makes the rule idempotent across firewall reloads.
|
||||
extraCommands = ''
|
||||
iptables -t nat -C POSTROUTING -s ${vars.lanCidr} -o tailscale0 -j MASQUERADE 2>/dev/null || \
|
||||
iptables -t nat -A POSTROUTING -s ${vars.lanCidr} -o tailscale0 -j MASQUERADE
|
||||
'';
|
||||
extraStopCommands = ''
|
||||
iptables -t nat -D POSTROUTING -s ${vars.lanCidr} -o tailscale0 -j MASQUERADE 2>/dev/null || true
|
||||
'';
|
||||
};
|
||||
# Forwarded subnet-router traffic arrives on tailscale0 already
|
||||
# tailscale-authenticated -- the firewall's normal per-port allow-list
|
||||
# would otherwise drop it. Standard NixOS/Tailscale subnet-router guidance.
|
||||
networking.firewall.trustedInterfaces = [ "tailscale0" ];
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{ pkgs, vars, ... }:
|
||||
{ pkgs, ... }:
|
||||
|
||||
{
|
||||
# virtualisation.docker.enable = true;
|
||||
@@ -15,7 +15,7 @@
|
||||
# experimental = true;
|
||||
# };
|
||||
};
|
||||
users.users.${vars.primaryUser}.extraGroups = [ "docker" ];
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
docker-compose
|
||||
docker-buildx
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
# Cluster-wide HA config shared by both ha-server nodes.
|
||||
#
|
||||
# Covers everything that is identical on both nodes and references cluster
|
||||
# topology (node IPs, hostnames, DRBD resource). Per-node identity
|
||||
# (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix.
|
||||
#
|
||||
# Corosync authkey:
|
||||
# /etc/corosync/authkey (mode 0400) is managed by sops-nix below.
|
||||
# Bootstrap: run scripts/ha/cluster-init.sh on node1 to generate the key,
|
||||
# then encrypt it with: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey
|
||||
# Both host keys must be registered via sync-host-keys.sh first so both nodes can decrypt it.
|
||||
#
|
||||
# DRBD fencing:
|
||||
# Production setting is resource-only: DRBD waits for the STONITH fence
|
||||
# agent to confirm the peer is dead before promoting to Primary. This
|
||||
# requires a working fence_pve_ssh STONITH resource in Pacemaker
|
||||
# (see scripts/ha/cluster-enable-stonith.sh). On a fresh cluster with
|
||||
# no fence device yet, temporarily change to dont-care and run
|
||||
# cluster-enable-stonith.sh once the fence key is deployed.
|
||||
{ lib, vars, ... }:
|
||||
{
|
||||
services.drbd = {
|
||||
enable = true;
|
||||
config = ''
|
||||
global {
|
||||
usage-count yes;
|
||||
}
|
||||
|
||||
common {
|
||||
net {
|
||||
protocol C;
|
||||
ping-int 1;
|
||||
verify-alg sha256;
|
||||
after-sb-0pri discard-zero-changes;
|
||||
after-sb-1pri discard-secondary;
|
||||
}
|
||||
disk {
|
||||
fencing resource-only;
|
||||
}
|
||||
}
|
||||
|
||||
resource ha-data {
|
||||
volume 0 {
|
||||
device /dev/drbd0;
|
||||
disk /dev/sdb;
|
||||
meta-disk internal;
|
||||
}
|
||||
|
||||
on ${vars.haServer1Host} {
|
||||
address ${vars.haServer1StorageIp}:${toString vars.ports.haServerDrbd};
|
||||
}
|
||||
|
||||
on ${vars.haServer2Host} {
|
||||
address ${vars.haServer2StorageIp}:${toString vars.ports.haServerDrbd};
|
||||
}
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
# /etc/corosync/authkey — sops binary secret, identical on both nodes.
|
||||
# Decryptable by both ha-server host keys (added by sync-host-keys.sh).
|
||||
sops.secrets.corosync_authkey = {
|
||||
sopsFile = ../../secrets/ha-corosync-authkey;
|
||||
format = "binary";
|
||||
path = "/etc/corosync/authkey";
|
||||
mode = "0400";
|
||||
restartUnits = [ "corosync.service" ];
|
||||
};
|
||||
|
||||
# NixOS common config enables NetworkManager by default; HA cluster nodes
|
||||
# need stable static IPs with predictable interface names — NM is not suitable.
|
||||
networking.networkmanager.enable = lib.mkForce false;
|
||||
|
||||
# services.corosync.enable is set by modules/ha/pacemaker-stack.nix.
|
||||
services.corosync = {
|
||||
clusterName = "ha-cluster";
|
||||
nodelist = [
|
||||
{ nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1StorageIp ]; }
|
||||
{ nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2StorageIp ]; }
|
||||
];
|
||||
};
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [
|
||||
vars.ports.haServerIscsi
|
||||
vars.ports.haServerPacemakerRemoted
|
||||
vars.ports.haServerPcsd
|
||||
vars.ports.haServerDrbd
|
||||
vars.ports.nfsRpcbind
|
||||
vars.ports.nfsd
|
||||
vars.ports.nfsMountd
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
vars.ports.haServerCorosync1
|
||||
vars.ports.haServerCorosync2
|
||||
vars.ports.haServerCorosyncCrypto
|
||||
vars.ports.nfsRpcbind
|
||||
vars.ports.nfsd
|
||||
vars.ports.nfsMountd
|
||||
];
|
||||
extraCommands = ''
|
||||
iptables -A INPUT -s ${vars.haServer1Ip}/32 -j ACCEPT
|
||||
iptables -A INPUT -s ${vars.haServer2Ip}/32 -j ACCEPT
|
||||
iptables -A INPUT -s ${vars.haStorageCidr} -j ACCEPT
|
||||
'';
|
||||
};
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
# LIO iSCSI target service (targetctl) for NixOS HA clusters.
|
||||
#
|
||||
# Provides the targetctl.service that saves/restores LIO configuration from
|
||||
# /etc/target/saveconfig.json. Pacemaker manages this service via its
|
||||
# systemd resource agent (class="systemd" type="targetctl").
|
||||
#
|
||||
# Why ExecStop is not simply "targetctl save":
|
||||
# targetctl save writes the LIO config to JSON but does NOT remove the LIO
|
||||
# target from the kernel's configfs. As a result, any fileio backing store
|
||||
# that LIO has open (e.g. iscsi-lun.img on an XFS-over-DRBD filesystem)
|
||||
# stays referenced in the kernel. The subsequent XFS umount from the
|
||||
# Filesystem OCF resource then returns EBUSY and either hangs for the full
|
||||
# op-stop timeout or fails outright, blocking the entire failover.
|
||||
#
|
||||
# The ExecStop script here additionally tears down the kernel LIO state
|
||||
# via rtslib_fb after saving, so the backing-store file descriptor is
|
||||
# released and umount succeeds immediately.
|
||||
#
|
||||
# Empty-config guard:
|
||||
# The save step is skipped when no iSCSI targets are currently active.
|
||||
# This prevents the secondary node (where LIO was never started) from
|
||||
# overwriting a valid saveconfig.json with an empty one when Pacemaker
|
||||
# stops the iscsi-target resource as part of a failover or cleanup.
|
||||
{ pkgs, ... }:
|
||||
|
||||
let
|
||||
python3 = pkgs.python3.withPackages (ps: [ ps.rtslib-fb ]);
|
||||
targetctl = "${pkgs.targetcli-fb}/bin/targetctl";
|
||||
|
||||
targetctlStop = pkgs.writeScript "targetctl-stop" ''
|
||||
#!${python3}/bin/python3
|
||||
import subprocess, sys
|
||||
import rtslib_fb
|
||||
|
||||
root = rtslib_fb.RTSRoot()
|
||||
targets = list(root.targets)
|
||||
if targets:
|
||||
subprocess.run(
|
||||
["${targetctl}", "save", "/etc/target/saveconfig.json"],
|
||||
capture_output=True,
|
||||
)
|
||||
print(f"saved {len(targets)} iSCSI target(s)")
|
||||
else:
|
||||
print("no active LIO targets — saveconfig.json unchanged")
|
||||
|
||||
for target in targets:
|
||||
try:
|
||||
for tpg in list(target.tpgs):
|
||||
tpg.enable = False
|
||||
target.delete()
|
||||
except Exception as e:
|
||||
print(f"warn (target): {e}", file=sys.stderr)
|
||||
for so in list(root.storage_objects):
|
||||
try:
|
||||
so.delete()
|
||||
except Exception as e:
|
||||
print(f"warn (backstore): {e}", file=sys.stderr)
|
||||
print("LIO kernel target cleared")
|
||||
'';
|
||||
in
|
||||
{
|
||||
boot.kernelModules = [
|
||||
"target_core_mod"
|
||||
"iscsi_target_mod"
|
||||
"target_core_file"
|
||||
"target_core_pscsi"
|
||||
"target_core_user"
|
||||
"configfs"
|
||||
];
|
||||
|
||||
systemd = {
|
||||
mounts = [{
|
||||
where = "/sys/kernel/config";
|
||||
what = "configfs";
|
||||
type = "configfs";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [ "targetctl.service" ];
|
||||
}];
|
||||
services.targetctl = {
|
||||
description = "LIO iSCSI target config save/restore";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "sys-kernel-config.mount" "network.target" ];
|
||||
requires = [ "sys-kernel-config.mount" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = "${targetctl} restore /etc/target/saveconfig.json";
|
||||
ExecStop = "${targetctlStop}";
|
||||
};
|
||||
unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
|
||||
};
|
||||
tmpfiles.rules = [
|
||||
"d /etc/target 0750 root root -"
|
||||
"f /etc/target/saveconfig.json 0640 root root -"
|
||||
];
|
||||
};
|
||||
|
||||
environment.systemPackages = [ pkgs.targetcli-fb ];
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
# Pacemaker + Corosync HA stack for NixOS with known-good workarounds.
|
||||
#
|
||||
# Issues fixed here (confirmed through live testing on NixOS 25.11):
|
||||
#
|
||||
# 1. StateDirectory ownership reset: systemd's StateDirectory=pacemaker
|
||||
# creates /var/lib/pacemaker owned root:root. pacemaker-based (the CIB
|
||||
# daemon) runs as the hacluster user and calls pcmk__daemon_can_write,
|
||||
# which requires the CIB directory to be owned by hacluster or be
|
||||
# group-writable by haclient. Workaround: remove StateDirectory and let
|
||||
# ExecStartPre create every required subdirectory with correct ownership.
|
||||
#
|
||||
# 2. HA_SBIN_DIR wrong path: ocf-shellfuncs sets HA_SBIN_DIR to the Nix
|
||||
# store path of the resource-agents derivation's /sbin, which doesn't
|
||||
# exist. The DRBD OCF agent uses ${HA_SBIN_DIR}/crm_master, so it exits
|
||||
# 127 without this override. Fix: export HA_SBIN_DIR=/run/current-system/sw/bin.
|
||||
#
|
||||
# 3. Broad PATH for OCF agents: the resource executor (pacemaker-execd) runs
|
||||
# OCF agent scripts as children. NixOS provides no implicit PATH for
|
||||
# system services; without an explicit PATH the agents can't find ip, ss,
|
||||
# mount, umount, drbdadm, etc.
|
||||
#
|
||||
# 4. FUSER=true: the Filesystem OCF agent calls check_binary $FUSER (default:
|
||||
# fuser from psmisc), which is not installed. Setting FUSER=true makes
|
||||
# check_binary succeed (true is always in PATH) and the subsequent
|
||||
# "$FUSER -km $mountpoint" becomes a no-op. Pair with force_unmount=false
|
||||
# on each Filesystem resource unless you want lazy unmount behaviour.
|
||||
{ lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
ocfBinPath = lib.concatStringsSep ":" [
|
||||
"${pkgs.iproute2}/bin"
|
||||
"${pkgs.iproute2}/sbin"
|
||||
"${pkgs.iputils}/bin"
|
||||
"${pkgs.util-linux}/bin"
|
||||
"${pkgs.util-linux}/sbin"
|
||||
"${pkgs.gawk}/bin"
|
||||
"${pkgs.gnugrep}/bin"
|
||||
"${pkgs.gnused}/bin"
|
||||
"${pkgs.coreutils}/bin"
|
||||
"${pkgs.bash}/bin"
|
||||
"${pkgs.procps}/bin"
|
||||
"${pkgs.xfsprogs}/bin"
|
||||
"${pkgs.drbd}/bin"
|
||||
"${pkgs.python3}/bin"
|
||||
"/run/current-system/sw/bin"
|
||||
"/run/current-system/sw/sbin"
|
||||
"/usr/local/sbin"
|
||||
"/usr/local/bin"
|
||||
"/usr/sbin"
|
||||
"/usr/bin"
|
||||
"/sbin"
|
||||
"/bin"
|
||||
];
|
||||
|
||||
# Single pre-start script: schemas symlink + directory ownership.
|
||||
# Runs before pacemakerd so pacemaker-based finds hacluster-owned dirs.
|
||||
preStartCmd = "${pkgs.bash}/bin/bash -c '"
|
||||
+ "ln -sfn ${pkgs.pacemaker}/share/pacemaker /var/lib/pacemaker/schemas; "
|
||||
+ "for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores "
|
||||
+ "/var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox "
|
||||
+ "/var/lib/pacemaker/hostcache; do "
|
||||
+ "mkdir -p \"\\$d\" && chown hacluster:pacemaker \"\\$d\" && chmod 2770 \"\\$d\"; "
|
||||
+ "done'";
|
||||
|
||||
ocfEnv = {
|
||||
PATH = lib.mkForce ocfBinPath;
|
||||
OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
|
||||
HA_SBIN_DIR = "/run/current-system/sw/bin";
|
||||
FUSER = "true";
|
||||
};
|
||||
in
|
||||
{
|
||||
users.groups.haclient = { };
|
||||
|
||||
services.corosync.enable = true;
|
||||
services.pacemaker.enable = true;
|
||||
|
||||
systemd.services = {
|
||||
pacemaker = {
|
||||
serviceConfig = {
|
||||
StateDirectory = lib.mkForce "";
|
||||
ExecStartPre = lib.mkBefore [ preStartCmd ];
|
||||
};
|
||||
environment = ocfEnv;
|
||||
};
|
||||
pacemaker-execd.environment = ocfEnv;
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
corosync
|
||||
pacemaker
|
||||
ocf-resource-agents
|
||||
];
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
# Fully declarative FreeIPA domain membership.
|
||||
#
|
||||
# Configures security.ipa (SSSD, Kerberos, PAM, NSSwitch) and places a
|
||||
# pre-provisioned host keytab via sops-nix so no imperative ipa-client-install
|
||||
# step is needed after deployment.
|
||||
#
|
||||
# Usage (in a host.nix imports list):
|
||||
# (import ../../modules/ipa/client.nix {
|
||||
# keytabSopsFile = ../../secrets/<hostname>.keytab;
|
||||
# caCertFile = ../../certs/ipa-ca.crt; # already committed — do not re-fetch
|
||||
# })
|
||||
#
|
||||
# The host.nix networking block must also set:
|
||||
# networking.domain = vars.homeDomain; # needed for Kerberos FQDN
|
||||
# networking.nameservers = [ vars.domainControllerIp ]; # IPA DNS
|
||||
#
|
||||
# One-time operator setup per host (do this BEFORE deploying):
|
||||
#
|
||||
# 0. Generate SSH host keys and the host's age key for sops:
|
||||
# scripts/secrets/sync-host-keys.sh <flake-target>
|
||||
# This must run before step 1 so the host age key is in .sops.yaml
|
||||
# and the keytab can be encrypted for the host to read at boot.
|
||||
#
|
||||
# 1. Add the IPA host account and produce the sops-encrypted keytab:
|
||||
# scripts/ipa/create-nixos-ipa-host-account.sh [--ip <addr>] <hostname>
|
||||
# The script handles ipa host-add, ipa-getkeytab, .sops.yaml patching,
|
||||
# and sops encryption in one step. See the script header for details.
|
||||
#
|
||||
# 2. Wire up the host (see "Usage" above), then deploy:
|
||||
# nixos-rebuild switch (or create-proxmox-resource.sh)
|
||||
# No further manual enrollment steps are required after deployment.
|
||||
#
|
||||
# Manual fallback (if the script isn't usable):
|
||||
# a. On the FreeIPA server: ipa host-add <fqdn> [--ip-address=<ip>] --force
|
||||
# b. On the FreeIPA server: ipa-getkeytab -s <ipa-server> -p host/<fqdn> -k /tmp/<host>.keytab
|
||||
# c. From the repo root (path must match for sops creation rule to apply):
|
||||
# cp /tmp/<host>.keytab secrets/<host>.keytab
|
||||
# sops -e --input-type binary -i secrets/<host>.keytab
|
||||
# d. Commit secrets/<host>.keytab and the updated .sops.yaml, then deploy.
|
||||
#
|
||||
# vars dependencies: homeDomain, ipaServer, domainControllerIp
|
||||
|
||||
{ keytabSopsFile, caCertFile }:
|
||||
{ config, lib, pkgs, vars, ... }:
|
||||
|
||||
let
|
||||
realm = lib.strings.toUpper vars.homeDomain;
|
||||
fqdn = "${config.networking.hostName}.${vars.homeDomain}";
|
||||
# "sweet.home" -> "dc=sweet,dc=home"
|
||||
basedn = lib.strings.concatMapStringsSep "," (c: "dc=${c}") (lib.strings.splitString "." vars.homeDomain);
|
||||
# security.ipa.certificate expects a derivation (package), not a raw path.
|
||||
caCertPkg = pkgs.writeText "ipa-ca.crt" (builtins.readFile caCertFile);
|
||||
in
|
||||
{
|
||||
security.ipa = {
|
||||
enable = true;
|
||||
domain = vars.homeDomain;
|
||||
realm = realm;
|
||||
server = vars.ipaServer;
|
||||
certificate = caCertPkg;
|
||||
basedn = basedn;
|
||||
ipaHostname = fqdn;
|
||||
offlinePasswords = true;
|
||||
cacheCredentials = true;
|
||||
};
|
||||
|
||||
# Fetch SSH public keys from IPA so users can log in with the key stored
|
||||
# in their IPA profile rather than needing ~/.ssh/authorized_keys on every
|
||||
# host. sss_ssh_authorizedkeys queries SSSD (which queries IPA LDAP).
|
||||
#
|
||||
# /nix/store is 1775 (group-writable by nixbld). OpenSSH 10.0+ rejects
|
||||
# AuthorizedKeysCommand binaries whose path contains any group-writable
|
||||
# component, silently skipping the command. Copy to /usr/local/bin (all
|
||||
# components root-owned, 755) so the path passes sshd's safety check.
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /usr/local 0755 root root - -"
|
||||
"d /usr/local/bin 0755 root root - -"
|
||||
"C+ /usr/local/bin/sss_ssh_authorizedkeys 0555 root root - ${pkgs.sssd}/bin/sss_ssh_authorizedkeys"
|
||||
];
|
||||
|
||||
services.openssh.extraConfig = ''
|
||||
AuthorizedKeysCommand /usr/local/bin/sss_ssh_authorizedkeys %u
|
||||
AuthorizedKeysCommandUser nobody
|
||||
'';
|
||||
|
||||
# Create the home directory on first login if it doesn't exist yet.
|
||||
# IPA users have no pre-created home on the host; without this sshd
|
||||
# opens a session to a non-existent directory and resets the connection.
|
||||
security.pam.services.sshd.makeHomeDir = true;
|
||||
|
||||
# Host keytab: pre-provisioned on the IPA server, sops-encrypted binary.
|
||||
# Placed at /etc/krb5.keytab before SSSD starts so the host authenticates
|
||||
# to IPA without running ipa-client-install.
|
||||
sops.secrets."ipa-host-keytab" = {
|
||||
sopsFile = keytabSopsFile;
|
||||
format = "binary";
|
||||
path = "/etc/krb5.keytab";
|
||||
owner = "root";
|
||||
group = "root";
|
||||
mode = "0600";
|
||||
restartUnits = [ "sssd.service" ];
|
||||
};
|
||||
}
|
||||
+17
-16
@@ -63,22 +63,23 @@ in
|
||||
# back to decide `pct create`'s --unprivileged flag, so the two stay
|
||||
# in sync).
|
||||
#
|
||||
# Any lxc-* host with an NFS fileSystem must be privileged: the kernel's
|
||||
# NFS client doesn't set FS_USERNS_MOUNT, so mounting NFS from inside
|
||||
# *any* non-init user namespace -- which is exactly what an unprivileged
|
||||
# container's UID-mapped root runs in -- is rejected at the VFS layer
|
||||
# with EPERM, no matter what Proxmox's own `mount=nfs;nfs4` container
|
||||
# feature allows at the AppArmor layer (confirmed live: TCP to the NFS
|
||||
# server succeeds, the server's export table matches the container's IP,
|
||||
# and `mount.nfs: Operation not permitted` still fires immediately with
|
||||
# no corresponding denial anywhere in the server's logs -- a kernel-level
|
||||
# rejection, not a network or export-permission one). Deriving this from
|
||||
# fileSystems rather than a per-host override keeps it self-consistent:
|
||||
# any new lxc-* host that declares an NFS mount automatically gets the
|
||||
# privilege level it needs without a separate manual flag.
|
||||
privileged = builtins.any
|
||||
(fs: fs.fsType == "nfs" || fs.fsType == "nfs4")
|
||||
(builtins.attrValues config.fileSystems);
|
||||
# lxc-docker is the one exception: the kernel's NFS client doesn't set
|
||||
# FS_USERNS_MOUNT, so mounting NFS from inside *any* non-init user
|
||||
# namespace -- which is exactly what an unprivileged container's
|
||||
# UID-mapped root runs in -- is rejected at the VFS layer with EPERM,
|
||||
# no matter what Proxmox's own `mount=nfs;nfs4` container feature
|
||||
# allows at the AppArmor layer (confirmed live: TCP to the NFS server
|
||||
# succeeds, the server's export table matches the container's IP, and
|
||||
# `mount.nfs: Operation not permitted` still fires immediately with no
|
||||
# corresponding denial anywhere in the server's logs -- a kernel-level
|
||||
# rejection, not a network or export-permission one). Keying off
|
||||
# hostName rather than something docker-build-type-specific because
|
||||
# modules/build-types/docker.nix is also composed for linode-docker/
|
||||
# proxmox-docker, which don't import proxmox-lxc.nix at all --setting
|
||||
# this option there would break their eval with "option does not
|
||||
# exist" regardless of any mkIf guard, since mkIf only makes a value
|
||||
# conditional, not whether the option needs to exist somewhere.
|
||||
privileged = config.networking.hostName == "docker";
|
||||
};
|
||||
|
||||
boot.loader = {
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
{ config, lib, vars, ... }:
|
||||
|
||||
let
|
||||
# Use the same FQDN approach as docker/mount-data.nix — a bare hostname is
|
||||
# unreliable: systemd-resolved only tries LLMNR for single-label names, and
|
||||
# a global search domain causes it to skip the interface-scoped LAN DNS.
|
||||
nfsServer = "${vars.nfsServerHost}.${vars.homeDomain}";
|
||||
in
|
||||
{
|
||||
fileSystems.${vars.nfsShares.pxebootImages.mountpoint} = {
|
||||
device = "${nfsServer}:${vars.storageRoot}/${vars.nfsShares.pxebootImages.subpath}";
|
||||
fsType = "nfs";
|
||||
options = [
|
||||
"_netdev"
|
||||
"noatime"
|
||||
] ++ (if config.boot.isContainer
|
||||
# NFSv4 requires rpc_pipefs (sunrpc filesystem), which Proxmox LXC
|
||||
# containers block unless `features: mount=nfs` is set. Use NFSv3+nolock
|
||||
# instead: no rpc_pipefs dependency at the protocol level, and rpcbind
|
||||
# on the server handles port resolution without needing client-side
|
||||
# sunrpc infrastructure. nofail keeps boot clean if server is unreachable.
|
||||
then [ "nfsvers=3" "proto=tcp" "nolock" "nofail" ]
|
||||
else [ "nfsvers=4.2" "x-systemd.automount" ]);
|
||||
};
|
||||
|
||||
# NixOS pulls var-lib-nfs-rpc_pipefs.mount (the sunrpc filesystem) into
|
||||
# nfs-client.target for any nfs fileSystems entry. In LXC containers the
|
||||
# sunrpc mount is blocked by Proxmox's AppArmor profile, causing it to fail
|
||||
# and the activation to report an error even though our mount uses nofail.
|
||||
# Add ConditionVirtualization=!container via drop-in so systemd skips the
|
||||
# unit entirely in containers (skip = inactive, not failed), which keeps
|
||||
# nfs-client.target green and activation clean.
|
||||
systemd.units = lib.mkIf config.boot.isContainer {
|
||||
"var-lib-nfs-rpc_pipefs.mount" = {
|
||||
overrideStrategy = "asDropin";
|
||||
text = ''
|
||||
[Unit]
|
||||
ConditionVirtualization=!container
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
{ pkgs, ... }:
|
||||
_:
|
||||
|
||||
{
|
||||
imports = [ ./enable-service.nix ];
|
||||
@@ -12,24 +12,4 @@
|
||||
# instead of relaying through DERP.
|
||||
openFirewall = true;
|
||||
};
|
||||
|
||||
# Tailscale recommends these ethtool flags on the uplink interface to get
|
||||
# full UDP GRO throughput on subnet routers (https://tailscale.com/s/ethtool-config-udp-gro).
|
||||
# The interface is derived from the default route so it works regardless of
|
||||
# what the NIC is named on a given host.
|
||||
systemd.services.tailscale-udp-gro = {
|
||||
description = "Enable UDP GRO forwarding on uplink for Tailscale subnet router";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [ pkgs.ethtool pkgs.iproute2 ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = pkgs.writeShellScript "tailscale-udp-gro" ''
|
||||
NETDEV=$(ip -o route get 8.8.8.8 | cut -f 5 -d " ")
|
||||
ethtool -K "$NETDEV" rx-udp-gro-forwarding on rx-gro-list off
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
{ vars, ... }:
|
||||
|
||||
{
|
||||
# Run dnsmasq on the LAN interface as a forwarding-only resolver for
|
||||
# *.ts.net (Tailscale MagicDNS names). FreeIPA's bind-dyndb-ldap
|
||||
# cannot reach 100.100.100.100 (Tailscale's internal resolver) directly
|
||||
# because the DC is not a Tailscale node. This host IS a Tailscale node
|
||||
# and can reach 100.100.100.100 via its tailscale0 interface, so it
|
||||
# acts as an intermediary: FreeIPA has a conditional forward zone for
|
||||
# ts.net pointing here (vars.tailscaleRouterIp), and this dnsmasq
|
||||
# instance forwards those queries onward to Tailscale's resolver.
|
||||
#
|
||||
# Configure FreeIPA once after deploying this host:
|
||||
# kinit admin
|
||||
# ipa dnsforwardzone-add ${vars.tailnetDomain} \
|
||||
# --forwarder=${vars.tailscaleRouterIp} \
|
||||
# --forward-policy=only
|
||||
# Note: IPA refuses to shadow ts.net (a real public TLD); use the
|
||||
# tailnet-specific subdomain (vars.tailnetDomain) instead.
|
||||
services.dnsmasq = {
|
||||
enable = true;
|
||||
# NixOS's dnsmasq module defaults resolveLocalQueries to true, which adds
|
||||
# 127.0.0.1 to networking.nameservers and makes dnsmasq bind to
|
||||
# listen-address=127.0.0.1. This instance is not the host's local
|
||||
# resolver — it only serves IPA's conditional forwarder for tailnet names.
|
||||
# The host uses domainControllerIp directly (networking.nameservers in
|
||||
# host.nix). Without this, all host DNS goes through dnsmasq, which has
|
||||
# no upstream for general queries (no-resolv=true), breaking resolution.
|
||||
resolveLocalQueries = false;
|
||||
settings = {
|
||||
# Listen only on the LAN interface — not tailscale0 or loopback.
|
||||
# bind-interfaces prevents dnsmasq from binding to 0.0.0.0 and
|
||||
# then filtering by interface later; combined with `interface` this
|
||||
# ensures it genuinely listens only on eth0.
|
||||
bind-interfaces = true;
|
||||
interface = [ vars.lxcLanInterface ];
|
||||
|
||||
# Forward-only: no local /etc/hosts or /etc/resolv.conf reading,
|
||||
# no negative caching of NXDOMAIN for names this instance doesn't
|
||||
# serve. All ts.net queries come from FreeIPA's conditional forwarder
|
||||
# and must be answered by Tailscale's resolver.
|
||||
no-hosts = true;
|
||||
no-resolv = true;
|
||||
|
||||
# Tailscale's internal "Quad100" resolver — reachable from any
|
||||
# Tailscale node via the tailscale0 interface. Scoped to the
|
||||
# specific tailnet subdomain (vars.tailnetDomain) rather than
|
||||
# all of ts.net: FreeIPA refuses to shadow ts.net (a real public
|
||||
# TLD with DNSimple nameservers) so the conditional forward zone
|
||||
# in FreeIPA must use the tailnet-specific subdomain instead:
|
||||
# ipa dnsforwardzone-add ${vars.tailnetDomain} \
|
||||
# --forwarder=${vars.tailscaleRouterIp} \
|
||||
# --forward-policy=only
|
||||
server = [ "/${vars.tailnetDomain}/100.100.100.100" ];
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedUDPPorts = [ 53 ];
|
||||
networking.firewall.allowedTCPPorts = [ 53 ];
|
||||
}
|
||||
@@ -88,17 +88,6 @@ export NIX_CACHE_HOST
|
||||
: "${LAN_DOMAIN:=gitea.lan.ddnsgeek.com}"
|
||||
export LAN_DOMAIN
|
||||
|
||||
# Matches variables.nix's homeDomain -- the base LAN domain for service
|
||||
# subdomains, FreeIPA Kerberos realm, and host FQDNs.
|
||||
: "${HOME_DOMAIN:=sweet.home}"
|
||||
export HOME_DOMAIN
|
||||
|
||||
# Matches variables.nix's ipaServer -- the FreeIPA server hostname.
|
||||
# scripts/ipa/create-nixos-ipa-host-account.sh SSHes here to run
|
||||
# ipa host-add and ipa-getkeytab.
|
||||
: "${IPA_SERVER:=domain-controller.sweet.home}"
|
||||
export IPA_SERVER
|
||||
|
||||
# nix_extra_opts: call as a plain statement (NOT inside $(...)/<(...) --
|
||||
# that forks a subshell, and the whole point is exporting a decision back
|
||||
# into *this* shell) to populate the global NIX_OPTS array with whatever
|
||||
|
||||
@@ -1,167 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# acceptance-tests.sh — HA cluster acceptance tests (T1–T7)
|
||||
#
|
||||
# Run from a host with SSH access to both HA nodes (or from node1 itself).
|
||||
# All 7 tests must pass before considering the cluster production-ready.
|
||||
# Test values below must match variables.nix haServer* values.
|
||||
set -euo pipefail
|
||||
|
||||
# ── Configuration ─────────────────────────────────────────────────────────
|
||||
NODE1="ha-server-1"
|
||||
NODE2="ha-server-2"
|
||||
NODE1_IP="192.168.2.200" # vars.haServer1Ip
|
||||
NODE2_IP="192.168.2.201" # vars.haServer2Ip
|
||||
VIP="192.168.2.202" # vars.haServerVip
|
||||
XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot
|
||||
ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
RESULTS=()
|
||||
|
||||
pass() { echo " PASS: $1"; ((PASS++)); RESULTS+=("PASS $1"); }
|
||||
fail() { echo " FAIL: $1"; ((FAIL++)); RESULTS+=("FAIL $1"); }
|
||||
|
||||
n1() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null; }
|
||||
n2() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null; }
|
||||
|
||||
echo "════════════════════════════════════════════════════"
|
||||
echo " HA Cluster Acceptance Tests — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "════════════════════════════════════════════════════"
|
||||
|
||||
# ── T1: Corosync quorum established ──────────────────────────────────────
|
||||
echo ""
|
||||
echo "[T1] Corosync quorum"
|
||||
if n1 "corosync-quorumtool -s" 2>/dev/null | grep -q "Quorate:.*Yes"; then
|
||||
pass "cluster has quorum"
|
||||
else
|
||||
fail "cluster does not have quorum — check corosync on both nodes"
|
||||
fi
|
||||
|
||||
# ── T2: DRBD Primary on node1, Secondary on node2 ────────────────────────
|
||||
echo ""
|
||||
echo "[T2] DRBD roles"
|
||||
DRBD_ROLE=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
|
||||
if [[ "$DRBD_ROLE" == "Primary/Secondary" || "$DRBD_ROLE" == "Primary" ]]; then
|
||||
pass "DRBD Primary on $NODE1 ($DRBD_ROLE)"
|
||||
else
|
||||
fail "unexpected DRBD role on $NODE1: $DRBD_ROLE (expected Primary/Secondary)"
|
||||
fi
|
||||
|
||||
DRBD_DSTATE=$(n1 "drbdadm dstate ha-data" 2>/dev/null || echo "unknown")
|
||||
if echo "$DRBD_DSTATE" | grep -q "UpToDate"; then
|
||||
pass "DRBD disk state UpToDate ($DRBD_DSTATE)"
|
||||
else
|
||||
fail "DRBD disk not UpToDate: $DRBD_DSTATE"
|
||||
fi
|
||||
|
||||
# ── T3: XFS mounted at haStorageRoot on the Active node ──────────────────
|
||||
echo ""
|
||||
echo "[T3] XFS mount"
|
||||
if n1 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||||
pass "XFS mounted at ${XFS_MOUNT} on $NODE1"
|
||||
else
|
||||
fail "XFS not mounted at ${XFS_MOUNT} on $NODE1"
|
||||
fi
|
||||
|
||||
if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||||
fail "XFS unexpectedly mounted on $NODE2 (should only be on Active node)"
|
||||
else
|
||||
pass "XFS not mounted on $NODE2 (correct — Secondary)"
|
||||
fi
|
||||
|
||||
# ── T4: iSCSI target visible on both nodes ────────────────────────────────
|
||||
echo ""
|
||||
echo "[T4] iSCSI target"
|
||||
IQN_COUNT=$(n1 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
|
||||
if [[ "$IQN_COUNT" -ge 1 ]]; then
|
||||
pass "iSCSI IQN active on $NODE1 ($IQN_COUNT target(s))"
|
||||
else
|
||||
fail "no iSCSI IQN active on $NODE1"
|
||||
fi
|
||||
|
||||
# iSCSI discovery from node2 via VIP
|
||||
if n2 "iscsiadm -m discovery -t sendtargets -p '${VIP}' 2>/dev/null | grep -q '${ISCSI_IQN}'"; then
|
||||
pass "iSCSI target discoverable from $NODE2 via VIP ${VIP}"
|
||||
else
|
||||
fail "iSCSI target not discoverable from $NODE2 via ${VIP}"
|
||||
fi
|
||||
|
||||
# ── T5: Failover — standby node1, verify resources move to node2 ──────────
|
||||
echo ""
|
||||
echo "[T5] Failover (standby $NODE1)"
|
||||
MYNODE=$(n1 "crm_node -n" 2>/dev/null || echo "")
|
||||
n1 "crm_standby -N '${MYNODE}' -v on" 2>/dev/null || true
|
||||
echo " Waiting up to 30 s for resources to move to $NODE2..."
|
||||
MOVED=false
|
||||
for i in $(seq 1 30); do
|
||||
if n2 "mountpoint -q '${XFS_MOUNT}'" 2>/dev/null; then
|
||||
MOVED=true
|
||||
echo " Resources moved in ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if $MOVED; then
|
||||
pass "XFS mounted on $NODE2 after failover"
|
||||
IQN_ON_N2=$(n2 "ls /sys/kernel/config/target/iscsi/ 2>/dev/null | grep -c iqn" || echo "0")
|
||||
[[ "$IQN_ON_N2" -ge 1 ]] \
|
||||
&& pass "iSCSI target active on $NODE2 after failover" \
|
||||
|| fail "iSCSI target NOT active on $NODE2 after failover"
|
||||
else
|
||||
fail "XFS did not mount on $NODE2 within 30 s — failover incomplete"
|
||||
fi
|
||||
|
||||
# ── T6: Data integrity — file written pre-failover readable post-failover ─
|
||||
echo ""
|
||||
echo "[T6] Data integrity"
|
||||
# Write a test file on node2 (now Active) and verify its content
|
||||
TEST_FILE="${XFS_MOUNT}/.acceptance-test-$$"
|
||||
TEST_CONTENT="ha-acceptance-test-$(date +%s)"
|
||||
n2 "echo '${TEST_CONTENT}' > '${TEST_FILE}'" 2>/dev/null || true
|
||||
READBACK=$(n2 "cat '${TEST_FILE}' 2>/dev/null" || echo "")
|
||||
if [[ "$READBACK" == "$TEST_CONTENT" ]]; then
|
||||
pass "test file written and read back correctly on $NODE2"
|
||||
else
|
||||
fail "data integrity check failed (wrote: '$TEST_CONTENT', read: '$READBACK')"
|
||||
fi
|
||||
n2 "rm -f '${TEST_FILE}'" 2>/dev/null || true
|
||||
|
||||
# ── T7: Node rejoin — un-standby node1, verify cluster is healthy ─────────
|
||||
echo ""
|
||||
echo "[T7] Node rejoin"
|
||||
n1 "crm_standby -N '${MYNODE}' -v off" 2>/dev/null || true
|
||||
n1 "crm_resource --cleanup" 2>/dev/null || true
|
||||
sleep 5
|
||||
|
||||
ONLINE_NODES=$(n2 "crm_mon -1 2>/dev/null | grep -c 'Online:'" || echo "0")
|
||||
if n1 "corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'"; then
|
||||
pass "$NODE1 rejoined — cluster has quorum"
|
||||
else
|
||||
fail "$NODE1 did not rejoin with quorum"
|
||||
fi
|
||||
|
||||
DRBD_ROLE_AFTER=$(n1 "drbdadm role ha-data" 2>/dev/null || echo "unknown")
|
||||
if echo "$DRBD_ROLE_AFTER" | grep -q "Secondary"; then
|
||||
pass "$NODE1 is DRBD Secondary after rejoin ($DRBD_ROLE_AFTER)"
|
||||
else
|
||||
fail "unexpected DRBD role on $NODE1 after rejoin: $DRBD_ROLE_AFTER"
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "════════════════════════════════════════════════════"
|
||||
echo " Results: ${PASS} PASS, ${FAIL} FAIL"
|
||||
echo "════════════════════════════════════════════════════"
|
||||
for r in "${RESULTS[@]}"; do echo " $r"; done
|
||||
echo ""
|
||||
|
||||
if [[ "$FAIL" -eq 0 ]]; then
|
||||
echo "ALL PASS — cluster is production-ready."
|
||||
exit 0
|
||||
else
|
||||
echo "SOME TESTS FAILED — investigate before deploying."
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,86 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# cluster-enable-stonith.sh — enable STONITH fence agent after the fence SSH
|
||||
# key is deployed to both nodes and authorised on the Proxmox host.
|
||||
#
|
||||
# Run from ha-server-1 as root AFTER:
|
||||
# - /etc/pacemaker/fence_pve_ssh exists on both nodes (chmod +x)
|
||||
# (copy from scripts/ha/fence-pve-ssh.py)
|
||||
# - /etc/fence-pve-ssh-key (SSH private key) exists on both nodes
|
||||
# - The corresponding public key is in authorized_keys on PVE_HOST
|
||||
# - VMID_NODE1 / VMID_NODE2 filled in below
|
||||
set -euo pipefail
|
||||
|
||||
# ── Configuration ─────────────────────────────────────────────────────────
|
||||
NODE1="ha-server-1"
|
||||
NODE2="ha-server-2"
|
||||
VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1
|
||||
VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2
|
||||
PVE_HOST="pve1.sweet.home"
|
||||
PVE_USER="wayne"
|
||||
FENCE_KEY="/etc/fence-pve-ssh-key"
|
||||
FENCE_SCRIPT="/etc/pacemaker/fence_pve_ssh"
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
log() { echo "[stonith-setup] $*"; }
|
||||
die() { echo "[stonith-setup] ERROR: $*" >&2; exit 1; }
|
||||
|
||||
[[ $(id -u) -eq 0 ]] || die "must run as root"
|
||||
[[ -n "$VMID_NODE1" ]] || die "VMID_NODE1 not set — edit this script"
|
||||
[[ -n "$VMID_NODE2" ]] || die "VMID_NODE2 not set — edit this script"
|
||||
[[ -f "$FENCE_KEY" ]] || die "fence key not found at $FENCE_KEY"
|
||||
[[ -f "$FENCE_SCRIPT" ]] || die "fence script not found at $FENCE_SCRIPT"
|
||||
|
||||
log "Verifying fence agent can reach ${PVE_HOST}..."
|
||||
ssh -i "$FENCE_KEY" -o BatchMode=yes -o ConnectTimeout=10 \
|
||||
-o StrictHostKeyChecking=no "${PVE_USER}@${PVE_HOST}" \
|
||||
"sudo /usr/sbin/qm list" &>/dev/null \
|
||||
|| die "Cannot SSH to ${PVE_USER}@${PVE_HOST} — check authorized_keys and sudo"
|
||||
log "Fence agent SSH connectivity confirmed"
|
||||
|
||||
log "Creating Pacemaker STONITH resources..."
|
||||
cibadmin --create --scope resources --xml-text "
|
||||
<primitive id=\"stonith-${NODE1}\" class=\"stonith\" type=\"external/fence_pve_ssh\">
|
||||
<instance_attributes id=\"stonith-${NODE1}-attrs\">
|
||||
<nvpair id=\"stonith-${NODE1}-plug\" name=\"plug\" value=\"${NODE1}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-vmid1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-vmid2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
|
||||
<nvpair id=\"stonith-${NODE1}-host-list\" name=\"pcmk_host_list\" value=\"${NODE1}\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"stonith-${NODE1}-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
" 2>/dev/null || true
|
||||
|
||||
cibadmin --create --scope resources --xml-text "
|
||||
<primitive id=\"stonith-${NODE2}\" class=\"stonith\" type=\"external/fence_pve_ssh\">
|
||||
<instance_attributes id=\"stonith-${NODE2}-attrs\">
|
||||
<nvpair id=\"stonith-${NODE2}-plug\" name=\"plug\" value=\"${NODE2}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-pve-host\" name=\"pve_host\" value=\"${PVE_HOST}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-pve-user\" name=\"pve_user\" value=\"${PVE_USER}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-key-file\" name=\"key_file\" value=\"${FENCE_KEY}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-vmid1\" name=\"vmid_node1\" value=\"${VMID_NODE1}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-vmid2\" name=\"vmid_node2\" value=\"${VMID_NODE2}\"/>
|
||||
<nvpair id=\"stonith-${NODE2}-host-list\" name=\"pcmk_host_list\" value=\"${NODE2}\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"stonith-${NODE2}-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"30s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
" 2>/dev/null || true
|
||||
|
||||
log "Enabling STONITH and restoring quorum policy..."
|
||||
crm_attribute -t crm_config -n stonith-enabled -v true
|
||||
crm_attribute -t crm_config -n no-quorum-policy -v stop
|
||||
|
||||
log "DRBD fencing mode must also be updated to resource-only (already the"
|
||||
log "default in cluster-config.nix; confirm with: cat /etc/drbd.d/ha-data.conf)"
|
||||
|
||||
log "Testing fence agent..."
|
||||
stonith_admin --list-devices && log "Fence devices listed successfully." \
|
||||
|| warn "stonith_admin --list-devices failed — check config"
|
||||
|
||||
log "STONITH enabled. Cluster is now fully HA."
|
||||
@@ -1,284 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# cluster-init.sh — one-time HA cluster initialisation script
|
||||
#
|
||||
# Run ONCE from ha-server-1 as root AFTER both VMs are booted and have SSH
|
||||
# access. It:
|
||||
# 1. Generates and distributes the corosync authkey
|
||||
# 2. Waits for corosync quorum and pacemaker
|
||||
# 3. Initialises DRBD metadata, promotes node1 to primary
|
||||
# 4. Creates XFS on /dev/drbd0 and mounts it
|
||||
# 5. Creates the directory tree and iSCSI LUN backing file
|
||||
# 6. Configures LIO iSCSI target (file-backed LUN)
|
||||
# 7. Configures Pacemaker resources: DRBD → XFS → iSCSI → NFS → VIP
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Both VMs booted with the ha-server config (nixos-rebuild done)
|
||||
# - SSH key access from node1 to root@NODE2_IP
|
||||
# - VMID_NODE1 / VMID_NODE2 filled in below (needed for STONITH setup;
|
||||
# cluster starts without STONITH, which you enable separately via
|
||||
# scripts/ha/cluster-enable-stonith.sh)
|
||||
# - Run as root on ha-server-1
|
||||
set -euo pipefail
|
||||
|
||||
# ── Configuration ─────────────────────────────────────────────────────────
|
||||
# These must match variables.nix haServer* values and the Proxmox VMID
|
||||
# assignments. Update before running.
|
||||
NODE1="ha-server-1"
|
||||
NODE2="ha-server-2"
|
||||
NODE1_IP="192.168.2.200" # vars.haServer1Ip
|
||||
NODE2_IP="192.168.2.201" # vars.haServer2Ip
|
||||
VIP="192.168.2.202" # vars.haServerVip
|
||||
XFS_MOUNT="/srv/ha-data" # vars.haStorageRoot
|
||||
ISCSI_IQN="iqn.2026-01.home.sweet:ha-storage" # vars.haIscsiIqn
|
||||
ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
|
||||
ISCSI_LUN_SIZE="10G"
|
||||
DRBD_DEVICE="/dev/drbd0"
|
||||
VMID_NODE1="" # FILL IN: Proxmox VMID for ha-server-1
|
||||
VMID_NODE2="" # FILL IN: Proxmox VMID for ha-server-2
|
||||
PVE_HOST="pve1.sweet.home"
|
||||
PVE_USER="wayne"
|
||||
|
||||
# NFS dataset subdirectories to create under XFS_MOUNT.
|
||||
# Must mirror vars.nfsShares subpath values in variables.nix.
|
||||
NFS_SUBDIRS=(
|
||||
"docker/config"
|
||||
"docker/volumes"
|
||||
"docker/databases"
|
||||
"docker/nextcloud-data"
|
||||
"raspi/volumes"
|
||||
"proxmox/iso"
|
||||
"proxmox/lxc"
|
||||
"pxe-boot/images"
|
||||
)
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
log() { echo "[cluster-init] $*"; }
|
||||
die() { echo "[cluster-init] ERROR: $*" >&2; exit 1; }
|
||||
warn() { echo "[cluster-init] WARNING: $*" >&2; }
|
||||
|
||||
[[ $(id -u) -eq 0 ]] || die "must run as root"
|
||||
[[ "$(hostname)" == "$NODE1" ]] || die "must run on $NODE1"
|
||||
|
||||
# ── 0. Corosync authkey ───────────────────────────────────────────────────
|
||||
AUTHKEY="/etc/corosync/authkey"
|
||||
mkdir -p /etc/corosync
|
||||
if [[ ! -f "$AUTHKEY" ]]; then
|
||||
log "Generating corosync authkey..."
|
||||
corosync-keygen -k "$AUTHKEY"
|
||||
chmod 0400 "$AUTHKEY"
|
||||
fi
|
||||
log "Distributing authkey to $NODE2..."
|
||||
ssh "root@${NODE2_IP}" "mkdir -p /etc/corosync"
|
||||
scp -q "$AUTHKEY" "root@${NODE2_IP}:${AUTHKEY}"
|
||||
ssh "root@${NODE2_IP}" "chmod 0400 '${AUTHKEY}'"
|
||||
|
||||
log "Restarting corosync on both nodes..."
|
||||
systemctl restart corosync
|
||||
ssh "root@${NODE2_IP}" "systemctl restart corosync"
|
||||
sleep 3
|
||||
|
||||
# ── 1. Corosync quorum ────────────────────────────────────────────────────
|
||||
log "Waiting for corosync quorum..."
|
||||
for i in $(seq 1 30); do
|
||||
if corosync-quorumtool -s 2>/dev/null | grep -q 'Quorate:.*Yes'; then
|
||||
log "Quorum established"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 30 ]] && die "corosync quorum not established after 60 s"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log "Waiting for pacemaker..."
|
||||
for i in $(seq 1 30); do
|
||||
if crm_mon -1 &>/dev/null; then
|
||||
log "Pacemaker running"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 30 ]] && die "pacemaker not running after 60 s"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# ── 2. DRBD initialisation ────────────────────────────────────────────────
|
||||
log "Initialising DRBD metadata on $NODE1..."
|
||||
if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then
|
||||
drbdadm create-md ha-data --force
|
||||
fi
|
||||
|
||||
log "Initialising DRBD metadata on $NODE2..."
|
||||
ssh "root@${NODE2_IP}" "
|
||||
if ! drbdadm dstate ha-data 2>/dev/null | grep -q 'UpToDate\|Inconsistent\|Diskless'; then
|
||||
drbdadm create-md ha-data --force
|
||||
fi
|
||||
"
|
||||
|
||||
log "Bringing up DRBD on both nodes..."
|
||||
drbdadm up ha-data 2>/dev/null || true
|
||||
ssh "root@${NODE2_IP}" "drbdadm up ha-data 2>/dev/null" || true
|
||||
|
||||
log "Forcing $NODE1 to DRBD Primary for initial sync..."
|
||||
drbdadm primary ha-data --force
|
||||
|
||||
log "Waiting for DRBD to finish initial sync (this may take several minutes)..."
|
||||
for i in $(seq 1 300); do
|
||||
state=$(drbdadm dstate ha-data 2>/dev/null || echo "unknown")
|
||||
if echo "$state" | grep -q "UpToDate/UpToDate"; then
|
||||
log "DRBD sync complete: $state"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 300 ]] && warn "DRBD not UpToDate after 300 s — continuing anyway (check drbdadm status)"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# ── 3. XFS filesystem ─────────────────────────────────────────────────────
|
||||
log "Creating XFS on ${DRBD_DEVICE}..."
|
||||
if ! xfs_info "${DRBD_DEVICE}" &>/dev/null; then
|
||||
mkfs.xfs -f "${DRBD_DEVICE}"
|
||||
fi
|
||||
|
||||
log "Mounting ${DRBD_DEVICE} at ${XFS_MOUNT}..."
|
||||
mkdir -p "${XFS_MOUNT}"
|
||||
mount "${DRBD_DEVICE}" "${XFS_MOUNT}"
|
||||
|
||||
# ── 4. NFS dataset directories ────────────────────────────────────────────
|
||||
log "Creating NFS dataset directories..."
|
||||
for subdir in "${NFS_SUBDIRS[@]}"; do
|
||||
mkdir -p "${XFS_MOUNT}/${subdir}"
|
||||
done
|
||||
|
||||
# ── 5. iSCSI LUN backing file ─────────────────────────────────────────────
|
||||
log "Creating iSCSI LUN backing file ${ISCSI_LUN_FILE} (${ISCSI_LUN_SIZE})..."
|
||||
if [[ ! -f "${ISCSI_LUN_FILE}" ]]; then
|
||||
fallocate -l "${ISCSI_LUN_SIZE}" "${ISCSI_LUN_FILE}"
|
||||
fi
|
||||
|
||||
# ── 6. LIO iSCSI target ───────────────────────────────────────────────────
|
||||
log "Configuring LIO iSCSI target via targetcli..."
|
||||
targetcli <<EOF
|
||||
/backstores/fileio create name=ha-lun0 file_or_dev=${ISCSI_LUN_FILE} size=0 write_back=false
|
||||
/iscsi create ${ISCSI_IQN}
|
||||
/iscsi/${ISCSI_IQN}/tpg1/luns create /backstores/fileio/ha-lun0
|
||||
/iscsi/${ISCSI_IQN}/tpg1/portals create ${VIP}
|
||||
/iscsi/${ISCSI_IQN}/tpg1 set attribute authentication=0
|
||||
/iscsi/${ISCSI_IQN}/tpg1 set attribute demo_mode_write_protect=0
|
||||
saveconfig /etc/target/saveconfig.json
|
||||
EOF
|
||||
|
||||
log "Distributing iSCSI saveconfig to $NODE2..."
|
||||
scp -q /etc/target/saveconfig.json "root@${NODE2_IP}:/etc/target/saveconfig.json"
|
||||
|
||||
log "Unmounting ${XFS_MOUNT} — Pacemaker manages it..."
|
||||
umount "${XFS_MOUNT}"
|
||||
|
||||
log "Demoting DRBD to Secondary — Pacemaker manages primary role..."
|
||||
drbdadm secondary ha-data
|
||||
|
||||
# ── 7. Pacemaker resources ────────────────────────────────────────────────
|
||||
log "Configuring Pacemaker cluster properties..."
|
||||
crm_attribute -t crm_config -n stonith-enabled -v false
|
||||
crm_attribute -t crm_config -n no-quorum-policy -v ignore
|
||||
|
||||
log "Creating DRBD promotable clone resource..."
|
||||
cibadmin --replace --scope resources --xml-text "
|
||||
<resources>
|
||||
<clone id=\"ms-drbd0\" globally-unique=\"false\">
|
||||
<meta_attributes id=\"ms-drbd0-meta\">
|
||||
<nvpair id=\"ms-drbd0-promotable\" name=\"promotable\" value=\"true\"/>
|
||||
<nvpair id=\"ms-drbd0-master-max\" name=\"master-max\" value=\"1\"/>
|
||||
<nvpair id=\"ms-drbd0-master-node-max\" name=\"master-node-max\" value=\"1\"/>
|
||||
<nvpair id=\"ms-drbd0-clone-max\" name=\"clone-max\" value=\"2\"/>
|
||||
<nvpair id=\"ms-drbd0-clone-node-max\" name=\"clone-node-max\" value=\"1\"/>
|
||||
<nvpair id=\"ms-drbd0-notify\" name=\"notify\" value=\"true\"/>
|
||||
<nvpair id=\"ms-drbd0-interleave\" name=\"interleave\" value=\"true\"/>
|
||||
</meta_attributes>
|
||||
<primitive id=\"drbd0\" class=\"ocf\" type=\"drbd\" provider=\"linbit\">
|
||||
<instance_attributes id=\"drbd0-attrs\">
|
||||
<nvpair id=\"drbd0-resource\" name=\"drbd_resource\" value=\"ha-data\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"drbd0-start\" name=\"start\" interval=\"0\" timeout=\"240s\"/>
|
||||
<op id=\"drbd0-stop\" name=\"stop\" interval=\"0\" timeout=\"120s\"/>
|
||||
<op id=\"drbd0-promote\" name=\"promote\" interval=\"0\" timeout=\"90s\"/>
|
||||
<op id=\"drbd0-demote\" name=\"demote\" interval=\"0\" timeout=\"90s\"/>
|
||||
<op id=\"drbd0-monitor-master\" name=\"monitor\" interval=\"20s\" timeout=\"20s\" role=\"Promoted\"/>
|
||||
<op id=\"drbd0-monitor-slave\" name=\"monitor\" interval=\"30s\" timeout=\"20s\" role=\"Unpromoted\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
</clone>
|
||||
<group id=\"ha-group\">
|
||||
<primitive id=\"xfs-data\" class=\"ocf\" type=\"Filesystem\" provider=\"heartbeat\">
|
||||
<instance_attributes id=\"xfs-data-attrs\">
|
||||
<nvpair id=\"xfs-data-device\" name=\"device\" value=\"${DRBD_DEVICE}\"/>
|
||||
<nvpair id=\"xfs-data-directory\" name=\"directory\" value=\"${XFS_MOUNT}\"/>
|
||||
<nvpair id=\"xfs-data-fstype\" name=\"fstype\" value=\"xfs\"/>
|
||||
<nvpair id=\"xfs-data-options\" name=\"options\" value=\"defaults\"/>
|
||||
<nvpair id=\"xfs-data-force_unmount\" name=\"force_unmount\" value=\"false\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"xfs-data-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"xfs-data-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"xfs-data-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
<primitive id=\"iscsi-target\" class=\"systemd\" type=\"targetctl\">
|
||||
<operations>
|
||||
<op id=\"iscsi-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"iscsi-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"iscsi-monitor\" name=\"monitor\" interval=\"20s\" timeout=\"40s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
<primitive id=\"nfs-server\" class=\"systemd\" type=\"nfs-server\">
|
||||
<operations>
|
||||
<op id=\"nfs-start\" name=\"start\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"nfs-stop\" name=\"stop\" interval=\"0\" timeout=\"60s\"/>
|
||||
<op id=\"nfs-monitor\" name=\"monitor\" interval=\"30s\" timeout=\"40s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
<primitive id=\"vip\" class=\"ocf\" type=\"IPaddr2\" provider=\"heartbeat\">
|
||||
<instance_attributes id=\"vip-attrs\">
|
||||
<nvpair id=\"vip-ip\" name=\"ip\" value=\"${VIP}\"/>
|
||||
<nvpair id=\"vip-cidr\" name=\"cidr_netmask\" value=\"24\"/>
|
||||
</instance_attributes>
|
||||
<operations>
|
||||
<op id=\"vip-start\" name=\"start\" interval=\"0\" timeout=\"20s\"/>
|
||||
<op id=\"vip-stop\" name=\"stop\" interval=\"0\" timeout=\"20s\"/>
|
||||
<op id=\"vip-monitor\" name=\"monitor\" interval=\"10s\" timeout=\"20s\"/>
|
||||
</operations>
|
||||
</primitive>
|
||||
</group>
|
||||
</resources>
|
||||
"
|
||||
|
||||
log "Adding ordering and colocation constraints..."
|
||||
cibadmin --create --scope constraints --xml-text "
|
||||
<constraints>
|
||||
<rsc_order id=\"order-drbd-group\" first=\"ms-drbd0\" first-action=\"promote\" then=\"ha-group\" then-action=\"start\"/>
|
||||
<rsc_colocation id=\"coloc-group-with-drbd\" rsc=\"ha-group\" with-rsc=\"ms-drbd0\" with-rsc-role=\"Master\" score=\"INFINITY\"/>
|
||||
</constraints>
|
||||
"
|
||||
|
||||
log "Waiting for resources to start..."
|
||||
for i in $(seq 1 60); do
|
||||
if crm_resource -r vip --locate 2>/dev/null | grep -q "running on"; then
|
||||
log "VIP is up: $(crm_resource -r vip --locate)"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 60 ]] && { warn "VIP not up after 120 s — check: crm_mon -1"; break; }
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log ""
|
||||
log "═══════════════════════════════════════════════════════════════"
|
||||
log " HA cluster initialised."
|
||||
log ""
|
||||
log " crm_mon -1 — cluster status"
|
||||
log " iscsiadm -m discovery -t st -p ${VIP} — verify iSCSI target"
|
||||
log " showmount -e ${VIP} — verify NFS exports"
|
||||
log ""
|
||||
log " To enable STONITH (after deploying fence SSH key):"
|
||||
log " 1. Fill in VMID_NODE1 / VMID_NODE2 in cluster-enable-stonith.sh"
|
||||
log " 2. Copy scripts/ha/fence-pve-ssh.py to /etc/pacemaker/fence_pve_ssh"
|
||||
log " on both nodes (chmod +x)"
|
||||
log " 3. Generate and distribute the fence SSH key"
|
||||
log " (see docs or cluster-enable-stonith.sh header)"
|
||||
log " 4. bash scripts/ha/cluster-enable-stonith.sh"
|
||||
log "═══════════════════════════════════════════════════════════════"
|
||||
@@ -1,179 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
fence_pve_ssh - Proxmox VE SSH fence agent for Pacemaker.
|
||||
|
||||
Uses SSH to reach the Proxmox host and run 'qm stop/start <vmid>'.
|
||||
Deploy to /etc/pacemaker/fence_pve_ssh on both HA nodes (chmod +x).
|
||||
|
||||
Configuration (as pacemaker stonith resource attributes):
|
||||
pve_host Proxmox host to SSH to (default: pve1.sweet.home)
|
||||
pve_user SSH user (default: wayne)
|
||||
key_file SSH private key path (default: /etc/fence-pve-ssh-key)
|
||||
vmid_node1 VMID for ha-server-1
|
||||
vmid_node2 VMID for ha-server-2
|
||||
plug Node name to act on (set by pacemaker: ha-server-1 or ha-server-2)
|
||||
action Action: off|on|reboot|status|list|metadata
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
|
||||
|
||||
METADATA = """<?xml version="1.0" ?>
|
||||
<resource-agent name="fence_pve_ssh" shortdesc="Proxmox VE SSH fence agent (test lab)">
|
||||
<longdesc>Fences a VM on a Proxmox VE host by SSHing to the PVE host and
|
||||
running qm stop/start. For test use only.</longdesc>
|
||||
<vendor-url>https://proxmox.com</vendor-url>
|
||||
<parameters>
|
||||
<parameter name="action" required="1" unique="0">
|
||||
<getopt mixed="-a, --action=[action]"/>
|
||||
<content type="string" default="reboot"/>
|
||||
<shortdesc lang="en">Fencing action: off|on|reboot|status|list</shortdesc>
|
||||
</parameter>
|
||||
<parameter name="plug" required="0" unique="0">
|
||||
<getopt mixed="-n, --plug=[nodename]"/>
|
||||
<content type="string"/>
|
||||
<shortdesc lang="en">Cluster node name to fence</shortdesc>
|
||||
</parameter>
|
||||
<parameter name="pve_host" required="0" unique="0">
|
||||
<getopt mixed="--pve-host=[host]"/>
|
||||
<content type="string" default="pve1.sweet.home"/>
|
||||
<shortdesc lang="en">Proxmox VE host to SSH to</shortdesc>
|
||||
</parameter>
|
||||
<parameter name="pve_user" required="0" unique="0">
|
||||
<getopt mixed="--pve-user=[user]"/>
|
||||
<content type="string" default="wayne"/>
|
||||
<shortdesc lang="en">SSH user on the Proxmox host</shortdesc>
|
||||
</parameter>
|
||||
<parameter name="key_file" required="0" unique="0">
|
||||
<getopt mixed="--key-file=[path]"/>
|
||||
<content type="string" default="/etc/fence-pve-ssh-key"/>
|
||||
<shortdesc lang="en">SSH private key file path</shortdesc>
|
||||
</parameter>
|
||||
<parameter name="vmid_node1" required="1" unique="0">
|
||||
<getopt mixed="--vmid-node1=[vmid]"/>
|
||||
<content type="string"/>
|
||||
<shortdesc lang="en">VMID for ha-test-node1</shortdesc>
|
||||
</parameter>
|
||||
<parameter name="vmid_node2" required="1" unique="0">
|
||||
<getopt mixed="--vmid-node2=[vmid]"/>
|
||||
<content type="string"/>
|
||||
<shortdesc lang="en">VMID for ha-test-node2</shortdesc>
|
||||
</parameter>
|
||||
</parameters>
|
||||
<actions>
|
||||
<action name="off" timeout="60s"/>
|
||||
<action name="on" timeout="60s"/>
|
||||
<action name="reboot" timeout="60s"/>
|
||||
<action name="status" timeout="30s"/>
|
||||
<action name="list" timeout="10s"/>
|
||||
<action name="metadata" timeout="5s"/>
|
||||
</actions>
|
||||
</resource-agent>
|
||||
"""
|
||||
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser(add_help=False)
|
||||
p.add_argument("-a", "--action", default="reboot")
|
||||
p.add_argument("-n", "--plug")
|
||||
p.add_argument("--pve-host", default="pve1.sweet.home")
|
||||
p.add_argument("--pve-user", default="wayne")
|
||||
p.add_argument("--key-file", default="/etc/fence-pve-ssh-key")
|
||||
p.add_argument("--vmid-node1")
|
||||
p.add_argument("--vmid-node2")
|
||||
# Allow remaining unknown args (pacemaker may pass extra ones)
|
||||
return p.parse_known_args()[0]
|
||||
|
||||
|
||||
def ssh(pve_host, pve_user, key_file, cmd):
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ssh",
|
||||
"-i", key_file,
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "ConnectTimeout=10",
|
||||
f"{pve_user}@{pve_host}",
|
||||
cmd,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def get_vmid(args):
|
||||
node = args.plug
|
||||
if not node:
|
||||
print("ERROR: --plug not specified", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
mapping = {
|
||||
"ha-server-1": args.vmid_node1,
|
||||
"ha-server-2": args.vmid_node2,
|
||||
}
|
||||
vmid = mapping.get(node)
|
||||
if not vmid:
|
||||
print(f"ERROR: unknown node '{node}'", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return vmid
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
action = args.action.lower()
|
||||
|
||||
if action == "metadata":
|
||||
print(METADATA)
|
||||
sys.exit(0)
|
||||
|
||||
if action == "list":
|
||||
if args.vmid_node1:
|
||||
print("ha-server-1")
|
||||
if args.vmid_node2:
|
||||
print("ha-server-2")
|
||||
sys.exit(0)
|
||||
|
||||
vmid = get_vmid(args)
|
||||
|
||||
if not os.path.exists(args.key_file):
|
||||
print(f"ERROR: SSH key not found at {args.key_file}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if action in ("off", "reboot"):
|
||||
print(f"Stopping VM {vmid} ({args.plug}) on {args.pve_host}...")
|
||||
r = ssh(args.pve_host, args.pve_user, args.key_file,
|
||||
f"sudo /usr/sbin/qm stop {vmid}")
|
||||
if r.returncode != 0:
|
||||
print(f"ERROR stopping VM: {r.stderr}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"VM {vmid} stopped")
|
||||
|
||||
if action in ("on", "reboot"):
|
||||
print(f"Starting VM {vmid} ({args.plug}) on {args.pve_host}...")
|
||||
r = ssh(args.pve_host, args.pve_user, args.key_file,
|
||||
f"sudo /usr/sbin/qm start {vmid}")
|
||||
if r.returncode != 0:
|
||||
print(f"ERROR starting VM: {r.stderr}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"VM {vmid} started")
|
||||
|
||||
if action == "status":
|
||||
r = ssh(args.pve_host, args.pve_user, args.key_file,
|
||||
f"sudo /usr/sbin/qm status {vmid}")
|
||||
if r.returncode != 0:
|
||||
print(f"ERROR querying VM status: {r.stderr}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
# qm status returns "status: running" or "status: stopped"
|
||||
status_line = r.stdout.strip()
|
||||
print(status_line)
|
||||
if "stopped" in status_line:
|
||||
sys.exit(2) # pacemaker interprets exit 2 as "off"
|
||||
sys.exit(0) # running = exit 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,295 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Add a NixOS host to the FreeIPA domain and produce a sops-encrypted keytab
|
||||
# at secrets/<hostname>.keytab, ready for modules/ipa/client.nix.
|
||||
#
|
||||
# One command replaces three error-prone manual steps:
|
||||
# 1. ipa host-add on the domain controller
|
||||
# 2. ipa-getkeytab on the domain controller + SCP back
|
||||
# 3. sops encrypt in-place (must be at secrets/<hostname>.keytab for
|
||||
# the creation rule to match -- the common mistake that breaks sops)
|
||||
#
|
||||
# Usage:
|
||||
# scripts/ipa/create-nixos-ipa-host-account.sh [options] <hostname>
|
||||
#
|
||||
# Arguments:
|
||||
# <hostname> Short hostname, e.g. "tailscale-router". The FQDN is
|
||||
# derived as <hostname>.<HOME_DOMAIN>.
|
||||
#
|
||||
# Options:
|
||||
# --ip <addr> Register this IP with the IPA host record (optional).
|
||||
# --dc <host> SSH to this host to run IPA commands.
|
||||
# Default: $IPA_SERVER (from env.sh / environment).
|
||||
# --dc-user <u> SSH user on the domain controller. Default: wayne.
|
||||
# --dry-run Print what would be done without making any changes.
|
||||
# -h, --help Show this message.
|
||||
#
|
||||
# Prereqs:
|
||||
# 1. Run from the repo root (so .sops.yaml and secrets/ are found).
|
||||
# 2. SSH access to the domain controller as --dc-user (default: wayne)
|
||||
# with passwordless sudo (or sudo cached). IPA commands and kinit run
|
||||
# as root via sudo so the Kerberos ticket is in root's cache where all
|
||||
# ipa tools expect it. If there's no valid ticket, the script runs
|
||||
# `sudo kinit admin` interactively — you'll be prompted for the IPA
|
||||
# admin password once. The password never touches this script.
|
||||
# 3. The host's age key(s) must already be in .sops.yaml. Run
|
||||
# scripts/secrets/sync-host-keys.sh <flake-target> first so the host
|
||||
# can decrypt its own keytab on boot. This script adds the .sops.yaml
|
||||
# creation rule for secrets/<hostname>.keytab automatically, but the
|
||||
# host age key anchor (&lxc-<hostname> etc.) must already exist —
|
||||
# otherwise only the admin key can decrypt the keytab and the deployed
|
||||
# host will fail to read it.
|
||||
# 4. sops in PATH, or Nix available to run it via `nix run`.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
# shellcheck source=../env.sh
|
||||
source "${SCRIPT_DIR}/../env.sh"
|
||||
|
||||
# --- Argument parsing ---
|
||||
|
||||
DC_HOST="${IPA_SERVER}"
|
||||
DC_USER="wayne"
|
||||
IP_ADDR=""
|
||||
DRY_RUN=false
|
||||
TARGET=""
|
||||
|
||||
usage() {
|
||||
sed -n '/^# Usage:/,/^[^#]/{ /^#/{ s/^# \?//; p } }' "$0"
|
||||
exit "${1:-0}"
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--ip) IP_ADDR="$2"; shift 2 ;;
|
||||
--dc) DC_HOST="$2"; shift 2 ;;
|
||||
--dc-user) DC_USER="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help) usage 0 ;;
|
||||
-*) echo "Unknown flag: $1" >&2; usage 1 ;;
|
||||
*)
|
||||
if [[ -n "${TARGET}" ]]; then echo "Unexpected argument: $1" >&2; usage 1; fi
|
||||
TARGET="$1"; shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "${TARGET}" ]]; then
|
||||
echo "Error: hostname required." >&2
|
||||
usage 1
|
||||
fi
|
||||
|
||||
FQDN="${TARGET}.${HOME_DOMAIN}"
|
||||
KEYTAB_SECRET="${REPO_ROOT}/secrets/${TARGET}.keytab"
|
||||
# Temp path on the domain controller — use a name that won't collide.
|
||||
DC_TMP="/tmp/nixos-keytab-${TARGET}-$$.keytab"
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
log() { echo "==> $*"; }
|
||||
logn() { echo " $*"; }
|
||||
|
||||
run() {
|
||||
if $DRY_RUN; then
|
||||
echo "[dry-run] $*"
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
dc_run() {
|
||||
# Run a command string on the domain controller via SSH.
|
||||
if $DRY_RUN; then
|
||||
echo "[dry-run] ssh ${DC_USER}@${DC_HOST} $*"
|
||||
else
|
||||
ssh "${DC_USER}@${DC_HOST}" "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Locate sops ---
|
||||
|
||||
if command -v sops &>/dev/null; then
|
||||
SOPS_CMD=(sops)
|
||||
else
|
||||
log "sops not in PATH — will use 'nix run nixpkgs#sops'"
|
||||
SOPS_CMD=(nix run "nixpkgs#sops" --)
|
||||
fi
|
||||
|
||||
# --- Preflight checks ---
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
[[ -f .sops.yaml ]] || { echo "Error: .sops.yaml not found — run from repo root." >&2; exit 1; }
|
||||
[[ -d secrets ]] || { echo "Error: secrets/ not found — run from repo root." >&2; exit 1; }
|
||||
|
||||
# --- Step 1: Ensure .sops.yaml has a creation rule for this keytab ---
|
||||
#
|
||||
# sops matches creation rules against the PATH of the file being encrypted,
|
||||
# not the output path. To match secrets/<hostname>.keytab, the file must
|
||||
# already be at that path when sops -e -i is called. The creation rule must
|
||||
# also exist at that point or sops will refuse with "no matching creation
|
||||
# rules found."
|
||||
|
||||
log "Checking .sops.yaml for creation rule: secrets/${TARGET}.keytab"
|
||||
|
||||
RULE_EXISTS=false
|
||||
# Match "path_regex: secrets/<hostname>...keytab" — using .*keytab rather
|
||||
# than \.keytab because the file stores the regex verbatim (\.keytab = two
|
||||
# chars: backslash + dot), which a BRE \. (= escaped literal dot) won't span.
|
||||
if grep -q "path_regex: secrets/${TARGET}.*keytab" .sops.yaml 2>/dev/null; then
|
||||
RULE_EXISTS=true
|
||||
logn "Rule already exists — skipping addition."
|
||||
fi
|
||||
|
||||
if ! $RULE_EXISTS; then
|
||||
# Collect which platform-variant age anchors exist in .sops.yaml for this
|
||||
# hostname. The keytab is platform-agnostic (same FQDN regardless of
|
||||
# whether lxc/proxmox/linode variant is deployed), so all platform anchors
|
||||
# that have been registered get added as recipients.
|
||||
RECIPIENTS=("*admin")
|
||||
for platform in lxc proxmox linode; do
|
||||
anchor="${platform}-${TARGET}"
|
||||
if grep -q "^ - &${anchor} " .sops.yaml; then
|
||||
RECIPIENTS+=("*${anchor}")
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${#RECIPIENTS[@]} -eq 1 ]]; then
|
||||
echo "Warning: no platform age keys found for '${TARGET}' in .sops.yaml." >&2
|
||||
echo " Run scripts/secrets/sync-host-keys.sh <flake-target> first," >&2
|
||||
echo " otherwise only the admin key can decrypt the keytab and the" >&2
|
||||
echo " deployed host won't be able to read it at boot." >&2
|
||||
echo " Continuing with admin-only encryption..." >&2
|
||||
fi
|
||||
|
||||
# Build the indented recipient list for the YAML block.
|
||||
RECIPIENT_YAML=""
|
||||
for r in "${RECIPIENTS[@]}"; do
|
||||
RECIPIENT_YAML+=" - ${r}"$'\n'
|
||||
done
|
||||
RECIPIENT_YAML="${RECIPIENT_YAML%$'\n'}" # strip trailing newline
|
||||
|
||||
NEW_RULE="
|
||||
# Host keytab for ${TARGET} FreeIPA enrollment (binary sops file).
|
||||
# Generated by scripts/ipa/create-nixos-ipa-host-account.sh.
|
||||
- path_regex: secrets/${TARGET}\\.keytab\$
|
||||
key_groups:
|
||||
- age:
|
||||
${RECIPIENT_YAML}"
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo "[dry-run] Would append to .sops.yaml:"
|
||||
echo "${NEW_RULE}"
|
||||
else
|
||||
logn "Adding creation rule (recipients: ${RECIPIENTS[*]})"
|
||||
printf '%s\n' "${NEW_RULE}" >> .sops.yaml
|
||||
logn "Added."
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Step 2: Add IPA host account (idempotent) ---
|
||||
|
||||
log "Adding FreeIPA host account: ${FQDN}"
|
||||
|
||||
# Ensure there's a valid admin Kerberos ticket on the DC.
|
||||
# ipa host-add and ipa-getkeytab both need one. All IPA commands run via
|
||||
# sudo so the ticket must be in root's cache — check and refresh as root.
|
||||
# ssh -t allocates a PTY so kinit (and sudo if needed) can prompt normally;
|
||||
# no password ever touches this script or the shell history.
|
||||
if ! $DRY_RUN; then
|
||||
if ! ssh "${DC_USER}@${DC_HOST}" "sudo klist -s" &>/dev/null; then
|
||||
log "No valid Kerberos ticket on ${DC_HOST} — running sudo kinit admin"
|
||||
ssh -t "${DC_USER}@${DC_HOST}" "sudo kinit admin"
|
||||
if ! ssh "${DC_USER}@${DC_HOST}" "sudo klist -s" &>/dev/null; then
|
||||
echo "Error: kinit admin failed or produced no valid ticket." >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
logn "Kerberos ticket on ${DC_HOST} is valid."
|
||||
fi
|
||||
fi
|
||||
|
||||
IP_FLAG=""
|
||||
[[ -n "${IP_ADDR}" ]] && IP_FLAG="--ip-address=${IP_ADDR}"
|
||||
|
||||
# --force: create the host record even if DNS doesn't resolve it yet.
|
||||
if $DRY_RUN; then
|
||||
echo "[dry-run] ssh ${DC_USER}@${DC_HOST} sudo ipa host-add '${FQDN}' ${IP_FLAG} --force"
|
||||
else
|
||||
HOST_ADD_OUT=$(ssh "${DC_USER}@${DC_HOST}" "sudo ipa host-add '${FQDN}' ${IP_FLAG} --force 2>&1") \
|
||||
&& HOST_ADD_RC=0 || HOST_ADD_RC=$?
|
||||
if [[ $HOST_ADD_RC -eq 0 ]]; then
|
||||
echo "${HOST_ADD_OUT}"
|
||||
elif echo "${HOST_ADD_OUT}" | grep -q "already exists"; then
|
||||
logn "(host already registered)"
|
||||
else
|
||||
echo "Error: ipa host-add failed (exit ${HOST_ADD_RC}):" >&2
|
||||
echo "${HOST_ADD_OUT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Step 3: Fetch the keytab from the domain controller ---
|
||||
|
||||
log "Fetching keytab for host/${FQDN}"
|
||||
|
||||
# Remove the plaintext keytab if the script aborts before encryption completes.
|
||||
# The trap is cleared at the end of step 4 once sops has encrypted it in-place.
|
||||
trap 'rm -f "${KEYTAB_SECRET}"' EXIT
|
||||
|
||||
dc_run "sudo ipa-getkeytab -s '${IPA_SERVER}' -p 'host/${FQDN}' -k '${DC_TMP}'"
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo "[dry-run] Would stream ${DC_USER}@${DC_HOST}:${DC_TMP} → secrets/${TARGET}.keytab"
|
||||
else
|
||||
logn "Streaming keytab from ${DC_HOST}:${DC_TMP} → secrets/${TARGET}.keytab"
|
||||
# scp can't read a root-owned temp file as ${DC_USER}; pipe through sudo cat instead.
|
||||
ssh "${DC_USER}@${DC_HOST}" "sudo cat '${DC_TMP}'" > "${KEYTAB_SECRET}"
|
||||
|
||||
logn "Removing temp file on ${DC_HOST}"
|
||||
dc_run "sudo rm -f '${DC_TMP}'"
|
||||
fi
|
||||
|
||||
# --- Step 4: Encrypt in-place ---
|
||||
#
|
||||
# The file must already be at secrets/<hostname>.keytab (done above) so
|
||||
# sops matches the creation rule by path. Using -i (in-place) rather than
|
||||
# stdout redirect keeps the path intact through the encrypt call.
|
||||
|
||||
log "Encrypting secrets/${TARGET}.keytab in-place with sops"
|
||||
run "${SOPS_CMD[@]}" -e --input-type binary -i "${KEYTAB_SECRET}"
|
||||
|
||||
# Encryption succeeded — the file is now sops-encrypted; cancel the cleanup trap.
|
||||
trap - EXIT
|
||||
|
||||
# --- Done ---
|
||||
|
||||
if ! $DRY_RUN; then
|
||||
echo ""
|
||||
echo "Done. secrets/${TARGET}.keytab is sops-encrypted and ready."
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. Verify: grep '\"data\": \"ENC' secrets/${TARGET}.keytab"
|
||||
echo " 2. Stage and commit:"
|
||||
echo " git add secrets/${TARGET}.keytab .sops.yaml"
|
||||
echo " git commit -m 'secrets: add IPA keytab for ${TARGET}'"
|
||||
echo " 3. Add to hosts/${TARGET}/host.nix (networking block and imports):"
|
||||
echo ""
|
||||
echo " networking = {"
|
||||
echo " hostName = \"${TARGET}\";"
|
||||
echo " domain = vars.homeDomain; # required for Kerberos FQDN"
|
||||
echo " nameservers = [ vars.domainControllerIp ]; # IPA DNS"
|
||||
echo " ..."
|
||||
echo " };"
|
||||
echo ""
|
||||
echo " imports = ["
|
||||
echo " (import ../../modules/ipa/client.nix {"
|
||||
echo " keytabSopsFile = ../../secrets/${TARGET}.keytab;"
|
||||
echo " caCertFile = ../../certs/ipa-ca.crt;"
|
||||
echo " })"
|
||||
echo " ];"
|
||||
echo ""
|
||||
echo " 4. Deploy: nixos-rebuild switch (or create-proxmox-resource.sh)"
|
||||
fi
|
||||
@@ -302,12 +302,6 @@ platform_prefix="lxc"
|
||||
[[ -z "$cores" ]] && cores="$PROXMOX_DEFAULT_CORES"
|
||||
[[ -z "$memory" ]] && memory="$PROXMOX_DEFAULT_MEMORY_MB"
|
||||
|
||||
if [[ "$type" == "vm" && -n "$disk_size" ]]; then
|
||||
echo "WARNING: --disk-size is LXC-only for create mode and is ignored for VMs." >&2
|
||||
echo " VM disk size comes from proxmoxImageSize in variables.nix (currently ${disk_size}G was requested)." >&2
|
||||
echo " To expand after creation, use: --modify --vmid <n> --grow-disk <GB>" >&2
|
||||
fi
|
||||
|
||||
# --- discover / resolve the flake target from --host --------------------
|
||||
# Emits "<target>\t<hostName>" pairs for every ${platform_prefix}-* flake
|
||||
# target -- the one source both --list and the --host lookup below read
|
||||
@@ -867,17 +861,17 @@ if [[ "$type" == "lxc" ]]; then
|
||||
local_swap="${swap:-$memory}"
|
||||
# --unprivileged: read back from modules/platforms/lxc.nix's own
|
||||
# proxmoxLXC.privileged (via flake_target_lxc_privileged) rather than
|
||||
# hardcoded. lxc.nix derives this automatically: any lxc-* host whose
|
||||
# config.fileSystems has an NFS entry gets privileged=true, because the
|
||||
# kernel's NFS client (FS_USERNS_MOUNT not set) rejects NFS mounts from
|
||||
# inside any non-init user namespace -- exactly what an unprivileged
|
||||
# container's UID-mapped root lives in -- with EPERM at the VFS layer,
|
||||
# regardless of AppArmor (see lxc.nix's own comment). The NixOS config
|
||||
# bakes in cgroup/capability/mount expectations matching whichever value
|
||||
# it was built with, so this must stay in sync -- `pct create`'s CLI
|
||||
# default is privileged (unlike the web UI, which defaults the other
|
||||
# way), so leaving it unset would create a privileged container running
|
||||
# a NixOS config that assumes unprivileged, a real mismatch.
|
||||
# hardcoded, since that's no longer the same for every lxc-* target --
|
||||
# lxc-docker sets it true so the container's NFS mounts work at all (the
|
||||
# kernel's NFS client can't mount from inside any unprivileged
|
||||
# container's user namespace, no matter what AppArmor allows -- see that
|
||||
# option's own comment). The NixOS config inside the image bakes in
|
||||
# cgroup/capability/mount expectations matching whichever value it was
|
||||
# built with, so this must stay in sync with it -- `pct create`'s own
|
||||
# CLI default for this flag is privileged (unlike the web UI, which
|
||||
# defaults its checkbox the other way), so leaving it unset would create
|
||||
# a privileged container running a NixOS config that assumes
|
||||
# unprivileged for every target except lxc-docker, a real mismatch.
|
||||
privileged_eval="$(flake_target_lxc_privileged "$repo_root" "$flake_target")"
|
||||
unprivileged_flag=1
|
||||
[[ "$privileged_eval" == "true" ]] && unprivileged_flag=0
|
||||
|
||||
@@ -85,10 +85,6 @@ ensure_admin_decrypt_key() {
|
||||
fi
|
||||
|
||||
local key_file="$DEFAULT_SOPS_AGE_KEY_FILE"
|
||||
# Expand a leading ~ that survived variable substitution without tilde
|
||||
# expansion (happens when SOPS_AGE_KEY_FILE or XDG_CONFIG_HOME is set with
|
||||
# a literal ~ in the caller's environment).
|
||||
key_file="${key_file/#~\//$HOME/}"
|
||||
|
||||
if [[ -s "$key_file" ]]; then
|
||||
echo "Found existing sops age key at ${key_file}."
|
||||
@@ -97,23 +93,36 @@ ensure_admin_decrypt_key() {
|
||||
|
||||
if [[ "$dry_run" -eq 1 ]]; then
|
||||
echo "[dry-run] No sops age decryption key found (checked \$SOPS_AGE_KEY, \$SOPS_AGE_KEY_FILE, ${key_file})."
|
||||
echo "[dry-run] Continuing dry run without one -- any 'would re-encrypt' output below"
|
||||
echo "[dry-run] couldn't actually run for real until a key is present."
|
||||
echo "[dry-run] Would generate a new one here -- continuing the dry run without one; any"
|
||||
echo "[dry-run] 'would re-encrypt' output below couldn't actually run for real yet."
|
||||
return
|
||||
fi
|
||||
|
||||
cat >&2 <<EOF
|
||||
No sops age decryption key found (checked \$SOPS_AGE_KEY, \$SOPS_AGE_KEY_FILE, ${key_file}).
|
||||
echo "No sops age decryption key found (checked \$SOPS_AGE_KEY, \$SOPS_AGE_KEY_FILE, ${key_file})."
|
||||
echo "Generating a new one at ${key_file}..."
|
||||
mkdir -p "$(dirname "$key_file")"
|
||||
nix-shell "${NIX_OPTS[@]}" -p age --run "age-keygen -o '${key_file}'" 2>&1 | grep -v "^Public key:" || true
|
||||
local new_pub
|
||||
new_pub="$(age_pubkey_from_identity_file "$key_file")"
|
||||
|
||||
Place your admin age private key at ${key_file}, or set SOPS_AGE_KEY (inline
|
||||
key) or SOPS_AGE_KEY_FILE (path to a different key file) and re-run.
|
||||
cat <<EOF
|
||||
|
||||
If the key is truly missing (not just mislocated), this is a manual recovery
|
||||
situation -- generating a brand-new admin key won't help, since it cannot
|
||||
decrypt anything already encrypted for the old one. Each secrets/*.yaml is
|
||||
also encrypted for its respective host key(s), so a running deployed host can
|
||||
still decrypt what it needs -- but the admin key is required for re-encryption
|
||||
(e.g. adding new recipients via sops updatekeys).
|
||||
A brand-new age key was just generated -- it cannot decrypt anything that
|
||||
already exists in secrets/*.yaml, since nothing was ever encrypted for it.
|
||||
That trust can't be bootstrapped automatically (nobody can decrypt a file
|
||||
for a recipient that didn't exist when it was last encrypted).
|
||||
|
||||
To actually use this key:
|
||||
1. Have someone who currently CAN decrypt replace the &admin entry in
|
||||
.sops.yaml with this public key:
|
||||
${new_pub}
|
||||
2. They re-encrypt every secrets/*.yaml:
|
||||
sops updatekeys --yes secrets/common.yaml
|
||||
sops updatekeys --yes secrets/nix-cache.yaml
|
||||
sops updatekeys --yes secrets/server.yaml
|
||||
3. Re-run this script.
|
||||
|
||||
Exiting without making any other changes.
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
STUB: run cluster-init.sh to generate, then: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey
|
||||
@@ -1,6 +0,0 @@
|
||||
# STUB — not yet encrypted with sops.
|
||||
# Bootstrap:
|
||||
# bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-1
|
||||
# sops updatekeys secrets/common.yaml (allows ha-server-1 to decrypt shared secrets)
|
||||
# sops secrets/ha-server-1.yaml (create with: beszel-token)
|
||||
beszel-token: REPLACE
|
||||
@@ -1,6 +0,0 @@
|
||||
# STUB — not yet encrypted with sops.
|
||||
# Bootstrap:
|
||||
# bash scripts/secrets/sync-host-keys.sh proxmox-ha-server-2
|
||||
# sops updatekeys secrets/common.yaml (allows ha-server-2 to decrypt shared secrets)
|
||||
# sops secrets/ha-server-2.yaml (create with: beszel-token)
|
||||
beszel-token: REPLACE
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:z2bazlADvtWkGcY4an7NgZgIDSDq8KmYpSq8tcBleW/33HlElMNhdmje8m8hd7WxgrVvCQDdlAfodBLj2mjbgSwBptlPX2zbwcmJdm0g0Ope+YyQZucIio7NnBaYa+OK8meWqmxO2WEeOk+hnNfNPbt8MkzF/FMMbP8xv9wSTKP9CDdmO7s8rdg8G1JxVfn9dyaSDUTuA2JGNtbh+osd4CtNCV/bc84MjXDZTgTfT0W/uv8VH3iPNT7BhQbU9FIY4dsFRYBf758VtJ+3mTBiQkR38IevJBvhFB7Wiqn118LSBgjdTKJnBwmGo6lMdo0LRYHEgUnb2ejhWMNie19x+LEAErDki0RBmjc2BbnOF908kUlW2fUPo5fz/hlfqlhDLS1uOxESKhfnKt7s5qzWiMX/nohpknVfyMbmu9j/M571z9sgFnJAylpW3NLp53lX8N9mINwRGXsyL/ijbNE=,iv:M/MNquAGN+lIIyVVvFRgKR6DKAhLclc/OY1HV5Xwoog=,tag:eEuURMErSqsoh+l9ZaHY9Q==,type:str]",
|
||||
"sops": {
|
||||
"age": [
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqZlUxRG0zR3IzMlpDL3Vl\nNmRWQXUvc1NDek5wbWtKVEFTcTlxL3ZoWldNCmJVMkFtbDkxN1FGQ2VLTlhBUW5Q\nZ3VzL0UydUQ2YnlUMmtqa1Z0d0FSVFUKLS0tIGJFelc4Rm84T3ZHSGZDdnNUbUFM\nRXZHV3N3aVlSOXhUZk04UFRMWU5TbFUKhPU80PVYuDFUCxu1CA8+W8bqkr+Ne2fh\n+nBUPJbGxfN9TyD9tUC77AMbcL1R2L5x+SSAh0bEgSWE24/RjsuVKw==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
|
||||
},
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3M1o3S2FibHV6N2tIOFEx\nRytJRDFIUU05azNDbWlpaDdFQXFRWlBUNnlFCnBwNVB0SEVwL3NWNS9aeG4xSUI1\nV3NOcUFxUUhaODYyWXdFMEhWeFpvancKLS0tIHY4NHFQRGJwUURQbWZ2cGlSdm9s\nOFJodGZkekk3UnBRRUwxb2YrQ1ZPcU0K0tCapb1hfVHFSNpmESXexYa5k9OE9Tha\n51QpU4mZHKGrnWPK/kyj7rHiz95TLsmwUdA8Q2hOiYNSxEaVBjYDQw==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age1jcx3yajjhghn8qh8za3yeu8nxykzlg3p4nrv03vnfvzl0mzayg2qmg940e"
|
||||
},
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkTmNrbUFoVFVGcytzQkk1\nMUxDdmJGL0xDcUNZaDFRQnUwNXoyMU90SEVvCi9EaTlVOGlMeUZtQzRJS1J2TTJY\neWdkMGNFMWZzQXVKOU5KcXVvalk1ekkKLS0tIDN2MlRQcG5aQnN3bUMrSDdvc09X\nM29hQ0pJRzk1amRHaS9mRlhrb2FMZG8KEWkSSP+MqGRU75qo7ctOCL7qHhyCM3l4\nL+ga1XOKxRsQkKQPozCi5Bm+k28W9hIDaC34Rw9difxICM9Z1rAbOA==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age1ufg390ydrmma849t9xfkxxl5xvdkk6mngnlzhmy7mvuaje8sgcmsmnq6l7"
|
||||
},
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1TUtxVjJ2STFpU3RNdWpJ\nVjcwUHVORVN6L1pKVC9TeWt4cTNoR2pPWVRBCkR4QVJ1bHFUa2Z5aWw1bG0vQWVy\ncjNGRDl5REZzbzdZWTFMRkU1eHFTc1kKLS0tIHNLVDFiRkRKTHhrSEx2S0J1R3cx\nSkdEQWJya1ptTTRqUjJZT3FFaFhkd0kKgecHrnzW9+Eb+b7c0z1yR+Y0Czsr9Kjh\nx1UUOfleHntJUCs8oYcOogKknEnhBJJoJ5oe/gWruRSwle1fs3cBqQ==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age1jlltcv5jcnm40z5k0q6hv053k2rqpqvemtuecdwn527uw8uqz4es3x7m68"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2026-07-27T21:22:58Z",
|
||||
"mac": "ENC[AES256_GCM,data:VG2ygV4X6yMxGlJgq3sN4GAzgVqiD06noyMxB2yb/FCmiFVYZAH/9LqF//G7MTInfIjFrTDyoHAny+m/ztUrJkHXI1Fzg0U8R/zYyQ6wj/GguL0gyhA70uriQhvHsRHefHPa7Km61Blo9cME6CBJWejtRvh1IMw9itAioVrSIZE=,iv:zjxNG6IYxkcNqLk08NADoDiIbS6at86y136SBzadW1U=,tag:sj4ZRR7ixU8h7aOUJmTCYg==,type:str]",
|
||||
"version": "3.13.2"
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:+/1AAVha+86abQIX8tebqPJ6BIcCCiTJgPe0OfFOwA2Mcx0NJdiPtQgyeo3G8fAolIQOFkk5reL+GUhlOz4iEbPOCWwFCckdH0tXiCiVcD35qQNMRurY8HmpM5FWFnyzBkKuyNZ8okPuo2+fA3NQh3gs94rpm2kBsfS18xvrlv9b8u1JvAxlH6GRMN1uUgFGmOXlpGAVjDUFiKUHN8tRSZpsxG0aKKPBZ82JdKgYfCiCQifS1366gIFrsjGriUC+P8wkadRnD1JE2ZAGSL7wJLaRmzA4LAMGXFGbitOsFxKxX3q8VWEXaaL+9h4rTe0WCrvmgDM9NUeVHpfRiJrLvQgrFZMUxuZF66vQVsLybjIWpYYJcjC6nfR0U3lMa9gjzOTMPxm8HdGKC53FhLM1HiPoe1a2Nno283fV0uaByMPmo257O6l1CXqb6KoZOGFccm7fKR/VnFAS1AInF+szI7/r+UcaTW6LwYeV3OwLE4a8b4OiqTLgCu6v,iv:kUlVXF4Yl5HGvoLwu9loiuuDVtz0kARRiUU3K+BPL2A=,tag:pVaGzQAX9Etoqc5XMGmNeg==,type:str]",
|
||||
"sops": {
|
||||
"age": [
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6OUVUdkoxM1M5THlYZ1ky\nYWJ5RjhtRGoxTldwRkI2T1p6dHR4Y3gwR1ZrCjRDS1hTM01lZ0FSalBYQ1B0bENs\nUytXNmlrUkV4TmJkMUM1QW1ZaExIcGMKLS0tIHdHKzdEcUxvM0ZYUEp0a2d6SHp5\nYXQydy9uNG15V2FhUUd4NEFTMTNNMEUKkIzKEoYzoVs+nhnpkHFgDkQqrWykatND\ntsNxcr1SXSKeEW1m/QpXZnn/aW3zSQR09PqOHf7PYU47/AdkwrUaAg==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
|
||||
},
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByZENrck42dWJjR2hYUVBl\ncERtRjR6dTdING5nWXlBc3o3eXFhaSs5NkFRCk43QXFUeTBJR0U5dG9YTDRmQ0Yx\naEo2blUvUXNZY3dpbEZRTUx1elNzdkkKLS0tIHJwNjRNM3V6WktWZ1BlUUtLRTNI\nMTVEbCtYbllIbTdxTGozWUluS3pIRXMKZCruDIkD/JofdAHWgPuaaKTDsz408ZkY\n77mhO8J+kd03qwt6qhFC5KF1lyjhwEnqrOE195+R/8Yl7hA/DsL2ZQ==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age1k7d2du5mejsmv5rzavm4xwgpthqvcfsehduquv28nzs53zppa3kqngfxq2"
|
||||
},
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxaC95RjYybzVlU2tYdHdU\naGJ4M2IvNkJwVlNIU0VCZEFUZk5VaksrVnpFCmJXS3luTG9WejR0Rk9DbHA3dngy\nRElFWjU1N2xuc3JaKzQ0L0U2Yktib1kKLS0tIFlrK3F0ZFh5Ync3ejVWMzRKTUx2\nZDhxdEMwa3B4TFZUcWdTdktDeGt1QUEKfgYnK2lW3cZuJGaw+bAKDipLuC4S5vK2\nxK2eJB5TP/xXrp0F3lx9sc2b1FOY9Vt9IQ7zVlBqJFkzJrcw8zYJJw==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age1zhfyuzlq40reuqlr34gf77852nhs3t6mqfzrqmas8z6sxk7tcfhsungrm0"
|
||||
},
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwOGFkSXNxTUIzLzR1Skk5\nMGZveHFDN3YwVmpwT3VTYkppWEU1aGVpdkhRCjZtRC92bVphd0ZFbzFxVzRUcGk4\nMUd1aWdEbFRaM01FT25JSVRoMTRsNUUKLS0tIDAybmcrYjFVWkFYaGY5TFZjcjFO\nYlZoQjJjN2lFNzd4RlJuSHFlRlNCOFEKgUIPnL2/OJgz9oMYt86/llHa7adTkhs6\n8yGYGV1wtU9aYtUMIR907SfYyZ6M4z8jH2wwzpQLbwQQMLgkejl1jg==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age1f7usptjx9rv4rxauasve200gxtdt9jkqhhdqstlf20wvlm7u75rsjfw50m"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2026-07-27T23:14:51Z",
|
||||
"mac": "ENC[AES256_GCM,data:CYhzR0Y29GXvWUBbb13s16v9hDQ4k4Xmd4FUpcyNtHe0L/IrDGkx1WHzAohNGZjGMNB9W3BgiEH85OoOo8r1F82El4/8s2prEJi8AARvQU3+2cmRjjhNCCYQWmJkoF/cZNpw6nVyWzZdfUpPvHjbuf6utlI3rtR0qiCpSo/32S0=,iv:XLKTU2Ute4jMkfRAbXiVGxqP9+fjSs6HwRv1JfTTe7w=,tag:oIqS/1HWQZ7mLWaO7GwLFA==,type:str]",
|
||||
"version": "3.13.2"
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
beszel-token: ENC[AES256_GCM,data:qg2eb7Eq/w7vIuiiDeK/s6h9nmHWuooyrd93GvdLD2hGOvka,iv:FWRBQTIY9PzV2V8J7z5ftEb5dvjN/S0hP0H9JV+Uuq8=,tag:WP7DHOu4elfqdLoW7vo9qg==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCZmozK3VjcENGb3l2MDNj
|
||||
QVE3VVI3Z3R0MjduSlJGL3BFWkg5NHZmZms0CkNBTWZKZFJVNkJMZ0tpNXhXWW5q
|
||||
WndvbmhuOVh5ZHhJMmJwenh0M2dEVlUKLS0tIGpXQ3dhRXVWUDZzdGg4RHN4V3p0
|
||||
WDZnK1lMUHVIS09hRUEzVHRHQ0EwcWsKEd5NtstHSPH8okXcFLW7BId5xPGmgyC3
|
||||
dAodIGkLBlJlaL1bZuRy6Vaac+mJZL5KJYD/o1kYSXRiHJfNpaulTg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXWWhYOFU3VHZ3S1VuRTF2
|
||||
QlZvMkJUNDZmT1RDWWZsMkxnZjJldjdrRG44CkU0NEJTR0NhQUpjMEY4RXBLSVBJ
|
||||
bDNtZ1UrRDNUbzA0NDJ1cWJKdlBBMWMKLS0tIE9vTmEzTUxpODF4SG1MQ1czM1N5
|
||||
dVgrMERMRXJwVmhtMldTZUlHNUJVTEkKzUf9cETbo0K9bCPkrlbUkgkhRIPvjCHR
|
||||
GFVwgb2fcaOzZHcDwlLDLaP6HbRPtzJfVbLpQ1XU0N/xijSr9ENB6g==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1f7usptjx9rv4rxauasve200gxtdt9jkqhhdqstlf20wvlm7u75rsjfw50m
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlOVdWN0tHNWZmaWlWQ2hJ
|
||||
MVFYd0pMNmJTdTlJQ3h1c0dzbDE3U2VqdzNzCitCcFdhbjRPYnl3Q214WnRxa2Jq
|
||||
eTFnc2VBcXlGcTVYdWpWTFZiWXBMUDAKLS0tIC9YdVkrTU5EeXNlNzliLzBlYUR2
|
||||
Qk1ITW9tMElzUHF4MVJyWS9uRGs1QncKj8OgxWYabf8MccWD6zYLxl+FvYPAvLCW
|
||||
p/FJ8uMa9YWp0RjICvWIwyO0gK7oaxTh/JzfBjABngEZnZazNYP7Uw==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1k7d2du5mejsmv5rzavm4xwgpthqvcfsehduquv28nzs53zppa3kqngfxq2
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsbzFKckFQVDg0VEpJMFlv
|
||||
WSs1ODVoYlZZK3YrSXlMU1hIQjdLTUhNdHhJCkRnbHY5Z2NGQVMvOHV4R2hTd0xu
|
||||
LzdheXdZVVBsTjhVOU8zZGFKN0lPWjQKLS0tIFdaaG5SeElDN3YvMjFxdHVLWlNX
|
||||
M1BQQmtJTUdsK2x5TEo5TmNrMEZidkEKhi4jNKhrkBAPhJoeYNg1d60LQTqmJt+G
|
||||
Z/2X5uWzh12zF5keL8cNHH6sD0z0MOChtd4UF37yr+lb/LFJkia6LQ==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1zhfyuzlq40reuqlr34gf77852nhs3t6mqfzrqmas8z6sxk7tcfhsungrm0
|
||||
lastmodified: "2026-07-26T01:12:11Z"
|
||||
mac: ENC[AES256_GCM,data:0r68y6XmdiW7p/h9QmTR5h37TxKp+0wF44LP8QfG6cGP0tvaasuvv+V9EC3msnREYjwJhlnHjRd8hJ31HGVkOHbYWI4V5cF4ZnaiGgrZJhsGcJvOJSroAvQwev+vIlXZpAPgZz3Z35FDvOpgb71l5c0so0l4EJhc70JOh2FxhvQ=,iv:Or6l/v/E01+9wtZwyl+pCttiuReR6V0kmBzE9KhsfgQ=,tag:cAEUgj/9tKDToKFr00g5UQ==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
+5
-63
@@ -4,20 +4,8 @@
|
||||
homeDomain = "sweet.home"; # base LAN domain for service subdomains (pve., docker.)
|
||||
tailnetDomain = "tail13f623.ts.net"; # Tailscale MagicDNS suffix
|
||||
lanCidr = "192.168.2.0/24"; # LAN subnet
|
||||
lanGateway = "192.168.2.254"; # LAN default gateway (router)
|
||||
lanPrefixLength = 24; # LAN subnet prefix length (/24 = 255.255.255.0)
|
||||
lxcLanInterface = "eth0"; # LAN NIC name in LXC containers (set by Proxmox --net0 name=eth0)
|
||||
vmLanInterface = "ens18"; # LAN NIC name in Proxmox VMs (virtio, first NIC)
|
||||
vmStorageInterface = "ens19"; # storage NIC name in HA server VMs (virtio, second NIC on vmbr1)
|
||||
pxeServerIp = "192.168.2.223"; # pxe-boot LXC container LAN IP
|
||||
nixCacheIp = "192.168.2.224"; # nix-cache LXC container LAN IP
|
||||
tailscaleRouterIp = "192.168.2.222"; # tailscale-router LXC container LAN IP
|
||||
torRelayIp = "192.168.2.221"; # tor-relay LXC container LAN IP
|
||||
serverIp = "192.168.2.226"; # server (NFS/ZFS) Proxmox VM LAN IP
|
||||
dockerIp = "192.168.2.225"; # docker Proxmox VM LAN IP
|
||||
pbsIp = "192.168.2.244"; # Proxmox Backup Server LAN IP (not NixOS-managed)
|
||||
domainControllerIp = "192.168.2.253"; # FreeIPA domain controller / primary DNS (not NixOS-managed)
|
||||
ipaServer = "domain-controller.sweet.home"; # FreeIPA server hostname (used by security.ipa and Kerberos; must be a resolvable FQDN, not an IP)
|
||||
pxeServerIp = "192.168.2.247"; # pxe-boot host's LAN IP
|
||||
pbsIp = "192.168.2.108"; # Proxmox Backup Server LAN IP
|
||||
|
||||
# Cross-host references (LAN hostnames/users other hosts reach over the network)
|
||||
nixCacheHost = "nix-cache"; # substituter/remote-builder hostname
|
||||
@@ -80,24 +68,6 @@
|
||||
# one-line change.
|
||||
primaryUser = "nixos";
|
||||
|
||||
# HA file server cluster
|
||||
# LAN IPs (vmbr0 / ens18) — client-facing: iSCSI initiators, NFS, management.
|
||||
# Storage IPs (vmbr1 / ens19) — isolated internal bridge, used for DRBD
|
||||
# replication and Corosync heartbeat only; never leaves pve1.
|
||||
# haServerVip: floating virtual IP managed by Pacemaker's IPaddr2 resource;
|
||||
# NFS and iSCSI clients connect here regardless of which node is Active.
|
||||
haServer1Host = "ha-server-1";
|
||||
haServer2Host = "ha-server-2";
|
||||
haServer1Ip = "192.168.2.228"; # LAN IP, node 1
|
||||
haServer2Ip = "192.168.2.227"; # LAN IP, node 2
|
||||
haServerVip = "192.168.2.229"; # floating VIP (Pacemaker IPaddr2)
|
||||
haServer1StorageIp = "192.168.4.228"; # storage-net IP, node 1 (vmbr1 / ens19)
|
||||
haServer2StorageIp = "192.168.4.227"; # storage-net IP, node 2 (vmbr1 / ens19)
|
||||
haStorageCidr = "192.168.4.0/29"; # storage subnet — internal to pve1 only
|
||||
haStoragePrefixLength = 29; # storage subnet prefix length (/29)
|
||||
haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node
|
||||
haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage";
|
||||
|
||||
# Storage
|
||||
storageRoot = "/tank"; # ZFS pool root on `server`
|
||||
|
||||
@@ -113,7 +83,6 @@
|
||||
# dataset or moving where it's mounted only needs changing it here — the
|
||||
# export and every client reference follow automatically.
|
||||
nfsShares = {
|
||||
options = "(rw,sync,no_subtree_check,no_root_squash)";
|
||||
dockerConfig = {
|
||||
subpath = "docker/config";
|
||||
mountpoint = "/mnt/docker/config";
|
||||
@@ -134,18 +103,6 @@
|
||||
subpath = "raspi/volumes";
|
||||
mountpoint = "/mnt/raspi-backup";
|
||||
};
|
||||
proxmoxIsos = {
|
||||
subpath = "proxmox/iso";
|
||||
mountpoint = "/mnt/iso";
|
||||
};
|
||||
proxmoxLxcImages = {
|
||||
subpath = "proxmox/lxc";
|
||||
mountpoint = "/mnt/lxc";
|
||||
};
|
||||
pxebootImages = {
|
||||
subpath = "pxe-boot/images";
|
||||
mountpoint = "/mnt/pxe-images";
|
||||
};
|
||||
};
|
||||
|
||||
# The Raspberry Pi's own NFS export — not under storageRoot/nfsServerHost,
|
||||
@@ -172,25 +129,10 @@
|
||||
# (modules/build-types/pxe-boot.nix).
|
||||
pxeBootTftp = 69;
|
||||
|
||||
# `server`'s NFS exports: portmapper (rpcbind), NFS data, and the
|
||||
# mountd RPC service (used by showmount/NFSv3 mount protocol).
|
||||
# Mountd listens on a fixed port so the firewall can whitelist it
|
||||
# explicitly rather than opening all of rpcbind's dynamic range.
|
||||
# All three need both TCP and UDP (modules/build-types/server.nix and
|
||||
# modules/build-types/ha-server.nix).
|
||||
# `server`'s NFS exports need both the portmapper (rpcbind) and the
|
||||
# NFS data port itself opened (modules/build-types/server.nix).
|
||||
nfsRpcbind = 111;
|
||||
nfsd = 2049;
|
||||
nfsMountd = 20048;
|
||||
|
||||
# HA cluster ports opened on ha-server-1 and ha-server-2
|
||||
# (modules/build-types/ha-server.nix / modules/ha/cluster-config.nix).
|
||||
haServerDrbd = 7789; # DRBD replication (TCP)
|
||||
haServerIscsi = 3260; # iSCSI target (TCP)
|
||||
haServerCorosync1 = 5404; # Corosync totem ring (UDP)
|
||||
haServerCorosync2 = 5405; # Corosync totem ring (UDP)
|
||||
haServerCorosyncCrypto = 5407; # Corosync crypto sync (UDP)
|
||||
haServerPacemakerRemoted = 3121; # pacemaker-remoted (TCP)
|
||||
haServerPcsd = 2224; # pcsd cluster daemon (TCP)
|
||||
|
||||
# Opened on the docker host's firewall for the Traefik-fronted
|
||||
# container stack (docker-compose config lives in the separate
|
||||
@@ -224,7 +166,7 @@
|
||||
# build (modules/disko/proxmox.nix, config.system.build.diskoImagesScript
|
||||
# — see docs/proxmox-images.md). Root fills whatever's left after the ESP
|
||||
# and swap partitions within this total.
|
||||
proxmoxImageSize = "50G";
|
||||
proxmoxImageSize = "20G";
|
||||
|
||||
# nix-cache's Nix store garbage collection retention
|
||||
# (modules/nix-cache/server.nix).
|
||||
|
||||
Reference in New Issue
Block a user