diff --git a/docs/ip-addressing.md b/docs/ip-addressing.md index f719995..0c85d61 100644 --- a/docs/ip-addressing.md +++ b/docs/ip-addressing.md @@ -2,17 +2,22 @@ ## Subnets -| Subnet | CIDR | Purpose | Routed? | -|---|---|---|---| -| LAN | `192.168.2.0/24` | General LAN — clients and infrastructure | Yes (gateway .254) | -| Cluster | `192.168.4.224/29` | HA file server DRBD replication + Corosync heartbeat | No — internal `vmbr1` only, no uplink | -| Storage client | `192.168.5.0/24` | HA file server iSCSI + NFS — docker and server access VIP here | No — internal `vmbr2` only, no uplink | +| Subnet | VLAN | CIDR | Purpose | Routed? | +|---|---|---|---|---| +| LAN | 2 (native/untagged) | `192.168.2.0/24` | General LAN — clients and infrastructure | Yes (gateway .254) | +| Cluster | 10 | `192.168.10.224/29` | HA file server DRBD replication + Corosync heartbeat | No — internal `vmbr1` only, no uplink | +| Storage client | 20 | `192.168.20.0/24` | HA file server iSCSI — docker and server access VIP here | No — internal `vmbr2` only, no uplink | The cluster and storage-client subnets never leave pve1. `vmbr1` and `vmbr2` are Proxmox Linux bridges with no physical port attached; traffic between guests on each bridge stays in-kernel. -The host octet is consistent across all subnets — e.g. ha-node1 is always `.228`: -`192.168.2.228` (LAN), `192.168.4.228` (cluster), `192.168.5.228` (storage client). +VLAN IDs match the third octet of each subnet (VLAN 2 → 192.168.**2**.x, VLAN 10 → 192.168.**10**.x, +VLAN 20 → 192.168.**20**.x). The host octet is consistent across all subnets — e.g. ha-node1 +is always `.228`: `192.168.2.228` (LAN), `192.168.10.228` (cluster), `192.168.20.228` (storage client). + +**Protocol separation** (enforced by firewall on HA nodes): +- iSCSI (port 3260): storage-client subnet (VLAN 20) only — docker and server +- NFS (ports 111, 2049, 20048): LAN subnet (VLAN 2) only — pxe-boot and other LAN clients --- @@ -79,14 +84,14 @@ MAC address churn. --- -## Cluster network — 192.168.4.224/29 +## Cluster network — VLAN 10 — 192.168.10.224/29 Internal to pve1 only. Proxmox bridge `vmbr1`, no physical NIC attached. | IP | Hostname | Interface role | |---|---|---| -| `192.168.4.228` | ha-node1 | DRBD replication + Corosync ring0 (primary heartbeat) | -| `192.168.4.227` | ha-node2 | DRBD replication + Corosync ring0 (primary heartbeat) | +| `192.168.10.228` | ha-node1 | DRBD replication + Corosync ring0 (primary heartbeat) | +| `192.168.10.227` | ha-node2 | DRBD replication + Corosync ring0 (primary heartbeat) | | — | no gateway | Isolated — not routed to LAN or internet | Corosync ring1 (backup heartbeat only) uses the LAN IPs (`192.168.2.228` / `192.168.2.227`) @@ -94,21 +99,22 @@ over `vmbr0` — no additional bridge needed, and DRBD traffic never crosses rin --- -## Storage-client network — 192.168.5.0/24 +## Storage-client network — VLAN 20 — 192.168.20.0/24 Internal to pve1 only. Proxmox bridge `vmbr2`, no physical NIC attached. | IP | Hostname | Interface / role | |---|---|---| -| `192.168.5.229` | ha-vip | Pacemaker floating VIP — iSCSI portal + NFS endpoint | -| `192.168.5.228` | ha-node1 | Storage-client NIC (ens20 / vmbr2) | -| `192.168.5.227` | ha-node2 | Storage-client NIC (ens20 / vmbr2) | -| `192.168.5.226` | server | Storage-client NIC (ens19 / vmbr2) — for final rsync before cutover | -| `192.168.5.225` | docker | Storage-client NIC (eth1 / vmbr2) — iSCSI initiator + NFS client | +| `192.168.20.229` | ha-vip-storage | Pacemaker floating VIP — iSCSI endpoint (port 3260) | +| `192.168.20.228` | ha-node1 | Storage-client NIC (ens20 / vmbr2) | +| `192.168.20.227` | ha-node2 | Storage-client NIC (ens20 / vmbr2) | +| `192.168.20.226` | server | Storage-client NIC (ens19 / vmbr2) — temporary, for final rsync before decommission | +| `192.168.20.225` | docker | Storage-client NIC (eth1 / vmbr2) — iSCSI initiator | | — | no gateway | Isolated — not routed to LAN or internet | -iSCSI initiators and NFS clients connect exclusively to the VIP (`192.168.5.229`) so sessions -survive failover transparently. The portal is bound to the VIP only (not `[::0]`). +iSCSI initiators connect to the VIP (`192.168.20.229:3260`) so sessions survive failover +transparently. Portal listens on `[::0]:3260`; firewall restricts port 3260 to +`192.168.20.0/24` source only — LAN clients cannot reach iSCSI. --- @@ -123,10 +129,10 @@ Current → target IP for every host being renumbered. | pve1 | `192.168.2.250` | `192.168.2.245` | `/etc/network/interfaces` on Proxmox host | | pbs | `192.168.2.108` | `192.168.2.244` | static config on PBS host | | nixos workstation | `192.168.2.119` | `192.168.2.243` | `networking.interfaces` / NetworkManager on guest | -| ha-node1 | — | `192.168.2.228` (LAN), `192.168.4.228` (cluster), `192.168.5.228` (storage) | active | -| ha-node2 | — | `192.168.2.227` (LAN), `192.168.4.227` (cluster), `192.168.5.227` (storage) | active | -| ha-vip-lan | — | `192.168.2.229` (vmbr0 / Pacemaker `vip-lan`) — LAN iSCSI + NFS | active | -| ha-vip-storage | — | `192.168.5.229` (vmbr2 / Pacemaker `vip-storage`) — storage-client iSCSI + NFS | active | +| ha-node1 | — | `192.168.2.228` (LAN), `192.168.10.228` (cluster/VLAN 10), `192.168.20.228` (storage/VLAN 20) | active | +| ha-node2 | — | `192.168.2.227` (LAN), `192.168.10.227` (cluster/VLAN 10), `192.168.20.227` (storage/VLAN 20) | active | +| ha-vip-lan | — | `192.168.2.229` (vmbr0 / Pacemaker `vip-lan`) — NFS endpoint for LAN clients | active | +| ha-vip-storage | — | `192.168.20.229` (vmbr2 / Pacemaker `vip-storage`) — iSCSI endpoint for VLAN 20 clients | active | | server | `192.168.2.252` | `192.168.2.226` | static config on guest | | docker | `192.168.2.249` | `192.168.2.225` | static config on guest | | nix-cache | `192.168.2.120` | `192.168.2.224` | static config on guest | diff --git a/modules/build-types/ha-server.nix b/modules/build-types/ha-server.nix index 3d54111..5af6c51 100644 --- a/modules/build-types/ha-server.nix +++ b/modules/build-types/ha-server.nix @@ -17,14 +17,12 @@ { lib, pkgs, vars, ... }: let - # Generates /etc/exports lines for all nfsShares data entries. Each share - # gets two export lines — one for LAN clients (pxe-boot, proxmox host) and - # one for storage-client-subnet clients (docker, server via vmbr2). + # Generates /etc/exports lines for all nfsShares data entries. + # NFS is LAN-only (VLAN 2). Storage-client subnet (VLAN 20) uses iSCSI only. mkNfsExports = storageRoot: lib.concatMapStrings (share: - " ${storageRoot}/${share.subpath} ${vars.lanCidr}${vars.nfsShares.options}\n" + - " ${storageRoot}/${share.subpath} ${vars.haClientCidr}${vars.nfsShares.options}\n") + " ${storageRoot}/${share.subpath} ${vars.lanCidr}${vars.nfsShares.options}\n") (lib.filter builtins.isAttrs (lib.attrValues vars.nfsShares)); in { diff --git a/modules/ha/cluster-config.nix b/modules/ha/cluster-config.nix index dfc3be4..e7a6003 100644 --- a/modules/ha/cluster-config.nix +++ b/modules/ha/cluster-config.nix @@ -131,27 +131,31 @@ in networking.firewall = { allowedTCPPorts = [ - vars.ports.haServerIscsi vars.ports.haServerPacemakerRemoted vars.ports.haServerPcsd vars.ports.haServerDrbd - vars.ports.nfsRpcbind - vars.ports.nfsd - vars.ports.nfsMountd ]; allowedUDPPorts = [ vars.ports.haServerCorosync1 vars.ports.haServerCorosync2 vars.ports.haServerCorosyncCrypto - vars.ports.nfsRpcbind - vars.ports.nfsd - vars.ports.nfsMountd ]; + # Protocol separation: iSCSI (VLAN 20 / storage clients only), + # NFS (VLAN 2 / LAN only). Cluster-internal subnets accepted wholesale + # since they are isolated bridges with no external uplink. extraCommands = '' - iptables -A INPUT -s ${vars.haServer1Ip}/32 -j ACCEPT - iptables -A INPUT -s ${vars.haServer2Ip}/32 -j ACCEPT - iptables -A INPUT -s ${vars.haStorageCidr} -j ACCEPT - iptables -A INPUT -s ${vars.haClientCidr} -j ACCEPT + iptables -A nixos-fw -s ${vars.haServer1Ip}/32 -j nixos-fw-accept + iptables -A nixos-fw -s ${vars.haServer2Ip}/32 -j nixos-fw-accept + iptables -A nixos-fw -s ${vars.haStorageCidr} -j nixos-fw-accept + + iptables -A nixos-fw -p tcp -s ${vars.haClientCidr} --dport ${toString vars.ports.haServerIscsi} -j nixos-fw-accept + + iptables -A nixos-fw -p tcp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsRpcbind} -j nixos-fw-accept + iptables -A nixos-fw -p udp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsRpcbind} -j nixos-fw-accept + iptables -A nixos-fw -p tcp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsd} -j nixos-fw-accept + iptables -A nixos-fw -p udp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsd} -j nixos-fw-accept + iptables -A nixos-fw -p tcp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsMountd} -j nixos-fw-accept + iptables -A nixos-fw -p udp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsMountd} -j nixos-fw-accept ''; }; } diff --git a/scripts/ha/acceptance-tests.sh b/scripts/ha/acceptance-tests.sh index f4ba646..860b9fd 100755 --- a/scripts/ha/acceptance-tests.sh +++ b/scripts/ha/acceptance-tests.sh @@ -12,7 +12,7 @@ NODE1="${NODE1:-ha-server-1}" NODE2="${NODE2:-ha-server-2}" NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip -VIP="${VIP:-192.168.5.229}" # vars.haServerVip +VIP="${VIP:-192.168.20.229}" # vars.haServerVip XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}" # vars.haIscsiIqn # ────────────────────────────────────────────────────────────────────────── diff --git a/scripts/ha/cluster-init.sh b/scripts/ha/cluster-init.sh index 657aad7..4d91e71 100755 --- a/scripts/ha/cluster-init.sh +++ b/scripts/ha/cluster-init.sh @@ -26,7 +26,7 @@ NODE1="${NODE1:-ha-server-1}" NODE2="${NODE2:-ha-server-2}" NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip -VIP="${VIP:-192.168.5.229}" # vars.haServerVip (storage-client, vmbr2) +VIP="${VIP:-192.168.20.229}" # vars.haServerVip (storage-client, vmbr2, VLAN 20) VIP_LAN="${VIP_LAN:-192.168.2.229}" # vars.haServerLanVip (LAN, vmbr0) XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}" # vars.haIscsiIqn @@ -419,7 +419,7 @@ cibadmin --replace --scope resources --xml-text ' - + diff --git a/scripts/ha/deploy.sh b/scripts/ha/deploy.sh index 58821db..b5aa029 100755 --- a/scripts/ha/deploy.sh +++ b/scripts/ha/deploy.sh @@ -84,9 +84,9 @@ NODE1_HOST="ha-server-1" NODE2_HOST="ha-server-2" NODE1_IP="192.168.2.228" NODE2_IP="192.168.2.227" -STORAGE_IP1="192.168.4.228" -STORAGE_IP2="192.168.4.227" -STORAGE_CIDR="192.168.4.0/29" +STORAGE_IP1="192.168.10.228" +STORAGE_IP2="192.168.10.227" +STORAGE_CIDR="192.168.10.224/29" SSH_USER="${PROXMOX_SSH_USER:-wayne}" # ── Argument parsing ────────────────────────────────────────────────────────── @@ -452,7 +452,7 @@ if ! $SKIP_CLUSTER_INIT; then ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${NODE1_IP}" \ "sudo env NODE1=${NODE1_HOST} NODE2=${NODE2_HOST} \ NODE1_IP=${NODE1_IP} NODE2_IP=${NODE2_IP} \ - VIP=192.168.5.229 XFS_MOUNT=/srv/ha-data \ + VIP=192.168.20.229 XFS_MOUNT=/srv/ha-data \ ISCSI_IQN=iqn.2026-01.home.sweet:ha-storage \ VMID_NODE1=${VMID1} VMID_NODE2=${VMID2} \ HA_USER=${HA_USER} HA_KEY=/root/.ssh/cluster-init-key \ @@ -491,7 +491,7 @@ if ! $SKIP_TESTS; then else NODE1="$NODE1_HOST" NODE2="$NODE2_HOST" \ NODE1_IP="$NODE1_IP" NODE2_IP="$NODE2_IP" \ - VIP="192.168.5.229" \ + VIP="192.168.20.229" \ bash "${REPO_ROOT}/scripts/ha/acceptance-tests.sh" fi fi diff --git a/scripts/ha/failover.sh b/scripts/ha/failover.sh index 24e97cd..2caded7 100755 --- a/scripts/ha/failover.sh +++ b/scripts/ha/failover.sh @@ -19,7 +19,7 @@ NODE1="${NODE1:-ha-server-1}" NODE2="${NODE2:-ha-server-2}" NODE1_IP="${NODE1_IP:-192.168.2.228}" NODE2_IP="${NODE2_IP:-192.168.2.227}" -VIP="${VIP:-192.168.5.229}" +VIP="${VIP:-192.168.20.229}" XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" HA_USER="${HA_USER:-nixos}" # ────────────────────────────────────────────────────────────────────────── diff --git a/scripts/ha/health.sh b/scripts/ha/health.sh index 7366193..f245721 100755 --- a/scripts/ha/health.sh +++ b/scripts/ha/health.sh @@ -11,7 +11,7 @@ NODE1="${NODE1:-ha-server-1}" NODE2="${NODE2:-ha-server-2}" NODE1_IP="${NODE1_IP:-192.168.2.228}" # vars.haServer1Ip NODE2_IP="${NODE2_IP:-192.168.2.227}" # vars.haServer2Ip -VIP="${VIP:-192.168.5.229}" # vars.haServerVip +VIP="${VIP:-192.168.20.229}" # vars.haServerVip XFS_MOUNT="${XFS_MOUNT:-/srv/ha-data}" # vars.haStorageRoot HA_USER="${HA_USER:-nixos}" # ────────────────────────────────────────────────────────────────────────── diff --git a/variables.nix b/variables.nix index 68c1248..1a4f328 100644 --- a/variables.nix +++ b/variables.nix @@ -103,27 +103,29 @@ # HA file server cluster # LAN IPs (vmbr0 / ens18) — management only after storage migration. - # Cluster IPs (vmbr1 / ens19) — isolated internal bridge, DRBD replication - # and Corosync heartbeat only; never leaves pve1. - # Storage-client IPs (vmbr2 / ens20) — isolated internal bridge for iSCSI - # and NFS; docker and server VMs connect here instead of crossing vmbr0. + # Cluster IPs (vmbr1 / ens19) — VLAN 10 (192.168.10.x), isolated internal bridge, + # DRBD replication and Corosync heartbeat only; never leaves pve1. + # Storage-client IPs (vmbr2 / ens20) — VLAN 20 (192.168.20.x), isolated internal + # bridge for iSCSI; docker and server VMs connect here instead of crossing vmbr0. # haServerVip: floating virtual IP on vmbr2, managed by Pacemaker IPaddr2; - # all iSCSI and NFS clients connect here regardless of which node is Active. + # iSCSI clients connect here regardless of which node is Active. + # Protocol separation: iSCSI on storage-client subnet (VLAN 20) only; + # NFS on LAN subnet (VLAN 2) only. Enforced by firewall on the HA nodes. haServer1Host = "ha-server-1"; haServer2Host = "ha-server-2"; haServer1Ip = "192.168.2.228"; # LAN IP, node 1 (vmbr0 / ens18) haServer2Ip = "192.168.2.227"; # LAN IP, node 2 (vmbr0 / ens18) - haServer1StorageIp = "192.168.4.228"; # cluster-net IP, node 1 (vmbr1 / ens19) - haServer2StorageIp = "192.168.4.227"; # cluster-net IP, node 2 (vmbr1 / ens19) - haStorageCidr = "192.168.4.224/29"; # cluster subnet — internal to pve1 only + haServer1StorageIp = "192.168.10.228"; # cluster-net IP, node 1 (vmbr1 / ens19, VLAN 10) + haServer2StorageIp = "192.168.10.227"; # cluster-net IP, node 2 (vmbr1 / ens19, VLAN 10) + haStorageCidr = "192.168.10.224/29"; # cluster subnet — VLAN 10, internal to pve1 only haStoragePrefixLength = 29; # cluster subnet prefix length (/29) - haServer1ClientIp = "192.168.5.228"; # storage-client IP, node 1 (vmbr2 / ens20) - haServer2ClientIp = "192.168.5.227"; # storage-client IP, node 2 (vmbr2 / ens20) - haServerVip = "192.168.5.229"; # storage-client floating VIP on vmbr2 (Pacemaker IPaddr2 vip-storage) - haServerLanVip = "192.168.2.229"; # LAN floating VIP on vmbr0 (Pacemaker IPaddr2 vip-lan) — LAN iSCSI/NFS access - dockerStorageIp = "192.168.5.225"; # docker CT storage-client IP (vmbr2 / eth1) - serverStorageIp = "192.168.5.226"; # server VM storage-client IP (vmbr2 / ens19) - haClientCidr = "192.168.5.0/24"; # storage-client subnet — internal to pve1 only + haServer1ClientIp = "192.168.20.228"; # storage-client IP, node 1 (vmbr2 / ens20, VLAN 20) + haServer2ClientIp = "192.168.20.227"; # storage-client IP, node 2 (vmbr2 / ens20, VLAN 20) + haServerVip = "192.168.20.229"; # storage-client floating VIP on vmbr2 (Pacemaker IPaddr2 vip-storage, VLAN 20) + haServerLanVip = "192.168.2.229"; # LAN floating VIP on vmbr0 (Pacemaker IPaddr2 vip-lan) — NFS access + dockerStorageIp = "192.168.20.225"; # docker CT storage-client IP (vmbr2 / eth1, VLAN 20) + serverStorageIp = "192.168.20.226"; # server VM storage-client IP (vmbr2 / ens19, VLAN 20) — temporary, for final rsync + haClientCidr = "192.168.20.0/24"; # storage-client subnet — VLAN 20, internal to pve1 only haClientPrefixLength = 24; # storage-client subnet prefix length (/24) haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage";