Archived
feat(ha): promote HA file server to production flake targets
Adds proxmox-ha-server-1 and proxmox-ha-server-2 as real mkTarget entries
alongside the existing proxmox-server, backed by a new ha-server build type.
New modules
modules/ha/cluster-config.nix — DRBD resource + corosync nodelist sourced
from vars (haServer1Host/Ip, haServer2Host/Ip); resource-only fencing for
production STONITH; HA port firewall rules for DRBD, iSCSI, Corosync, pcsd
modules/build-types/ha-server.nix — imports pacemaker-stack + iscsi-target
+ cluster-config + beszel; NFS exports from vars.haStorageRoot (XFS-over-DRBD
mount); nfs-server.service.wantedBy force-cleared so Pacemaker controls
start/stop on the Active node only
New hosts
hosts/ha-server-{1,2}/host.nix — static IP from vars, unique hostId; sops
secrets (beszel, corosync authkey) are TODOs pending sync-host-keys.sh
variables.nix
haServer1/2Host, haServer1/2Ip, haServerVip, haStorageRoot, haIscsiIqn
ports.haServerDrbd/Iscsi/Corosync{1,2,Crypto}/PacemakerRemoted/Pcsd
scripts/ha/ (migrated + updated from test-lab/ha/)
cluster-init.sh — generates corosync authkey, initialises DRBD/XFS/iSCSI,
creates NFS dataset dirs, configures Pacemaker with DRBD + XFS + iSCSI
+ nfs-server + VIP; STONITH disabled initially (enable separately)
cluster-enable-stonith.sh — enables fence_pve_ssh STONITH after key deploy
fence-pve-ssh.py — Proxmox SSH fence agent (node names updated to ha-server-1/2)
acceptance-tests.sh — T1–T7 production acceptance tests
test-lab/ha/ removed — all Nix config moved to modules/ha/ and
modules/build-types/; scripts moved to scripts/ha/
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
This commit is contained in:
Claude Sonnet 4.6
@@ -0,0 +1,43 @@
|
||||
# HA file server build type: DRBD + XFS + LIO iSCSI + NFS, managed by
|
||||
# Corosync + Pacemaker. Both ha-server-1 and ha-server-2 use this type.
|
||||
#
|
||||
# NFS start/stop:
|
||||
# services.nfs.server.enable = true configures /etc/exports, wires up
|
||||
# rpcbind, and loads kernel modules — but nfs-server.service.wantedBy is
|
||||
# force-cleared so systemd does NOT auto-start it at boot. Pacemaker's
|
||||
# ha-group resource group (configured by scripts/ha/cluster-init.sh)
|
||||
# starts and stops nfs-server as part of the failover sequence after the
|
||||
# XFS mount and iSCSI target are brought up on the new Active node.
|
||||
#
|
||||
# Beszel agent:
|
||||
# Enabled here via enable-agent.nix. The agent KEY (used to pair with
|
||||
# the Beszel hub) is not set yet — add it to hosts/ha-server-{1,2}/host.nix
|
||||
# under services.beszel.agent.environment.KEY once the hub accepts the
|
||||
# new agents, following the pattern in hosts/server/host.nix.
|
||||
{ lib, vars, ... }:
|
||||
{
|
||||
imports = [
|
||||
../ha/pacemaker-stack.nix
|
||||
../ha/iscsi-target.nix
|
||||
../ha/cluster-config.nix
|
||||
../beszel/enable-agent.nix
|
||||
];
|
||||
|
||||
services.nfs.server = {
|
||||
enable = true;
|
||||
exports = ''
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerConfig.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.dockerDatabases.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.nextcloudData.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.raspiVolumes.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.proxmoxIsos.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.proxmoxLxcImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
${vars.haStorageRoot}/${vars.nfsShares.pxebootImages.subpath} ${vars.lanCidr}${vars.nfsShares.options}
|
||||
'';
|
||||
};
|
||||
|
||||
# Pacemaker controls nfs-server — prevent systemd from starting it at boot
|
||||
# on both nodes (only the Active node should be serving NFS).
|
||||
systemd.services.nfs-server.wantedBy = lib.mkForce [ ];
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
# Cluster-wide HA config shared by both ha-server nodes.
|
||||
#
|
||||
# Covers everything that is identical on both nodes and references cluster
|
||||
# topology (node IPs, hostnames, DRBD resource). Per-node identity
|
||||
# (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix.
|
||||
#
|
||||
# Corosync authkey:
|
||||
# /etc/corosync/authkey must be present (mode 0400) for corosync to start.
|
||||
# It is NOT managed declaratively here — the initial deploy uses
|
||||
# scripts/ha/cluster-init.sh to generate it via corosync-keygen and
|
||||
# distribute it to both nodes.
|
||||
# TODO: once both hosts have their sops keys registered via
|
||||
# scripts/secrets/sync-host-keys.sh, add a sops secret here so the
|
||||
# authkey survives nixos-rebuild.
|
||||
#
|
||||
# DRBD fencing:
|
||||
# Production setting is resource-only: DRBD waits for the STONITH fence
|
||||
# agent to confirm the peer is dead before promoting to Primary. This
|
||||
# requires a working fence_pve_ssh STONITH resource in Pacemaker
|
||||
# (see scripts/ha/cluster-enable-stonith.sh). On a fresh cluster with
|
||||
# no fence device yet, temporarily change to dont-care and run
|
||||
# cluster-enable-stonith.sh once the fence key is deployed.
|
||||
{ vars, ... }:
|
||||
{
|
||||
services.drbd = {
|
||||
enable = true;
|
||||
config = ''
|
||||
global {
|
||||
usage-count yes;
|
||||
}
|
||||
|
||||
common {
|
||||
net {
|
||||
protocol C;
|
||||
ping-int 1;
|
||||
verify-alg sha256;
|
||||
after-sb-0pri discard-zero-changes;
|
||||
after-sb-1pri discard-secondary;
|
||||
}
|
||||
disk {
|
||||
fencing resource-only;
|
||||
}
|
||||
}
|
||||
|
||||
resource ha-data {
|
||||
volume 0 {
|
||||
device /dev/drbd0;
|
||||
disk /dev/sdb;
|
||||
meta-disk internal;
|
||||
}
|
||||
|
||||
on ${vars.haServer1Host} {
|
||||
address ${vars.haServer1Ip}:${toString vars.ports.haServerDrbd};
|
||||
}
|
||||
|
||||
on ${vars.haServer2Host} {
|
||||
address ${vars.haServer2Ip}:${toString vars.ports.haServerDrbd};
|
||||
}
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
# services.corosync.enable is set by modules/ha/pacemaker-stack.nix.
|
||||
services.corosync = {
|
||||
clusterName = "ha-cluster";
|
||||
nodelist = [
|
||||
{ nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1Ip ]; }
|
||||
{ nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2Ip ]; }
|
||||
];
|
||||
};
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [
|
||||
vars.ports.haServerIscsi
|
||||
vars.ports.haServerPacemakerRemoted
|
||||
vars.ports.haServerPcsd
|
||||
vars.ports.haServerDrbd
|
||||
vars.ports.nfsRpcbind
|
||||
vars.ports.nfsd
|
||||
vars.ports.nfsMountd
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
vars.ports.haServerCorosync1
|
||||
vars.ports.haServerCorosync2
|
||||
vars.ports.haServerCorosyncCrypto
|
||||
vars.ports.nfsRpcbind
|
||||
vars.ports.nfsd
|
||||
vars.ports.nfsMountd
|
||||
];
|
||||
extraCommands = ''
|
||||
iptables -A INPUT -s ${vars.haServer1Ip}/32 -j ACCEPT
|
||||
iptables -A INPUT -s ${vars.haServer2Ip}/32 -j ACCEPT
|
||||
'';
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user