Archived
beatzaplentyandClaude Sonnet 4.6
23634134f0
- Remove before/after-resync-target LVM handlers (snapshot-resync-target-lvm.sh doesn't exist in NixOS; exit code 127 caused DRBD to drop connections on sync) - Remove split-brain handler pointing to /usr/lib/drbd/ (Debian path) - Fix testAuthKey from 126 to 128 bytes (corosync minimum is 1024 bits) - Fix cluster-init.sh quorum check: corosync-quorumtool has no -q flag; use -s | grep 'Quorate: Yes' instead Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
262 lines
9.7 KiB
Nix
262 lines
9.7 KiB
Nix
# Shared HA stack config for both test nodes.
|
|
# These are throwaway test VMs — not production hosts.
|
|
# No sops-nix, no clan, no home-manager.
|
|
{ lib, pkgs, vars, ... }:
|
|
|
|
let
|
|
node1Ip = "192.168.2.200";
|
|
node2Ip = "192.168.2.201";
|
|
drbdPort = 7789;
|
|
|
|
# Test-only corosync authkey (128 bytes = 1024 bits minimum for corosync).
|
|
# Not secret — this is a disposable test cluster, not production.
|
|
testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
|
|
|
|
# OCF agent path (from pacemaker's --with-ocfdir).
|
|
# We expose this on PATH so custom scripts can find each other.
|
|
ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d";
|
|
|
|
# Concatenated PATH that includes all binaries OCF agents and pacemaker
|
|
# lrmd children need. This is the workaround for nixpkgs#207891 (PATH
|
|
# not set correctly for OCF agent child processes).
|
|
ocfBinPath = lib.concatStringsSep ":" [
|
|
"${pkgs.iproute2}/bin"
|
|
"${pkgs.iproute2}/sbin"
|
|
"${pkgs.iputils}/bin"
|
|
"${pkgs.util-linux}/bin"
|
|
"${pkgs.util-linux}/sbin"
|
|
"${pkgs.gawk}/bin"
|
|
"${pkgs.gnugrep}/bin"
|
|
"${pkgs.gnused}/bin"
|
|
"${pkgs.coreutils}/bin"
|
|
"${pkgs.bash}/bin"
|
|
"${pkgs.procps}/bin"
|
|
"${pkgs.xfsprogs}/bin"
|
|
"${pkgs.drbd}/bin"
|
|
"${pkgs.targetcli-fb}/bin"
|
|
"${pkgs.python3}/bin"
|
|
"/run/current-system/sw/bin"
|
|
"/run/current-system/sw/sbin"
|
|
"/usr/local/sbin"
|
|
"/usr/local/bin"
|
|
"/usr/sbin"
|
|
"/usr/bin"
|
|
"/sbin"
|
|
"/bin"
|
|
];
|
|
in
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# ── Hardware (Proxmox VM) ──────────────────────────────────────────────
|
|
imports = [
|
|
../../modules/hardware-configuration/vm/proxmox.nix
|
|
../../modules/boot/efi.nix
|
|
];
|
|
|
|
# ── Nix settings ──────────────────────────────────────────────────────
|
|
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
|
|
|
# ── SSH ───────────────────────────────────────────────────────────────
|
|
services.openssh = {
|
|
enable = true;
|
|
settings.PermitRootLogin = "yes";
|
|
};
|
|
users.users.root.openssh.authorizedKeys.keys = [
|
|
vars.adminSshKey
|
|
# Claude Code session key (this machine) — test-lab only
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
|
|
];
|
|
|
|
# Allow QEMU guest exec for key injection fallback
|
|
services.qemuGuest.enable = true;
|
|
|
|
# ── Networking ────────────────────────────────────────────────────────
|
|
networking.useDHCP = false;
|
|
networking.defaultGateway = "192.168.2.1";
|
|
networking.nameservers = [ "192.168.2.1" "8.8.8.8" ];
|
|
|
|
# ── DRBD ──────────────────────────────────────────────────────────────
|
|
services.drbd.enable = true;
|
|
services.drbd.config = ''
|
|
global {
|
|
usage-count yes;
|
|
}
|
|
|
|
common {
|
|
net {
|
|
protocol C;
|
|
ping-int 1;
|
|
verify-alg sha256;
|
|
after-sb-0pri discard-zero-changes;
|
|
after-sb-1pri discard-secondary;
|
|
}
|
|
disk {
|
|
# resource-only: DRBD itself won't fence (Pacemaker handles STONITH);
|
|
# the DRBD resource agent uses fencing to guard primary promotion.
|
|
fencing resource-only;
|
|
}
|
|
handlers {
|
|
# NOTE: LVM-specific before/after-resync-target handlers omitted
|
|
# (raw /dev/sdb, no LVM). split-brain handler also omitted —
|
|
# Pacemaker STONITH manages split-brain fencing for the test cluster.
|
|
}
|
|
}
|
|
|
|
resource ha-data {
|
|
volume 0 {
|
|
device /dev/drbd0;
|
|
disk /dev/sdb; # scsi1 in Proxmox VM → sdb
|
|
meta-disk internal;
|
|
}
|
|
|
|
on ha-test-node1 {
|
|
address ${node1Ip}:${toString drbdPort};
|
|
}
|
|
|
|
on ha-test-node2 {
|
|
address ${node2Ip}:${toString drbdPort};
|
|
}
|
|
}
|
|
'';
|
|
|
|
# ── Corosync ──────────────────────────────────────────────────────────
|
|
services.corosync = {
|
|
enable = true;
|
|
clusterName = "ha-test";
|
|
nodelist = [
|
|
{ nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; }
|
|
{ nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; }
|
|
];
|
|
};
|
|
|
|
# Corosync authkey (test-only, not secret — generated with
|
|
# `corosync-keygen` for production).
|
|
environment.etc."corosync/authkey" = {
|
|
source = builtins.toFile "authkey" testAuthKey;
|
|
mode = "0400";
|
|
};
|
|
|
|
# ── Pacemaker ─────────────────────────────────────────────────────────
|
|
services.pacemaker.enable = true;
|
|
|
|
# Fix for nixpkgs#207891:
|
|
# 1. Ensure CIB directories are owned by hacluster before starting.
|
|
# The stock module sets StateDirectory=pacemaker (owned by root);
|
|
# pacemaker internally drops to hacluster but needs to write there.
|
|
# 2. Set PATH so OCF agent child processes can find all required binaries.
|
|
systemd.services.pacemaker.serviceConfig = {
|
|
ExecStartPre = [
|
|
"${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'"
|
|
];
|
|
};
|
|
systemd.services.pacemaker.environment = {
|
|
PATH = lib.mkForce ocfBinPath;
|
|
# Expose OCF root so pacemaker and lrmd agree on where agents live.
|
|
OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
|
|
};
|
|
|
|
# pacemaker-execd is the local resource executor that calls OCF agents.
|
|
# Give it the same PATH so OCF scripts can find all required binaries.
|
|
systemd.services.pacemaker-execd.environment = {
|
|
PATH = lib.mkForce ocfBinPath;
|
|
OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
|
|
};
|
|
|
|
# ── LIO / iSCSI target ────────────────────────────────────────────────
|
|
# targetcli-fb is the management tool; actual kernel support is via
|
|
# the LIO modules. We add a systemd service that saves/restores the
|
|
# target configuration so Pacemaker can trigger it via a systemd-class
|
|
# resource.
|
|
boot.kernelModules = [
|
|
"target_core_mod"
|
|
"iscsi_target_mod"
|
|
"target_core_file"
|
|
"target_core_pscsi"
|
|
"target_core_user"
|
|
"configfs"
|
|
];
|
|
|
|
# configfs must be mounted for rtslib/targetcli to work
|
|
systemd.mounts = [{
|
|
where = "/sys/kernel/config";
|
|
what = "configfs";
|
|
type = "configfs";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "targetctl.service" ];
|
|
}];
|
|
|
|
# targetctl: save/restore LIO configuration (mirrors Debian's package)
|
|
systemd.services.targetctl = {
|
|
description = "LIO iSCSI target config save/restore";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "sys-kernel-config.mount" "network.target" ];
|
|
requires = [ "sys-kernel-config.mount" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json";
|
|
ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json";
|
|
};
|
|
unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
|
|
};
|
|
|
|
# ── Packages ──────────────────────────────────────────────────────────
|
|
environment.systemPackages = with pkgs; [
|
|
# HA stack
|
|
corosync # corosync-cfgtool, corosync-quorumtool
|
|
pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI
|
|
drbd # drbdadm, drbdsetup, drbdmon
|
|
ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …)
|
|
|
|
# Storage
|
|
xfsprogs # mkfs.xfs, xfs_admin, xfs_info
|
|
targetcli-fb # targetcli shell + targetctl
|
|
|
|
# Networking / debug
|
|
iproute2 # ip, ss
|
|
iputils # ping
|
|
tcpdump
|
|
lsof
|
|
|
|
# Scripting / config
|
|
python3
|
|
curl
|
|
jq
|
|
vim
|
|
htop
|
|
];
|
|
|
|
# ── Firewall ──────────────────────────────────────────────────────────
|
|
networking.firewall = {
|
|
enable = true;
|
|
allowedTCPPorts = [
|
|
22 # SSH
|
|
3260 # iSCSI
|
|
3121 # pacemaker-remoted
|
|
2224 # pcsd
|
|
drbdPort # DRBD replication
|
|
];
|
|
allowedUDPPorts = [
|
|
5404 # corosync cluster
|
|
5405 # corosync cluster
|
|
5407 # corosync crypto
|
|
];
|
|
# Corosync uses ports 5404-5407 UDP; allow them on the cluster net
|
|
extraCommands = ''
|
|
iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT
|
|
iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT
|
|
'';
|
|
};
|
|
|
|
# ── tmpfiles: target config dir ────────────────────────────────────────
|
|
systemd.tmpfiles.rules = [
|
|
"d /etc/target 0750 root root -"
|
|
"f /etc/target/saveconfig.json 0640 root root -"
|
|
];
|
|
|
|
# ── Locale / time ─────────────────────────────────────────────────────
|
|
time.timeZone = vars.timeZone;
|
|
i18n.defaultLocale = "en_AU.UTF-8";
|
|
}
|