# Shared HA stack config for both test nodes. # These are throwaway test VMs — not production hosts. # No sops-nix, no clan, no home-manager. { lib, pkgs, vars, ... }: let node1Ip = "192.168.2.200"; node2Ip = "192.168.2.201"; drbdPort = 7789; # Test-only corosync authkey (128 bytes = 1024 bits minimum for corosync). # Not secret — this is a disposable test cluster, not production. testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; # OCF agent path (from pacemaker's --with-ocfdir). # We expose this on PATH so custom scripts can find each other. ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d"; # Concatenated PATH that includes all binaries OCF agents and pacemaker # lrmd children need. This is the workaround for nixpkgs#207891 (PATH # not set correctly for OCF agent child processes). ocfBinPath = lib.concatStringsSep ":" [ "${pkgs.iproute2}/bin" "${pkgs.iproute2}/sbin" "${pkgs.iputils}/bin" "${pkgs.util-linux}/bin" "${pkgs.util-linux}/sbin" "${pkgs.gawk}/bin" "${pkgs.gnugrep}/bin" "${pkgs.gnused}/bin" "${pkgs.coreutils}/bin" "${pkgs.bash}/bin" "${pkgs.procps}/bin" "${pkgs.xfsprogs}/bin" "${pkgs.drbd}/bin" "${pkgs.targetcli-fb}/bin" "${pkgs.python3}/bin" "/run/current-system/sw/bin" "/run/current-system/sw/sbin" "/usr/local/sbin" "/usr/local/bin" "/usr/sbin" "/usr/bin" "/sbin" "/bin" ]; in { system.stateVersion = "26.05"; # ── Hardware (Proxmox VM) ────────────────────────────────────────────── imports = [ ../../modules/hardware-configuration/vm/proxmox.nix ../../modules/boot/efi.nix ]; # ── Nix settings ────────────────────────────────────────────────────── nix.settings.experimental-features = [ "nix-command" "flakes" ]; # ── SSH ─────────────────────────────────────────────────────────────── services.openssh = { enable = true; settings.PermitRootLogin = "yes"; }; users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey # Claude Code session key (this machine) — test-lab only "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos" ]; # Allow QEMU guest exec for key injection fallback services.qemuGuest.enable = true; # ── Networking ──────────────────────────────────────────────────────── networking.useDHCP = false; networking.defaultGateway = "192.168.2.1"; networking.nameservers = [ "192.168.2.1" "8.8.8.8" ]; # ── DRBD ────────────────────────────────────────────────────────────── services.drbd.enable = true; services.drbd.config = '' global { usage-count yes; } common { net { protocol C; ping-int 1; verify-alg sha256; after-sb-0pri discard-zero-changes; after-sb-1pri discard-secondary; } disk { # resource-only: DRBD itself won't fence (Pacemaker handles STONITH); # the DRBD resource agent uses fencing to guard primary promotion. fencing resource-only; } handlers { # NOTE: LVM-specific before/after-resync-target handlers omitted # (raw /dev/sdb, no LVM). split-brain handler also omitted — # Pacemaker STONITH manages split-brain fencing for the test cluster. } } resource ha-data { volume 0 { device /dev/drbd0; disk /dev/sdb; # scsi1 in Proxmox VM → sdb meta-disk internal; } on ha-test-node1 { address ${node1Ip}:${toString drbdPort}; } on ha-test-node2 { address ${node2Ip}:${toString drbdPort}; } } ''; # ── Corosync ────────────────────────────────────────────────────────── services.corosync = { enable = true; clusterName = "ha-test"; nodelist = [ { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; } { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; } ]; }; # Corosync authkey (test-only, not secret — generated with # `corosync-keygen` for production). environment.etc."corosync/authkey" = { source = builtins.toFile "authkey" testAuthKey; mode = "0400"; }; # ── Pacemaker ───────────────────────────────────────────────────────── services.pacemaker.enable = true; # Fix for nixpkgs#207891: # 1. Ensure CIB directories are owned by hacluster before starting. # The stock module sets StateDirectory=pacemaker (owned by root); # pacemaker internally drops to hacluster but needs to write there. # 2. Set PATH so OCF agent child processes can find all required binaries. systemd.services.pacemaker.serviceConfig = { ExecStartPre = [ "${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'" ]; }; systemd.services.pacemaker.environment = { PATH = lib.mkForce ocfBinPath; # Expose OCF root so pacemaker and lrmd agree on where agents live. OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; }; # pacemaker-execd is the local resource executor that calls OCF agents. # Give it the same PATH so OCF scripts can find all required binaries. systemd.services.pacemaker-execd.environment = { PATH = lib.mkForce ocfBinPath; OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf"; }; # ── LIO / iSCSI target ──────────────────────────────────────────────── # targetcli-fb is the management tool; actual kernel support is via # the LIO modules. We add a systemd service that saves/restores the # target configuration so Pacemaker can trigger it via a systemd-class # resource. boot.kernelModules = [ "target_core_mod" "iscsi_target_mod" "target_core_file" "target_core_pscsi" "target_core_user" "configfs" ]; # configfs must be mounted for rtslib/targetcli to work systemd.mounts = [{ where = "/sys/kernel/config"; what = "configfs"; type = "configfs"; wantedBy = [ "multi-user.target" ]; before = [ "targetctl.service" ]; }]; # targetctl: save/restore LIO configuration (mirrors Debian's package) systemd.services.targetctl = { description = "LIO iSCSI target config save/restore"; wantedBy = [ "multi-user.target" ]; after = [ "sys-kernel-config.mount" "network.target" ]; requires = [ "sys-kernel-config.mount" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json"; ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json"; }; unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json"; }; # ── Packages ────────────────────────────────────────────────────────── environment.systemPackages = with pkgs; [ # HA stack corosync # corosync-cfgtool, corosync-quorumtool pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI drbd # drbdadm, drbdsetup, drbdmon ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …) # Storage xfsprogs # mkfs.xfs, xfs_admin, xfs_info targetcli-fb # targetcli shell + targetctl # Networking / debug iproute2 # ip, ss iputils # ping tcpdump lsof # Scripting / config python3 curl jq vim htop ]; # ── Firewall ────────────────────────────────────────────────────────── networking.firewall = { enable = true; allowedTCPPorts = [ 22 # SSH 3260 # iSCSI 3121 # pacemaker-remoted 2224 # pcsd drbdPort # DRBD replication ]; allowedUDPPorts = [ 5404 # corosync cluster 5405 # corosync cluster 5407 # corosync crypto ]; # Corosync uses ports 5404-5407 UDP; allow them on the cluster net extraCommands = '' iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT ''; }; # ── tmpfiles: target config dir ──────────────────────────────────────── systemd.tmpfiles.rules = [ "d /etc/target 0750 root root -" "f /etc/target/saveconfig.json 0640 root root -" ]; # ── Locale / time ───────────────────────────────────────────────────── time.timeZone = vars.timeZone; i18n.defaultLocale = "en_AU.UTF-8"; }