nixos/modules/ha/cluster-config.nix

# Cluster-wide HA config shared by both ha-server nodes.
#
# Covers everything that is identical on both nodes and references cluster
# topology (node IPs, hostnames, DRBD resource).  Per-node identity
# (hostname, static IP, stateVersion) lives in hosts/ha-server-{1,2}/host.nix.
#
# Corosync authkey:
#   /etc/corosync/authkey (mode 0400) is managed by sops-nix below.
#   Bootstrap: run scripts/ha/cluster-init.sh on node1 to generate the key,
#   then encrypt it with: sops -e --input-type binary /etc/corosync/authkey > secrets/ha-corosync-authkey
#   Both host keys must be registered via sync-host-keys.sh first so both nodes can decrypt it.
#
# DRBD fencing:
#   resource-only with crm-fence-peer.sh: DRBD calls the Pacemaker-aware
#   crm-fence-peer.sh handler before promoting.  The handler checks the CIB
#   to confirm the peer's DRBD resource is stopped and returns 7 (successfully
#   fenced), allowing safe promotion without requiring power-fencing (STONITH).
#   The unfence handler crm-unfence-peer.sh clears the outdate flag when the
#   peer reconnects.  This is the correct setting for Pacemaker+DRBD clusters
#   with STONITH disabled; crm-fence-peer.sh replaces the need for a separate
#   STONITH device during the testing phase.  Switch to resource-and-stonith
#   once the fence_pve_ssh STONITH resource is active (see
#   scripts/ha/cluster-enable-stonith.sh).
#
#   PATH wrapper: when the DRBD kernel module invokes the fence-peer handler
#   via the UMH (User Mode Helper) mechanism it provides a minimal PATH that
#   omits /run/current-system/sw/bin.  crm-fence-peer.sh calls cibadmin,
#   crm_mon etc.; if those aren't found a pipeline in the script breaks with
#   SIGPIPE.  A process killed by signal has WEXITSTATUS() == 0, so the kernel
#   sees exit code 0 and logs "fence-peer helper broken, returned 0", looping
#   forever.  The writeShellScript wrappers below prepend the NixOS sw path
#   before exec-ing the real handler, giving it a working Pacemaker toolchain.
{ lib, pkgs, vars, ... }:
let
  fencePeerWrapper = pkgs.writeShellScript "drbd-fence-peer" ''
    export PATH="/run/current-system/sw/bin:/run/current-system/sw/sbin:$PATH"
    exec /run/current-system/sw/lib/drbd/crm-fence-peer.sh "$@"
  '';
  unfencePeerWrapper = pkgs.writeShellScript "drbd-unfence-peer" ''
    export PATH="/run/current-system/sw/bin:/run/current-system/sw/sbin:$PATH"
    exec /run/current-system/sw/lib/drbd/crm-unfence-peer.sh "$@"
  '';
in
{
  # Root SSH access — same key set as the nixos user so all admin keys can reach root.
  users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey ] ++ vars.extraAdminSshKeys;

  # Passwordless sudo for wheel — operator SSHes as nixos and uses sudo for
  # cluster management commands (drbdadm, crm*, pcs, etc.)
  security.sudo.wheelNeedsPassword = lib.mkForce false;

  # DRBD lock-file directory (drbd-utils checks for it; missing → harmless but noisy warnings).
  systemd.tmpfiles.rules = [ "d /var/lib/drbd 0750 root root -" ];

  # Prevent drbd.service from auto-starting at boot / nixos-rebuild switch.
  # Pacemaker's OCF drbd agent calls drbdadm up/down directly when managing
  # the resource.  If drbd.service also runs drbdadm up all while DRBD is
  # already Primary under Pacemaker, apply-al fails with "device busy" (exit 20).
  systemd.services.drbd.wantedBy = lib.mkForce [ ];

  services.drbd = {
    enable = true;
    config = ''
      global {
        usage-count yes;
      }

      common {
        net {
          protocol C;
          ping-int 1;
          verify-alg sha256;
          after-sb-0pri discard-zero-changes;
          after-sb-1pri discard-secondary;
        }
        disk {
          fencing resource-only;
        }
        handlers {
          fence-peer   "${fencePeerWrapper}";
          unfence-peer "${unfencePeerWrapper}";
        }
      }

      resource ha-data {
        volume 0 {
          device    /dev/drbd0;
          disk      ${vars.haServerDrbdDisk};
          meta-disk internal;
        }

        on ${vars.haServer1Host} {
          address ${vars.haServer1StorageIp}:${toString vars.ports.haServerDrbd};
        }

        on ${vars.haServer2Host} {
          address ${vars.haServer2StorageIp}:${toString vars.ports.haServerDrbd};
        }
      }
    '';
  };

  # /etc/corosync/authkey — sops binary secret, identical on both nodes.
  # Decryptable by both ha-server host keys (added by sync-host-keys.sh).
  sops.secrets.corosync_authkey = {
    sopsFile = ../../secrets/ha-corosync-authkey;
    format = "binary";
    path = "/etc/corosync/authkey";
    mode = "0400";
    restartUnits = [ "corosync.service" ];
  };

  # NixOS common config enables NetworkManager by default; HA cluster nodes
  # need stable static IPs with predictable interface names — NM is not suitable.
  networking.networkmanager.enable = lib.mkForce false;

  # services.corosync.enable is set by modules/ha/pacemaker-stack.nix.
  services.corosync = {
    clusterName = "ha-cluster";
    nodelist = [
      # ring0: cluster-internal vmbr1 (primary heartbeat + DRBD path)
      # ring1: LAN vmbr0 (backup heartbeat only — never carries DRBD)
      { nodeid = 1; name = vars.haServer1Host; ring_addrs = [ vars.haServer1StorageIp vars.haServer1Ip ]; }
      { nodeid = 2; name = vars.haServer2Host; ring_addrs = [ vars.haServer2StorageIp vars.haServer2Ip ]; }
    ];
  };

  networking.firewall = {
    allowedTCPPorts = [
      vars.ports.haServerPacemakerRemoted
      vars.ports.haServerPcsd
      vars.ports.haServerDrbd
    ];
    allowedUDPPorts = [
      vars.ports.haServerCorosync1
      vars.ports.haServerCorosync2
      vars.ports.haServerCorosyncCrypto
    ];
    # Protocol separation: iSCSI (VLAN 20 / storage clients only),
    # NFS (VLAN 2 / LAN only).  Cluster-internal subnets accepted wholesale
    # since they are isolated bridges with no external uplink.
    extraCommands = ''
      iptables -A nixos-fw -s ${vars.haServer1Ip}/32 -j nixos-fw-accept
      iptables -A nixos-fw -s ${vars.haServer2Ip}/32 -j nixos-fw-accept
      iptables -A nixos-fw -s ${vars.haStorageCidr} -j nixos-fw-accept

      iptables -A nixos-fw -p tcp -s ${vars.haClientCidr} --dport ${toString vars.ports.haServerIscsi} -j nixos-fw-accept

      iptables -A nixos-fw -p tcp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsRpcbind} -j nixos-fw-accept
      iptables -A nixos-fw -p udp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsRpcbind} -j nixos-fw-accept
      iptables -A nixos-fw -p tcp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsd} -j nixos-fw-accept
      iptables -A nixos-fw -p udp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsd} -j nixos-fw-accept
      iptables -A nixos-fw -p tcp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsMountd} -j nixos-fw-accept
      iptables -A nixos-fw -p udp -s ${vars.lanCidr} --dport ${toString vars.ports.nfsMountd} -j nixos-fw-accept

      iptables -A nixos-fw -p tcp -s ${vars.haClientCidr} --dport ${toString vars.ports.nfsRpcbind} -j nixos-fw-accept
      iptables -A nixos-fw -p udp -s ${vars.haClientCidr} --dport ${toString vars.ports.nfsRpcbind} -j nixos-fw-accept
      iptables -A nixos-fw -p tcp -s ${vars.haClientCidr} --dport ${toString vars.ports.nfsd} -j nixos-fw-accept
      iptables -A nixos-fw -p udp -s ${vars.haClientCidr} --dport ${toString vars.ports.nfsd} -j nixos-fw-accept
      iptables -A nixos-fw -p tcp -s ${vars.haClientCidr} --dport ${toString vars.ports.nfsMountd} -j nixos-fw-accept
      iptables -A nixos-fw -p udp -s ${vars.haClientCidr} --dport ${toString vars.ports.nfsMountd} -j nixos-fw-accept
    '';
  };
}