nixos/test-lab/ha/common.nix

# Shared HA stack config for both test nodes.
# These are throwaway test VMs — not production hosts.
# No sops-nix, no clan, no home-manager.
{ lib, pkgs, vars, ... }:

let
  node1Ip = "192.168.2.200";
  node2Ip = "192.168.2.201";
  drbdPort = 7789;

  # Test-only corosync authkey (128 bytes minimum).
  # Not secret — this is a disposable test cluster, not production.
  testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";

  # OCF agent path (from pacemaker's --with-ocfdir).
  # We expose this on PATH so custom scripts can find each other.
  ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d";

  # Concatenated PATH that includes all binaries OCF agents and pacemaker
  # lrmd children need.  This is the workaround for nixpkgs#207891 (PATH
  # not set correctly for OCF agent child processes).
  ocfBinPath = lib.concatStringsSep ":" [
    "${pkgs.iproute2}/bin"
    "${pkgs.iproute2}/sbin"
    "${pkgs.iputils}/bin"
    "${pkgs.util-linux}/bin"
    "${pkgs.util-linux}/sbin"
    "${pkgs.gawk}/bin"
    "${pkgs.gnugrep}/bin"
    "${pkgs.gnused}/bin"
    "${pkgs.coreutils}/bin"
    "${pkgs.bash}/bin"
    "${pkgs.procps}/bin"
    "${pkgs.xfsprogs}/bin"
    "${pkgs.drbd}/bin"
    "${pkgs.targetcli-fb}/bin"
    "${pkgs.python3}/bin"
    "/run/current-system/sw/bin"
    "/run/current-system/sw/sbin"
    "/usr/local/sbin"
    "/usr/local/bin"
    "/usr/sbin"
    "/usr/bin"
    "/sbin"
    "/bin"
  ];
in
{
  system.stateVersion = "26.05";

  # ── Hardware (Proxmox VM) ──────────────────────────────────────────────
  imports = [
    ../../modules/hardware-configuration/vm/proxmox.nix
    ../../modules/boot/efi.nix
  ];

  # ── Nix settings ──────────────────────────────────────────────────────
  nix.settings.experimental-features = [ "nix-command" "flakes" ];

  # ── SSH ───────────────────────────────────────────────────────────────
  services.openssh = {
    enable = true;
    settings.PermitRootLogin = "yes";
  };
  users.users.root.openssh.authorizedKeys.keys = [
    vars.adminSshKey
    # Claude Code session key (this machine) — test-lab only
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
  ];

  # Allow QEMU guest exec for key injection fallback
  services.qemuGuest.enable = true;

  # ── Networking ────────────────────────────────────────────────────────
  networking.useDHCP = false;
  networking.defaultGateway = "192.168.2.1";
  networking.nameservers = [ "192.168.2.1" "8.8.8.8" ];

  # ── DRBD ──────────────────────────────────────────────────────────────
  services.drbd.enable = true;
  services.drbd.config = ''
    global {
      usage-count yes;
    }

    common {
      net {
        protocol C;
        ping-int 1;
        verify-alg sha256;
        after-sb-0pri discard-zero-changes;
        after-sb-1pri discard-secondary;
      }
      disk {
        # resource-only: DRBD itself won't fence (Pacemaker handles STONITH);
        # the DRBD resource agent uses fencing to guard primary promotion.
        fencing resource-only;
      }
      handlers {
        # Called by DRBD when a split-brain is detected and this node is
        # in the secondary role — notifies pacemaker to fence the split node.
        split-brain "/usr/lib/drbd/notify-split-brain.sh root";
        before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k";
        after-resync-target "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh";
      }
    }

    resource ha-data {
      volume 0 {
        device    /dev/drbd0;
        disk      /dev/sdb;   # scsi1 in Proxmox VM → sdb
        meta-disk internal;
      }

      on ha-test-node1 {
        address ${node1Ip}:${toString drbdPort};
      }

      on ha-test-node2 {
        address ${node2Ip}:${toString drbdPort};
      }
    }
  '';

  # ── Corosync ──────────────────────────────────────────────────────────
  services.corosync = {
    enable = true;
    clusterName = "ha-test";
    nodelist = [
      { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; }
      { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; }
    ];
  };

  # Corosync authkey (test-only, not secret — generated with
  # `corosync-keygen` for production).
  environment.etc."corosync/authkey" = {
    source = builtins.toFile "authkey" testAuthKey;
    mode = "0400";
  };

  # ── Pacemaker ─────────────────────────────────────────────────────────
  services.pacemaker.enable = true;

  # Fix for nixpkgs#207891:
  # 1. Ensure CIB directories are owned by hacluster before starting.
  #    The stock module sets StateDirectory=pacemaker (owned by root);
  #    pacemaker internally drops to hacluster but needs to write there.
  # 2. Set PATH so OCF agent child processes can find all required binaries.
  systemd.services.pacemaker.serviceConfig = {
    ExecStartPre = [
      "${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'"
    ];
  };
  systemd.services.pacemaker.environment = {
    PATH = lib.mkForce ocfBinPath;
    # Expose OCF root so pacemaker and lrmd agree on where agents live.
    OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
  };

  # pacemaker-execd is the local resource executor that calls OCF agents.
  # Give it the same PATH so OCF scripts can find all required binaries.
  systemd.services.pacemaker-execd.environment = {
    PATH = lib.mkForce ocfBinPath;
    OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
  };

  # ── LIO / iSCSI target ────────────────────────────────────────────────
  # targetcli-fb is the management tool; actual kernel support is via
  # the LIO modules.  We add a systemd service that saves/restores the
  # target configuration so Pacemaker can trigger it via a systemd-class
  # resource.
  boot.kernelModules = [
    "target_core_mod"
    "iscsi_target_mod"
    "target_core_file"
    "target_core_pscsi"
    "target_core_user"
    "configfs"
  ];

  # configfs must be mounted for rtslib/targetcli to work
  systemd.mounts = [{
    where = "/sys/kernel/config";
    what = "configfs";
    type = "configfs";
    wantedBy = [ "multi-user.target" ];
    before = [ "targetctl.service" ];
  }];

  # targetctl: save/restore LIO configuration (mirrors Debian's package)
  systemd.services.targetctl = {
    description = "LIO iSCSI target config save/restore";
    wantedBy = [ "multi-user.target" ];
    after = [ "sys-kernel-config.mount" "network.target" ];
    requires = [ "sys-kernel-config.mount" ];
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
      ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json";
      ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json";
    };
    unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
  };

  # ── Packages ──────────────────────────────────────────────────────────
  environment.systemPackages = with pkgs; [
    # HA stack
    corosync        # corosync-cfgtool, corosync-quorumtool
    pacemaker       # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI
    drbd            # drbdadm, drbdsetup, drbdmon
    ocf-resource-agents  # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …)

    # Storage
    xfsprogs        # mkfs.xfs, xfs_admin, xfs_info
    targetcli-fb    # targetcli shell + targetctl

    # Networking / debug
    iproute2        # ip, ss
    iputils         # ping
    tcpdump
    lsof

    # Scripting / config
    python3
    curl
    jq
    vim
    htop
  ];

  # ── Firewall ──────────────────────────────────────────────────────────
  networking.firewall = {
    enable = true;
    allowedTCPPorts = [
      22      # SSH
      3260    # iSCSI
      3121    # pacemaker-remoted
      2224    # pcsd
      drbdPort # DRBD replication
    ];
    allowedUDPPorts = [
      5404    # corosync cluster
      5405    # corosync cluster
      5407    # corosync crypto
    ];
    # Corosync uses ports 5404-5407 UDP; allow them on the cluster net
    extraCommands = ''
      iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT
      iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT
    '';
  };

  # ── tmpfiles: target config dir ────────────────────────────────────────
  systemd.tmpfiles.rules = [
    "d /etc/target 0750 root root -"
    "f /etc/target/saveconfig.json 0640 root root -"
  ];

  # ── Locale / time ─────────────────────────────────────────────────────
  time.timeZone = vars.timeZone;
  i18n.defaultLocale = "en_AU.UTF-8";
}