+x on scripts and remove beszel from ha for now

At first boot, pacemaker fails with 'Dependency failed' because ipa-activation
fails (IPA not enrolled yet) before corosync/authkey are ready, causing a
systemd boot-ordering race.  The service recovers fine when started manually.
Add an explicit 'systemctl start pacemaker' on both nodes immediately after
restarting corosync so cluster-init doesn't time out waiting for it.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8

hosts/ha-server-1/host.nix

@@ -24,7 +24,7 @@
   };
 
   # Set KEY after pairing this host with the beszel hub; the token is sops-managed.
-  services.beszel.agent.environment.KEY = "";
+  # services.beszel.agent.environment.KEY = "";
 
   system.stateVersion = "26.05";
 }

hosts/ha-server-2/host.nix

@@ -24,7 +24,7 @@
   };
 
   # Set KEY after pairing this host with the beszel hub; the token is sops-managed.
-  services.beszel.agent.environment.KEY = "";
+  # services.beszel.agent.environment.KEY = "";
 
   system.stateVersion = "26.05";
 }

modules/build-types/gui.nix

@@ -81,4 +81,70 @@
   programs.firefox.enable = true;
 
   nixpkgs.config.allowUnfree = true;
+
+  # GUI-specific Home Manager additions for the IPA primary user, extending
+  # the baseline in modules/ipa/client.nix with desktop apps and services
+  # that only make sense on a graphical workstation.
+  home-manager.users.${vars.ipaUser} = { pkgs, ... }: {
+    home = {
+      packages = with pkgs; [
+        git
+        vim
+        nextcloud-client
+        chromium
+        claude-code
+        fish
+        sops
+      ];
+      sessionVariables = {
+        EDITOR = "vim";
+        SOPS_AGE_KEY_FILE = "/home/${vars.ipaUser}/.config/sops/age/keys.txt";
+      };
+      file = {
+        ".local/share/applications/proxmox-chromium-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox (Chromium)
+          Exec=chromium --app=https://pve.${vars.homeDomain}:${toString vars.ports.pveWeb} --window-size=1920,1080 --window-position=0,0
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=Hypervisor;
+          StartupWMClass=PVE
+        '';
+        ".local/share/applications/pbs-chromium-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox Backup Server (Chromium)
+          Exec=chromium --app=https://${vars.pbsIp}:${toString vars.ports.pbsWeb} --window-size=1920,1080 --window-position=0,0
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=backup;
+        '';
+        ".local/share/applications/proxmox-firefox-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox (Firefox)
+          Exec=firefox --new-instance https://pve.${vars.homeDomain}:${toString vars.ports.pveWeb} --profile ProxmoxWebApp --window-size=1920,1080 --class ProxmoxWebApp
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=Hypervisor;
+          StartupWMClass=PVE
+        '';
+        ".local/share/applications/pbs-firefox-app.desktop".text = ''
+          [Desktop Entry]
+          Type=Application
+          Name=Proxmox Backup Server (Firefox)
+          Exec=firefox --new-window https://${vars.pbsIp}:${toString vars.ports.pbsWeb} --profile PbsWebApp --window-size=1920,1080 --class PbsWebApp
+          Icon=/home/${vars.ipaUser}/.local/share/icons/proxmox.png
+          Terminal=false
+          Categories=backup;
+          StartupWMClass=PBS
+        '';
+      };
+    };
+    services.nextcloud-client = {
+      enable = true;
+      startInBackground = true;
+    };
+  };
 }

modules/build-types/ha-server.nix

@@ -30,7 +30,7 @@ in
     ../ha/pacemaker-stack.nix
     ../ha/iscsi-target.nix
     ../ha/cluster-config.nix
-    ../beszel/enable-agent.nix
+    # ../beszel/enable-agent.nix
   ];
 
   # xfsprogs: mkfs.xfs/xfs_info needed by cluster-init.sh.

modules/docker/enable-service.nix

@@ -1,5 +1,8 @@
 { lib, pkgs, vars, ... }:
 
+let
+  gid = toString vars.dockerAccessGid;
+in
 {
   virtualisation.docker = {
     enable = true;
@@ -16,4 +19,27 @@
     docker-compose
     docker-buildx
   ];
+
+  # NixOS's group activation uses plain `groupmod` without --non-unique.
+  # When SSSD is active it exposes the IPA "docker-access" group at
+  # vars.dockerAccessGid via NSS, so groupmod sees that GID as already in
+  # use and silently skips the change (warning: "not applying GID change").
+  # This script runs after the normal "groups" step and applies the change
+  # with --non-unique (which lets the local docker group share the GID with
+  # the SSSD-provided IPA group). If the GID actually changed it also
+  # restarts docker.socket so the socket is recreated with the new GID.
+  system.activationScripts.docker-group-gid = {
+    deps = [ "groups" ];
+    text = ''
+      current=$(grep "^docker:" /etc/group | cut -d: -f3)
+      if [ "$current" != "${gid}" ]; then
+        ${pkgs.shadow}/bin/groupmod --non-unique -g ${gid} docker
+        if ${pkgs.systemd}/bin/systemctl is-active --quiet docker.socket; then
+          ${pkgs.systemd}/bin/systemctl stop docker.service docker.socket
+          rm -f /var/run/docker.sock
+          ${pkgs.systemd}/bin/systemctl start docker.socket docker.service
+        fi
+      fi
+    '';
+  };
 }

modules/ipa/client.nix

@@ -60,6 +60,15 @@ lib.mkIf enabled {
     pam.services = {
       sshd.makeHomeDir = true;
       lightdm.makeHomeDir = true;
+
+      # pam_unix returns PAM_AUTHINFO_UNAVAIL without prompting when the local
+      # stub has "!" in shadow (account locked), so PAM_AUTHTOK is never set
+      # and pam_sss's use_first_pass fails with "No authentication token".
+      # Changing to try_first_pass makes pam_sss prompt independently when no
+      # prior module has set the token, restoring IPA password login via
+      # LightDM and su.
+      login.rules.auth.sss.settings = lib.mkForce { try_first_pass = true; };
+      su.rules.auth.sss.settings = lib.mkForce { try_first_pass = true; };
     };
 
     # HM with useUserPackages = true (flake.nix) sets users.users.${ipaUser}.packages,
@@ -170,6 +179,13 @@ lib.mkIf enabled {
     group = "users";
     extraGroups = [ "wheel" ];
     createHome = false;
+    # "!" is not a password hash — it is the standard "account locked" marker.
+    # It cannot authenticate anyone locally. It exists solely so NixOS generates
+    # a shadow entry for this stub user; without one pam_unix returns
+    # PAM_AUTHINFO_UNAVAIL before prompting, which means PAM_AUTHTOK is never
+    # set and the subsequent pam_sss use_first_pass call has nothing to work
+    # with — blocking LightDM and su logins even when IPA/SSSD auth succeeds.
+    hashedPassword = "!";
   };
 
   # Home Manager config for the IPA primary user, applied on every enrolled
@@ -186,7 +202,7 @@ lib.mkIf enabled {
       homeDirectory = lib.mkForce "/home/${vars.ipaUser}";
       stateVersion = "26.05";
       packages = with pkgs; [ tmux sshfs ];
-      sessionVariables.EDITOR = "nano";
+      sessionVariables.EDITOR = lib.mkDefault "nano";
     };
     programs.home-manager.enable = true;
     programs.bash.enable = true;

scripts/ha/cluster-init.sh

@@ -103,11 +103,16 @@ n2_ssh "mkdir -p /etc/corosync"
 n2_scp "$AUTHKEY" "$AUTHKEY"
 n2_ssh "chmod 0400 '${AUTHKEY}'"
 
-log "Restarting corosync on both nodes..."
+log "Restarting corosync and pacemaker on both nodes..."
 systemctl restart corosync
 n2_ssh "systemctl restart corosync"
 sleep 3
 
+log "Starting pacemaker on both nodes (may have failed at boot before authkey was placed)..."
+systemctl start pacemaker 2>/dev/null || systemctl restart pacemaker 2>/dev/null || true
+n2_ssh "systemctl start pacemaker 2>/dev/null || systemctl restart pacemaker 2>/dev/null || true"
+sleep 2
+
 # ── 1. Corosync quorum ────────────────────────────────────────────────────
 log "Waiting for corosync quorum..."
 for i in $(seq 1 30); do

scripts/ha/deploy.sh

@@ -355,58 +355,6 @@ if ! $SKIP_CLUSTER_INIT; then
       "sudo mkdir -p /root/.ssh && sudo cp /tmp/cluster-init-key /root/.ssh/cluster-init-key && \
        sudo chmod 600 /root/.ssh/cluster-init-key && rm -f /tmp/cluster-init-key"
 
-    # Fix targetctl.service on both nodes: the iscsi-target.nix module bakes
-    # pkgs.targetcli-fb for the targetctl binary, but targetctl is actually in
-    # rtslib-fb (a different store path).  Apply a runtime dropin that corrects
-    # both ExecStart and ExecStop before Pacemaker ever touches the service.
-    # The fixed iscsi-target.nix module will make this redundant on next rebuild.
-    logn "Patching targetctl.service on both nodes..."
-    _patch_targetctl() {
-      local ip="$1"
-      ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "${HA_USER}@${ip}" sudo bash << 'PATCH'
-set -euo pipefail
-TC=$(find /nix/store -maxdepth 4 -path '*/python3*env/bin/targetctl' 2>/dev/null | head -1)
-PY=$(find /nix/store -maxdepth 4 -path '*/python3*env/bin/python3' -name 'python3' 2>/dev/null | \
-  while IFS= read -r p; do "$p" -c "import rtslib_fb" 2>/dev/null && echo "$p" && break; done | head -1)
-[[ -n "$TC" && -n "$PY" ]] || { echo "targetctl or python3+rtslib_fb not found"; exit 1; }
-# Write stop script that saves LIO config then tears down kernel state
-"$PY" - "$TC" "$PY" << 'PYEOF'
-import sys, os, stat
-tc, py = sys.argv[1], sys.argv[2]
-script = f"""#!{py}
-import subprocess, sys, rtslib_fb
-root = rtslib_fb.RTSRoot()
-targets = list(root.targets)
-if targets:
-    r = subprocess.run(["{tc}", "save", "/etc/target/saveconfig.json"], capture_output=True)
-    print(f"saved {{len(targets)}} target(s); rc={{r.returncode}}")
-else:
-    print("no active LIO targets")
-for t in targets:
-    try:
-        for tpg in list(t.tpgs): tpg.enable = False
-        t.delete()
-    except Exception as e: print(f"warn: {{e}}", file=sys.stderr)
-for so in list(root.storage_objects):
-    try: so.delete()
-    except Exception as e: print(f"warn: {{e}}", file=sys.stderr)
-print("LIO kernel target cleared")
-"""
-path = "/run/ha-targetctl-stop.py"
-with open(path, "w") as f: f.write(script)
-os.chmod(path, 0o755)
-print(f"wrote {path}")
-PYEOF
-mkdir -p /run/systemd/system/targetctl.service.d
-printf '[Service]\nExecStart=\nExecStart=%s restore /etc/target/saveconfig.json\nExecStop=\nExecStop=/run/ha-targetctl-stop.py\n' \
-  "$TC" > /run/systemd/system/targetctl.service.d/fix-exec.conf
-systemctl daemon-reload
-echo "patched on $(hostname)"
-PATCH
-    }
-    _patch_targetctl "${NODE1_IP}"
-    _patch_targetctl "${NODE2_IP}"
-
     logn "Uploading cluster-init.sh to ${NODE1_HOST}..."
     scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
       "$CLUSTER_INIT" "${HA_USER}@${NODE1_IP}:/tmp/cluster-init.sh"

secrets/ha-corosync-authkey

@@ -1,22 +1,22 @@
 {
-	"data": "ENC[AES256_GCM,data:olqZm/3yxtVnlDeYzAWnXjUP9pgkruFEpeS6qONHSfdrj0lJOch44zn1pbG6Jg5QYcksny0rHki9PJEZSbRIyrZ2mSKfTJing3n9M5ODm9xPMbNYn8cpVB4AzZmCSAH7mN4p/98+5hJ1qzopGwCgKTcOOJ9DMUl9AStXORY=,iv:R2K3OehIfyKZXRFH3Y9KYnvEp6CjHPT4Ki2P1YrC6+I=,tag:1FM/E0n9teARAZ1HZM4b3A==,type:str]",
+	"data": "ENC[AES256_GCM,data:5sUi7wWm5dkv2rLtFUswiWm/NAjIlDmsD+WRxI1iuhHltaPrTR0u3x7Ceh3FZv6PeL5LGaNq2cgfvECPyL9vbIf938bzoGr3hQsWE3m5tlclE8ptdfyleG6MQyv/QRP7gX8B03R+Pggl0ci6nSdnzRGigVwFLaXKFuFEV6StW4fpVZCqNWYpAUXeYi65/sByuEO2NM44RE8syHcfnrd01NaV9cEohro93meLkcHiU+K/iMMjQOA99PnRs6w//ATTTo7bp+Ja2qJeaYPzQtXWotjeXqtBZ6Tqx9yiSVhuhugWZxIljXgsvhugKZd9HNvkU4ApTjV5IHZ7OXCmEPvhdw==,iv:XWWplv6uq7fKUHX6As6On27GL2A1j/k5OQmwlL1p6rw=,tag:hogRMGolftuLDrDcPOVY4Q==,type:str]",
 	"sops": {
 		"age": [
 			{
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3RFRoRW1rRklxbXYybUtu\nL1lEOXJBMDNCN1paU1k5N0hQRTJYQ1dQcTBVCk9OYnloOFZad2toOHo0UXFYSzNR\nZDUwV00vKzVnckMwSE9MMnE4RUozZTgKLS0tIFByL3Z3SzJFeEVNZjhuRDNTMzFo\nNU5tZ2tQU1ZORE5qNm1JUEJrTGdzUHMKypYeJz6BeUUY4aPKazB1nxncOA3DGkal\nfemLr9uwCJw+D3xfXzrIKrI0w3OH7bu2LmWqrNDSz3Bwa5VDuHoLqQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPM1VaUXNpbUZNR09kd3hM\ncEtscFIzQS9NdG1CWlpNZWFVZzdSdkUzNzNvCk1MREo3ODFzRXZTRU5ZeUxYREo3\nbndncHRDdlVRQmdta2d2ZzZWR3VmdUEKLS0tICtGem14SUt1RUhzcEFKRDZqME1q\nb1NmTmFtbGNTdkVrdE1HQUpFMno1N2sKR016kjx9mrOelYQOkYt7B+PkD/Kk9Sqb\niTKqm1KPgadi1w9V9iorMJHQb1gCoVJogt6LBybNZUOvixJZpND9GQ==\n-----END AGE ENCRYPTED FILE-----\n",
 				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
 			},
 			{
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqTzhtOHM1Q1lWRXdadHB6\nZ1VrMXFsQWZEbDBMMnQvYnZSN090VnpyQnd3CkdJaHBsbzQ4SERmU0dBS2RZMUpi\nZVYrZmtSNXhiaDNTZkNMVEZud3ZzT3cKLS0tIFhETER5eWJRK05ValFhenRnSGg1\nc1prdWVTVmMyWWt2UkNXWVRLcWRHcm8KjVn1faGmsWiFzcNg4PnxZQfeQONFKz/i\nyJGJc7w2KSZ3La+44jfitMziJQ3AFRUlAhGDX25kWZJ7PtjmFvYwuQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5Z3duRmU4b01XWFA2RUVE\nN2d0bVFnckJ1dGZPc2w5OUQ4UHd6WUZrQmpjCjRBZnFqSDRFZkxYT3BmZHo4bHpQ\nSW5rWllHajk2SENyTjZzRVNTb0prM1UKLS0tIG0ra1ZSdi8xcWZlSGRaY0pBbzF6\ncVRTSEhkY2hyU2g0eDNTeHp1MkFiNDQKI0FPp/gKAS4zD1KRZBJoX/8UD+6E/N+h\nIzCFcuARU/0nCPGhaF8O61j1BSjaoYmko79P67Nx9LqBTiNFED+zyw==\n-----END AGE ENCRYPTED FILE-----\n",
 				"recipient": "age1nxlnrevqs2msdatze562vz6ym4pgydt2zndye83lwqauy6flggtqrevyw5"
 			},
 			{
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkVXFQazlvUEx2NzVUNlF1\nN3FQNTdFL1FtSDhCdmNQclRMdG1vRVhvcEhzClF4Ykh1ZS9JekFzOHkvOVphejVy\nd2s0a0IrZ1d4V3diOGNvMlBHUVlwTmMKLS0tIEFhTE13Q2F1di9ESXdJVDhJUVl4\na1kzNDBLcHhlRGg2Y3hSSmdEc2k5MlEKpU2sIv5Pq0LaKkvqA/fRkiB9PcvsSjvq\no4iEFzGNDi63QzXqftWnzengEy/6nWGNA31fzScQftESesGlKfYYlA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsKytKc0tiWEl1RUJtYjFX\nY1QwdTA1Zmdqd21zUHNPTWt6K2lKdWlqaFIwCnZRZnFqaEIrU1FuK3FiZkJMS1pT\nL2NnOHliWFlQUGtWMGgrcnBhY2xaRzAKLS0tIGZJNndKQW1JdU02Qk5sd1FjbGty\nTHFCckEyMXV6Q1Y0U3hQOGNRSFplTFkKBvxMLidbviVipkhjgHvzQzBi/SBE4/wa\ndoKWNXOMkODG2cRF30cWBezH1dk6tUWuok3/o7ywKIba+3gMsYBl/A==\n-----END AGE ENCRYPTED FILE-----\n",
 				"recipient": "age1scfc8p53q5aq2a87tcmsazmj8sfeft0s8kxg0et4nm3ucxyp3c3s6te82y"
 			}
 		],
-		"lastmodified": "2026-07-28T06:46:00Z",
-		"mac": "ENC[AES256_GCM,data:xJHyh4JuY+qnKljS3mtfN4Cy0pGPUszhU4ud2L8JxTFGUHTXhZzwQ3s1vM88efLTwM7yDK7oBMBUH2xligAzglDnqkE1T0l7R9k+hyldm4nYqOK6Y8GyPt/aald6oLYGCx3J7Ap0Oqq/crU+s4NiY9enf+R+08jliFyrPBHuRSY=,iv:FGtURIfSrf1tRGqpPfn40GqybhNYFBcvR7n1YkcAMGY=,tag:ir0TU7YuxpPuew+nyC9/eg==,type:str]",
+		"lastmodified": "2026-07-28T10:35:12Z",
+		"mac": "ENC[AES256_GCM,data:x/Urm0+CkFPVqc5FeyW1kh2iTZGqFdzLQLGz2BqT4ruBFRAZ8B17dTtbk/hkmEWgctGgsBeHxc+pLBgRV7DVFk9oj79p1wFdxOaiGi/24Aj0GVIV5rYsaaJ1BOOZgYnsWz0Zt2xY1u5MUGAcikdWaVme8QOdsIc+0Tvg1r+upDY=,iv:Z5jmJ/fI9FmGOHI8rWHVDxnQ3kotrpi02YE9wAiHdkA=,tag:JBfjMw9GFZdJsIZwVzTr0A==,type:str]",
 		"version": "3.13.3"
 	}
 }