Merge pull request #1 from beatz174-bit/codex/refactor-nix-cache-for-binary-cache-and-remote-builder

Add Nix cache / remote-builder modules, wire hosts, and harden maintenance scripts

common/configuration.nix

@@ -49,17 +49,9 @@ users.users.root = {
 
   #Enable flakes
 
-  nix.settings.experimental-features = "nix-command flakes";
-
   nix.settings = {
-  substituters = [
+    experimental-features = [ "nix-command" "flakes" ];
-    "http://nix-cache"
+    auto-optimise-store = true;
-#    "https://cache.nixos.org/"
-  ];
-  trusted-public-keys = [
-    "cache.local-1:usoWYanY3Kpq2+kDIS2nhWoLZiRxanmdysdzqCFBHW4="
-    "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
-  ];
   };
 
 

docs/nix-cache.md

@@ -0,0 +1,48 @@
+# nix-cache architecture
+
+This repository configures `nix-cache` as a **binary cache server** and a **remote builder** for other hosts.
+
+## Important design notes
+
+- This is **not** a shared `/nix/store` setup.
+- Every machine still keeps and uses its own local `/nix/store`.
+- Clients prefer `http://nix-cache` for substitutes and keep `https://cache.nixos.org/` as fallback.
+- Clients can offload builds to `nix-cache` through SSH (`nix.distributedBuilds`).
+
+## Binary cache signing keys (on nix-cache)
+
+```bash
+sudo install -d -m 0700 /etc/nix
+sudo nix-store --generate-binary-cache-key nix-cache-1   /etc/nix/cache-priv.pem   /etc/nix/cache-pub.pem
+sudo chmod 0600 /etc/nix/cache-priv.pem
+sudo chmod 0644 /etc/nix/cache-pub.pem
+cat /etc/nix/cache-pub.pem
+```
+
+Do not commit private keys.
+
+## Remote builder SSH keys
+
+On each client, install the private key used to authenticate as `nixremote`:
+
+```bash
+sudo install -d -m 0700 /root/.ssh
+sudo install -m 0600 ./nixremote /root/.ssh/nixremote
+sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version
+```
+
+On `nix-cache`, install the matching public key used by `nixremote` authorized keys.
+
+## Manual verification
+
+After deployment:
+
+```bash
+curl http://nix-cache/nix-cache-info
+nix store ping --store http://nix-cache
+nix show-config | grep -E 'substituters|trusted-public-keys|builders-use-substitutes'
+sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version
+nix build nixpkgs#hello --builders 'ssh://nixremote@nix-cache x86_64-linux /root/.ssh/nixremote 4 2 big-parallel,kvm,nixos-test,benchmark' -L
+nix path-info -r nixpkgs#hello
+curl -I "http://nix-cache/$(basename "$(nix path-info nixpkgs#hello)").narinfo"
+```

flake.nix

@@ -1,11 +1,7 @@
 {
   
   description = "LAN NixOS configs";
-      nixConfig = {
+  # GitHub tokens must be provided outside this repository (local nix.conf, env, or deployment secrets).
-        access-tokens = [
-          "github.com=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"
-        ];
-      };          
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
     nixos-conf-editor.url = "github:snowfallorg/nixos-conf-editor";

hosts/docker/configuration.nix

@@ -15,6 +15,8 @@ in
   imports =
     [ # Include the results of the hardware scan.
       ../../common/configuration.nix
+      ../../modules/nix/cache-client.nix
+      ../../modules/nix/remote-builder-client.nix
     ];
 
   networking.hostName = "docker"; # Define your hostname.

hosts/kuma/configuration.nix

@@ -15,6 +15,8 @@ in
   imports =
     [ # Include the results of the hardware scan.
       ../../common/configuration.nix
+      ../../modules/nix/cache-client.nix
+      ../../modules/nix/remote-builder-client.nix
     ];
 
   networking.hostName = "kuma"; # Define your hostname.

hosts/nix-cache/configuration.nix

@@ -8,26 +8,16 @@
   imports =
     [ # Include the results of the hardware scan.
       ../../common/configuration.nix
+      ../../modules/nix/cache-server.nix
     ];
 
   networking.hostName = "nix-cache"; # Define your hostname.
 
-  services.nix-serve.enable = true;
-  services.nix-serve.secretKeyFile = "/etc/nix/cache-priv.pem";
-
-  services.nginx = {
-    enable = true;
-    recommendedProxySettings = true;
-    virtualHosts."cache.local" = {
-      locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}";
-    };
-  };
 services.prometheus.exporters.node = {
   enable = true;
   openFirewall = true;
 };
 
-  networking.firewall.allowedTCPPorts = [ config.services.nginx.defaultHTTPListenPort ];
   # Open ports in the firewall.
 #  networking.firewall.allowedTCPPorts = [ 80 8080 443 ];
   # networking.firewall.allowedUDPPorts = [ ... ];

hosts/nix-minimal/configuration.nix

@@ -8,6 +8,8 @@
   imports =
     [ # Include the results of the hardware scan.
       ../../common/configuration.nix
+      ../../modules/nix/cache-client.nix
+      ../../modules/nix/remote-builder-client.nix
     ];
 
   networking.hostName = "nix-minimal"; # Define your hostname.

hosts/nixos/configuration.nix

@@ -28,6 +28,8 @@ in {
   imports =
     [ # Include the results of the hardware scan.
       ../../common/configuration.nix
+      ../../modules/nix/cache-client.nix
+      ../../modules/nix/remote-builder-client.nix
     ];
 
   # Bootloader.
@@ -94,8 +96,6 @@ in {
 
   system.stateVersion = "25.05"; # Did you read the comment?
 
-
-  nix.settings.experimental-features = "nix-command flakes";
 services.xrdp.enable = true;
 services.xrdp.defaultWindowManager = "cinnamon-session";
 services.xrdp.openFirewall = true;

hosts/server/configuration.nix

@@ -8,6 +8,8 @@
   imports =
     [ # Include the results of the hardware scan.
       ../../common/configuration.nix
+      ../../modules/nix/cache-client.nix
+      ../../modules/nix/remote-builder-client.nix
     ];
 
   networking.hostName = "server"; # Define your hostname.

modules/nix/cache-client.nix

@@ -0,0 +1,15 @@
+{ ... }:
+
+{
+  nix.settings = {
+    substituters = [
+      "http://nix-cache"
+      "https://cache.nixos.org/"
+    ];
+    trusted-public-keys = [
+      "cache.local-1:usoWYanY3Kpq2+kDIS2nhWoLZiRxanmdysdzqCFBHW4="
+      "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+    ];
+    auto-optimise-store = true;
+  };
+}

modules/nix/cache-server.nix

@@ -0,0 +1,58 @@
+{ config, pkgs, ... }:
+
+{
+  # Generate the binary cache key pair on the nix-cache host:
+  #   sudo install -d -m 0700 /etc/nix
+  #   sudo nix-store --generate-binary-cache-key nix-cache-1 \
+  #     /etc/nix/cache-priv.pem \
+  #     /etc/nix/cache-pub.pem
+  #   sudo chmod 0600 /etc/nix/cache-priv.pem
+  #   sudo chmod 0644 /etc/nix/cache-pub.pem
+  #   cat /etc/nix/cache-pub.pem
+  services.nix-serve = {
+    enable = true;
+    secretKeyFile = "/etc/nix/cache-priv.pem";
+  };
+
+  services.nginx = {
+    enable = true;
+    recommendedProxySettings = true;
+    virtualHosts."nix-cache" = {
+      locations."/" = {
+        proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}";
+      };
+    };
+  };
+
+  networking.firewall.allowedTCPPorts = [ 80 ];
+
+  users.groups.nixremote = {};
+
+  users.users.nixremote = {
+    isSystemUser = true;
+    group = "nixremote";
+    createHome = true;
+    home = "/var/lib/nixremote";
+    shell = pkgs.bashInteractive;
+    # Provide remote builder public keys here (safe to commit public keys only):
+    # openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAA... client@host" ];
+    #
+    # Avoid absolute keyFiles paths here because they break pure flake evaluation.
+    openssh.authorizedKeys.keys = [ ];
+  };
+
+  services.openssh.enable = true;
+
+  nix.settings = {
+    trusted-users = [ "root" "nixremote" ];
+    experimental-features = [ "nix-command" "flakes" ];
+    auto-optimise-store = true;
+    builders-use-substitutes = true;
+  };
+
+  nix.gc = {
+    automatic = true;
+    dates = "weekly";
+    options = "--delete-older-than 30d";
+  };
+}

modules/nix/remote-builder-client.nix

@@ -0,0 +1,26 @@
+{ pkgs, ... }:
+
+{
+  # Install the remote builder key on each client host (do not commit private keys):
+  #   sudo install -d -m 0700 /root/.ssh
+  #   sudo install -m 0600 ./nixremote /root/.ssh/nixremote
+  #   sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version
+  nix.distributedBuilds = true;
+
+  nix.buildMachines = [
+    {
+      hostName = "nix-cache";
+      sshUser = "nixremote";
+      sshKey = "/root/.ssh/nixremote";
+      system = pkgs.stdenv.hostPlatform.system;
+      maxJobs = 4;
+      speedFactor = 2;
+      supportedFeatures = [ "nixos-test" "benchmark" "big-parallel" "kvm" ];
+    }
+  ];
+
+  nix.settings = {
+    builders-use-substitutes = true;
+    max-jobs = "auto";
+  };
+}

scripts/codex-maintenance.sh

@@ -9,7 +9,22 @@ warn-dirty = false
 
 MODE="${1:-validate}"
 
-hosts_json="$(nix eval --json --no-accept-flake-config .#nixosConfigurations --apply builtins.attrNames)"
+ensure_nix_profile() {
+  if [ -f /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh ]; then
+    . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh
+  elif [ -f "$HOME/.nix-profile/etc/profile.d/nix.sh" ]; then
+    . "$HOME/.nix-profile/etc/profile.d/nix.sh"
+  fi
+}
+
+ensure_nix_profile
+
+if ! command -v nix >/dev/null 2>&1; then
+  echo "ERROR: nix is not available in PATH. Run bash scripts/codex-setup.sh first." >&2
+  exit 127
+fi
+
+hosts_json="$(nix eval --json --no-use-registries --no-accept-flake-config .#nixosConfigurations --apply builtins.attrNames)"
 hosts="$(echo "$hosts_json" | jq -r '.[]')"
 
 echo "Hosts:"
@@ -29,17 +44,17 @@ fi
 
 echo
 echo "Checking Nix formatting with nixpkgs-fmt..."
-nix run --no-accept-flake-config nixpkgs#nixpkgs-fmt -- --check .
+nix run --no-use-registries --no-accept-flake-config github:NixOS/nixpkgs/nixos-25.11#nixpkgs-fmt -- --check .
 
 echo
 echo "Running statix lint..."
-nix run --no-accept-flake-config nixpkgs#statix -- check .
+nix run --no-use-registries --no-accept-flake-config github:NixOS/nixpkgs/nixos-25.11#statix -- check .
 
 echo
 echo "Evaluating host toplevel derivations..."
 for host in $hosts; do
   echo "==> $host"
-  nix eval --raw --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath"
+  nix eval --raw --no-use-registries --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath"
 done
 
 if [[ "$MODE" == "dry-run" ]]; then
@@ -47,7 +62,7 @@ if [[ "$MODE" == "dry-run" ]]; then
   echo "Running dry-run builds for all hosts. This will not create result symlinks."
   for host in $hosts; do
     echo "==> Dry-run build: $host"
-    nix build --dry-run --no-link --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel"
+    nix build --dry-run --no-link --no-use-registries --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel"
   done
 fi
 

scripts/codex-setup.sh

@@ -76,13 +76,13 @@ if ! command -v jq >/dev/null 2>&1; then
 fi
 
 echo "Available NixOS hosts:"
-hosts="$(nix eval --json --no-accept-flake-config .#nixosConfigurations --apply builtins.attrNames | jq -r '.[]')"
+hosts="$(nix eval --json --no-use-registries --no-accept-flake-config .#nixosConfigurations --apply builtins.attrNames | jq -r '.[]')"
 echo "$hosts"
 
 echo "Evaluating all host toplevel derivations..."
 for host in $hosts; do
   echo "==> Evaluating $host"
-  nix eval --raw --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath"
+  nix eval --raw --no-use-registries --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath"
 done
 
 echo "Codex setup complete."