diff --git a/common/configuration.nix b/common/configuration.nix index 0d8eaf4..30dc808 100644 --- a/common/configuration.nix +++ b/common/configuration.nix @@ -49,18 +49,10 @@ users.users.root = { #Enable flakes - nix.settings.experimental-features = "nix-command flakes"; - -nix.settings = { - substituters = [ - "http://nix-cache" -# "https://cache.nixos.org/" - ]; - trusted-public-keys = [ - "cache.local-1:usoWYanY3Kpq2+kDIS2nhWoLZiRxanmdysdzqCFBHW4=" - "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" - ]; -}; + nix.settings = { + experimental-features = [ "nix-command" "flakes" ]; + auto-optimise-store = true; + }; programs.git = { diff --git a/docs/nix-cache.md b/docs/nix-cache.md new file mode 100644 index 0000000..8dbcaf5 --- /dev/null +++ b/docs/nix-cache.md @@ -0,0 +1,48 @@ +# nix-cache architecture + +This repository configures `nix-cache` as a **binary cache server** and a **remote builder** for other hosts. + +## Important design notes + +- This is **not** a shared `/nix/store` setup. +- Every machine still keeps and uses its own local `/nix/store`. +- Clients prefer `http://nix-cache` for substitutes and keep `https://cache.nixos.org/` as fallback. +- Clients can offload builds to `nix-cache` through SSH (`nix.distributedBuilds`). + +## Binary cache signing keys (on nix-cache) + +```bash +sudo install -d -m 0700 /etc/nix +sudo nix-store --generate-binary-cache-key nix-cache-1 /etc/nix/cache-priv.pem /etc/nix/cache-pub.pem +sudo chmod 0600 /etc/nix/cache-priv.pem +sudo chmod 0644 /etc/nix/cache-pub.pem +cat /etc/nix/cache-pub.pem +``` + +Do not commit private keys. + +## Remote builder SSH keys + +On each client, install the private key used to authenticate as `nixremote`: + +```bash +sudo install -d -m 0700 /root/.ssh +sudo install -m 0600 ./nixremote /root/.ssh/nixremote +sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version +``` + +On `nix-cache`, install the matching public key used by `nixremote` authorized keys. + +## Manual verification + +After deployment: + +```bash +curl http://nix-cache/nix-cache-info +nix store ping --store http://nix-cache +nix show-config | grep -E 'substituters|trusted-public-keys|builders-use-substitutes' +sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version +nix build nixpkgs#hello --builders 'ssh://nixremote@nix-cache x86_64-linux /root/.ssh/nixremote 4 2 big-parallel,kvm,nixos-test,benchmark' -L +nix path-info -r nixpkgs#hello +curl -I "http://nix-cache/$(basename "$(nix path-info nixpkgs#hello)").narinfo" +``` diff --git a/flake.nix b/flake.nix index 7f90c35..72a231d 100644 --- a/flake.nix +++ b/flake.nix @@ -1,11 +1,7 @@ { description = "LAN NixOS configs"; - nixConfig = { - access-tokens = [ - "github.com=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" - ]; - }; + # GitHub tokens must be provided outside this repository (local nix.conf, env, or deployment secrets). inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; nixos-conf-editor.url = "github:snowfallorg/nixos-conf-editor"; diff --git a/hosts/docker/configuration.nix b/hosts/docker/configuration.nix index 631df71..e4f0452 100644 --- a/hosts/docker/configuration.nix +++ b/hosts/docker/configuration.nix @@ -15,6 +15,8 @@ in imports = [ # Include the results of the hardware scan. ../../common/configuration.nix + ../../modules/nix/cache-client.nix + ../../modules/nix/remote-builder-client.nix ]; networking.hostName = "docker"; # Define your hostname. diff --git a/hosts/kuma/configuration.nix b/hosts/kuma/configuration.nix index 2bf73d7..0cddbe2 100644 --- a/hosts/kuma/configuration.nix +++ b/hosts/kuma/configuration.nix @@ -15,6 +15,8 @@ in imports = [ # Include the results of the hardware scan. ../../common/configuration.nix + ../../modules/nix/cache-client.nix + ../../modules/nix/remote-builder-client.nix ]; networking.hostName = "kuma"; # Define your hostname. diff --git a/hosts/nix-cache/configuration.nix b/hosts/nix-cache/configuration.nix index d845094..c7d0e04 100644 --- a/hosts/nix-cache/configuration.nix +++ b/hosts/nix-cache/configuration.nix @@ -8,26 +8,16 @@ imports = [ # Include the results of the hardware scan. ../../common/configuration.nix + ../../modules/nix/cache-server.nix ]; networking.hostName = "nix-cache"; # Define your hostname. - services.nix-serve.enable = true; - services.nix-serve.secretKeyFile = "/etc/nix/cache-priv.pem"; - - services.nginx = { - enable = true; - recommendedProxySettings = true; - virtualHosts."cache.local" = { - locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}"; - }; - }; services.prometheus.exporters.node = { enable = true; openFirewall = true; }; - networking.firewall.allowedTCPPorts = [ config.services.nginx.defaultHTTPListenPort ]; # Open ports in the firewall. # networking.firewall.allowedTCPPorts = [ 80 8080 443 ]; # networking.firewall.allowedUDPPorts = [ ... ]; diff --git a/hosts/nix-minimal/configuration.nix b/hosts/nix-minimal/configuration.nix index d963a3d..1dc9f8e 100644 --- a/hosts/nix-minimal/configuration.nix +++ b/hosts/nix-minimal/configuration.nix @@ -8,6 +8,8 @@ imports = [ # Include the results of the hardware scan. ../../common/configuration.nix + ../../modules/nix/cache-client.nix + ../../modules/nix/remote-builder-client.nix ]; networking.hostName = "nix-minimal"; # Define your hostname. diff --git a/hosts/nixos/configuration.nix b/hosts/nixos/configuration.nix index e7f1d15..d51f61c 100644 --- a/hosts/nixos/configuration.nix +++ b/hosts/nixos/configuration.nix @@ -28,6 +28,8 @@ in { imports = [ # Include the results of the hardware scan. ../../common/configuration.nix + ../../modules/nix/cache-client.nix + ../../modules/nix/remote-builder-client.nix ]; # Bootloader. @@ -94,8 +96,6 @@ in { system.stateVersion = "25.05"; # Did you read the comment? - - nix.settings.experimental-features = "nix-command flakes"; services.xrdp.enable = true; services.xrdp.defaultWindowManager = "cinnamon-session"; services.xrdp.openFirewall = true; diff --git a/hosts/server/configuration.nix b/hosts/server/configuration.nix index 9a2a8f6..1b08c25 100644 --- a/hosts/server/configuration.nix +++ b/hosts/server/configuration.nix @@ -8,6 +8,8 @@ imports = [ # Include the results of the hardware scan. ../../common/configuration.nix + ../../modules/nix/cache-client.nix + ../../modules/nix/remote-builder-client.nix ]; networking.hostName = "server"; # Define your hostname. diff --git a/modules/nix/cache-client.nix b/modules/nix/cache-client.nix new file mode 100644 index 0000000..ab60f0c --- /dev/null +++ b/modules/nix/cache-client.nix @@ -0,0 +1,15 @@ +{ ... }: + +{ + nix.settings = { + substituters = [ + "http://nix-cache" + "https://cache.nixos.org/" + ]; + trusted-public-keys = [ + "cache.local-1:usoWYanY3Kpq2+kDIS2nhWoLZiRxanmdysdzqCFBHW4=" + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + ]; + auto-optimise-store = true; + }; +} diff --git a/modules/nix/cache-server.nix b/modules/nix/cache-server.nix new file mode 100644 index 0000000..40a1d2c --- /dev/null +++ b/modules/nix/cache-server.nix @@ -0,0 +1,58 @@ +{ config, pkgs, ... }: + +{ + # Generate the binary cache key pair on the nix-cache host: + # sudo install -d -m 0700 /etc/nix + # sudo nix-store --generate-binary-cache-key nix-cache-1 \ + # /etc/nix/cache-priv.pem \ + # /etc/nix/cache-pub.pem + # sudo chmod 0600 /etc/nix/cache-priv.pem + # sudo chmod 0644 /etc/nix/cache-pub.pem + # cat /etc/nix/cache-pub.pem + services.nix-serve = { + enable = true; + secretKeyFile = "/etc/nix/cache-priv.pem"; + }; + + services.nginx = { + enable = true; + recommendedProxySettings = true; + virtualHosts."nix-cache" = { + locations."/" = { + proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}"; + }; + }; + }; + + networking.firewall.allowedTCPPorts = [ 80 ]; + + users.groups.nixremote = {}; + + users.users.nixremote = { + isSystemUser = true; + group = "nixremote"; + createHome = true; + home = "/var/lib/nixremote"; + shell = pkgs.bashInteractive; + # Provide remote builder public keys here (safe to commit public keys only): + # openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAA... client@host" ]; + # + # Avoid absolute keyFiles paths here because they break pure flake evaluation. + openssh.authorizedKeys.keys = [ ]; + }; + + services.openssh.enable = true; + + nix.settings = { + trusted-users = [ "root" "nixremote" ]; + experimental-features = [ "nix-command" "flakes" ]; + auto-optimise-store = true; + builders-use-substitutes = true; + }; + + nix.gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 30d"; + }; +} diff --git a/modules/nix/remote-builder-client.nix b/modules/nix/remote-builder-client.nix new file mode 100644 index 0000000..1adad42 --- /dev/null +++ b/modules/nix/remote-builder-client.nix @@ -0,0 +1,26 @@ +{ pkgs, ... }: + +{ + # Install the remote builder key on each client host (do not commit private keys): + # sudo install -d -m 0700 /root/.ssh + # sudo install -m 0600 ./nixremote /root/.ssh/nixremote + # sudo ssh -i /root/.ssh/nixremote nixremote@nix-cache nix-store --version + nix.distributedBuilds = true; + + nix.buildMachines = [ + { + hostName = "nix-cache"; + sshUser = "nixremote"; + sshKey = "/root/.ssh/nixremote"; + system = pkgs.stdenv.hostPlatform.system; + maxJobs = 4; + speedFactor = 2; + supportedFeatures = [ "nixos-test" "benchmark" "big-parallel" "kvm" ]; + } + ]; + + nix.settings = { + builders-use-substitutes = true; + max-jobs = "auto"; + }; +} diff --git a/scripts/codex-maintenance.sh b/scripts/codex-maintenance.sh old mode 100644 new mode 100755 index 843eb38..5c4c2bf --- a/scripts/codex-maintenance.sh +++ b/scripts/codex-maintenance.sh @@ -9,7 +9,22 @@ warn-dirty = false MODE="${1:-validate}" -hosts_json="$(nix eval --json --no-accept-flake-config .#nixosConfigurations --apply builtins.attrNames)" +ensure_nix_profile() { + if [ -f /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh ]; then + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh + elif [ -f "$HOME/.nix-profile/etc/profile.d/nix.sh" ]; then + . "$HOME/.nix-profile/etc/profile.d/nix.sh" + fi +} + +ensure_nix_profile + +if ! command -v nix >/dev/null 2>&1; then + echo "ERROR: nix is not available in PATH. Run bash scripts/codex-setup.sh first." >&2 + exit 127 +fi + +hosts_json="$(nix eval --json --no-use-registries --no-accept-flake-config .#nixosConfigurations --apply builtins.attrNames)" hosts="$(echo "$hosts_json" | jq -r '.[]')" echo "Hosts:" @@ -29,17 +44,17 @@ fi echo echo "Checking Nix formatting with nixpkgs-fmt..." -nix run --no-accept-flake-config nixpkgs#nixpkgs-fmt -- --check . +nix run --no-use-registries --no-accept-flake-config github:NixOS/nixpkgs/nixos-25.11#nixpkgs-fmt -- --check . echo echo "Running statix lint..." -nix run --no-accept-flake-config nixpkgs#statix -- check . +nix run --no-use-registries --no-accept-flake-config github:NixOS/nixpkgs/nixos-25.11#statix -- check . echo echo "Evaluating host toplevel derivations..." for host in $hosts; do echo "==> $host" - nix eval --raw --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath" + nix eval --raw --no-use-registries --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath" done if [[ "$MODE" == "dry-run" ]]; then @@ -47,7 +62,7 @@ if [[ "$MODE" == "dry-run" ]]; then echo "Running dry-run builds for all hosts. This will not create result symlinks." for host in $hosts; do echo "==> Dry-run build: $host" - nix build --dry-run --no-link --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel" + nix build --dry-run --no-link --no-use-registries --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel" done fi diff --git a/scripts/codex-setup.sh b/scripts/codex-setup.sh old mode 100644 new mode 100755 index ad78490..a0e4e20 --- a/scripts/codex-setup.sh +++ b/scripts/codex-setup.sh @@ -76,13 +76,13 @@ if ! command -v jq >/dev/null 2>&1; then fi echo "Available NixOS hosts:" -hosts="$(nix eval --json --no-accept-flake-config .#nixosConfigurations --apply builtins.attrNames | jq -r '.[]')" +hosts="$(nix eval --json --no-use-registries --no-accept-flake-config .#nixosConfigurations --apply builtins.attrNames | jq -r '.[]')" echo "$hosts" echo "Evaluating all host toplevel derivations..." for host in $hosts; do echo "==> Evaluating $host" - nix eval --raw --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath" + nix eval --raw --no-use-registries --no-accept-flake-config ".#nixosConfigurations.${host}.config.system.build.toplevel.drvPath" done echo "Codex setup complete."