Add central Nix builder and binary cache

This commit is contained in:
2026-08-31 08:07:48 +09:00
parent fda29cd08d
commit ec3770502d
21 changed files with 438 additions and 39 deletions
+3
View File
@@ -0,0 +1,3 @@
{
description = "Fleet build and deploy command-line tools";
}
+63
View File
@@ -0,0 +1,63 @@
{
inputs,
pkgs,
primaryUser,
...
}:
let
system = pkgs.stdenv.hostPlatform.system;
deployRs = inputs.deploy-rs.packages.${system}.default;
fleetBuild = pkgs.writeShellApplication {
name = "fleet-build";
runtimeInputs = [
pkgs.jq
pkgs.nix
];
text = ''
flake_ref="''${FLAKE:-/home/${primaryUser}/dotfiles}"
if (( $# == 0 )); then
# Keep this pipeline inside command substitution so pipefail and
# writeShellApplication's errexit propagate evaluation failures.
host_lines="$(
nix eval --json "$flake_ref#nixosConfigurations" \
--apply 'configs: builtins.attrNames configs' |
jq -r '.[] | select(. != "installer")'
)"
if [[ -z "$host_lines" ]]; then
echo "No deployable NixOS hosts found in $flake_ref" >&2
exit 1
fi
mapfile -t hosts <<< "$host_lines"
else
hosts=("$@")
fi
for host in "''${hosts[@]}"; do
nix build \
--out-link "/var/lib/nix-fleet/roots/build/$host" \
"$flake_ref#nixosConfigurations.$host.config.system.build.toplevel"
done
'';
};
fleetDeploy = pkgs.writeShellApplication {
name = "fleet-deploy";
runtimeInputs = [ deployRs ];
text = ''
cd "''${FLAKE:-/home/${primaryUser}/dotfiles}" || exit 1
exec deploy \
--keep-result \
--result-path /var/lib/nix-fleet/roots/deploy \
"$@"
'';
};
in
{
environment.systemPackages = [
fleetBuild
fleetDeploy
];
}
+10
View File
@@ -39,10 +39,12 @@ required on every supported host.
| `workload.development` | NixOS, macOS with Home Manager |
| `workload.game` | NixOS, macOS |
| `workload.machine-learning` | NixOS with Home Manager |
| `workload.nix-builder` | NixOS central build and binary-cache VM |
| `workload.personal` | NixOS, macOS with Home Manager |
| `workload.remote-access` | NixOS, macOS |
| `workload.camera` | NixOS |
| `workload.server` | NixOS, macOS with Home Manager |
| `networking.homelab-cache-client` | NixOS, macOS with access to nix-builder |
| `networking.tailscale-client` | NixOS, macOS |
| `networking.tailscale-subnet-router` | NixOS |
| `security.fingerprint` | NixOS, macOS |
@@ -63,6 +65,14 @@ support does not implicitly select an interface or workload.
`workload.machine-learning` provides the Hugging Face Hub CLI for hosts used
to download and publish machine learning models and datasets.
`workload.nix-builder` provides the central build policy, persistent fleet GC
roots, deploy-rs tooling, SOPS integration, and Harmonia binary cache. Network
reachability and remote shell access remain independent host selections.
`networking.homelab-cache-client` adds the internal Harmonia substituter and
its trusted public key. It requires the public key generated during
`hosts/nix-builder/README.md` bootstrap.
`workload.personal` provides Pear Desktop on both NixOS and macOS. Home Manager
enables performance improvements, synced lyrics, tracker blocking, the album
color theme, and custom output-device selection while preserving user-owned
@@ -0,0 +1,5 @@
{
description = "Use the homelab Harmonia binary cache";
includes = [ "systems.nix.homelab-cache" ];
}
@@ -0,0 +1,6 @@
{
sops.secrets."users/moons/hashedPassword" = {
sopsFile = ../../../secrets/common/system.yaml;
neededForUsers = true;
};
}
@@ -0,0 +1,10 @@
{
description = "Central Nix builder, deploy controller, and binary cache";
includes = [
"applications.nix-fleet"
"services.harmonia"
"systems.nix.build-server"
"systems.sops"
];
}
+3
View File
@@ -0,0 +1,3 @@
{
description = "Harmonia binary cache backed by the local Nix store";
}
+19
View File
@@ -0,0 +1,19 @@
let
signingKeyPath = "/run/secrets/harmonia/signing-key";
in
{
services.harmonia.cache = {
enable = true;
signKeyPaths = [ signingKeyPath ];
settings = {
bind = "0.0.0.0:5000";
priority = 30;
};
};
# Keep activation usable while the host-specific SOPS secret is bootstrapped.
# Once the secret exists, starting the socket also starts Harmonia on demand.
systemd.sockets.harmonia.unitConfig.ConditionPathExists = signingKeyPath;
systemd.services.harmonia.unitConfig.ConditionPathExists = signingKeyPath;
}
@@ -0,0 +1,3 @@
{
description = "Central Nix build server policy and persistent fleet roots";
}
@@ -0,0 +1,33 @@
{ primaryUser, ... }:
let
GiB = 1024 * 1024 * 1024;
in
{
nix = {
nrBuildUsers = 64;
settings = {
# Limit concurrent derivations so build scratch and memory usage remain
# bounded. Each derivation may still use every vCPU exposed to the VM.
max-jobs = 2;
cores = 0;
# Keep enough room for large desktop, browser, and CUDA closures.
min-free = 64 * GiB;
max-free = 128 * GiB;
};
};
systemd.services.nix-daemon.serviceConfig = {
MemoryAccounting = true;
MemoryMax = "90%";
OOMScoreAdjust = 500;
};
systemd.tmpfiles.rules = [
"d /var/lib/nix-fleet 0750 ${primaryUser} users - -"
"d /var/lib/nix-fleet/roots 0750 ${primaryUser} users - -"
"d /var/lib/nix-fleet/roots/build 0750 ${primaryUser} users - -"
"d /var/lib/nix-fleet/roots/deploy 0750 ${primaryUser} users - -"
];
}
@@ -0,0 +1,22 @@
{ lib, ... }:
let
publicKeyFile = ./public-key;
hasPublicKey = builtins.pathExists publicKeyFile;
publicKey = if hasPublicKey then lib.removeSuffix "\n" (builtins.readFile publicKeyFile) else "";
in
{
assertions = [
{
assertion = hasPublicKey;
message = ''
systems.nix.homelab-cache requires
modules/systems/nix/homelab-cache/public-key
'';
}
];
nix.settings = lib.mkIf hasPublicKey {
extra-substituters = [ "http://nix-builder:5000" ];
extra-trusted-public-keys = [ publicKey ];
};
}
@@ -0,0 +1,3 @@
{
description = "Homelab Harmonia binary-cache client settings";
}
-5
View File
@@ -1,8 +1,3 @@
{
services.pcscd.enable = true;
sops.secrets."users/moons/hashedPassword" = {
sopsFile = ../../../secrets/common/system.yaml;
neededForUsers = true;
};
}