moons-14/dotfiles
1
0
Fork 0
mirror of https://github.com/moons-14/dotfiles.git synced 2026-10-06 06:08:11 +09:00
Code Issues Actions 1 Packages Projects Releases 4 Wiki Activity

Add central Nix builder and binary cache

Browse Source
This commit is contained in:
moons-14
2026-08-31 08:07:48 +09:00
parent fda29cd08d
commit ec3770502d
21 changed files with 438 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AGENTS.md
+2
View File
@@ -357,6 +357,7 @@ modules/profiles/
│ ├── labwc/
│ └── niri/
├── networking/
│ ├── homelab-cache-client/
│ ├── tailscale-client/
│ └── tailscale-subnet-router/
├── platform/
@@ -371,6 +372,7 @@ modules/profiles/
│ ├── development/
│ ├── game/
│ ├── machine-learning/
│ ├── nix-builder/
│ ├── personal/
│ ├── server/
│ └── remote-access/
flake.lock
Generated
+106 -34
View File
@@ -268,6 +268,28 @@
"type": "github"
}
},
"deploy-rs": {
"inputs": {
"flake-compat": "flake-compat_2",
"nixpkgs": [
"nixpkgs"
],
"utils": "utils"
},
"locked": {
"lastModified": 1786361680,
"narHash": "sha256-IxaZkb9rCGEZ+yGndxKXONeIEcKMzoFUsvLTB5G/caw=",
"owner": "serokell",
"repo": "deploy-rs",
"rev": "16901271e5b30b591e56f7a84f25f186fb20f3e1",
"type": "github"
},
"original": {
"owner": "serokell",
"repo": "deploy-rs",
"type": "github"
}
},
"disko": {
"inputs": {
"nixpkgs": [
@@ -323,11 +345,11 @@
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1767039857,
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
"lastModified": 1733328505,
"narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
"rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
"type": "github"
},
"original": {
@@ -341,13 +363,13 @@
"locked": {
"lastModified": 1767039857,
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
"owner": "NixOS",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
"type": "github"
},
"original": {
"owner": "NixOS",
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
@@ -369,6 +391,22 @@
}
},
"flake-compat_5": {
"flake": false,
"locked": {
"lastModified": 1767039857,
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
"owner": "NixOS",
"repo": "flake-compat",
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_6": {
"flake": false,
"locked": {
"lastModified": 1767039857,
@@ -384,7 +422,7 @@
"type": "github"
}
},
"flake-compat_6": {
"flake-compat_7": {
"flake": false,
"locked": {
"lastModified": 1767039857,
@@ -579,10 +617,10 @@
},
"ghostty": {
"inputs": {
"flake-compat": "flake-compat_2",
"flake-compat": "flake-compat_3",
"home-manager": "home-manager_2",
"nixpkgs": "nixpkgs_4",
"systems": "systems_4",
"systems": "systems_5",
"zig": "zig",
"zon2nix": "zon2nix"
},
@@ -625,7 +663,7 @@
},
"git-hooks-nix": {
"inputs": {
"flake-compat": "flake-compat_3",
"flake-compat": "flake-compat_4",
"nixpkgs": "nixpkgs_5"
},
"locked": {
@@ -791,7 +829,7 @@
"bun2nix": "bun2nix_2",
"flake-parts": "flake-parts_4",
"nixpkgs": "nixpkgs_6",
"systems": "systems_5",
"systems": "systems_6",
"treefmt-nix": "treefmt-nix_3"
},
"locked": {
@@ -996,7 +1034,7 @@
},
"nixos-wsl": {
"inputs": {
"flake-compat": "flake-compat_5",
"flake-compat": "flake-compat_6",
"nixpkgs": "nixpkgs_10"
},
"locked": {
@@ -1312,7 +1350,7 @@
"nixpkgs": [
"nixpkgs"
],
"systems": "systems_6"
"systems": "systems_7"
},
"locked": {
"lastModified": 1787536726,
@@ -1397,7 +1435,7 @@
},
"pre-commit": {
"inputs": {
"flake-compat": "flake-compat_4",
"flake-compat": "flake-compat_5",
"nixpkgs": [
"lanzaboote",
"nixpkgs"
@@ -1422,6 +1460,7 @@
"browser-previews": "browser-previews",
"codex-desktop-linux": "codex-desktop-linux",
"codex-session-usage": "codex-session-usage",
"deploy-rs": "deploy-rs",
"disko": "disko",
"flake-parts": "flake-parts_3",
"ghostty": "ghostty",
@@ -1444,7 +1483,7 @@
"skills": "skills",
"sops-nix": "sops-nix",
"stylix": "stylix",
"systems": "systems_8",
"systems": "systems_9",
"treefmt-nix": "treefmt-nix_4",
"vicinae": "vicinae",
"vicinae-extensions": "vicinae-extensions"
@@ -1579,7 +1618,7 @@
"nixpkgs"
],
"nur": "nur",
"systems": "systems_7",
"systems": "systems_8",
"tinted-kitty": "tinted-kitty",
"tinted-schemes": "tinted-schemes",
"tinted-tmux": "tinted-tmux",
@@ -1629,6 +1668,21 @@
"type": "github"
}
},
"systems_11": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"systems_2": {
"locked": {
"lastModified": 1681028828,
@@ -1660,7 +1714,6 @@
}
},
"systems_4": {
"flake": false,
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
@@ -1676,6 +1729,7 @@
}
},
"systems_5": {
"flake": false,
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
@@ -1706,6 +1760,21 @@
}
},
"systems_7": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"systems_8": {
"locked": {
"lastModified": 1774449309,
"narHash": "sha256-brhZ8DmuGtzkCYHJg4HEd602amKm89Y9ytsFZ5uWD1w=",
@@ -1721,21 +1790,6 @@
"type": "github"
}
},
"systems_8": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"systems_9": {
"locked": {
"lastModified": 1681028828,
@@ -1899,12 +1953,30 @@
"type": "github"
}
},
"utils": {
"inputs": {
"systems": "systems_4"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"vicinae": {
"inputs": {
"nixpkgs": "nixpkgs_14",
"numen": "numen",
"soulver-cpp": "soulver-cpp",
"systems": "systems_9"
"systems": "systems_10"
},
"locked": {
"lastModified": 1787956866,
@@ -1922,11 +1994,11 @@
},
"vicinae-extensions": {
"inputs": {
"flake-compat": "flake-compat_6",
"flake-compat": "flake-compat_7",
"nixpkgs": [
"nixpkgs"
],
"systems": "systems_10",
"systems": "systems_11",
"vicinae": "vicinae_2"
},
"locked": {
flake.nix
+5
View File
@@ -60,6 +60,11 @@
inputs.nixpkgs.follows = "nixpkgs";
};
deploy-rs = {
url = "github:serokell/deploy-rs";
inputs.nixpkgs.follows = "nixpkgs";
};
# Disk management
disko = {
url = "github:nix-community/disko";
flake/default.nix
+1
View File
@@ -1,5 +1,6 @@
{
imports = [
./deploy.nix
./formatter.nix
./git-hooks.nix
./registry.nix
flake/deploy.nix
+28
View File
@@ -0,0 +1,28 @@
{
inputs,
self,
...
}:
{
flake.deploy = {
nodes.nix-builder = {
hostname = "nix-builder";
sshUser = "moons";
user = "root";
interactiveSudo = true;
remoteBuild = true;
autoRollback = true;
magicRollback = true;
profiles.system.path = inputs.deploy-rs.lib.x86_64-linux.activate.nixos self.nixosConfigurations.nix-builder;
};
};
perSystem =
{ system, ... }:
{
apps.deploy = inputs.deploy-rs.apps.${system}.default;
checks = inputs.deploy-rs.lib.${system}.deployChecks self.deploy;
};
}
hosts/default.nix
+2
View File
@@ -8,7 +8,9 @@
profiles = [
"base"
"interface.cli"
"networking.tailscale-client"
"platform.vm"
"workload.nix-builder"
"workload.remote-access"
];
};
hosts/nix-builder/README.md
+101
View File
@@ -0,0 +1,101 @@
# nix-builder bootstrap
The host configuration can be built before its cache signing secret exists.
Harmonia's socket remains stopped until SOPS installs the signing key at
`/run/secrets/harmonia/signing-key`.
## Proxmox storage layout
The host configuration expects three filesystems. Keep the build scratch space
separate from the store so a large build cannot fill the root filesystem.
| Mount point | Suggested size | Contents |
| -------------------- | -------------- | ------------------------------- |
| `/` | 48 GiB | NixOS and mutable system state |
| `/var/lib/nix-build` | 192 GiB | Disposable build scratch space |
| `/nix/store` | 1 TiB | Fleet closures and binary cache |
The build-server policy starts emergency store GC below 64 GiB free and aims
for 128 GiB free. Persistent roots under `/var/lib/nix-fleet/roots` protect the
latest fleet builds from that GC. It also limits Nix to two concurrent
derivations while allowing each derivation to use every vCPU assigned to the
VM.
For the two dedicated ext4 data filesystems, remove the default root-reserved
blocks once after formatting; keep the root filesystem's reserve intact:
```bash
sudo tune2fs -m 0 /dev/disk/by-label/nix-build
sudo tune2fs -m 0 /dev/disk/by-label/nix-store
```
## Initial deployment
Once the VM is reachable as `moons@nix-builder`, deploy it from the repository:
```bash
nix run .#deploy -- .#nix-builder
```
deploy-rs uses the target's `ssh-ng` store, so the system closure is built on
the builder rather than copied from the laptop. Automatic and magic rollback
remain enabled.
## Add the host SOPS recipient
After the VM has a stable SSH host key, derive its age recipient:
```bash
ssh-keyscan -t ed25519 nix-builder 2>/dev/null | ssh-to-age
```
Add the recipient to `.sops.yaml` and add a creation rule for
`secrets/hosts/nix-builder/*.yaml`. The admin YubiKey recipient should remain in
the same key group for recovery.
## Generate the cache signing key
Run this on a trusted Nix machine, preferably with the temporary files on a
tmpfs:
```bash
nix-store --generate-binary-cache-key \
cache.app.homelabs.run-1 \
harmonia.private \
harmonia.public
```
Create `secrets/hosts/nix-builder/system.yaml` with SOPS and store the complete
contents of `harmonia.private` at `harmonia.signing-key`:
```yaml
harmonia:
signing-key: cache.app.homelabs.run-1:REDACTED
```
Copy the complete contents of `harmonia.public` to
`modules/systems/nix/homelab-cache/public-key`. The private plaintext file must
not be committed or retained.
After committing both encrypted/public files, select
`networking.homelab-cache-client` on each client host.
Redeploy the builder and verify the cache after installing the secret:
```bash
nix run .#deploy -- .#nix-builder
curl --fail http://nix-builder:5000/nix-cache-info
```
## Normal operation
Run `fleet-build` on the builder to build and root every NixOS host, or pass a
list of host names to build only those hosts. Run `fleet-deploy` with the normal
deploy-rs target syntax when additional fleet nodes have been added to
`flake/deploy.nix`:
```bash
fleet-build
fleet-build x1g13 galleria
fleet-deploy .#nix-builder
```
hosts/nix-builder/nixos.nix
+13
View File
@@ -1,6 +1,19 @@
{ lib, ... }:
let
hostSecrets = ../../secrets/hosts/nix-builder/system.yaml;
in
{
imports = [
./filesystem.nix
./hardware-configuration.nix
];
sops.secrets = lib.mkIf (builtins.pathExists hostSecrets) {
"harmonia/signing-key" = {
sopsFile = hostSecrets;
restartUnits = [ "harmonia.service" ];
};
};
networking.firewall.interfaces."tailscale0".allowedTCPPorts = [ 5000 ];
}
modules/applications/nix-fleet/meta.nix
+3
View File
@@ -0,0 +1,3 @@
{
description = "Fleet build and deploy command-line tools";
}
modules/applications/nix-fleet/nixos.nix
+63
View File
@@ -0,0 +1,63 @@
{
inputs,
pkgs,
primaryUser,
...
}:
let
system = pkgs.stdenv.hostPlatform.system;
deployRs = inputs.deploy-rs.packages.${system}.default;
fleetBuild = pkgs.writeShellApplication {
name = "fleet-build";
runtimeInputs = [
pkgs.jq
pkgs.nix
];
text = ''
flake_ref="''${FLAKE:-/home/${primaryUser}/dotfiles}"
if (( $# == 0 )); then
# Keep this pipeline inside command substitution so pipefail and
# writeShellApplication's errexit propagate evaluation failures.
host_lines="$(
nix eval --json "$flake_ref#nixosConfigurations" \
--apply 'configs: builtins.attrNames configs' |
jq -r '.[] | select(. != "installer")'
)"
if [[ -z "$host_lines" ]]; then
echo "No deployable NixOS hosts found in $flake_ref" >&2
exit 1
fi
mapfile -t hosts <<< "$host_lines"
else
hosts=("$@")
fi
for host in "''${hosts[@]}"; do
nix build \
--out-link "/var/lib/nix-fleet/roots/build/$host" \
"$flake_ref#nixosConfigurations.$host.config.system.build.toplevel"
done
'';
};
fleetDeploy = pkgs.writeShellApplication {
name = "fleet-deploy";
runtimeInputs = [ deployRs ];
text = ''
cd "''${FLAKE:-/home/${primaryUser}/dotfiles}" || exit 1
exec deploy \
--keep-result \
--result-path /var/lib/nix-fleet/roots/deploy \
"$@"
'';
};
in
{
environment.systemPackages = [
fleetBuild
fleetDeploy
];
}
modules/profiles/README.md
+10
View File
@@ -39,10 +39,12 @@ required on every supported host.
| `workload.development` | NixOS, macOS with Home Manager |
| `workload.game` | NixOS, macOS |
| `workload.machine-learning` | NixOS with Home Manager |
| `workload.nix-builder` | NixOS central build and binary-cache VM |
| `workload.personal` | NixOS, macOS with Home Manager |
| `workload.remote-access` | NixOS, macOS |
| `workload.camera` | NixOS |
| `workload.server` | NixOS, macOS with Home Manager |
| `networking.homelab-cache-client` | NixOS, macOS with access to nix-builder |
| `networking.tailscale-client` | NixOS, macOS |
| `networking.tailscale-subnet-router` | NixOS |
| `security.fingerprint` | NixOS, macOS |
@@ -63,6 +65,14 @@ support does not implicitly select an interface or workload.
`workload.machine-learning` provides the Hugging Face Hub CLI for hosts used
to download and publish machine learning models and datasets.
`workload.nix-builder` provides the central build policy, persistent fleet GC
roots, deploy-rs tooling, SOPS integration, and Harmonia binary cache. Network
reachability and remote shell access remain independent host selections.
`networking.homelab-cache-client` adds the internal Harmonia substituter and
its trusted public key. It requires the public key generated during
`hosts/nix-builder/README.md` bootstrap.
`workload.personal` provides Pear Desktop on both NixOS and macOS. Home Manager
enables performance improvements, synced lyrics, tracker blocking, the album
color theme, and custom output-device selection while preserving user-owned
modules/profiles/networking/homelab-cache-client/meta.nix
+5
View File
@@ -0,0 +1,5 @@
{
description = "Use the homelab Harmonia binary cache";
includes = [ "systems.nix.homelab-cache" ];
}
modules/profiles/security/secrets/nixos.nix
+6
View File
@@ -0,0 +1,6 @@
{
sops.secrets."users/moons/hashedPassword" = {
sopsFile = ../../../secrets/common/system.yaml;
neededForUsers = true;
};
}
modules/profiles/workload/nix-builder/meta.nix
+10
View File
@@ -0,0 +1,10 @@
{
description = "Central Nix builder, deploy controller, and binary cache";
includes = [
"applications.nix-fleet"
"services.harmonia"
"systems.nix.build-server"
"systems.sops"
];
}
modules/services/harmonia/meta.nix
+3
View File
@@ -0,0 +1,3 @@
{
description = "Harmonia binary cache backed by the local Nix store";
}
modules/services/harmonia/nixos.nix
+19
View File
@@ -0,0 +1,19 @@
let
signingKeyPath = "/run/secrets/harmonia/signing-key";
in
{
services.harmonia.cache = {
enable = true;
signKeyPaths = [ signingKeyPath ];
settings = {
bind = "0.0.0.0:5000";
priority = 30;
};
};
# Keep activation usable while the host-specific SOPS secret is bootstrapped.
# Once the secret exists, starting the socket also starts Harmonia on demand.
systemd.sockets.harmonia.unitConfig.ConditionPathExists = signingKeyPath;
systemd.services.harmonia.unitConfig.ConditionPathExists = signingKeyPath;
}
modules/systems/nix/build-server/meta.nix
+3
View File
@@ -0,0 +1,3 @@
{
description = "Central Nix build server policy and persistent fleet roots";
}
modules/systems/nix/build-server/nixos.nix
+33
View File
@@ -0,0 +1,33 @@
{ primaryUser, ... }:
let
GiB = 1024 * 1024 * 1024;
in
{
nix = {
nrBuildUsers = 64;
settings = {
# Limit concurrent derivations so build scratch and memory usage remain
# bounded. Each derivation may still use every vCPU exposed to the VM.
max-jobs = 2;
cores = 0;
# Keep enough room for large desktop, browser, and CUDA closures.
min-free = 64 * GiB;
max-free = 128 * GiB;
};
};
systemd.services.nix-daemon.serviceConfig = {
MemoryAccounting = true;
MemoryMax = "90%";
OOMScoreAdjust = 500;
};
systemd.tmpfiles.rules = [
"d /var/lib/nix-fleet 0750 ${primaryUser} users - -"
"d /var/lib/nix-fleet/roots 0750 ${primaryUser} users - -"
"d /var/lib/nix-fleet/roots/build 0750 ${primaryUser} users - -"
"d /var/lib/nix-fleet/roots/deploy 0750 ${primaryUser} users - -"
];
}
modules/systems/nix/homelab-cache/common.nix
+22
View File
@@ -0,0 +1,22 @@
{ lib, ... }:
let
publicKeyFile = ./public-key;
hasPublicKey = builtins.pathExists publicKeyFile;
publicKey = if hasPublicKey then lib.removeSuffix "\n" (builtins.readFile publicKeyFile) else "";
in
{
assertions = [
{
assertion = hasPublicKey;
message = ''
systems.nix.homelab-cache requires
modules/systems/nix/homelab-cache/public-key
'';
}
];
nix.settings = lib.mkIf hasPublicKey {
extra-substituters = [ "http://nix-builder:5000" ];
extra-trusted-public-keys = [ publicKey ];
};
}
modules/systems/nix/homelab-cache/meta.nix
+3
View File
@@ -0,0 +1,3 @@
{
description = "Homelab Harmonia binary-cache client settings";
}
modules/systems/sops/nixos.nix
-5
View File
@@ -1,8 +1,3 @@
{
services.pcscd.enable = true;
sops.secrets."users/moons/hashedPassword" = {
sopsFile = ../../../secrets/common/system.yaml;
neededForUsers = true;
};
}