From 4ff5ae8883548329519ec2a3fb9835eb7258d4cb Mon Sep 17 00:00:00 2001 From: moons Date: Wed, 15 Jul 2026 14:38:59 +0900 Subject: [PATCH] feat --- .gitea/scripts/publish-nix-cache.sh | 772 ----------------------- .gitea/workflows/nix-cache-bootstrap.yml | 57 -- .gitea/workflows/nix-cache-update.yml | 60 -- docs/gitea-binary-cache.md | 118 ---- modules/features/dev/default.nix | 1 + modules/features/dev/drawio.nix | 20 + profiles/workloads/dev.nix | 1 + 7 files changed, 22 insertions(+), 1007 deletions(-) delete mode 100755 .gitea/scripts/publish-nix-cache.sh delete mode 100644 .gitea/workflows/nix-cache-bootstrap.yml delete mode 100644 .gitea/workflows/nix-cache-update.yml delete mode 100644 docs/gitea-binary-cache.md create mode 100644 modules/features/dev/drawio.nix diff --git a/.gitea/scripts/publish-nix-cache.sh b/.gitea/scripts/publish-nix-cache.sh deleted file mode 100755 index d16b240..0000000 --- a/.gitea/scripts/publish-nix-cache.sh +++ /dev/null @@ -1,772 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Gitea Releases are used as an append-only object store. The cache-latest -# release contains the HTTP binary-cache index, while generation releases -# contain immutable NAR payloads. - -mode=${CACHE_MODE:-} -server_url=${CACHE_SERVER_URL:-} -api_server_url=${CACHE_API_SERVER_URL:-} -repository=${CACHE_REPOSITORY:-} -commit=${CACHE_COMMIT:-} -ref_name=${CACHE_REF_NAME:-unknown} -index_tag=${CACHE_INDEX_TAG:-cache-latest} -generation_prefix=${CACHE_GENERATION_PREFIX:-nix-cache-generation-} -upload_jobs=${CACHE_UPLOAD_JOBS:-4} -max_upload_bytes=${CACHE_MAX_UPLOAD_BYTES:-} -key_file=${NIX_CACHE_KEY_FILE:-} -api_is_internal=0 -api_curl_options=() - -for command in curl jq nix awk comm sed find sort; do - if ! command -v "$command" >/dev/null 2>&1; then - echo "Required command is unavailable: $command" >&2 - exit 1 - fi -done - -if [[ $mode != bootstrap && $mode != update ]]; then - echo "CACHE_MODE must be either 'bootstrap' or 'update'." >&2 - exit 1 -fi - -if [[ -z $server_url || -z $repository || -z $commit ]]; then - echo "CACHE_SERVER_URL, CACHE_REPOSITORY, and CACHE_COMMIT are required." >&2 - exit 1 -fi - -if [[ -z ${GITEA_TOKEN:-} ]]; then - echo "GITEA_TOKEN is required." >&2 - exit 1 -fi - -if [[ ! -s $key_file ]]; then - echo "NIX_CACHE_KEY_FILE must point to a non-empty signing key." >&2 - exit 1 -fi - -if [[ ! $upload_jobs =~ ^[1-9][0-9]*$ ]]; then - echo "CACHE_UPLOAD_JOBS must be a positive integer." >&2 - exit 1 -fi - -if [[ -n $max_upload_bytes && ! $max_upload_bytes =~ ^[0-9]+$ ]]; then - echo "CACHE_MAX_UPLOAD_BYTES must be zero or a positive integer." >&2 - exit 1 -fi - -server_url=${server_url%/} -api_server_url=${api_server_url:-$server_url} -api_server_url=${api_server_url%/} -api_base="${api_server_url}/api/v1/repos/${repository}" -download_base="${server_url}/${repository}/releases/download" -cache_uri="${download_base}/${index_tag}" -manifest_url="${cache_uri}/cache-manifest.json" -public_key_url="${cache_uri}/cache-public-key" -public_key=$(nix key convert-secret-to-public <"$key_file") -key_name=${public_key%%:*} - -work_dir=$(mktemp -d "${RUNNER_TEMP:-/tmp}/nix-release-cache.XXXXXX") -cache_dir="${work_dir}/cache" -rewritten_dir="${work_dir}/narinfo" -manifest_file="${work_dir}/manifest.json" -all_releases_file="${work_dir}/all-releases.json" -generation_release_file="${work_dir}/generation-release.json" -object_updates_file="${work_dir}/object-updates.jsonl" -narinfo_updates_file="${work_dir}/narinfo-updates.jsonl" -skipped_updates_file="${work_dir}/skipped-updates.jsonl" -nar_upload_queue="${work_dir}/nar-upload-queue" -narinfo_upload_queue="${work_dir}/narinfo-upload-queue" -mkdir -p "$cache_dir" "$rewritten_dir" -: >"$object_updates_file" -: >"$narinfo_updates_file" -: >"$skipped_updates_file" -: >"$nar_upload_queue" -: >"$narinfo_upload_queue" -trap 'rm -rf "$work_dir"' EXIT - -api_request() { - local method=$1 - local path=$2 - shift 2 - - curl "${api_curl_options[@]}" --fail-with-body --silent --show-error \ - --retry 5 --retry-delay 2 --retry-all-errors \ - --request "$method" \ - --header "Authorization: token ${GITEA_TOKEN}" \ - --header "Accept: application/json" \ - "$@" \ - "${api_base}${path}" -} - -api_get_optional() { - local path=$1 - local output=$2 - local status - - status=$(curl "${api_curl_options[@]}" --silent --show-error \ - --retry 5 --retry-delay 2 --retry-all-errors \ - --output "$output" --write-out '%{http_code}' \ - --header "Authorization: token ${GITEA_TOKEN}" \ - --header "Accept: application/json" \ - "${api_base}${path}") - - case "$status" in - 200) - return 0 - ;; - 404) - rm -f "$output" - return 1 - ;; - *) - echo "Gitea API request failed with HTTP ${status}: ${path}" >&2 - cat "$output" >&2 - return 2 - ;; - esac -} - -create_release() { - local tag=$1 - local name=$2 - local body=$3 - local prerelease=$4 - - jq -n \ - --arg tag "$tag" \ - --arg name "$name" \ - --arg body "$body" \ - --arg target "$commit" \ - --argjson prerelease "$prerelease" \ - '{ - tag_name: $tag, - target_commitish: $target, - name: $name, - body: $body, - draft: false, - prerelease: $prerelease - }' | api_request POST /releases \ - --header 'Content-Type: application/json' \ - --data-binary @- -} - -delete_asset() { - local release_id=$1 - local asset_id=$2 - api_request DELETE "/releases/${release_id}/assets/${asset_id}" >/dev/null -} - -upload_asset() { - local release_id=$1 - local file=$2 - local name=$3 - local response_file - local status - local curl_status - local size - local size_mib - - response_file=$(mktemp "${work_dir}/upload-response.XXXXXX") - size=$(stat -c '%s' "$file") - size_mib=$(((size + 1048575) / 1048576)) - - if status=$(curl "${api_curl_options[@]}" --silent --show-error \ - --retry 5 --retry-delay 2 --retry-all-errors \ - --request POST \ - --header "Authorization: token ${GITEA_TOKEN}" \ - --form "attachment=@${file};type=application/octet-stream" \ - --output "$response_file" --write-out '%{http_code}' \ - "${api_base}/releases/${release_id}/assets?name=${name}"); then - curl_status=0 - else - curl_status=$? - fi - - if [[ $status == 200 || $status == 201 ]]; then - rm -f "$response_file" - return 0 - fi - - echo "Asset upload failed: name=${name} size=${size}B (${size_mib}MiB) HTTP=${status:-000} curl=${curl_status}" >&2 - if [[ $status == 413 ]]; then - echo "The upload endpoint rejected this NAR as too large. Set the Gitea Actions variable NIX_CACHE_API_SERVER_URL to an origin URL that bypasses Cloudflare, and verify the origin proxy and Gitea release size limits." >&2 - fi - sed -n '1,20p' "$response_file" >&2 - rm -f "$response_file" - return 1 -} - -upload_asset_response() { - local release_id=$1 - local file=$2 - local name=$3 - - curl "${api_curl_options[@]}" --fail-with-body --silent --show-error \ - --retry 5 --retry-delay 2 --retry-all-errors \ - --request POST \ - --header "Authorization: token ${GITEA_TOKEN}" \ - --form "attachment=@${file};type=application/octet-stream" \ - "${api_base}/releases/${release_id}/assets?name=${name}" -} - -rename_asset() { - local release_id=$1 - local asset_id=$2 - local name=$3 - - jq -n --arg name "$name" '{name: $name}' | api_request PATCH \ - "/releases/${release_id}/assets/${asset_id}" \ - --header 'Content-Type: application/json' \ - --data-binary @- >/dev/null -} - -upload_queue() { - local release_id=$1 - local queue_file=$2 - local file - local name - local pid - local failed=0 - local -a pids=() - - if [[ ! -s $queue_file ]]; then - return - fi - - while IFS=$'\t' read -r file name; do - upload_asset "$release_id" "$file" "$name" & - pids+=("$!") - - if ((${#pids[@]} == upload_jobs)); then - for pid in "${pids[@]}"; do - if ! wait "$pid"; then - failed=1 - fi - done - pids=() - fi - done <"$queue_file" - - for pid in "${pids[@]}"; do - if ! wait "$pid"; then - failed=1 - fi - done - if ((failed)); then - return 1 - fi -} - -list_all_releases() { - local page=1 - local page_file="${work_dir}/releases-page.json" - local releases_jsonl="${work_dir}/releases.jsonl" - local count - : >"$releases_jsonl" - - while :; do - api_request GET "/releases?draft=false&pre-release=true&limit=50&page=${page}" >"$page_file" - count=$(jq 'length' "$page_file") - if ((count == 0)); then - break - fi - jq -c '.[]' "$page_file" >>"$releases_jsonl" - ((page += 1)) - done - - jq -s '.' "$releases_jsonl" >"$all_releases_file" -} - -configure_api_transport() { - local candidate - local candidate_url - local public_authority - local version_json - - if [[ -n ${CACHE_API_SERVER_URL:-} ]]; then - api_is_internal=1 - echo "Using configured Gitea origin API: ${api_server_url}" - elif [[ $server_url == https://* ]]; then - public_authority=${server_url#https://} - public_authority=${public_authority%%/*} - - for candidate in host.containers.internal host.docker.internal; do - if version_json=$(curl --silent --show-error \ - --noproxy '*' --connect-timeout 3 --max-time 5 \ - --connect-to "${public_authority}:443:${candidate}:443" \ - "${server_url}/api/v1/version" 2>/dev/null) && - jq -e '.version | type == "string"' <<<"$version_json" >/dev/null; then - api_curl_options=( - --noproxy '*' - --connect-to "${public_authority}:443:${candidate}:443" - ) - api_is_internal=1 - echo "Using direct Gitea HTTPS transport through ${candidate}:443; Cloudflare is bypassed." - break - fi - done - - if ((api_is_internal == 0)); then - for candidate in host.containers.internal host.docker.internal; do - candidate_url="http://${candidate}:3000" - if version_json=$(curl --silent --show-error \ - --noproxy '*' --connect-timeout 3 --max-time 5 \ - "${candidate_url}/api/v1/version" 2>/dev/null) && - jq -e '.version | type == "string"' <<<"$version_json" >/dev/null; then - api_server_url=$candidate_url - api_base="${api_server_url}/api/v1/repos/${repository}" - api_curl_options=(--noproxy '*') - api_is_internal=1 - echo "Using direct Gitea API through ${candidate}:3000; Cloudflare is bypassed." - break - fi - done - fi - fi - - if [[ -z $max_upload_bytes ]]; then - if ((api_is_internal)); then - max_upload_bytes=0 - echo "No workflow-side NAR size limit is applied on the internal API transport." - else - max_upload_bytes=90000000 - echo "No internal Gitea API was reachable; using the public endpoint and limiting NAR uploads to ${max_upload_bytes} bytes." - fi - fi -} - -initialize_manifest() { - jq -n \ - --arg uri "$cache_uri" \ - --arg manifest "$manifest_url" \ - --arg public_key "$public_key" \ - --arg public_key_url "$public_key_url" \ - '{ - schemaVersion: 1, - cache: { - uri: $uri, - nixCacheInfo: ($uri + "/nix-cache-info"), - manifest: $manifest, - publicKey: $public_key, - publicKeyUrl: $public_key_url - }, - generatedAt: null, - generations: [], - objects: {}, - narinfos: {}, - skipped: {} - }' >"$manifest_file" -} - -configure_api_transport - -index_release_file="${work_dir}/index-release.json" -if api_get_optional "/releases/tags/${index_tag}" "$index_release_file"; then - if [[ $mode == bootstrap ]]; then - if jq -e '.assets[]? | select(.name == "cache-manifest.json")' \ - "$index_release_file" >/dev/null; then - echo "Release '${index_tag}' is already bootstrapped." >&2 - exit 1 - fi - echo "Resuming an interrupted cache bootstrap." - initialize_manifest - else - manifest_asset_url=$(jq -r ' - [ - .assets[]? - | select(.name == "cache-manifest.json") - ] - | last - | .browser_download_url // empty - ' "$index_release_file") - if [[ -z $manifest_asset_url ]]; then - manifest_asset_url=$(jq -r ' - [ - .assets[]? - | select(.name | test("^cache-manifest-[0-9a-f]+\\.json$")) - ] - | sort_by(.created_at) - | last - | .browser_download_url // empty - ' "$index_release_file") - if [[ -z $manifest_asset_url ]]; then - echo "The cache index has no recoverable manifest." >&2 - exit 1 - fi - echo "Recovering the cache index from a temporary manifest." - fi - - curl --fail-with-body --silent --show-error \ - --retry 5 --retry-delay 2 --retry-all-errors \ - --header "Authorization: token ${GITEA_TOKEN}" \ - --output "$manifest_file" \ - "$manifest_asset_url" - - if ! jq -e --arg public_key "$public_key" \ - '.schemaVersion == 1 and .cache.publicKey == $public_key' \ - "$manifest_file" >/dev/null; then - echo "The cache manifest is invalid or was signed by a different key." >&2 - exit 1 - fi - fi -else - optional_status=$? - if ((optional_status != 1)); then - exit "$optional_status" - fi - - if [[ $mode == update ]]; then - echo "Release '${index_tag}' is missing. Run the bootstrap workflow first." >&2 - exit 1 - fi - - create_release \ - "$index_tag" \ - "Nix binary cache index" \ - "Stable HTTP index for the release-backed Nix binary cache." \ - false >"$index_release_file" - initialize_manifest -fi - -index_release_id=$(jq -r '.id' "$index_release_file") -if [[ -z $index_release_id || $index_release_id == null ]]; then - echo "Could not determine the cache index release ID." >&2 - exit 1 -fi - -store_paths_before_file="${work_dir}/store-paths-before" -store_paths_after_file="${work_dir}/store-paths-after" -new_store_paths_file="${work_dir}/new-store-paths" -echo "Recording the Nix store state before evaluation and builds..." -nix path-info --all | sort -u >"$store_paths_before_file" - -echo "Evaluating NixOS hosts..." -hosts_file="${work_dir}/hosts" -nix eval --json '.#nixosConfigurations' \ - --apply 'configs: builtins.attrNames configs' | jq -r '.[]' >"$hosts_file" -mapfile -t hosts <"$hosts_file" - -if ((${#hosts[@]} == 0)); then - echo "No NixOS configurations were discovered." >&2 - exit 1 -fi - -echo "Building hosts: ${hosts[*]}" -roots_file="${work_dir}/roots" -: >"$roots_file" -failed_hosts=() -for host in "${hosts[@]}"; do - host_roots_file="${work_dir}/roots-${host}" - echo "Building host: ${host}" - if nix build --no-link --print-out-paths --print-build-logs \ - ".#nixosConfigurations.${host}.config.system.build.toplevel" | - sort -u >"$host_roots_file"; then - cat "$host_roots_file" >>"$roots_file" - else - echo "Host build failed; completed store paths will still be published: ${host}" >&2 - failed_hosts+=("$host") - fi -done -sort -u -o "$roots_file" "$roots_file" - -echo "Recording store paths completed during this job..." -nix path-info --all | sort -u >"$store_paths_after_file" -comm -13 "$store_paths_before_file" "$store_paths_after_file" >"$new_store_paths_file" - -export_paths_file="${work_dir}/export-paths" -cat "$roots_file" "$new_store_paths_file" | sort -u >"$export_paths_file" -if [[ ! -s $export_paths_file ]]; then - echo "No successfully completed store paths are available to publish." >&2 - exit 1 -fi - -successful_root_count=$(wc -l <"$roots_file") -new_store_path_count=$(wc -l <"$new_store_paths_file") -echo "Exporting ${successful_root_count} successful host roots and ${new_store_path_count} newly completed store paths..." -nix copy \ - --to "file://${cache_dir}?compression=zstd&compression-level=6&secret-key=${key_file}" \ - --stdin <"$export_paths_file" - -first_narinfo=$(find "$cache_dir" -maxdepth 1 -type f -name '*.narinfo' -print -quit) -if [[ -z $first_narinfo ]] || ! grep -Fq "Sig: ${key_name}:" "$first_narinfo"; then - echo "Generated narinfo files do not contain the expected cache signature." >&2 - exit 1 -fi - -list_all_releases - -# Recover immutable NAR objects left by an interrupted older run. A NAR asset's -# content-addressed filename is globally unique, so it can be reused safely. -discovered_objects_file="${work_dir}/discovered-objects.json" -jq --arg prefix "$generation_prefix" ' - reduce ( - .[] - | select(.tag_name | startswith($prefix)) as $release - | $release.assets[]? - | select(.name | test("\\.nar\\.(zst|xz|bz2|gz)$")) - | { - key: .name, - value: { - url: .browser_download_url, - generation: $release.tag_name, - size: .size - } - } - ) as $object ({}; .[$object.key] //= $object.value) -' "$all_releases_file" >"$discovered_objects_file" - -jq --slurpfile discovered "$discovered_objects_file" \ - '.objects = ($discovered[0] + .objects)' \ - "$manifest_file" >"${manifest_file}.new" -mv "${manifest_file}.new" "$manifest_file" - -generation_tag="${generation_prefix}${commit}" -if api_get_optional "/releases/tags/${generation_tag}" "$generation_release_file"; then - echo "Resuming generation release '${generation_tag}'." -else - optional_status=$? - if ((optional_status != 1)); then - exit "$optional_status" - fi - - create_release \ - "$generation_tag" \ - "Nix cache ${commit:0:12}" \ - "Branch: ${ref_name}\nCommit: ${commit}\nMode: ${mode}\nFailed hosts: ${failed_hosts[*]:-none}" \ - true >"$generation_release_file" -fi -generation_release_id=$(jq -r '.id' "$generation_release_file") - -declare -A known_narinfos=() -declare -A object_urls=() -declare -A object_sizes=() -declare -A queued_objects=() -declare -A index_asset_ids=() - -while IFS= read -r hash; do - known_narinfos["$hash"]=1 -done < <(jq -r '.narinfos | keys[]' "$manifest_file") - -while IFS=$'\t' read -r name url; do - object_urls["$name"]=$url -done < <( - jq -r '.objects | to_entries[] | [.key, .value.url] | @tsv' \ - "$manifest_file" -) - -while IFS=$'\t' read -r name id; do - index_asset_ids["$name"]=$id -done < <(jq -r '.assets[]? | [.name, (.id | tostring)] | @tsv' "$index_release_file") - -new_nar_count=0 -new_narinfo_count=0 -skipped_path_count=0 -while IFS= read -r -d '' narinfo_file; do - narinfo_name=$(basename "$narinfo_file") - store_hash=${narinfo_name%.narinfo} - - if [[ -n ${known_narinfos[$store_hash]:-} ]]; then - continue - fi - - nar_relative=$(sed -n 's/^URL: //p' "$narinfo_file") - store_path=$(sed -n 's/^StorePath: //p' "$narinfo_file") - if [[ $nar_relative != nar/* || -z $store_path ]]; then - echo "Malformed narinfo file: ${narinfo_file}" >&2 - exit 1 - fi - - nar_name=${nar_relative#nar/} - nar_file="${cache_dir}/${nar_relative}" - if [[ ! -f $nar_file ]]; then - echo "NAR payload is missing: ${nar_file}" >&2 - exit 1 - fi - - if [[ -z ${object_urls[$nar_name]:-} ]]; then - object_sizes["$nar_name"]=$(stat -c '%s' "$nar_file") - nar_size=${object_sizes[$nar_name]} - - if ((max_upload_bytes > 0 && nar_size > max_upload_bytes)); then - echo "Skipping oversized NAR: storePath=${store_path} name=${nar_name} size=${nar_size}B limit=${max_upload_bytes}B" >&2 - jq -cn \ - --arg key "$store_hash" \ - --arg store_path "$store_path" \ - --arg nar "$nar_name" \ - --arg reason "upload-size-limit" \ - --argjson size "$nar_size" \ - --argjson limit "$max_upload_bytes" \ - '{ - key: $key, - value: { - storePath: $store_path, - nar: $nar, - size: $size, - limit: $limit, - reason: $reason - } - }' >>"$skipped_updates_file" - ((skipped_path_count += 1)) - continue - fi - - object_urls["$nar_name"]="${download_base}/${generation_tag}/${nar_name}" - - if [[ -z ${queued_objects[$nar_name]:-} ]]; then - printf '%s\t%s\n' "$nar_file" "$nar_name" >>"$nar_upload_queue" - queued_objects["$nar_name"]=1 - ((new_nar_count += 1)) - fi - - jq -cn \ - --arg key "$nar_name" \ - --arg url "${object_urls[$nar_name]}" \ - --arg generation "$generation_tag" \ - --argjson size "${object_sizes[$nar_name]}" \ - '{key: $key, value: {url: $url, generation: $generation, size: $size}}' \ - >>"$object_updates_file" - fi - - rewritten_file="${rewritten_dir}/${narinfo_name}" - awk -v url="${object_urls[$nar_name]}" ' - BEGIN { replaced = 0 } - /^URL: / { - print "URL: " url - replaced = 1 - next - } - { print } - END { if (!replaced) exit 1 } - ' "$narinfo_file" >"$rewritten_file" - - if [[ -n ${index_asset_ids[$narinfo_name]:-} ]]; then - delete_asset "$index_release_id" "${index_asset_ids[$narinfo_name]}" - fi - printf '%s\t%s\n' "$rewritten_file" "$narinfo_name" >>"$narinfo_upload_queue" - - jq -cn \ - --arg key "$store_hash" \ - --arg url "${cache_uri}/${narinfo_name}" \ - --arg store_path "$store_path" \ - --arg nar "$nar_name" \ - '{key: $key, value: {url: $url, storePath: $store_path, nar: $nar}}' \ - >>"$narinfo_updates_file" - ((new_narinfo_count += 1)) -done < <(find "$cache_dir" -maxdepth 1 -type f -name '*.narinfo' -print0 | sort -z) - -echo "Uploading ${new_nar_count} new NAR objects to '${generation_tag}'..." -upload_queue "$generation_release_id" "$nar_upload_queue" - -echo "Uploading ${new_narinfo_count} new narinfo files to '${index_tag}'..." -upload_queue "$index_release_id" "$narinfo_upload_queue" - -now=$(date -u +%Y-%m-%dT%H:%M:%SZ) -generations_file="${work_dir}/generations.json" -jq --arg prefix "$generation_prefix" ' - [ - .[] - | select(.tag_name | startswith($prefix)) - | { - tag: .tag_name, - commit: .target_commitish, - createdAt: .created_at - } - ] -' "$all_releases_file" >"$generations_file" - -jq -s \ - --slurpfile object_updates "$object_updates_file" \ - --slurpfile narinfo_updates "$narinfo_updates_file" \ - --slurpfile skipped_updates "$skipped_updates_file" \ - --slurpfile generations "$generations_file" \ - --arg generation_tag "$generation_tag" \ - --arg commit "$commit" \ - --arg now "$now" \ - ' - .[0] - | .skipped = (.skipped // {}) - | reduce $object_updates[] as $update (.; .objects[$update.key] = $update.value) - | reduce $skipped_updates[] as $update (.; .skipped[$update.key] = $update.value) - | reduce $narinfo_updates[] as $update (.; - .narinfos[$update.key] = $update.value - | del(.skipped[$update.key]) - ) - | .generatedAt = $now - | .objects as $objects - | .generations = ( - reduce ( - $generations[0] + [{tag: $generation_tag, commit: $commit, createdAt: $now}] - )[] as $generation ( - {}; - .[$generation.tag] = $generation - ) - | [.[]] - | sort_by(.createdAt) - | map( - . as $generation - | . + { - objects: [ - $objects - | to_entries[] - | select(.value.generation == $generation.tag) - | .key - ] - } - ) - ) - ' "$manifest_file" >"${manifest_file}.new" -mv "${manifest_file}.new" "$manifest_file" - -if [[ $mode == bootstrap ]]; then - for root_asset_name in nix-cache-info cache-public-key; do - root_asset_id=${index_asset_ids[$root_asset_name]:-} - if [[ -n $root_asset_id ]]; then - delete_asset "$index_release_id" "$root_asset_id" - fi - done - upload_asset "$index_release_id" "${cache_dir}/nix-cache-info" nix-cache-info - printf '%s\n' "$public_key" >"${work_dir}/cache-public-key" - upload_asset "$index_release_id" "${work_dir}/cache-public-key" cache-public-key -fi - -manifest_asset_name=cache-manifest.json -old_manifest_asset_id=${index_asset_ids[$manifest_asset_name]:-} -temporary_manifest_name="cache-manifest-${commit}.json" -while IFS= read -r stale_temporary_asset_id; do - delete_asset "$index_release_id" "$stale_temporary_asset_id" -done < <( - jq -r ' - .assets[]? - | select(.name | test("^cache-manifest-[0-9a-f]+\\.json$")) - | .id - ' "$index_release_file" -) - -temporary_manifest_asset=$( - upload_asset_response "$index_release_id" "$manifest_file" "$temporary_manifest_name" -) -temporary_manifest_asset_id=$(jq -r '.id' <<<"$temporary_manifest_asset") -if [[ -z $temporary_manifest_asset_id || $temporary_manifest_asset_id == null ]]; then - echo "Could not determine the temporary manifest asset ID." >&2 - exit 1 -fi - -if [[ -n $old_manifest_asset_id ]]; then - delete_asset "$index_release_id" "$old_manifest_asset_id" -fi -rename_asset "$index_release_id" "$temporary_manifest_asset_id" "$manifest_asset_name" - -echo "Published Nix cache generation: ${generation_tag}" -echo "Cache URI: ${cache_uri}" -echo "Public key: ${public_key}" -if ((skipped_path_count > 0)); then - echo "Skipped ${skipped_path_count} store paths whose compressed NAR exceeded ${max_upload_bytes} bytes. They are listed in cache-manifest.json and will fall back to another substituter or a local build." >&2 -fi - -if ((${#failed_hosts[@]} > 0)); then - echo "Cache publication succeeded, but the following host builds failed:" >&2 - printf ' - %s\n' "${failed_hosts[@]}" >&2 - exit 1 -fi diff --git a/.gitea/workflows/nix-cache-bootstrap.yml b/.gitea/workflows/nix-cache-bootstrap.yml deleted file mode 100644 index fc74f3b..0000000 --- a/.gitea/workflows/nix-cache-bootstrap.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Bootstrap Nix binary cache -on: - workflow_dispatch: -permissions: - contents: write -concurrency: - group: nix-release-cache-publisher - cancel-in-progress: false -jobs: - bootstrap: - name: Build every host and bootstrap the cache - runs-on: ubuntu-24.04 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Nix - uses: cachix/install-nix-action@v31 - with: - extra_nix_config: | - experimental-features = nix-command flakes - accept-flake-config = true - - name: Prepare unsandboxed Nix builds - run: | - set -euo pipefail - if [[ -e /homeless-shelter ]]; then - if ((EUID == 0)); then - rm -rf --one-file-system -- /homeless-shelter - else - sudo -n rm -rf --one-file-system -- /homeless-shelter - fi - fi - - name: Build and publish the initial cache - env: - GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} - NIX_CACHE_PRIVATE_KEY: ${{ secrets.NIX_CACHE_PRIVATE_KEY }} - CACHE_MODE: bootstrap - CACHE_REPOSITORY: moons-14/dotfiles - CACHE_SERVER_URL: https://git.yutakobayashi.com - CACHE_API_SERVER_URL: ${{ vars.NIX_CACHE_API_SERVER_URL }} - CACHE_COMMIT: ${{ github.sha }} - CACHE_REF_NAME: ${{ github.ref_name }} - run: | - set -euo pipefail - - if [[ -z "${NIX_CACHE_PRIVATE_KEY:-}" ]]; then - echo "Repository secret NIX_CACHE_PRIVATE_KEY is required." >&2 - exit 1 - fi - - key_file="${RUNNER_TEMP:-/tmp}/nix-cache-private-key" - umask 077 - printf '%s\n' "$NIX_CACHE_PRIVATE_KEY" > "$key_file" - unset NIX_CACHE_PRIVATE_KEY - - export NIX_CACHE_KEY_FILE="$key_file" - trap 'rm -f "$key_file"' EXIT - ./.gitea/scripts/publish-nix-cache.sh diff --git a/.gitea/workflows/nix-cache-update.yml b/.gitea/workflows/nix-cache-update.yml deleted file mode 100644 index 4309d32..0000000 --- a/.gitea/workflows/nix-cache-update.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Update Nix binary cache -on: - push: - branches: - - "**" - workflow_dispatch: -permissions: - contents: write -concurrency: - group: nix-release-cache-publisher - cancel-in-progress: false -jobs: - update: - name: Build every host and publish new cache objects - runs-on: ubuntu-24.04 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Nix - uses: cachix/install-nix-action@v31 - with: - extra_nix_config: | - experimental-features = nix-command flakes - accept-flake-config = true - - name: Prepare unsandboxed Nix builds - run: | - set -euo pipefail - if [[ -e /homeless-shelter ]]; then - if ((EUID == 0)); then - rm -rf --one-file-system -- /homeless-shelter - else - sudo -n rm -rf --one-file-system -- /homeless-shelter - fi - fi - - name: Build and publish new cache objects - env: - GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} - NIX_CACHE_PRIVATE_KEY: ${{ secrets.NIX_CACHE_PRIVATE_KEY }} - CACHE_MODE: update - CACHE_REPOSITORY: moons-14/dotfiles - CACHE_SERVER_URL: https://git.yutakobayashi.com - CACHE_API_SERVER_URL: ${{ vars.NIX_CACHE_API_SERVER_URL }} - CACHE_COMMIT: ${{ github.sha }} - CACHE_REF_NAME: ${{ github.ref_name }} - run: | - set -euo pipefail - - if [[ -z "${NIX_CACHE_PRIVATE_KEY:-}" ]]; then - echo "Repository secret NIX_CACHE_PRIVATE_KEY is required." >&2 - exit 1 - fi - - key_file="${RUNNER_TEMP:-/tmp}/nix-cache-private-key" - umask 077 - printf '%s\n' "$NIX_CACHE_PRIVATE_KEY" > "$key_file" - unset NIX_CACHE_PRIVATE_KEY - - export NIX_CACHE_KEY_FILE="$key_file" - trap 'rm -f "$key_file"' EXIT - ./.gitea/scripts/publish-nix-cache.sh diff --git a/docs/gitea-binary-cache.md b/docs/gitea-binary-cache.md deleted file mode 100644 index bdd177c..0000000 --- a/docs/gitea-binary-cache.md +++ /dev/null @@ -1,118 +0,0 @@ -# Gitea Release-backed Nix binary cache - -The workflows in `.gitea/workflows/` publish the closures of every -`nixosConfigurations` host to Gitea Releases. - -- `nix-cache-bootstrap.yml` is a one-shot manual workflow that creates the - initial cache. -- `nix-cache-update.yml` runs on every branch push. It creates one immutable - generation release per commit and uploads only NAR content hashes that have - not appeared in an older generation. -- Hosts are built independently. If one host fails, successful host closures - and store paths completed during the failed build are published before the - job reports the build failure. -- The `cache-latest` release is the stable cache index. It contains - `nix-cache-info`, `cache-public-key`, `cache-manifest.json`, and every - `.narinfo` file. -- Each narinfo has an absolute `URL:` that points at the generation release - containing its immutable NAR. Rewriting `URL:` does not alter the signed - store-path fingerprint. - -The operational manifest enumerates all narinfo and NAR URLs. Nix itself does -not read that manifest: it requests `nix-cache-info` and -`.narinfo` directly from the cache URI. - -## One-time setup - -Generate a signing key on a trusted machine: - -```sh -umask 077 -nix key generate-secret --key-name dotfiles-gitea-cache-1 > cache-private-key -nix key convert-secret-to-public < cache-private-key -``` - -Add the complete contents of `cache-private-key` as the repository Actions -secret `NIX_CACHE_PRIVATE_KEY`. Do not commit this file. Ensure the repository -Actions token is allowed to write Releases, then run **Bootstrap Nix binary -cache** once from the Actions UI. - -The bootstrap log and the following stable asset expose the public key: - -```text -https://git.yutakobayashi.com/moons-14/dotfiles/releases/download/cache-latest/cache-public-key -``` - -The repository and its Release assets must be publicly readable for ordinary -Nix clients to use this as an unauthenticated substituter. The runner needs -enough disk for the Nix store plus one compressed copy of all host closures. -It also needs `bash`, `curl`, `jq`, and standard GNU userland tools. - -The workflows remove `/homeless-shelter` before building. Nix requires that -dummy home path not to exist when the runner performs builds without a sandbox. - -## Large release assets and Cloudflare - -`git.yutakobayashi.com` is proxied by Cloudflare. Large NAR uploads can receive -`413 Request Entity Too Large` before they reach Gitea. NAR files cannot be -split because the Nix binary-cache protocol downloads each NAR as one object. - -Create an HTTPS origin hostname that is DNS-only in Cloudflare, or use a -private Gitea URL reachable from the runner. Set that URL as the repository -Actions variable `NIX_CACHE_API_SERVER_URL`, for example: - -```text -https://git-origin.yutakobayashi.com -``` - -Only Gitea API calls and uploads use this variable. `CACHE_SERVER_URL` remains -the public URL, so the URLs written to narinfo and the client substituter stay -under `https://git.yutakobayashi.com`. - -The origin reverse proxy request-body limit and Gitea's -`[repository.release] FILE_MAX_SIZE` must also be larger than the largest NAR. -Protect an origin hostname with a firewall or another access control that still -allows the Actions runner to reach it. - -Without an explicit origin variable, the workflow automatically probes the -Podman/Docker host through `host.containers.internal` and -`host.docker.internal`. It first connects directly to host port 443 while -preserving the public hostname and TLS SNI, then tries Gitea's conventional -HTTP port 3000. A successful probe bypasses Cloudflare and disables the -workflow-side NAR size limit. - -If no internal route is reachable, the workflow uses the public endpoint and -enforces `CACHE_MAX_UPLOAD_BYTES=90000000`. NARs larger than that limit and -their narinfo files are not uploaded. They are recorded under `skipped` in -`cache-manifest.json`. Nix clients can still substitute every smaller store -path and obtain a skipped path from another substituter or build it locally. - -## NixOS client configuration - -After bootstrap, copy the exact value from `cache-public-key` into -`extra-trusted-public-keys`: - -```nix -{ - nix.settings = { - extra-substituters = [ - "https://git.yutakobayashi.com/moons-14/dotfiles/releases/download/cache-latest" - ]; - extra-trusted-public-keys = [ - "dotfiles-gitea-cache-1:REPLACE_WITH_THE_GENERATED_PUBLIC_KEY" - ]; - }; -} -``` - -The substituter value is the directory-like cache URI, not the manifest file -URL. A quick validation after bootstrap is: - -```sh -cache=https://git.yutakobayashi.com/moons-14/dotfiles/releases/download/cache-latest -curl --fail "$cache/nix-cache-info" -curl --fail "$cache/cache-manifest.json" | jq '.cache, (.objects | length), (.narinfos | length)' -``` - -Because every branch receives the signing secret, only trusted users should be -allowed to push branches or modify Actions workflows in this repository. diff --git a/modules/features/dev/default.nix b/modules/features/dev/default.nix index 6d2893a..9095601 100644 --- a/modules/features/dev/default.nix +++ b/modules/features/dev/default.nix @@ -3,6 +3,7 @@ ./agent.nix ./arduino.nix ./bun.nix + ./drawio.nix ./java.nix ./nix.nix ./python.nix diff --git a/modules/features/dev/drawio.nix b/modules/features/dev/drawio.nix new file mode 100644 index 0000000..f114068 --- /dev/null +++ b/modules/features/dev/drawio.nix @@ -0,0 +1,20 @@ +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.my.features.dev.drawio; +in +{ + options.my.features.dev.drawio = { + enable = lib.mkEnableOption "Draw.io diagram editor"; + }; + + config = lib.mkIf cfg.enable { + environment.systemPackages = with pkgs; [ + drawio # Diagram editor + ]; + }; +} diff --git a/profiles/workloads/dev.nix b/profiles/workloads/dev.nix index 6142d9a..fcae78b 100644 --- a/profiles/workloads/dev.nix +++ b/profiles/workloads/dev.nix @@ -6,6 +6,7 @@ nix.enable = true; python.enable = true; bun.enable = true; + drawio.enable = true; java.enable = true; }; gui.editor.enable = true;