From 703e24fcdfd32d98e0850aa3b433fc020b087132 Mon Sep 17 00:00:00 2001 From: moons Date: Tue, 14 Jul 2026 22:34:07 +0900 Subject: [PATCH] gitea update --- .gitea/scripts/publish-nix-cache.sh | 90 +++++++++++++++++++++--- .gitea/workflows/nix-cache-bootstrap.yml | 2 + .gitea/workflows/nix-cache-update.yml | 2 + docs/gitea-binary-cache.md | 30 ++++++++ 4 files changed, 114 insertions(+), 10 deletions(-) diff --git a/.gitea/scripts/publish-nix-cache.sh b/.gitea/scripts/publish-nix-cache.sh index f7f7bbc..62403a1 100755 --- a/.gitea/scripts/publish-nix-cache.sh +++ b/.gitea/scripts/publish-nix-cache.sh @@ -7,12 +7,14 @@ set -euo pipefail mode=${CACHE_MODE:-} server_url=${CACHE_SERVER_URL:-} +api_server_url=${CACHE_API_SERVER_URL:-} repository=${CACHE_REPOSITORY:-} commit=${CACHE_COMMIT:-} ref_name=${CACHE_REF_NAME:-unknown} index_tag=${CACHE_INDEX_TAG:-cache-latest} generation_prefix=${CACHE_GENERATION_PREFIX:-nix-cache-generation-} upload_jobs=${CACHE_UPLOAD_JOBS:-4} +max_upload_bytes=${CACHE_MAX_UPLOAD_BYTES:-90000000} key_file=${NIX_CACHE_KEY_FILE:-} for command in curl jq nix awk comm sed find sort; do @@ -47,8 +49,15 @@ if [[ ! $upload_jobs =~ ^[1-9][0-9]*$ ]]; then exit 1 fi +if [[ ! $max_upload_bytes =~ ^[0-9]+$ ]]; then + echo "CACHE_MAX_UPLOAD_BYTES must be zero or a positive integer." >&2 + exit 1 +fi + server_url=${server_url%/} -api_base="${server_url}/api/v1/repos/${repository}" +api_server_url=${api_server_url:-$server_url} +api_server_url=${api_server_url%/} +api_base="${api_server_url}/api/v1/repos/${repository}" download_base="${server_url}/${repository}/releases/download" cache_uri="${download_base}/${index_tag}" manifest_url="${cache_uri}/cache-manifest.json" @@ -64,11 +73,13 @@ all_releases_file="${work_dir}/all-releases.json" generation_release_file="${work_dir}/generation-release.json" object_updates_file="${work_dir}/object-updates.jsonl" narinfo_updates_file="${work_dir}/narinfo-updates.jsonl" +skipped_updates_file="${work_dir}/skipped-updates.jsonl" nar_upload_queue="${work_dir}/nar-upload-queue" narinfo_upload_queue="${work_dir}/narinfo-upload-queue" mkdir -p "$cache_dir" "$rewritten_dir" : >"$object_updates_file" : >"$narinfo_updates_file" +: >"$skipped_updates_file" : >"$nar_upload_queue" : >"$narinfo_upload_queue" trap 'rm -rf "$work_dir"' EXIT @@ -149,14 +160,40 @@ upload_asset() { local release_id=$1 local file=$2 local name=$3 + local response_file + local status + local curl_status + local size + local size_mib - curl --fail-with-body --silent --show-error \ + response_file=$(mktemp "${work_dir}/upload-response.XXXXXX") + size=$(stat -c '%s' "$file") + size_mib=$(((size + 1048575) / 1048576)) + + if status=$(curl --silent --show-error \ --retry 5 --retry-delay 2 --retry-all-errors \ --request POST \ --header "Authorization: token ${GITEA_TOKEN}" \ --form "attachment=@${file};type=application/octet-stream" \ - --output /dev/null \ - "${api_base}/releases/${release_id}/assets?name=${name}" + --output "$response_file" --write-out '%{http_code}' \ + "${api_base}/releases/${release_id}/assets?name=${name}"); then + curl_status=0 + else + curl_status=$? + fi + + if [[ $status == 200 || $status == 201 ]]; then + rm -f "$response_file" + return 0 + fi + + echo "Asset upload failed: name=${name} size=${size}B (${size_mib}MiB) HTTP=${status:-000} curl=${curl_status}" >&2 + if [[ $status == 413 ]]; then + echo "The upload endpoint rejected this NAR as too large. Set the Gitea Actions variable NIX_CACHE_API_SERVER_URL to an origin URL that bypasses Cloudflare, and verify the origin proxy and Gitea release size limits." >&2 + fi + sed -n '1,20p' "$response_file" >&2 + rm -f "$response_file" + return 1 } upload_asset_response() { @@ -207,9 +244,6 @@ upload_queue() { fi done pids=() - if ((failed)); then - return 1 - fi fi done <"$queue_file" @@ -261,7 +295,8 @@ initialize_manifest() { generatedAt: null, generations: [], objects: {}, - narinfos: {} + narinfos: {}, + skipped: {} }' >"$manifest_file" } @@ -465,6 +500,7 @@ done < <(jq -r '.assets[]? | [.name, (.id | tostring)] | @tsv' "$index_release_f new_nar_count=0 new_narinfo_count=0 +skipped_path_count=0 while IFS= read -r -d '' narinfo_file; do narinfo_name=$(basename "$narinfo_file") store_hash=${narinfo_name%.narinfo} @@ -488,8 +524,33 @@ while IFS= read -r -d '' narinfo_file; do fi if [[ -z ${object_urls[$nar_name]:-} ]]; then - object_urls["$nar_name"]="${download_base}/${generation_tag}/${nar_name}" object_sizes["$nar_name"]=$(stat -c '%s' "$nar_file") + nar_size=${object_sizes[$nar_name]} + + if ((max_upload_bytes > 0 && nar_size > max_upload_bytes)); then + echo "Skipping oversized NAR: storePath=${store_path} name=${nar_name} size=${nar_size}B limit=${max_upload_bytes}B" >&2 + jq -cn \ + --arg key "$store_hash" \ + --arg store_path "$store_path" \ + --arg nar "$nar_name" \ + --arg reason "upload-size-limit" \ + --argjson size "$nar_size" \ + --argjson limit "$max_upload_bytes" \ + '{ + key: $key, + value: { + storePath: $store_path, + nar: $nar, + size: $size, + limit: $limit, + reason: $reason + } + }' >>"$skipped_updates_file" + ((skipped_path_count += 1)) + continue + fi + + object_urls["$nar_name"]="${download_base}/${generation_tag}/${nar_name}" if [[ -z ${queued_objects[$nar_name]:-} ]]; then printf '%s\t%s\n' "$nar_file" "$nar_name" >>"$nar_upload_queue" @@ -556,14 +617,20 @@ jq --arg prefix "$generation_prefix" ' jq -s \ --slurpfile object_updates "$object_updates_file" \ --slurpfile narinfo_updates "$narinfo_updates_file" \ + --slurpfile skipped_updates "$skipped_updates_file" \ --slurpfile generations "$generations_file" \ --arg generation_tag "$generation_tag" \ --arg commit "$commit" \ --arg now "$now" \ ' .[0] + | .skipped = (.skipped // {}) | reduce $object_updates[] as $update (.; .objects[$update.key] = $update.value) - | reduce $narinfo_updates[] as $update (.; .narinfos[$update.key] = $update.value) + | reduce $skipped_updates[] as $update (.; .skipped[$update.key] = $update.value) + | reduce $narinfo_updates[] as $update (.; + .narinfos[$update.key] = $update.value + | del(.skipped[$update.key]) + ) | .generatedAt = $now | .objects as $objects | .generations = ( @@ -632,6 +699,9 @@ rename_asset "$index_release_id" "$temporary_manifest_asset_id" "$manifest_asset echo "Published Nix cache generation: ${generation_tag}" echo "Cache URI: ${cache_uri}" echo "Public key: ${public_key}" +if ((skipped_path_count > 0)); then + echo "Skipped ${skipped_path_count} store paths whose compressed NAR exceeded ${max_upload_bytes} bytes. They are listed in cache-manifest.json and will fall back to another substituter or a local build." >&2 +fi if ((${#failed_hosts[@]} > 0)); then echo "Cache publication succeeded, but the following host builds failed:" >&2 diff --git a/.gitea/workflows/nix-cache-bootstrap.yml b/.gitea/workflows/nix-cache-bootstrap.yml index 0e815f3..f57712c 100644 --- a/.gitea/workflows/nix-cache-bootstrap.yml +++ b/.gitea/workflows/nix-cache-bootstrap.yml @@ -36,6 +36,8 @@ jobs: CACHE_MODE: bootstrap CACHE_REPOSITORY: moons-14/dotfiles CACHE_SERVER_URL: https://git.yutakobayashi.com + CACHE_API_SERVER_URL: ${{ vars.NIX_CACHE_API_SERVER_URL }} + CACHE_MAX_UPLOAD_BYTES: "90000000" CACHE_COMMIT: ${{ github.sha }} CACHE_REF_NAME: ${{ github.ref_name }} run: | diff --git a/.gitea/workflows/nix-cache-update.yml b/.gitea/workflows/nix-cache-update.yml index caf3ce4..b3a3503 100644 --- a/.gitea/workflows/nix-cache-update.yml +++ b/.gitea/workflows/nix-cache-update.yml @@ -39,6 +39,8 @@ jobs: CACHE_MODE: update CACHE_REPOSITORY: moons-14/dotfiles CACHE_SERVER_URL: https://git.yutakobayashi.com + CACHE_API_SERVER_URL: ${{ vars.NIX_CACHE_API_SERVER_URL }} + CACHE_MAX_UPLOAD_BYTES: "90000000" CACHE_COMMIT: ${{ github.sha }} CACHE_REF_NAME: ${{ github.ref_name }} run: | diff --git a/docs/gitea-binary-cache.md b/docs/gitea-binary-cache.md index 22b758b..610db45 100644 --- a/docs/gitea-binary-cache.md +++ b/docs/gitea-binary-cache.md @@ -51,6 +51,36 @@ It also needs `bash`, `curl`, `jq`, and standard GNU userland tools. The workflows remove `/homeless-shelter` before building. Nix requires that dummy home path not to exist when the runner performs builds without a sandbox. +## Large release assets and Cloudflare + +`git.yutakobayashi.com` is proxied by Cloudflare. Large NAR uploads can receive +`413 Request Entity Too Large` before they reach Gitea. NAR files cannot be +split because the Nix binary-cache protocol downloads each NAR as one object. + +Create an HTTPS origin hostname that is DNS-only in Cloudflare, or use a +private Gitea URL reachable from the runner. Set that URL as the repository +Actions variable `NIX_CACHE_API_SERVER_URL`, for example: + +```text +https://git-origin.yutakobayashi.com +``` + +Only Gitea API calls and uploads use this variable. `CACHE_SERVER_URL` remains +the public URL, so the URLs written to narinfo and the client substituter stay +under `https://git.yutakobayashi.com`. + +The origin reverse proxy request-body limit and Gitea's +`[repository.release] FILE_MAX_SIZE` must also be larger than the largest NAR. +Protect an origin hostname with a firewall or another access control that still +allows the Actions runner to reach it. + +When server-side limits cannot be changed, the workflows enforce +`CACHE_MAX_UPLOAD_BYTES=90000000`. NARs larger than that limit and their +narinfo files are not uploaded. They are recorded under `skipped` in +`cache-manifest.json`. Nix clients can still substitute every smaller store +path and obtain a skipped path from another substituter or build it locally. +Set the value to `0` only when the upload path has no smaller request limit. + ## NixOS client configuration After bootstrap, copy the exact value from `cache-public-key` into