daily data update and add ci

scripts/update_daily_asmr_data.py

@@ -45,6 +45,11 @@ def parse_args() -> argparse.Namespace:
         help="Filtered ASMR JSONL output file.",
     )
     parser.add_argument("--data-dir", default="web/data", help="Directory containing web data assets.")
+    parser.add_argument(
+        "--skip-web-data",
+        action="store_true",
+        help="Update raw and filtered JSONL files without reading or writing web/data assets.",
+    )
     parser.add_argument("--order", default="create_date", help="ASMR API order parameter.")
     parser.add_argument("--sort", default="desc", help="ASMR API sort parameter.")
     parser.add_argument("--subtitle", default="0", help="ASMR API subtitle parameter.")
@@ -712,21 +717,27 @@ def main() -> int:
     existing_keys = {key for work in existing_works for key in identity_keys(work)}
     new_works, fetched_existing_by_key, scanned_pages = fetch_latest_works(args, existing_keys)
 
-    if not new_works and not args.force and not web_data_out_of_sync(filtered_path, Path(args.data_dir)):
+    needs_web_data_update = False if args.skip_web_data else web_data_out_of_sync(filtered_path, Path(args.data_dir))
+    if not new_works and not args.force and not needs_web_data_update:
         print(f"Scanned pages: {scanned_pages}")
-        print("No new works found. web/data update skipped.")
+        print("No new works found. update skipped.")
         return 0
 
     merged_works = merge_raw_works(existing_works, new_works, fetched_existing_by_key)
     filtered_works, filter_stats = filter_raw_works(args, merged_works)
-    web_stats = dry_run_web_stats(args, filtered_works) if args.dry_run else None
+    web_stats = None
+    if args.dry_run:
+        web_stats = {"embeddingDiffs": []} if args.skip_web_data else dry_run_web_stats(args, filtered_works)
 
     print(f"Scanned pages: {scanned_pages}")
     print(f"New raw works: {len(new_works)}")
     print(f"Raw works after merge: {len(merged_works)}")
     print(f"Filtered works: {filter_stats['output']} ({filter_stats['total_removed']} removed)")
     if web_stats:
-        print(f"web/data would reuse {web_stats['reused']} embeddings and request {web_stats['embedded']} embeddings")
+        if args.skip_web_data:
+            print("web/data would be skipped")
+        else:
+            print(f"web/data would reuse {web_stats['reused']} embeddings and request {web_stats['embedded']} embeddings")
         print_dry_run_diff(
             args=args,
             new_works=new_works,
@@ -743,9 +754,13 @@ def main() -> int:
 
     write_jsonl_atomic(input_path, merged_works)
     write_jsonl_atomic(filtered_path, filtered_works)
-    web_stats = build_incremental_web_data(args, filtered_works)
     print(f"Updated {input_path}")
     print(f"Updated {filtered_path}")
+    if args.skip_web_data:
+        print("Skipped web/data update")
+        return 0
+
+    web_stats = build_incremental_web_data(args, filtered_works)
     print(
         f"Updated {args.data_dir}: {web_stats['count']} works, "
         f"reused {web_stats['reused']} embeddings, requested {web_stats['embedded']} embeddings"