main.py - cdnV2 -

main.py (32926B)
      1 import gzip
      2 import hashlib
      3 import os
      4 import shutil
      5 import time
      6 import uuid
      7 import io
      8 import json
      9 import zipfile
     10 import tarfile
     11 import tempfile
     12 import asyncio
     13 import socket
     14 import ipaddress
     15 from urllib.parse import urlparse, unquote
     16 import httpx
     17 from collections import deque
     18 from pathlib import Path
     19 from pydantic import BaseModel
     20 from fastapi import FastAPI, Request, File, UploadFile, HTTPException, Form
     21 from fastapi.responses import StreamingResponse, HTMLResponse, Response, PlainTextResponse, RedirectResponse, FileResponse
     22 from .fast_html import FastTemplates
     23 from .fast_tree import tree_payload, shell_payload, json_response
     24 
     25 from .storage import (
     26     is_file,
     27     get_path_info,
     28     _get_full_tree,
     29     list_directory,
     30     list_files_recursive,
     31     upload_temp_file,
     32     upload_folder_scoped,
     33     repo_stats,
     34     search_files,
     35     write_batch_manifest,
     36     get_batch_manifest,
     37     HF_BUCKET_ID,
     38     bucket_url,
     39     auth_headers,
     40     format_size,
     41 )
     42 from .gh_oidc import verify_actions_token
     43 from .shortener import shorten_url, get_destination_url
     44 from .upload_guard import get_client_ip, inspect_upload, is_video_upload
     45 from .stats import router as stats_router
     46 from .admin import router as admin_router, record_upload, is_banned, gh_check, gh_record
     47 
     48 app = FastAPI()
     49 app.include_router(admin_router)
     50 app.include_router(stats_router)
     51 templates = FastTemplates(directory="app/templates")
     52 
     53 STATIC_DIR = Path(__file__).parent / "static"
     54 _NO_STORE_FILES = {"manifest.json", "sw.js"}
     55 _REVALIDATE_FILES = {"prefetch.js", "fasttree.js", "app.css", "app.js"}
     56 
     57 
     58 def _asset_version() -> str:
     59     """Short content hash of the UI's own stylesheet and script, used as ?v= so a deploy is never paired with stale files."""
     60     h = hashlib.sha1()
     61     for name in ("app.css", "app.js"):
     62         try:
     63             h.update((STATIC_DIR / name).read_bytes())
     64         except OSError:
     65             pass
     66     return h.hexdigest()[:10]
     67 
     68 
     69 templates.env.globals["asset_v"] = _asset_version()
     70 
     71 
     72 _GZ_CACHE: dict = {}
     73 
     74 
     75 def _gzipped(path: Path) -> bytes:
     76     """gzip of a static file, kept in memory (a few files, keyed by mtime so a redeploy is picked up)."""
     77     key = (path.name, path.stat().st_mtime_ns)
     78     data = _GZ_CACHE.get(key)
     79     if data is None:
     80         data = _GZ_CACHE[key] = gzip.compress(path.read_bytes(), 9)
     81     return data
     82 
     83 
     84 @app.get("/static/{filename}")
     85 async def static_no_cache_root(filename: str, request: Request):
     86     if filename in _NO_STORE_FILES:
     87         cache_control = "no-store, no-cache, must-revalidate, max-age=0"
     88     elif filename in _REVALIDATE_FILES:
     89         # Short cache: full page loads don't re-download it, and a deploy still shows up within minutes.
     90         cache_control = "public, max-age=300"
     91     else:
     92         raise HTTPException(status_code=404)
     93     file_path = STATIC_DIR / filename
     94     if not file_path.is_file():
     95         raise HTTPException(status_code=404)
     96     if filename in _REVALIDATE_FILES and "gzip" in request.headers.get("accept-encoding", "").lower():
     97         media = "text/css" if filename.endswith(".css") else "text/javascript"
     98         return Response(
     99             _gzipped(file_path),
    100             media_type=f"{media}; charset=utf-8",
    101             headers={"Cache-Control": cache_control, "Content-Encoding": "gzip", "Vary": "Accept-Encoding"},
    102         )
    103     return FileResponse(file_path, headers={"Cache-Control": cache_control})
    104 
    105 
    106 @app.get("/static/icons/{filename}")
    107 async def static_icons(filename: str):
    108     file_path = STATIC_DIR / "icons" / filename
    109     if not file_path.is_file():
    110         raise HTTPException(status_code=404)
    111     return FileResponse(
    112         file_path,
    113         headers={"Cache-Control": "public, max-age=86400"},
    114     )
    115 
    116 
    117 @app.get("/favicon.ico")
    118 async def favicon():
    119     return FileResponse(
    120         STATIC_DIR / "favicon.ico",
    121         headers={"Cache-Control": "public, max-age=86400"},
    122     )
    123 
    124 
    125 CDN_BASE_URL = os.getenv("CDN_BASE_URL", "https://cdn.amit.is-a.dev")
    126 RAW_DOMAIN = os.getenv("RAW_DOMAIN", "raw.cdn.amit.is-a.dev")
    127 RAW_BASE_URL = os.getenv("RAW_BASE_URL", f"https://{RAW_DOMAIN}")
    128 RAW_PREFIX = "raw/"
    129 
    130 
    131 async def render_context(extra: dict) -> dict:
    132     stats = await repo_stats()
    133     ctx = dict(extra)
    134     ctx["repo_file_count"] = stats["file_count"] if stats else None
    135     ctx["repo_size_str"] = stats["size_str"] if stats else None
    136     ctx.setdefault("cdn_base_url", CDN_BASE_URL)
    137     ctx.setdefault("raw_base_url", RAW_BASE_URL)
    138     return ctx
    139 
    140 
    141 @app.exception_handler(404)
    142 async def not_found_handler(request: Request, exc: HTTPException):
    143     if request.url.hostname == RAW_DOMAIN:
    144         return PlainTextResponse("404: File Not Found", status_code=404)
    145     ctx = await render_context({"page": "404"})
    146     return templates.TemplateResponse(request, "index.html", ctx, status_code=404)
    147 
    148 
    149 async def _proxy_stream(client: httpx.AsyncClient, r: httpx.Response):
    150     try:
    151         async for chunk in r.aiter_raw():
    152             yield chunk
    153     finally:
    154         await client.aclose()
    155 
    156 
    157 _WEB_ASSET_MIME_TYPES = {
    158     ".css": "text/css",
    159     ".js": "text/javascript",
    160     ".mjs": "text/javascript",
    161     ".json": "application/json",
    162     ".svg": "image/svg+xml",
    163     ".xml": "application/xml",
    164     ".woff": "font/woff",
    165     ".woff2": "font/woff2",
    166     ".ttf": "font/ttf",
    167     ".otf": "font/otf",
    168     ".wasm": "application/wasm",
    169     ".webmanifest": "application/manifest+json",
    170     ".map": "application/json",
    171 }
    172 
    173 def _resolve_content_type(filename: str, upstream_content_type: str | None) -> str | None:
    174     """
    175     Hugging Face's resolve/ endpoint (and GitHub's raw.githubusercontent.com, for
    176     that matter) often serves files it doesn't specially recognize back as
    177     text/plain or application/octet-stream. Combined with X-Content-Type-Options:
    178     nosniff below, that makes browsers refuse to apply a .css file as a stylesheet
    179     or run a .js file as a script - the Content-Type has to be right, not just
    180     close. Override it from the file extension for the asset types this actually
    181     breaks, rather than trusting whatever the upstream sent.
    182     """
    183     ext = "." + filename.rsplit(".", 1)[-1] if "." in filename else ""
    184     if ext in (".md", ".txt"):
    185         return "text/plain; charset=utf-8"
    186     mime = _WEB_ASSET_MIME_TYPES.get(ext)
    187     if mime:
    188         if mime.startswith("text/") or mime in ("application/json", "application/xml", "application/manifest+json"):
    189             return f"{mime}; charset=utf-8"
    190         return mime
    191     return upstream_content_type
    192 
    193 async def stream_raw(path: str, request: Request):
    194     hf_url = bucket_url(path)
    195     client = httpx.AsyncClient(follow_redirects=True)
    196     req_headers = dict(auth_headers())
    197     range_header = request.headers.get("range")
    198     if range_header:
    199         req_headers["Range"] = range_header
    200     req = client.build_request("GET", hf_url, headers=req_headers)
    201     r = await client.send(req, stream=True)
    202 
    203     if r.status_code not in (200, 206):
    204         await client.aclose()
    205         raise HTTPException(status_code=404, detail="File not found")
    206 
    207     headers = {
    208         "Access-Control-Allow-Origin": "*",
    209         "Cache-Control": "public, max-age=31536000",
    210         "X-Content-Type-Options": "nosniff",
    211         "Accept-Ranges": "bytes",
    212     }
    213 
    214     for h in ["Content-Type", "Content-Encoding", "Content-Length", "Etag", "Content-Range"]:
    215         if h in r.headers:
    216             headers[h] = r.headers[h]
    217 
    218     filename = path.split("/")[-1].lower()
    219     resolved_type = _resolve_content_type(filename, headers.get("Content-Type"))
    220     if resolved_type:
    221         headers["Content-Type"] = resolved_type
    222 
    223     return StreamingResponse(
    224         _proxy_stream(client, r),
    225         status_code=r.status_code,
    226         headers=headers,
    227     )
    228 
    229 
    230 @app.get("/api/get/{path:path}")
    231 async def download_file(path: str, request: Request):
    232     hf_url = bucket_url(path)
    233     client = httpx.AsyncClient(follow_redirects=True)
    234     req_headers = dict(auth_headers())
    235     range_header = request.headers.get("range")
    236     if range_header:
    237         req_headers["Range"] = range_header
    238     req = client.build_request("GET", hf_url, headers=req_headers)
    239     r = await client.send(req, stream=True)
    240 
    241     if r.status_code not in (200, 206):
    242         await client.aclose()
    243         raise HTTPException(status_code=404, detail="Not found")
    244 
    245     filename = path.split("/")[-1]
    246     headers = {
    247         "Content-Disposition": f'attachment; filename="{filename}"',
    248         "Content-Type": _resolve_content_type(filename.lower(), r.headers.get("Content-Type")) or "application/octet-stream",
    249         "Accept-Ranges": "bytes",
    250     }
    251 
    252     for h in ["Content-Length", "Content-Encoding", "Etag", "Content-Range"]:
    253         if h in r.headers:
    254             headers[h] = r.headers[h]
    255 
    256     return StreamingResponse(
    257         _proxy_stream(client, r),
    258         status_code=r.status_code,
    259         headers=headers,
    260     )
    261 
    262 
    263 UPLOAD_LIMIT_PER_HOUR = 200 * 1024 * 1024        # file uploads (/api/put), per IP
    264 URL_UPLOAD_LIMIT_PER_HOUR = 50 * 1024 * 1024     # uploads from a link (/api/put-url), per IP
    265 
    266 
    267 class _UploadRateLimiter:
    268     """Sliding one-hour window of uploaded bytes, per key (the client IP). No per-minute limit."""
    269 
    270     def __init__(self, per_hour: int, name: str = "Upload"):
    271         self.per_hour = per_hour
    272         self.name = name
    273         self._usage: dict[str, deque] = {}
    274 
    275     def check_and_record(self, key: str, size: int):
    276         now = time.time()
    277         if size > self.per_hour:  # can never fit, so don't say "try again later"
    278             raise HTTPException(
    279                 status_code=413,
    280                 detail=f"File too large (limit {format_size(self.per_hour)} per file).",
    281             )
    282         dq = self._usage.setdefault(key, deque())
    283         while dq and now - dq[0][0] > 3600:
    284             dq.popleft()
    285         if sum(s for _, s in dq) + size > self.per_hour:
    286             raise HTTPException(
    287                 status_code=429,
    288                 detail=f"{self.name} rate limit exceeded: {format_size(self.per_hour)}/hour. Try again later.",
    289             )
    290         dq.append((now, size))
    291         if len(self._usage) > 10000:  # drop idle clients so the table can't grow forever
    292             self._usage = {k: v for k, v in self._usage.items() if v and now - v[-1][0] <= 3600}
    293 
    294 
    295 _upload_limiter = _UploadRateLimiter(UPLOAD_LIMIT_PER_HOUR, "Upload")
    296 _url_upload_limiter = _UploadRateLimiter(URL_UPLOAD_LIMIT_PER_HOUR, "URL upload")
    297 
    298 
    299 class _CountRateLimiter:
    300     """Sliding-window request counter per key (e.g. per client IP)."""
    301 
    302     def __init__(self, per_minute: int, per_hour: int):
    303         self.per_minute = per_minute
    304         self.per_hour = per_hour
    305         self._hits: dict[str, deque] = {}
    306 
    307     def check_and_record(self, key: str):
    308         now = time.time()
    309         dq = self._hits.setdefault(key, deque())
    310         while dq and now - dq[0] > 3600:
    311             dq.popleft()
    312         if sum(1 for t in dq if now - t <= 60) >= self.per_minute:
    313             raise HTTPException(
    314                 status_code=429,
    315                 detail=f"Rate limit exceeded: {self.per_minute} requests/minute. Try again shortly.",
    316             )
    317         if len(dq) >= self.per_hour:
    318             raise HTTPException(
    319                 status_code=429,
    320                 detail=f"Rate limit exceeded: {self.per_hour} requests/hour. Try again later.",
    321             )
    322         dq.append(now)
    323         if len(self._hits) > 10000:
    324             self._hits = {k: v for k, v in self._hits.items() if v and now - v[-1] <= 3600}
    325 
    326 
    327 _shorten_limiter = _CountRateLimiter(per_minute=10, per_hour=50)
    328 SHORTEN_MAX_URL_LENGTH = 2048
    329 BATCH_MIN_FILES = 2
    330 ALLOWED_UPLOAD_FOLDERS = {"uploads", "third-party"}
    331 
    332 
    333 def _validate_folder(folder: str) -> str:
    334     folder = (folder or "uploads").strip().strip("/")
    335     if folder not in ALLOWED_UPLOAD_FOLDERS:
    336         raise HTTPException(
    337             status_code=400,
    338             detail=f"Invalid folder. Must be one of: {', '.join(sorted(ALLOWED_UPLOAD_FOLDERS))}.",
    339         )
    340     return folder
    341 
    342 
    343 @app.post("/api/put")
    344 async def handle_upload(
    345     request: Request,
    346     files: list[UploadFile] = File(...),
    347     folder: str = Form("uploads"),
    348 ):
    349     folder = _validate_folder(folder)
    350     client_ip = get_client_ip(request)
    351     results = []
    352 
    353     for file in files:
    354         temp_path = f"/tmp/{uuid.uuid4()}"
    355 
    356         with open(temp_path, "wb") as f:
    357             shutil.copyfileobj(file.file, f)
    358 
    359         size = os.path.getsize(temp_path)
    360 
    361         try:
    362             guard_ctx = await inspect_upload(request, file.filename, temp_path, size, folder, "file")
    363             _upload_limiter.check_and_record(client_ip, size)
    364         except HTTPException:
    365             os.remove(temp_path)
    366             raise
    367 
    368         try:
    369             hf_path = await upload_temp_file(temp_path, file.filename, folder)
    370         finally:
    371             if os.path.exists(temp_path):
    372                 os.remove(temp_path)
    373 
    374         await record_upload(guard_ctx, hf_path)
    375 
    376         results.append({
    377             "filename": file.filename,
    378             "hf_path": hf_path,
    379             "size": size,
    380             "cdn_url": f"{CDN_BASE_URL}/{hf_path}",
    381             "raw_url": f"{RAW_BASE_URL}/{hf_path}",
    382         })
    383 
    384     response = {"files": results}
    385 
    386     if len(results) >= BATCH_MIN_FILES:
    387         batch_id = uuid.uuid4().hex[:10]
    388         await write_batch_manifest(batch_id, results)
    389         response["batch_id"] = batch_id
    390         response["batch_url"] = f"{CDN_BASE_URL}/pack/{batch_id}"
    391 
    392     return response
    393 
    394 
    395 def _safe_extract_tar(tar: tarfile.TarFile, dest: str):
    396     dest_real = os.path.realpath(dest)
    397 
    398     for member in tar.getmembers():
    399         member_path = os.path.realpath(os.path.join(dest, member.name))
    400 
    401         if not member_path.startswith(dest_real + os.sep) and member_path != dest_real:
    402             raise HTTPException(
    403                 status_code=400,
    404                 detail="Archive contains an unsafe path.",
    405             )
    406 
    407         if member.issym() or member.islnk():
    408             raise HTTPException(
    409                 status_code=400,
    410                 detail="Archive contains symlinks, which are not allowed.",
    411             )
    412 
    413     tar.extractall(dest)
    414 
    415 
    416 @app.get("/gh-sync")
    417 async def gh_sync_docs(request: Request):
    418     return templates.TemplateResponse(
    419         request,
    420         "gh_sync_docs.html",
    421         {"page": "gh-sync"},
    422     )
    423 
    424 
    425 @app.get("/documentation")
    426 async def documentation(request: Request):
    427     return templates.TemplateResponse(
    428         request,
    429         "documentation.html",
    430         {"page": "documentation"},
    431     )
    432 
    433 
    434 @app.post("/api/gh-sync")
    435 async def gh_sync(
    436     request: Request,
    437     token: str = Form(...),
    438     archive: UploadFile = File(...),
    439 ):
    440     claims = verify_actions_token(token)
    441     owner, repo = claims["owner"], claims["repo"]
    442     client_ip = get_client_ip(request)
    443 
    444     if await is_banned(client_ip):
    445         raise HTTPException(status_code=403, detail="You are banned from this CDN.")
    446     await gh_check(owner, claims["owner_id"])
    447 
    448     archive.file.seek(0, os.SEEK_END)
    449     archive_size = archive.file.tell()
    450     archive.file.seek(0)
    451     if archive_size > GH_SYNC_MAX_ARCHIVE_BYTES:  # checked before anything is read into memory
    452         raise HTTPException(
    453             status_code=413,
    454             detail=f"Archive too large (limit {format_size(GH_SYNC_MAX_ARCHIVE_BYTES)} after compression).",
    455         )
    456 
    457     with tempfile.TemporaryDirectory() as tmp_upload, tempfile.TemporaryDirectory() as tmp_extract:
    458         archive_path = os.path.join(tmp_upload, "payload.tar.gz")
    459 
    460         with open(archive_path, "wb") as f:
    461             shutil.copyfileobj(archive.file, f)  # streamed to disk, never held in memory
    462 
    463         try:
    464             with tarfile.open(archive_path, "r:gz") as tar:
    465                 _safe_extract_tar(tar, tmp_extract)
    466         except tarfile.TarError:
    467             raise HTTPException(
    468                 status_code=400,
    469                 detail="Could not read archive (expected a .tar.gz).",
    470             )
    471 
    472         for root, _dirs, names in os.walk(tmp_extract):
    473             for name in names:
    474                 full = os.path.join(root, name)
    475                 if is_video_upload(name, full):
    476                     raise HTTPException(
    477                         status_code=415,
    478                         detail=f"Video files are not allowed on this CDN (found {name}). Nothing was synced.",
    479                     )
    480 
    481         dest_prefix = await upload_folder_scoped(
    482             tmp_extract,
    483             owner,
    484             repo,
    485         )
    486 
    487     await gh_record(owner, claims["owner_id"], repo, claims["actor"], client_ip, archive_size, dest_prefix)
    488 
    489     return {
    490         "synced_to": dest_prefix,
    491         "cdn_url": f"{CDN_BASE_URL}/{dest_prefix}/",
    492         "raw_url_prefix": f"{RAW_BASE_URL}/{dest_prefix}/",
    493         "note": "cdn_url is the browsable folder listing. raw_url_prefix isn't a link by itself - append an individual filename to it to get that file's direct raw URL.",
    494         "ref": claims["ref"],
    495     }
    496 
    497 
    498 @app.get("/shorten")
    499 async def url_shortener_page(request: Request):
    500     ctx = await render_context({"page": "shorten"})
    501     return templates.TemplateResponse(request, "url_shortener.html", ctx)
    502 
    503 
    504 class ShortenRequest(BaseModel):
    505     url: str
    506 
    507 
    508 @app.post("/api/shorten")
    509 async def api_shorten(request: Request, body: ShortenRequest):
    510     url = body.url.strip()
    511     if len(url) > SHORTEN_MAX_URL_LENGTH:
    512         raise HTTPException(
    513             status_code=400,
    514             detail=f"URL too long (max {SHORTEN_MAX_URL_LENGTH} characters).",
    515         )
    516     client_ip = get_client_ip(request)
    517     if await is_banned(client_ip):
    518         raise HTTPException(status_code=403, detail="You are banned from this CDN.")
    519     _shorten_limiter.check_and_record(client_ip)
    520     _assert_public_url(url)
    521     short_id = await shorten_url(url)
    522 
    523     return {
    524         "id": short_id,
    525         "masked_url": f"{CDN_BASE_URL}/mask/{short_id}",
    526     }
    527 
    528 
    529 @app.get("/mask/{short_id}")
    530 async def mask_redirect(request: Request, short_id: str):
    531     destination = await get_destination_url(short_id)
    532 
    533     if destination is None:
    534         raise HTTPException(
    535             status_code=404,
    536             detail="Short URL not found",
    537         )
    538 
    539     return RedirectResponse(
    540         destination,
    541         status_code=307,
    542         headers={"Referrer-Policy": "no-referrer"},
    543     )
    544 
    545 
    546 _MAX_URL_UPLOAD_BYTES = URL_UPLOAD_LIMIT_PER_HOUR
    547 GH_SYNC_MAX_ARCHIVE_BYTES = 200 * 1024 * 1024    # GitHub sync: compressed .tar.gz size (no rate limit)
    548 
    549 
    550 def _assert_public_url(url: str):
    551     parsed = urlparse(url)
    552 
    553     if parsed.scheme != "https":
    554         raise HTTPException(
    555             status_code=400,
    556             detail="Only https:// URLs are supported.",
    557         )
    558 
    559     if not parsed.hostname:
    560         raise HTTPException(
    561             status_code=400,
    562             detail="Invalid URL.",
    563         )
    564 
    565     hostname_lower = parsed.hostname.lower()
    566 
    567     if hostname_lower == "localhost" or hostname_lower.endswith(".localhost"):
    568         raise HTTPException(
    569             status_code=400,
    570             detail="URLs pointing to private/internal addresses are not allowed.",
    571         )
    572 
    573     try:
    574         addrinfos = socket.getaddrinfo(parsed.hostname, None)
    575     except socket.gaierror:
    576         raise HTTPException(
    577             status_code=400,
    578             detail="Could not resolve host.",
    579         )
    580 
    581     for family, _, _, _, sockaddr in addrinfos:
    582         ip = ipaddress.ip_address(sockaddr[0])
    583 
    584         if (
    585             ip.is_private
    586             or ip.is_loopback
    587             or ip.is_link_local
    588             or ip.is_reserved
    589             or ip.is_multicast
    590             or ip.is_unspecified
    591         ):
    592             raise HTTPException(
    593                 status_code=400,
    594                 detail="URLs pointing to private/internal addresses are not allowed.",
    595             )
    596 
    597 
    598 class UploadUrlRequest(BaseModel):
    599     url: str
    600     folder: str = "uploads"
    601 
    602 
    603 @app.post("/api/put-url")
    604 async def handle_upload_from_url(
    605     request: Request,
    606     body: UploadUrlRequest,
    607 ):
    608     folder = _validate_folder(body.folder)
    609     client_ip = get_client_ip(request)
    610     url = body.url.strip()
    611 
    612     _assert_public_url(url)
    613 
    614     parsed = urlparse(url)
    615     filename = (
    616         os.path.basename(unquote(parsed.path))
    617         or f"download-{uuid.uuid4().hex[:8]}"
    618     )
    619 
    620     temp_path = f"/tmp/{uuid.uuid4()}"
    621     downloaded = 0
    622 
    623     async with httpx.AsyncClient(
    624         follow_redirects=True,
    625         timeout=30.0,
    626     ) as client:
    627         try:
    628             async with client.stream("GET", url) as r:
    629                 if r.status_code != 200:
    630                     raise HTTPException(
    631                         status_code=400,
    632                         detail=f"Source returned status {r.status_code}.",
    633                     )
    634 
    635                 content_length = r.headers.get("Content-Length")
    636 
    637                 if content_length and int(content_length) > _MAX_URL_UPLOAD_BYTES:
    638                     raise HTTPException(
    639                         status_code=413,
    640                         detail=f"File too large (limit {format_size(_MAX_URL_UPLOAD_BYTES)} per fetch).",
    641                     )
    642 
    643                 cd = r.headers.get("Content-Disposition", "")
    644 
    645                 if "filename=" in cd:
    646                     filename = cd.split("filename=")[-1].strip('"; ') or filename
    647 
    648                 with open(temp_path, "wb") as f:
    649                     async for chunk in r.aiter_bytes():
    650                         downloaded += len(chunk)
    651 
    652                         if downloaded > _MAX_URL_UPLOAD_BYTES:
    653                             raise HTTPException(
    654                                 status_code=413,
    655                                 detail=f"File too large (limit {format_size(_MAX_URL_UPLOAD_BYTES)} per fetch).",
    656                             )
    657 
    658                         f.write(chunk)
    659 
    660         except httpx.RequestError:
    661             if os.path.exists(temp_path):
    662                 os.remove(temp_path)
    663 
    664             raise HTTPException(
    665                 status_code=400,
    666                 detail="Could not fetch the URL.",
    667             )
    668 
    669         except HTTPException:
    670             if os.path.exists(temp_path):
    671                 os.remove(temp_path)
    672             raise
    673 
    674     try:
    675         guard_ctx = await inspect_upload(
    676             request,
    677             filename,
    678             temp_path,
    679             downloaded,
    680             folder,
    681             "url",
    682         )
    683         _url_upload_limiter.check_and_record(
    684             client_ip,
    685             downloaded,
    686         )
    687     except HTTPException:
    688         os.remove(temp_path)
    689         raise
    690 
    691     try:
    692         hf_path = await upload_temp_file(
    693             temp_path,
    694             filename,
    695             folder,
    696         )
    697     finally:
    698         if os.path.exists(temp_path):
    699             os.remove(temp_path)
    700 
    701     await record_upload(guard_ctx, hf_path)
    702 
    703     return {
    704         "files": [{
    705             "filename": filename,
    706             "hf_path": hf_path,
    707             "size": downloaded,
    708             "cdn_url": f"{CDN_BASE_URL}/{hf_path}",
    709             "raw_url": f"{RAW_BASE_URL}/{hf_path}",
    710         }]
    711     }
    712 
    713 
    714 @app.get("/api/pack-info/{path:path}")
    715 async def zip_stats(path: str):
    716     clean_path = path.strip("/")
    717 
    718     try:
    719         files = await list_files_recursive(clean_path)
    720     except ValueError as e:
    721         raise HTTPException(
    722             status_code=413,
    723             detail=str(e),
    724         )
    725 
    726     if not files:
    727         raise HTTPException(
    728             status_code=404,
    729             detail="Folder is empty or not found",
    730         )
    731 
    732     total_size = sum(f["size"] for f in files)
    733 
    734     return {
    735         "file_count": len(files),
    736         "total_size": total_size,
    737         "size_str": format_size(total_size),
    738     }
    739 
    740 
    741 @app.get("/api/pack/{path:path}")
    742 async def download_zip(path: str):
    743     clean_path = path.strip("/")
    744 
    745     try:
    746         files = await list_files_recursive(clean_path)
    747     except ValueError as e:
    748         raise HTTPException(
    749             status_code=413,
    750             detail=str(e),
    751         )
    752 
    753     if not files:
    754         raise HTTPException(
    755             status_code=404,
    756             detail="Folder is empty or not found",
    757         )
    758 
    759     prefix_len = (
    760         len(clean_path.rstrip("/")) + 1
    761         if clean_path
    762         else 0
    763     )
    764 
    765     ZIP_FETCH_CONCURRENCY = 8
    766     semaphore = asyncio.Semaphore(ZIP_FETCH_CONCURRENCY)
    767 
    768     async def fetch(client: httpx.AsyncClient, f: dict):
    769         hf_url = bucket_url(f["path"])
    770 
    771         async with semaphore:
    772             r = await client.get(hf_url, headers=auth_headers())
    773 
    774         return f, r
    775 
    776     buffer = io.BytesIO()
    777 
    778     async with httpx.AsyncClient(
    779         follow_redirects=True,
    780     ) as client:
    781         results = await asyncio.gather(
    782             *(fetch(client, f) for f in files)
    783         )
    784 
    785         with zipfile.ZipFile(
    786             buffer,
    787             "w",
    788             zipfile.ZIP_DEFLATED,
    789         ) as zf:
    790             for f, r in results:
    791                 if r.status_code != 200:
    792                     continue
    793 
    794                 arcname = (
    795                     f["path"][prefix_len:]
    796                     if prefix_len
    797                     else f["path"]
    798                 )
    799 
    800                 zf.writestr(
    801                     arcname,
    802                     r.content,
    803                 )
    804 
    805     buffer.seek(0)
    806 
    807     zip_filename = (
    808         clean_path.rstrip("/").split("/")[-1]
    809         if clean_path
    810         else HF_BUCKET_ID.split("/")[-1]
    811     ) + ".zip"
    812 
    813     return StreamingResponse(
    814         buffer,
    815         media_type="application/zip",
    816         headers={
    817             "Content-Disposition": (
    818                 f'attachment; filename="{zip_filename}"'
    819             )
    820         },
    821     )
    822 
    823 
    824 @app.get("/api/find")
    825 async def api_search(q: str = ""):
    826     results = await search_files(q)
    827     return {"results": results}
    828 
    829 
    830 @app.get("/api/tree")
    831 async def api_tree(request: Request):
    832     """Every non-hidden file as [path, size], for instant folder navigation (static/fasttree.js)."""
    833     items = await _get_full_tree()
    834     if not items:
    835         # Storage hiccup (or an empty bucket): say so rather than hand the browser an empty tree
    836         # that would replace a good cached one.
    837         raise HTTPException(status_code=503, detail="File tree unavailable")
    838     payload, etag = tree_payload(items)
    839     return json_response(request, payload, etag)
    840 
    841 
    842 @app.get("/api/shell")
    843 async def api_shell(request: Request):
    844     """A folder page skeleton plus the HTML fragments the browser fills in locally."""
    845     ctx = {
    846         "cdn_base_url": CDN_BASE_URL,
    847         "raw_base_url": RAW_BASE_URL,
    848         "repo_file_count": None,   # the footer stats slot is filled in from the tree
    849         "repo_size_str": None,
    850     }
    851     payload, etag = shell_payload(templates.env, ctx)
    852     return json_response(request, payload, etag)
    853 
    854 
    855 @app.get("/pack/{batch_id}")
    856 async def batch_page(request: Request, batch_id: str):
    857     manifest = await get_batch_manifest(batch_id)
    858 
    859     if not manifest:
    860         raise HTTPException(
    861             status_code=404,
    862             detail="Batch not found",
    863         )
    864 
    865     batch_files = []
    866 
    867     for f in manifest.get("files", []):
    868         item = dict(f)
    869         item["size_str"] = format_size(
    870             f.get("size")
    871         )
    872         batch_files.append(item)
    873 
    874     ctx = await render_context({
    875         "page": "batch",
    876         "batch_id": batch_id,
    877         "batch_files": batch_files,
    878     })
    879 
    880     return templates.TemplateResponse(
    881         request,
    882         "index.html",
    883         ctx,
    884     )
    885 
    886 
    887 @app.get("/api/pack-info-batch/{batch_id}")
    888 async def zip_stats_batch(batch_id: str):
    889     manifest = await get_batch_manifest(batch_id)
    890 
    891     if not manifest:
    892         raise HTTPException(
    893             status_code=404,
    894             detail="Batch not found",
    895         )
    896 
    897     files = manifest.get("files", [])
    898 
    899     if not files:
    900         raise HTTPException(
    901             status_code=404,
    902             detail="Batch is empty",
    903         )
    904 
    905     total_size = sum(
    906         f.get("size", 0) or 0
    907         for f in files
    908     )
    909 
    910     return {
    911         "file_count": len(files),
    912         "total_size": total_size,
    913         "size_str": format_size(total_size),
    914     }
    915 
    916 
    917 @app.get("/api/pack-batch/{batch_id}")
    918 async def download_zip_batch(batch_id: str):
    919     manifest = await get_batch_manifest(batch_id)
    920 
    921     if not manifest:
    922         raise HTTPException(
    923             status_code=404,
    924             detail="Batch not found",
    925         )
    926 
    927     files = manifest.get("files", [])
    928 
    929     if not files:
    930         raise HTTPException(
    931             status_code=404,
    932             detail="Batch is empty",
    933         )
    934 
    935     ZIP_FETCH_CONCURRENCY = 8
    936     semaphore = asyncio.Semaphore(ZIP_FETCH_CONCURRENCY)
    937 
    938     async def fetch(client: httpx.AsyncClient, f: dict):
    939         hf_url = bucket_url(f["hf_path"])
    940 
    941         async with semaphore:
    942             r = await client.get(hf_url, headers=auth_headers())
    943 
    944         return f, r
    945 
    946     seen_names = set()
    947     buffer = io.BytesIO()
    948 
    949     async with httpx.AsyncClient(
    950         follow_redirects=True,
    951     ) as client:
    952         results = await asyncio.gather(
    953             *(fetch(client, f) for f in files)
    954         )
    955 
    956         with zipfile.ZipFile(
    957             buffer,
    958             "w",
    959             zipfile.ZIP_DEFLATED,
    960         ) as zf:
    961             for f, r in results:
    962                 if r.status_code != 200:
    963                     continue
    964 
    965                 arcname = (
    966                     f.get("filename")
    967                     or f["hf_path"].split("/")[-1]
    968                 )
    969 
    970                 if arcname in seen_names:
    971                     arcname = (
    972                         f"{f['hf_path'].split('/')[-1].split('-')[0]}"
    973                         f"-{arcname}"
    974                     )
    975 
    976                 seen_names.add(arcname)
    977 
    978                 zf.writestr(
    979                     arcname,
    980                     r.content,
    981                 )
    982 
    983     buffer.seek(0)
    984 
    985     return StreamingResponse(
    986         buffer,
    987         media_type="application/zip",
    988         headers={
    989             "Content-Disposition": (
    990                 f'attachment; filename="batch-{batch_id}.zip"'
    991             )
    992         },
    993     )
    994 
    995 
    996 @app.api_route(
    997     "/ping",
    998     methods=["GET", "HEAD"],
    999     response_class=PlainTextResponse,
   1000 )
   1001 async def ping():
   1002     return "Server is awake!"
   1003 
   1004 
   1005 @app.api_route(
   1006     "/{path:path}",
   1007     methods=["GET", "HEAD"],
   1008 )
   1009 async def serve(request: Request, path: str):
   1010     clean_path = path.strip("/")
   1011 
   1012     if request.url.hostname == RAW_DOMAIN:
   1013         if not clean_path:
   1014             return HTMLResponse(
   1015                 "Specify a file path.",
   1016                 status_code=200,
   1017             )
   1018 
   1019         try:
   1020             return await stream_raw(
   1021                 clean_path,
   1022                 request,
   1023             )
   1024         except HTTPException as e:
   1025             if e.status_code == 404:
   1026                 entries = await list_directory(clean_path)
   1027 
   1028                 if entries:
   1029                     return RedirectResponse(
   1030                         f"{CDN_BASE_URL}/{clean_path}/"
   1031                     )
   1032 
   1033             raise
   1034 
   1035     if (
   1036         clean_path == RAW_PREFIX.rstrip("/")
   1037         or clean_path.startswith(RAW_PREFIX)
   1038     ):
   1039         raw_path = clean_path[len(RAW_PREFIX):]
   1040 
   1041         if not raw_path:
   1042             return HTMLResponse(
   1043                 "/raw/   specify a file path after this prefix.",
   1044                 status_code=200,
   1045             )
   1046 
   1047         return RedirectResponse(
   1048             f"{RAW_BASE_URL}/{raw_path}"
   1049         )
   1050 
   1051     if clean_path == "new":
   1052         ctx = await render_context({"page": "upload"})
   1053         return templates.TemplateResponse(
   1054             request,
   1055             "index.html",
   1056             ctx,
   1057         )
   1058 
   1059     if clean_path == "find":
   1060         ctx = await render_context({"page": "search"})
   1061         return templates.TemplateResponse(
   1062             request,
   1063             "index.html",
   1064             ctx,
   1065         )
   1066 
   1067     if clean_path == "history":
   1068         ctx = await render_context({"page": "history"})
   1069         return templates.TemplateResponse(
   1070             request,
   1071             "index.html",
   1072             ctx,
   1073         )
   1074 
   1075     if clean_path:
   1076         info = await get_path_info(clean_path)
   1077 
   1078         if info["exists"]:
   1079             filename = clean_path.split("/")[-1]
   1080             ext = (
   1081                 filename.rsplit(".", 1)[-1].lower()
   1082                 if "." in filename
   1083                 else ""
   1084             )
   1085 
   1086             ctx = await render_context({
   1087                 "page": "file",
   1088                 "path": clean_path,
   1089                 "filename": filename,
   1090                 "raw_base_url": RAW_BASE_URL,
   1091                 "file_ext": ext,
   1092                 "file_size_str": format_size(info["size"]),
   1093             })
   1094 
   1095             return templates.TemplateResponse(
   1096                 request,
   1097                 "index.html",
   1098                 ctx,
   1099             )
   1100 
   1101     items = await list_directory(clean_path)
   1102 
   1103     if clean_path and not items:
   1104         raise HTTPException(
   1105             status_code=404,
   1106             detail="Not Found",
   1107         )
   1108 
   1109     extra_ctx = {
   1110         "page": "listing",
   1111         "path": clean_path,
   1112         "items": items,
   1113         "raw_base_url": RAW_BASE_URL,
   1114     }
   1115 
   1116     ctx = await render_context(extra_ctx)
   1117 
   1118     return templates.TemplateResponse(
   1119         request,
   1120         "index.html",
   1121         ctx,
   1122     )