commit 34b4fc1fa2321c61fbb8393b5485e0a582e02e66
parent 39ec30ed89e102a3f9a8c2b5ea37b646695db99a
Author: Amit Dutta <mail@amit.is-a.dev>
Date: Wed, 16 Sep 2026 22:58:27 +0530
Refactor HTTP exception handling and response formatting
Diffstat:
| M | app/main.py | | | 657 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------- |
1 file changed, 528 insertions(+), 129 deletions(-)
diff --git a/app/main.py b/app/main.py
@@ -10,7 +10,6 @@ import tempfile
import asyncio
import socket
import ipaddress
-import random
from urllib.parse import urlparse, unquote
import httpx
from collections import deque
@@ -44,6 +43,7 @@ templates = Jinja2Templates(directory="app/templates")
STATIC_DIR = Path(__file__).parent / "static"
_NO_STORE_FILES = {"manifest.json", "sw.js"}
+
@app.get("/static/{filename}")
async def static_no_cache_root(filename: str):
if filename not in _NO_STORE_FILES:
@@ -56,6 +56,7 @@ async def static_no_cache_root(filename: str):
headers={"Cache-Control": "no-store, no-cache, must-revalidate, max-age=0"},
)
+
@app.get("/static/icons/{filename}")
async def static_icons(filename: str):
file_path = STATIC_DIR / "icons" / filename
@@ -66,6 +67,7 @@ async def static_icons(filename: str):
headers={"Cache-Control": "public, max-age=86400"},
)
+
@app.get("/favicon.ico")
async def favicon():
return FileResponse(
@@ -73,11 +75,13 @@ async def favicon():
headers={"Cache-Control": "public, max-age=86400"},
)
+
CDN_BASE_URL = os.getenv("CDN_BASE_URL", "https://cdn.amit.is-a.dev")
RAW_DOMAIN = os.getenv("RAW_DOMAIN", "raw.cdn.amit.is-a.dev")
RAW_BASE_URL = os.getenv("RAW_BASE_URL", f"https://{RAW_DOMAIN}")
RAW_PREFIX = "raw/"
+
async def render_context(extra: dict) -> dict:
stats = await repo_stats()
ctx = dict(extra)
@@ -87,6 +91,7 @@ async def render_context(extra: dict) -> dict:
ctx.setdefault("raw_base_url", RAW_BASE_URL)
return ctx
+
@app.exception_handler(404)
async def not_found_handler(request: Request, exc: HTTPException):
if request.url.hostname == RAW_DOMAIN:
@@ -94,6 +99,7 @@ async def not_found_handler(request: Request, exc: HTTPException):
ctx = await render_context({"page": "404"})
return templates.TemplateResponse(request, "index.html", ctx, status_code=404)
+
async def _proxy_stream(client: httpx.AsyncClient, r: httpx.Response):
try:
async for chunk in r.aiter_raw():
@@ -101,6 +107,7 @@ async def _proxy_stream(client: httpx.AsyncClient, r: httpx.Response):
finally:
await client.aclose()
+
async def stream_raw(path: str, request: Request):
hf_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/{path}"
client = httpx.AsyncClient(follow_redirects=True)
@@ -110,28 +117,33 @@ async def stream_raw(path: str, request: Request):
req_headers["Range"] = range_header
req = client.build_request("GET", hf_url, headers=req_headers)
r = await client.send(req, stream=True)
-
+
if r.status_code not in (200, 206):
await client.aclose()
raise HTTPException(status_code=404, detail="File not found")
-
+
headers = {
"Access-Control-Allow-Origin": "*",
"Cache-Control": "public, max-age=31536000",
"X-Content-Type-Options": "nosniff",
"Accept-Ranges": "bytes",
}
-
+
for h in ["Content-Type", "Content-Encoding", "Content-Length", "Etag", "Content-Range"]:
if h in r.headers:
headers[h] = r.headers[h]
-
+
filename = path.split("/")[-1].lower()
if filename.endswith(".md") or filename.endswith(".txt"):
if headers.get("Content-Type", "application/octet-stream") == "application/octet-stream":
headers["Content-Type"] = "text/plain; charset=utf-8"
-
- return StreamingResponse(_proxy_stream(client, r), status_code=r.status_code, headers=headers)
+
+ return StreamingResponse(
+ _proxy_stream(client, r),
+ status_code=r.status_code,
+ headers=headers,
+ )
+
@app.get("/api/get/{path:path}")
async def download_file(path: str, request: Request):
@@ -143,26 +155,33 @@ async def download_file(path: str, request: Request):
req_headers["Range"] = range_header
req = client.build_request("GET", hf_url, headers=req_headers)
r = await client.send(req, stream=True)
-
+
if r.status_code not in (200, 206):
await client.aclose()
raise HTTPException(status_code=404, detail="Not found")
-
+
filename = path.split("/")[-1]
headers = {
"Content-Disposition": f'attachment; filename="{filename}"',
"Content-Type": r.headers.get("Content-Type", "application/octet-stream"),
"Accept-Ranges": "bytes",
}
+
for h in ["Content-Length", "Content-Encoding", "Etag", "Content-Range"]:
if h in r.headers:
headers[h] = r.headers[h]
-
- return StreamingResponse(_proxy_stream(client, r), status_code=r.status_code, headers=headers)
+
+ return StreamingResponse(
+ _proxy_stream(client, r),
+ status_code=r.status_code,
+ headers=headers,
+ )
+
UPLOAD_LIMIT_PER_MINUTE = 50 * 1024 * 1024
UPLOAD_LIMIT_PER_HOUR = 300 * 1024 * 1024
+
class _UploadRateLimiter:
def __init__(self):
self._usage: dict[str, deque] = {}
@@ -186,6 +205,7 @@ class _UploadRateLimiter:
status_code=429,
detail=f"Upload rate limit exceeded: {format_size(UPLOAD_LIMIT_PER_MINUTE)}/minute. Try again shortly.",
)
+
if hour_used + size > UPLOAD_LIMIT_PER_HOUR:
raise HTTPException(
status_code=429,
@@ -194,10 +214,12 @@ class _UploadRateLimiter:
dq.append((now, size))
+
_upload_limiter = _UploadRateLimiter()
BATCH_MIN_FILES = 2
ALLOWED_UPLOAD_FOLDERS = {"uploads", "third-party"}
+
def _validate_folder(folder: str) -> str:
folder = (folder or "uploads").strip().strip("/")
if folder not in ALLOWED_UPLOAD_FOLDERS:
@@ -207,6 +229,7 @@ def _validate_folder(folder: str) -> str:
)
return folder
+
@app.post("/api/put")
async def handle_upload(
request: Request,
@@ -219,8 +242,10 @@ async def handle_upload(
for file in files:
temp_path = f"/tmp/{uuid.uuid4()}-{file.filename}"
+
with open(temp_path, "wb") as f:
shutil.copyfileobj(file.file, f)
+
size = os.path.getsize(temp_path)
try:
@@ -244,6 +269,7 @@ async def handle_upload(
})
response = {"files": results}
+
if len(results) >= BATCH_MIN_FILES:
batch_id = uuid.uuid4().hex[:10]
await write_batch_manifest(batch_id, results)
@@ -252,38 +278,69 @@ async def handle_upload(
return response
+
GH_SYNC_LIMIT_PER_HOUR = 2 * 1024 * 1024 * 1024
_gh_sync_limiter = _UploadRateLimiter()
+
def _safe_extract_tar(tar: tarfile.TarFile, dest: str):
dest_real = os.path.realpath(dest)
+
for member in tar.getmembers():
member_path = os.path.realpath(os.path.join(dest, member.name))
+
if not member_path.startswith(dest_real + os.sep) and member_path != dest_real:
- raise HTTPException(status_code=400, detail="Archive contains an unsafe path.")
+ raise HTTPException(
+ status_code=400,
+ detail="Archive contains an unsafe path.",
+ )
+
if member.issym() or member.islnk():
- raise HTTPException(status_code=400, detail="Archive contains symlinks, which are not allowed.")
+ raise HTTPException(
+ status_code=400,
+ detail="Archive contains symlinks, which are not allowed.",
+ )
+
tar.extractall(dest)
+
@app.get("/gh-sync")
async def gh_sync_docs(request: Request):
- return templates.TemplateResponse(request, "gh_sync_docs.html", {"page": "gh-sync"})
+ return templates.TemplateResponse(
+ request,
+ "gh_sync_docs.html",
+ {"page": "gh-sync"},
+ )
+
@app.get("/documentation")
async def documentation(request: Request):
- return templates.TemplateResponse(request, "documentation.html", {"page": "documentation"})
+ return templates.TemplateResponse(
+ request,
+ "documentation.html",
+ {"page": "documentation"},
+ )
+
@app.post("/api/gh-sync")
-async def gh_sync(request: Request, token: str = Form(...), archive: UploadFile = File(...)):
+async def gh_sync(
+ request: Request,
+ token: str = Form(...),
+ archive: UploadFile = File(...),
+):
claims = verify_actions_token(token)
owner, repo = claims["owner"], claims["repo"]
client_ip = request.client.host if request.client else "unknown"
contents = await archive.read()
- _gh_sync_limiter.check_and_record(f"gh-sync:{owner}/{repo}:{client_ip}", len(contents))
+ _gh_sync_limiter.check_and_record(
+ f"gh-sync:{owner}/{repo}:{client_ip}",
+ len(contents),
+ )
with tempfile.TemporaryDirectory() as tmp_upload, tempfile.TemporaryDirectory() as tmp_extract:
archive_path = os.path.join(tmp_upload, "payload.tar.gz")
+
with open(archive_path, "wb") as f:
f.write(contents)
@@ -291,9 +348,16 @@ async def gh_sync(request: Request, token: str = Form(...), archive: UploadFile
with tarfile.open(archive_path, "r:gz") as tar:
_safe_extract_tar(tar, tmp_extract)
except tarfile.TarError:
- raise HTTPException(status_code=400, detail="Could not read archive (expected a .tar.gz).")
+ raise HTTPException(
+ status_code=400,
+ detail="Could not read archive (expected a .tar.gz).",
+ )
- dest_prefix = await upload_folder_scoped(tmp_extract, owner, repo)
+ dest_prefix = await upload_folder_scoped(
+ tmp_extract,
+ owner,
+ repo,
+ )
return {
"synced_to": dest_prefix,
@@ -303,14 +367,17 @@ async def gh_sync(request: Request, token: str = Form(...), archive: UploadFile
"ref": claims["ref"],
}
+
@app.get("/ytmusic")
async def url_ytmusic_page(request: Request):
ctx = await render_context({"page": "ytmusic"})
return templates.TemplateResponse(request, "ytmusic.html", ctx)
+
# ---------------------------------------------------------------------------------
# YTMusic routes: Cobalt API Proxy
# ---------------------------------------------------------------------------------
+
class YtRequest(BaseModel):
url: str
@@ -328,12 +395,6 @@ COBALT_PUBLIC_PREFIX = "/cobalt"
methods=["GET", "HEAD"],
)
async def cobalt_tunnel_proxy(path: str, request: Request):
- """
- Proxy Cobalt tunnel requests through the same Render service.
-
- Cobalt itself listens only on 127.0.0.1:9000, while this FastAPI
- application is the only publicly exposed process.
- """
cobalt_url = f"{COBALT_INTERNAL_URL}/{path}"
query_string = request.url.query
@@ -342,30 +403,52 @@ async def cobalt_tunnel_proxy(path: str, request: Request):
headers = {}
- for header in (
+ for header in [
"Range",
"If-Range",
"If-None-Match",
"If-Modified-Since",
- ):
+ ]:
value = request.headers.get(header)
if value:
headers[header] = value
client = httpx.AsyncClient(
follow_redirects=False,
- timeout=None,
+ timeout=60.0,
)
try:
- upstream = await client.send(
- client.build_request(
- request.method,
- cobalt_url,
- headers=headers,
- ),
- stream=True,
+ req = client.build_request(
+ request.method,
+ cobalt_url,
+ headers=headers,
+ )
+
+ res = await client.send(req, stream=True)
+
+ response_headers = {}
+
+ for header in [
+ "Content-Type",
+ "Content-Length",
+ "Content-Range",
+ "Accept-Ranges",
+ "Content-Disposition",
+ "Cache-Control",
+ "ETag",
+ "Last-Modified",
+ ]:
+ value = res.headers.get(header)
+ if value:
+ response_headers[header] = value
+
+ return StreamingResponse(
+ _proxy_stream(client, res),
+ status_code=res.status_code,
+ headers=response_headers,
)
+
except httpx.RequestError as exc:
await client.aclose()
raise HTTPException(
@@ -373,36 +456,6 @@ async def cobalt_tunnel_proxy(path: str, request: Request):
detail=f"Error connecting to Cobalt tunnel: {exc}",
)
- response_headers = {}
-
- for header in (
- "content-type",
- "content-length",
- "content-range",
- "accept-ranges",
- "content-disposition",
- "cache-control",
- "etag",
- "last-modified",
- "estimated-content-length",
- ):
- value = upstream.headers.get(header)
- if value:
- response_headers[header] = value
-
- async def stream_cobalt():
- try:
- async for chunk in upstream.aiter_raw():
- yield chunk
- finally:
- await client.aclose()
-
- return StreamingResponse(
- stream_cobalt(),
- status_code=upstream.status_code,
- headers=response_headers,
- )
-
@app.post("/api/yt/download")
async def yt_download_cobalt(request: Request, body: YtRequest):
@@ -486,13 +539,13 @@ async def yt_download_cobalt(request: Request, body: YtRequest):
print("========== COBALT ERROR ==========")
print(f"Full response: {data}")
-
+
if isinstance(error, dict):
print(f"Error code: {error.get('code')}")
print(f"Error context: {error.get('context')}")
else:
print(f"Error: {error}")
-
+
print("==================================")
if isinstance(error, dict):
@@ -526,8 +579,6 @@ async def yt_download_cobalt(request: Request, body: YtRequest):
),
)
- # Cobalt may return its own tunnel URL.
- # Convert it to our public FastAPI proxy URL.
if status == "tunnel":
parsed = urlparse(dl_url)
@@ -555,57 +606,170 @@ async def yt_download_cobalt(request: Request, body: YtRequest):
# ---------------------------------------------------------------------------------
+# TEMPORARY COBALT DIAGNOSTIC ENDPOINT
+# ---------------------------------------------------------------------------------
+
+@app.get("/api/yt/debug")
+async def yt_debug(url: str):
+ url = url.strip()
+
+ if not url:
+ raise HTTPException(
+ status_code=400,
+ detail="Missing YouTube URL.",
+ )
+
+ cobalt_api = f"{COBALT_INTERNAL_URL}/"
+
+ headers = {
+ "Accept": "application/json",
+ "Content-Type": "application/json",
+ "User-Agent": (
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/122.0.0.0 Safari/537.36"
+ ),
+ }
+
+ cobalt_api_token = os.getenv("COBALT_API_TOKEN")
+
+ if cobalt_api_token:
+ if " " not in cobalt_api_token:
+ headers["Authorization"] = f"Api-Key {cobalt_api_token}"
+ else:
+ headers["Authorization"] = cobalt_api_token
+
+ payload = {
+ "url": url,
+ "downloadMode": "audio",
+ "audioFormat": "mp3",
+ "audioBitrate": "128",
+ "filenameStyle": "basic",
+ }
+
+ async with httpx.AsyncClient(
+ timeout=60.0,
+ follow_redirects=False,
+ ) as client:
+ try:
+ res = await client.post(
+ cobalt_api,
+ json=payload,
+ headers=headers,
+ )
+
+ raw_text = res.text
+
+ try:
+ response_body = res.json()
+ except Exception:
+ response_body = raw_text
+
+ print("========== COBALT DEBUG ==========")
+ print(f"HTTP status: {res.status_code}")
+ print(f"Response headers: {dict(res.headers)}")
+ print(f"Response body: {response_body}")
+ print("===================================")
+
+ return {
+ "cobalt_url": cobalt_api,
+ "http_status": res.status_code,
+ "response_headers": dict(res.headers),
+ "response": response_body,
+ }
+
+ except httpx.RequestError as exc:
+ print("========== COBALT DEBUG ERROR ==========")
+ print(f"Request error: {repr(exc)}")
+ print("========================================")
+
+ raise HTTPException(
+ status_code=502,
+ detail={
+ "message": "Error connecting to Cobalt",
+ "error": repr(exc),
+ },
+ )
+
+
+# ---------------------------------------------------------------------------------
@app.get("/shorten")
async def url_shortener_page(request: Request):
ctx = await render_context({"page": "shorten"})
return templates.TemplateResponse(request, "url_shortener.html", ctx)
+
class ShortenRequest(BaseModel):
url: str
+
@app.post("/api/shorten")
async def api_shorten(request: Request, body: ShortenRequest):
url = body.url.strip()
_assert_public_url(url)
short_id = await shorten_url(url)
+
return {
"id": short_id,
"masked_url": f"{CDN_BASE_URL}/mask/{short_id}",
}
+
@app.get("/mask/{short_id}")
async def mask_redirect(request: Request, short_id: str):
destination = await get_destination_url(short_id)
+
if destination is None:
- raise HTTPException(status_code=404, detail="Short URL not found")
+ raise HTTPException(
+ status_code=404,
+ detail="Short URL not found",
+ )
+
return RedirectResponse(
destination,
status_code=307,
- headers={"Referrer-Policy": "no-referrer"}
+ headers={"Referrer-Policy": "no-referrer"},
)
+
_MAX_URL_UPLOAD_BYTES = UPLOAD_LIMIT_PER_HOUR
+
def _assert_public_url(url: str):
parsed = urlparse(url)
+
if parsed.scheme != "https":
- raise HTTPException(status_code=400, detail="Only https:// URLs are supported.")
+ raise HTTPException(
+ status_code=400,
+ detail="Only https:// URLs are supported.",
+ )
+
if not parsed.hostname:
- raise HTTPException(status_code=400, detail="Invalid URL.")
+ raise HTTPException(
+ status_code=400,
+ detail="Invalid URL.",
+ )
+
hostname_lower = parsed.hostname.lower()
+
if hostname_lower == "localhost" or hostname_lower.endswith(".localhost"):
raise HTTPException(
status_code=400,
detail="URLs pointing to private/internal addresses are not allowed.",
)
+
try:
addrinfos = socket.getaddrinfo(parsed.hostname, None)
except socket.gaierror:
- raise HTTPException(status_code=400, detail="Could not resolve host.")
-
+ raise HTTPException(
+ status_code=400,
+ detail="Could not resolve host.",
+ )
+
for family, _, _, _, sockaddr in addrinfos:
ip = ipaddress.ip_address(sockaddr[0])
+
if (
ip.is_private
or ip.is_loopback
@@ -619,33 +783,54 @@ def _assert_public_url(url: str):
detail="URLs pointing to private/internal addresses are not allowed.",
)
+
class UploadUrlRequest(BaseModel):
url: str
folder: str = "uploads"
+
@app.post("/api/put-url")
-async def handle_upload_from_url(request: Request, body: UploadUrlRequest):
+async def handle_upload_from_url(
+ request: Request,
+ body: UploadUrlRequest,
+):
folder = _validate_folder(body.folder)
client_ip = request.client.host if request.client else "unknown"
url = body.url.strip()
+
_assert_public_url(url)
+
parsed = urlparse(url)
- filename = os.path.basename(unquote(parsed.path)) or f"download-{uuid.uuid4().hex[:8]}"
+ filename = (
+ os.path.basename(unquote(parsed.path))
+ or f"download-{uuid.uuid4().hex[:8]}"
+ )
+
temp_path = f"/tmp/{uuid.uuid4()}-{filename}"
downloaded = 0
- async with httpx.AsyncClient(follow_redirects=True, timeout=30.0) as client:
+ async with httpx.AsyncClient(
+ follow_redirects=True,
+ timeout=30.0,
+ ) as client:
try:
async with client.stream("GET", url) as r:
if r.status_code != 200:
- raise HTTPException(status_code=400, detail=f"Source returned status {r.status_code}.")
+ raise HTTPException(
+ status_code=400,
+ detail=f"Source returned status {r.status_code}.",
+ )
+
content_length = r.headers.get("Content-Length")
+
if content_length and int(content_length) > _MAX_URL_UPLOAD_BYTES:
raise HTTPException(
status_code=413,
detail=f"File too large (limit {format_size(_MAX_URL_UPLOAD_BYTES)} per fetch).",
)
+
cd = r.headers.get("Content-Disposition", "")
+
if "filename=" in cd:
filename = cd.split("filename=")[-1].strip('"; ') or filename
temp_path = f"/tmp/{uuid.uuid4()}-{filename}"
@@ -653,29 +838,44 @@ async def handle_upload_from_url(request: Request, body: UploadUrlRequest):
with open(temp_path, "wb") as f:
async for chunk in r.aiter_bytes():
downloaded += len(chunk)
+
if downloaded > _MAX_URL_UPLOAD_BYTES:
raise HTTPException(
status_code=413,
detail=f"File too large (limit {format_size(_MAX_URL_UPLOAD_BYTES)} per fetch).",
)
+
f.write(chunk)
+
except httpx.RequestError:
if os.path.exists(temp_path):
os.remove(temp_path)
- raise HTTPException(status_code=400, detail="Could not fetch the URL.")
+
+ raise HTTPException(
+ status_code=400,
+ detail="Could not fetch the URL.",
+ )
+
except HTTPException:
if os.path.exists(temp_path):
os.remove(temp_path)
raise
try:
- _upload_limiter.check_and_record(client_ip, downloaded)
+ _upload_limiter.check_and_record(
+ client_ip,
+ downloaded,
+ )
except HTTPException:
os.remove(temp_path)
raise
try:
- hf_path = await upload_temp_file(temp_path, filename, folder)
+ hf_path = await upload_temp_file(
+ temp_path,
+ filename,
+ folder,
+ )
finally:
if os.path.exists(temp_path):
os.remove(temp_path)
@@ -690,176 +890,360 @@ async def handle_upload_from_url(request: Request, body: UploadUrlRequest):
}]
}
+
@app.get("/api/pack-info/{path:path}")
async def zip_stats(path: str):
clean_path = path.strip("/")
+
try:
files = await list_files_recursive(clean_path)
except ValueError as e:
- raise HTTPException(status_code=413, detail=str(e))
+ raise HTTPException(
+ status_code=413,
+ detail=str(e),
+ )
+
if not files:
- raise HTTPException(status_code=404, detail="Folder is empty or not found")
+ raise HTTPException(
+ status_code=404,
+ detail="Folder is empty or not found",
+ )
+
total_size = sum(f["size"] for f in files)
+
return {
"file_count": len(files),
"total_size": total_size,
"size_str": format_size(total_size),
}
+
@app.get("/api/pack/{path:path}")
async def download_zip(path: str):
clean_path = path.strip("/")
+
try:
files = await list_files_recursive(clean_path)
except ValueError as e:
- raise HTTPException(status_code=413, detail=str(e))
+ raise HTTPException(
+ status_code=413,
+ detail=str(e),
+ )
+
if not files:
- raise HTTPException(status_code=404, detail="Folder is empty or not found")
+ raise HTTPException(
+ status_code=404,
+ detail="Folder is empty or not found",
+ )
+
+ prefix_len = (
+ len(clean_path.rstrip("/")) + 1
+ if clean_path
+ else 0
+ )
- prefix_len = len(clean_path.rstrip("/")) + 1 if clean_path else 0
ZIP_FETCH_CONCURRENCY = 8
semaphore = asyncio.Semaphore(ZIP_FETCH_CONCURRENCY)
async def fetch(client: httpx.AsyncClient, f: dict):
- hf_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/{f['path']}"
+ hf_url = (
+ f"https://huggingface.co/datasets/"
+ f"{HF_REPO_ID}/resolve/main/{f['path']}"
+ )
+
async with semaphore:
r = await client.get(hf_url)
+
return f, r
buffer = io.BytesIO()
- async with httpx.AsyncClient(follow_redirects=True) as client:
- results = await asyncio.gather(*(fetch(client, f) for f in files))
- with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zf:
+
+ async with httpx.AsyncClient(
+ follow_redirects=True,
+ ) as client:
+ results = await asyncio.gather(
+ *(fetch(client, f) for f in files)
+ )
+
+ with zipfile.ZipFile(
+ buffer,
+ "w",
+ zipfile.ZIP_DEFLATED,
+ ) as zf:
for f, r in results:
if r.status_code != 200:
continue
- arcname = f["path"][prefix_len:] if prefix_len else f["path"]
- zf.writestr(arcname, r.content)
+
+ arcname = (
+ f["path"][prefix_len:]
+ if prefix_len
+ else f["path"]
+ )
+
+ zf.writestr(
+ arcname,
+ r.content,
+ )
buffer.seek(0)
- zip_filename = (clean_path.rstrip("/").split("/")[-1] if clean_path else HF_REPO_ID.split("/")[-1]) + ".zip"
+
+ zip_filename = (
+ clean_path.rstrip("/").split("/")[-1]
+ if clean_path
+ else HF_REPO_ID.split("/")[-1]
+ ) + ".zip"
+
return StreamingResponse(
buffer,
media_type="application/zip",
- headers={"Content-Disposition": f'attachment; filename="{zip_filename}"'},
+ headers={
+ "Content-Disposition": (
+ f'attachment; filename="{zip_filename}"'
+ )
+ },
)
+
@app.get("/api/find")
async def api_search(q: str = ""):
results = await search_files(q)
return {"results": results}
+
@app.get("/pack/{batch_id}")
async def batch_page(request: Request, batch_id: str):
manifest = await get_batch_manifest(batch_id)
+
if not manifest:
- raise HTTPException(status_code=404, detail="Batch not found")
+ raise HTTPException(
+ status_code=404,
+ detail="Batch not found",
+ )
+
batch_files = []
+
for f in manifest.get("files", []):
item = dict(f)
- item["size_str"] = format_size(f.get("size"))
+ item["size_str"] = format_size(
+ f.get("size")
+ )
batch_files.append(item)
+
ctx = await render_context({
"page": "batch",
"batch_id": batch_id,
"batch_files": batch_files,
})
- return templates.TemplateResponse(request, "index.html", ctx)
+
+ return templates.TemplateResponse(
+ request,
+ "index.html",
+ ctx,
+ )
+
@app.get("/api/pack-info-batch/{batch_id}")
async def zip_stats_batch(batch_id: str):
manifest = await get_batch_manifest(batch_id)
+
if not manifest:
- raise HTTPException(status_code=404, detail="Batch not found")
+ raise HTTPException(
+ status_code=404,
+ detail="Batch not found",
+ )
+
files = manifest.get("files", [])
+
if not files:
- raise HTTPException(status_code=404, detail="Batch is empty")
- total_size = sum(f.get("size", 0) or 0 for f in files)
+ raise HTTPException(
+ status_code=404,
+ detail="Batch is empty",
+ )
+
+ total_size = sum(
+ f.get("size", 0) or 0
+ for f in files
+ )
+
return {
"file_count": len(files),
"total_size": total_size,
"size_str": format_size(total_size),
}
+
@app.get("/api/pack-batch/{batch_id}")
async def download_zip_batch(batch_id: str):
manifest = await get_batch_manifest(batch_id)
+
if not manifest:
- raise HTTPException(status_code=404, detail="Batch not found")
+ raise HTTPException(
+ status_code=404,
+ detail="Batch not found",
+ )
+
files = manifest.get("files", [])
+
if not files:
- raise HTTPException(status_code=404, detail="Batch is empty")
+ raise HTTPException(
+ status_code=404,
+ detail="Batch is empty",
+ )
+
ZIP_FETCH_CONCURRENCY = 8
semaphore = asyncio.Semaphore(ZIP_FETCH_CONCURRENCY)
async def fetch(client: httpx.AsyncClient, f: dict):
- hf_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/{f['hf_path']}"
+ hf_url = (
+ f"https://huggingface.co/datasets/"
+ f"{HF_REPO_ID}/resolve/main/{f['hf_path']}"
+ )
+
async with semaphore:
r = await client.get(hf_url)
+
return f, r
seen_names = set()
buffer = io.BytesIO()
- async with httpx.AsyncClient(follow_redirects=True) as client:
- results = await asyncio.gather(*(fetch(client, f) for f in files))
- with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zf:
+
+ async with httpx.AsyncClient(
+ follow_redirects=True,
+ ) as client:
+ results = await asyncio.gather(
+ *(fetch(client, f) for f in files)
+ )
+
+ with zipfile.ZipFile(
+ buffer,
+ "w",
+ zipfile.ZIP_DEFLATED,
+ ) as zf:
for f, r in results:
if r.status_code != 200:
continue
- arcname = f.get("filename") or f["hf_path"].split("/")[-1]
+
+ arcname = (
+ f.get("filename")
+ or f["hf_path"].split("/")[-1]
+ )
+
if arcname in seen_names:
- arcname = f"{f['hf_path'].split('/')[-1].split('-')[0]}-{arcname}"
+ arcname = (
+ f"{f['hf_path'].split('/')[-1].split('-')[0]}"
+ f"-{arcname}"
+ )
+
seen_names.add(arcname)
- zf.writestr(arcname, r.content)
+
+ zf.writestr(
+ arcname,
+ r.content,
+ )
buffer.seek(0)
+
return StreamingResponse(
buffer,
media_type="application/zip",
- headers={"Content-Disposition": f'attachment; filename="batch-{batch_id}.zip"'},
+ headers={
+ "Content-Disposition": (
+ f'attachment; filename="batch-{batch_id}.zip"'
+ )
+ },
)
-@app.api_route("/ping", methods=["GET", "HEAD"], response_class=PlainTextResponse)
+
+@app.api_route(
+ "/ping",
+ methods=["GET", "HEAD"],
+ response_class=PlainTextResponse,
+)
async def ping():
return "Server is awake!"
-@app.api_route("/{path:path}", methods=["GET", "HEAD"])
+
+@app.api_route(
+ "/{path:path}",
+ methods=["GET", "HEAD"],
+)
async def serve(request: Request, path: str):
clean_path = path.strip("/")
-
+
if request.url.hostname == RAW_DOMAIN:
if not clean_path:
- return HTMLResponse("Specify a file path.", status_code=200)
+ return HTMLResponse(
+ "Specify a file path.",
+ status_code=200,
+ )
+
try:
- return await stream_raw(clean_path, request)
+ return await stream_raw(
+ clean_path,
+ request,
+ )
except HTTPException as e:
if e.status_code == 404:
entries = await list_directory(clean_path)
+
if entries:
- return RedirectResponse(f"{CDN_BASE_URL}/{clean_path}/")
+ return RedirectResponse(
+ f"{CDN_BASE_URL}/{clean_path}/"
+ )
+
raise
- if clean_path == RAW_PREFIX.rstrip("/") or clean_path.startswith(RAW_PREFIX):
+ if (
+ clean_path == RAW_PREFIX.rstrip("/")
+ or clean_path.startswith(RAW_PREFIX)
+ ):
raw_path = clean_path[len(RAW_PREFIX):]
+
if not raw_path:
- return HTMLResponse("/raw/ specify a file path after this prefix.", status_code=200)
- return RedirectResponse(f"{RAW_BASE_URL}/{raw_path}")
+ return HTMLResponse(
+ "/raw/ specify a file path after this prefix.",
+ status_code=200,
+ )
+
+ return RedirectResponse(
+ f"{RAW_BASE_URL}/{raw_path}"
+ )
if clean_path == "new":
ctx = await render_context({"page": "upload"})
- return templates.TemplateResponse(request, "index.html", ctx)
+ return templates.TemplateResponse(
+ request,
+ "index.html",
+ ctx,
+ )
+
if clean_path == "find":
ctx = await render_context({"page": "search"})
- return templates.TemplateResponse(request, "index.html", ctx)
+ return templates.TemplateResponse(
+ request,
+ "index.html",
+ ctx,
+ )
+
if clean_path == "history":
ctx = await render_context({"page": "history"})
- return templates.TemplateResponse(request, "index.html", ctx)
+ return templates.TemplateResponse(
+ request,
+ "index.html",
+ ctx,
+ )
if clean_path:
info = await get_file_info(clean_path)
+
if info["exists"]:
filename = clean_path.split("/")[-1]
- ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+ ext = (
+ filename.rsplit(".", 1)[-1].lower()
+ if "." in filename
+ else ""
+ )
+
ctx = await render_context({
"page": "file",
"path": clean_path,
@@ -868,20 +1252,35 @@ async def serve(request: Request, path: str):
"file_ext": ext,
"file_size_str": format_size(info["size"]),
})
- return templates.TemplateResponse(request, "index.html", ctx)
+
+ return templates.TemplateResponse(
+ request,
+ "index.html",
+ ctx,
+ )
items = await list_directory(clean_path)
+
if clean_path and not items:
- raise HTTPException(status_code=404, detail="Not Found")
-
+ raise HTTPException(
+ status_code=404,
+ detail="Not Found",
+ )
+
extra_ctx = {
"page": "listing",
"path": clean_path,
"items": items,
"raw_base_url": RAW_BASE_URL,
}
+
if not clean_path:
extra_ctx["recent_activity"] = await get_recent_activity()
-
+
ctx = await render_context(extra_ctx)
- return templates.TemplateResponse(request, "index.html", ctx)
+
+ return templates.TemplateResponse(
+ request,
+ "index.html",
+ ctx,
+ )
