commit f5afc3459d8f26fe2a23df9421fb1229f25af35a
parent 68c61e717bdb6ae057a8bf1078dd68b7b8c2089e
Author: Amit Dutta <mail@amit.is-a.dev>
Date: Tue, 6 Oct 2026 13:47:59 +0530
Merge pull request #29 from notamitgamer/migration
Migrate to HF Storage Buckets; add admin dashboard, GitHub sync allowlist, stats page and upload hardening
Diffstat:
14 files changed, 1033 insertions(+), 356 deletions(-)
diff --git a/app/admin.py b/app/admin.py
@@ -0,0 +1,467 @@
+"""Admin dashboard: who uploaded what, IP bans, and the GitHub-sync allowlist.
+
+State is kept in memory and saved as ONE encrypted blob in the Hugging Face
+bucket (_admin/state.enc), so it survives restarts and redeploys. The bucket can
+be public, so the blob is Fernet-encrypted; the key comes from ADMIN_DATA_KEY
+(preferred) or is derived from ADMIN_TOKEN. With no ADMIN_TOKEN the dashboard,
+tracking, bans and the allowlist are all off.
+"""
+from __future__ import annotations
+
+import asyncio
+import base64
+import hashlib
+import hmac
+import ipaddress
+import json
+import os
+import time
+import uuid
+from collections import deque
+from urllib.parse import unquote, urlparse
+
+import httpx
+from cryptography.fernet import Fernet, InvalidToken
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import HTMLResponse
+from pydantic import BaseModel
+
+from . import storage
+from .storage import bucket_url, auth_headers, _do_upload_bytes
+
+router = APIRouter()
+
+ADMIN_TOKEN = os.getenv("ADMIN_TOKEN", "")
+_SECRET = os.getenv("ADMIN_DATA_KEY", "") or ADMIN_TOKEN
+GH_ALWAYS_ALLOW = {o.strip().lower() for o in os.getenv("GH_ALWAYS_ALLOW", "notamitgamer").split(",") if o.strip()}
+STATE_PATH = "_admin/state.enc"
+MAX_LOG_ROWS = 2000
+SAVE_INTERVAL = 30
+
+_state: dict = {"log": [], "bans": {}, "gh": {}}
+_loaded = False
+_retry_at = 0.0
+_load_error = ""
+_dirty = False
+_load_lock = asyncio.Lock()
+_save_lock = asyncio.Lock()
+_loop_task = None
+_auth_fails: dict[str, deque] = {}
+
+
+def enabled() -> bool:
+ return bool(ADMIN_TOKEN)
+
+
+def _fernet() -> Fernet:
+ return Fernet(base64.urlsafe_b64encode(hashlib.sha256(_SECRET.encode()).digest()))
+
+
+# ---------------------------------------------------------------- persistence
+
+async def _fetch() -> dict | None:
+ """Saved state, {} if none exists yet, None if it could not be read."""
+ try:
+ async with httpx.AsyncClient(follow_redirects=True, timeout=20.0) as c:
+ r = await c.get(bucket_url(STATE_PATH), headers=auth_headers())
+ except httpx.HTTPError:
+ return None
+ if r.status_code == 404:
+ return {}
+ if r.status_code != 200:
+ return None
+ return json.loads(_fernet().decrypt(r.content))
+
+
+async def ensure_loaded() -> bool:
+ global _loaded, _load_error, _loop_task, _retry_at
+ if not enabled():
+ return False
+ if _loaded:
+ return True
+ if time.time() < _retry_at:
+ return False
+ async with _load_lock:
+ if _loaded:
+ return True
+ _retry_at = time.time() + 60 # don't hammer the bucket (or stall uploads) while it's failing
+ try:
+ saved = await _fetch()
+ except (InvalidToken, ValueError):
+ _load_error = "Saved state could not be decrypted - ADMIN_DATA_KEY/ADMIN_TOKEN changed? Saving is paused so it is not overwritten."
+ return False
+ if saved is None:
+ _load_error = "Could not reach the bucket; running in memory only and retrying."
+ return False
+ _state["log"] = saved.get("log", []) + _state["log"]
+ _state["bans"] = {**saved.get("bans", {}), **_state["bans"]}
+ _state["gh"] = {**saved.get("gh", {}), **_state["gh"]}
+ _loaded, _load_error, _retry_at = True, "", 0.0
+ if _loop_task is None:
+ _loop_task = asyncio.create_task(_save_loop())
+ return True
+
+
+async def save() -> bool:
+ global _dirty
+ if not (enabled() and _loaded):
+ return False
+ async with _save_lock:
+ _dirty = False
+ _state["log"] = _state["log"][-MAX_LOG_ROWS:]
+ blob = _fernet().encrypt(json.dumps(_state).encode())
+ try:
+ await asyncio.to_thread(_do_upload_bytes, blob, STATE_PATH)
+ return True
+ except Exception:
+ _dirty = True
+ return False
+
+
+async def _save_loop() -> None:
+ while True:
+ await asyncio.sleep(SAVE_INTERVAL)
+ if _dirty:
+ await save()
+
+
+# ------------------------------------------------------------------ recording
+
+def _row(ip: str, name: str, path: str, size: int, method: str, ua: str = "", status: str = "ok", extra: str = "") -> None:
+ global _dirty
+ _state["log"].append({
+ "id": uuid.uuid4().hex[:8], "t": int(time.time()), "ip": ip, "name": (name or "")[:200],
+ "path": path, "size": size, "method": method, "ua": (ua or "")[:200], "status": status, "extra": extra[:100],
+ })
+ _dirty = True
+
+
+async def record_upload(ctx, path: str) -> None:
+ if await ensure_loaded():
+ _row(ctx.ip, ctx.filename, path, ctx.size, ctx.method, ctx.user_agent)
+
+
+async def record_blocked(ctx, reason: str) -> None:
+ if await ensure_loaded():
+ _row(ctx.ip, ctx.filename, "", ctx.size, ctx.method, ctx.user_agent, status=reason)
+
+
+async def is_banned(ip: str) -> bool:
+ return enabled() and await ensure_loaded() and ip in _state["bans"]
+
+
+# ------------------------------------------------------- GitHub sync allowlist
+
+def _gh_key(owner: str, owner_id: str) -> str:
+ return owner_id or owner.lower()
+
+
+async def gh_check(owner: str, owner_id: str) -> None:
+ """Raise unless this GitHub account may sync. A new account gets one free sync."""
+ if not await ensure_loaded():
+ return
+ rec = _state["gh"].get(_gh_key(owner, owner_id))
+ if rec and rec["status"] == "banned":
+ raise HTTPException(status_code=403, detail="This GitHub account is banned from the CDN.")
+ if owner.lower() in GH_ALWAYS_ALLOW or not rec or rec["status"] == "allowed":
+ return
+ if rec["syncs"] >= 1:
+ raise HTTPException(
+ status_code=403,
+ detail=f"First sync from '{owner}' was accepted; further syncs need approval from the CDN owner.",
+ )
+
+
+async def gh_record(owner: str, owner_id: str, repo: str, actor: str, ip: str, size: int, dest: str) -> None:
+ global _dirty
+ if not await ensure_loaded():
+ return
+ now = int(time.time())
+ key = _gh_key(owner, owner_id)
+ status = "allowed" if owner.lower() in GH_ALWAYS_ALLOW else "pending"
+ rec = _state["gh"].setdefault(key, {"status": status, "first": now, "syncs": 0, "repos": []})
+ rec.update(owner=owner, owner_id=owner_id, last=now, last_ip=ip, last_actor=actor)
+ rec["syncs"] += 1
+ if repo not in rec["repos"]:
+ rec["repos"] = (rec["repos"] + [repo])[-20:]
+ _row(ip, f"{owner}/{repo}", dest, size, "github", status="ok", extra=f"actor {actor}")
+ _dirty = True
+ if rec["syncs"] == 1:
+ await save() # new account: persist right away so it shows up for approval
+
+
+# ------------------------------------------------------------------ dashboard
+
+def _require_admin(request: Request) -> None:
+ if not enabled():
+ raise HTTPException(status_code=404, detail="Not found")
+ from .upload_guard import get_client_ip # lazy: upload_guard imports this module
+ ip = get_client_ip(request)
+ now = time.time()
+ fails = _auth_fails.setdefault(ip, deque())
+ while fails and now - fails[0] > 60:
+ fails.popleft()
+ if len(fails) >= 8:
+ raise HTTPException(status_code=429, detail="Too many attempts. Wait a minute.")
+ if not hmac.compare_digest(request.headers.get("authorization", ""), f"Bearer {ADMIN_TOKEN}"):
+ fails.append(now)
+ raise HTTPException(status_code=401, detail="Unauthorized")
+
+
+class IpBody(BaseModel):
+ ip: str
+ note: str = ""
+
+
+class GhBody(BaseModel):
+ key: str
+ action: str # allow | revoke | ban
+
+
+class PathBody(BaseModel):
+ path: str
+
+
+@router.get("/admin", response_class=HTMLResponse)
+async def admin_page():
+ if not enabled():
+ raise HTTPException(status_code=404, detail="Not found")
+ return HTMLResponse(_PAGE, headers={"Cache-Control": "no-store", "X-Robots-Tag": "noindex"})
+
+
+def _search(q: str) -> list:
+ """Case-insensitive search over file name, CDN path, IP, GitHub repo/actor, method and status.
+ Every word must match. A pasted CDN link works too (only its path is used)."""
+ q = (q or "").strip()
+ if q.lower().startswith(("http://", "https://")):
+ q = unquote(urlparse(q).path).strip("/")
+ terms = q.lower().split()
+ if not terms:
+ return _state["log"]
+ out = []
+ for r in _state["log"]:
+ hay = " ".join((r["name"], r["path"], r["ip"], r["method"], r["status"], r.get("extra", ""))).lower()
+ if all(t in hay for t in terms):
+ out.append(r)
+ return out
+
+
+@router.get("/api/admin/state")
+async def admin_state(request: Request, q: str = ""):
+ _require_admin(request)
+ await ensure_loaded()
+ from .upload_guard import get_client_ip, is_cloudflare_ip, is_public_ip, TRUST_CF_CONNECTING_IP, TRUSTED_PROXY_HOPS
+ h = request.headers
+ me = get_client_ip(request)
+ rows = _search(q)
+ return {
+ "whoami": {
+ "detected": me, "ok": is_public_ip(me) and not is_cloudflare_ip(me),
+ "x_forwarded_for": h.get("x-forwarded-for"), "cf_connecting_ip": h.get("cf-connecting-ip"),
+ "true_client_ip": h.get("true-client-ip"), "socket_peer": request.client.host if request.client else None,
+ "trust_cf_header": TRUST_CF_CONNECTING_IP, "proxy_hops": TRUSTED_PROXY_HOPS,
+ },
+ "now": int(time.time()), "error": _load_error,
+ "log": rows[-300:][::-1], "matched": len(rows), "total": len(_state["log"]),
+ "blocked": sum(1 for r in _state["log"] if r["status"] not in ("ok", "deleted")), "bans": _state["bans"],
+ "gh": sorted(_state["gh"].items(), key=lambda kv: kv[1].get("last", 0), reverse=True),
+ "always_allow": sorted(GH_ALWAYS_ALLOW),
+ }
+
+
+async def _mutate_and_save() -> dict:
+ if not await save():
+ raise HTTPException(status_code=503, detail=_load_error or "Could not save to the bucket - try again.")
+ return {"ok": True}
+
+
+@router.post("/api/admin/ban")
+async def admin_ban(request: Request, body: IpBody):
+ _require_admin(request)
+ if not await ensure_loaded():
+ raise HTTPException(status_code=503, detail=_load_error)
+ try:
+ ip = str(ipaddress.ip_address(body.ip.strip()))
+ except ValueError: # also stops "unknown" (undetectable IP) from banning everyone
+ raise HTTPException(status_code=400, detail="Not a valid IP address.")
+ from .upload_guard import is_cloudflare_ip
+ if not ipaddress.ip_address(ip).is_global or is_cloudflare_ip(ip):
+ raise HTTPException(
+ status_code=400,
+ detail="That is an internal or Cloudflare address, not a real visitor (banning it could block many people). The server is not seeing visitor IPs correctly - see 'How the server sees you' at the top of the dashboard.",
+ )
+ _state["bans"][ip] = {"t": int(time.time()), "note": body.note[:100]}
+ return await _mutate_and_save()
+
+
+@router.post("/api/admin/unban")
+async def admin_unban(request: Request, body: IpBody):
+ _require_admin(request)
+ if not await ensure_loaded():
+ raise HTTPException(status_code=503, detail=_load_error)
+ _state["bans"].pop(body.ip.strip(), None)
+ return await _mutate_and_save()
+
+
+@router.post("/api/admin/gh")
+async def admin_gh(request: Request, body: GhBody):
+ _require_admin(request)
+ if not await ensure_loaded():
+ raise HTTPException(status_code=503, detail=_load_error)
+ rec = _state["gh"].get(body.key)
+ if not rec or body.action not in {"allow", "revoke", "ban"}:
+ raise HTTPException(status_code=400, detail="Unknown account or action.")
+ rec["status"] = {"allow": "allowed", "revoke": "pending", "ban": "banned"}[body.action]
+ return await _mutate_and_save()
+
+
+@router.post("/api/admin/delete")
+async def admin_delete(request: Request, body: PathBody):
+ _require_admin(request)
+ if not await ensure_loaded():
+ raise HTTPException(status_code=503, detail=_load_error)
+ path = body.path.strip().strip("/")
+ parts = path.split("/")
+ if len(parts) < 2 or ".." in parts or path.startswith("_"):
+ raise HTTPException(status_code=400, detail="Invalid path.")
+ await asyncio.to_thread(storage.delete_object, path)
+ for r in _state["log"]:
+ if r["path"] == path:
+ r["status"] = "deleted"
+ return await _mutate_and_save()
+
+
+_PAGE = """<!doctype html><html lang="en"><head><meta charset="utf-8">
+<meta name="viewport" content="width=device-width,initial-scale=1"><meta name="robots" content="noindex"><title>CDN admin</title>
+<style>
+:root{--bg:#0d1117;--sf:#161b22;--bd:#30363d;--ln:#21262d;--tx:#e6edf3;--dim:#9da7b3;--red:#ff7b72;--grn:#56d364;--amb:#d29922;--blu:#58a6ff}
+*{box-sizing:border-box}
+body{font:15px/1.45 system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;margin:0 auto;padding:14px 14px 64px;background:var(--bg);color:var(--tx);max-width:1200px;-webkit-text-size-adjust:100%}
+header.bar{display:flex;align-items:center;justify-content:space-between;gap:10px;margin-bottom:12px}
+h1{font-size:19px;margin:0}h2{font-size:15px;margin:24px 0 8px;color:var(--dim);font-weight:600;scroll-margin-top:12px}
+a{color:var(--blu);text-decoration:none}
+.links{display:flex;gap:8px;align-items:center}
+.tiles{display:grid;grid-template-columns:repeat(2,1fr);gap:8px}
+.tile{display:block;background:var(--sf);border:1px solid var(--bd);border-radius:10px;padding:10px 12px;color:var(--tx)}
+.tile b{display:block;font-size:21px;line-height:1.2}.tile span{font-size:12px;color:var(--dim)}.tile.amb{border-color:var(--amb)}.tile.amb b{color:var(--amb)}
+table{width:100%;border-collapse:collapse;font-size:13px}
+th,td{text-align:left;padding:8px;border-bottom:1px solid var(--ln);vertical-align:top}th{color:var(--dim);font-weight:600}
+.act{white-space:nowrap}.name{overflow-wrap:anywhere}
+button{font:inherit;font-size:13px;min-height:34px;padding:5px 12px;border-radius:8px;border:1px solid var(--bd);background:var(--sf);color:var(--tx);cursor:pointer}
+button.red{border-color:#da3633;color:var(--red)}button.green{border-color:#238636;color:var(--grn)}button:active{background:var(--ln)}
+input{font:inherit;font-size:16px;padding:10px 12px;border-radius:8px;border:1px solid var(--bd);background:var(--bg);color:var(--tx);width:100%;max-width:420px}
+.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px;overflow-wrap:break-word}.nw{white-space:nowrap}
+.dim{color:var(--dim)}.bad{color:var(--red)}.ok{color:var(--grn)}.warn{color:var(--amb)}
+.banner{background:#3d1d1d;border:1px solid #da3633;padding:9px 12px;border-radius:8px;margin-top:10px}
+.sec{border-radius:10px;scroll-margin-top:10px}.sec.attn{border-left:3px solid var(--amb);padding-left:10px}
+.flash{animation:flash 1.8s ease-out}@keyframes flash{0%{box-shadow:0 0 0 4px var(--amb);background:#2a2110}100%{box-shadow:0 0 0 4px transparent;background:transparent}}
+@media(prefers-reduced-motion:reduce){.flash{animation:none}}
+.who{margin-top:22px;background:var(--sf);border:1px solid var(--bd);border-radius:10px;padding:10px 12px}.who summary{cursor:pointer}
+.who table{margin-top:8px}.who p{margin:8px 0 0;font-size:12px}
+#login{display:flex;gap:8px;flex-wrap:wrap;align-items:center;margin-top:20px}#login input{flex:1 1 220px}
+#qrow{display:flex;gap:10px;align-items:center;flex-wrap:wrap;margin-bottom:8px}#qrow input{flex:1 1 260px}
+@media(min-width:721px){.tiles{grid-template-columns:repeat(4,1fr)}th,td{padding:8px 10px}}
+@media(max-width:720px){
+ body{padding:12px 12px 64px}
+ table.cards,table.cards tbody{display:block}
+ table.cards tr{display:block;background:var(--sf);border:1px solid var(--bd);border-radius:10px;padding:10px 12px;margin:0 0 10px}
+ table.cards tr:first-child{display:none}
+ table.cards td{display:flex;gap:14px;justify-content:space-between;align-items:baseline;border:0;padding:3px 0;text-align:right;overflow-wrap:anywhere}
+ table.cards td::before{content:attr(data-l);color:var(--dim);flex:none;text-align:left}
+ table.cards td.name{display:block;text-align:left;font-size:14px;font-weight:600;padding-bottom:6px}table.cards td.name::before,table.cards td.act::before{display:none}
+ table.cards td:empty{display:none}
+ table.cards td.empty{display:block;text-align:left;color:var(--dim)}
+ table.cards td.act{display:flex;justify-content:stretch;gap:8px;margin-top:8px;padding-top:10px;border-top:1px solid var(--ln)}
+ table.cards td.act button{flex:1;min-height:44px;font-size:14px}
+ table.kv tr{display:block;padding:6px 0;border-bottom:1px solid var(--ln)}table.kv td{display:block;border:0;padding:1px 0}table.kv td:first-child{color:var(--dim);font-size:12px}
+}
+</style></head><body>
+<header class="bar"><h1>CDN admin</h1><div class="links" id="nav" style="display:none"><a href="/stats">Stats</a><button onclick="logout()">Log out</button></div></header>
+<div id="login"><input id="tok" type="password" placeholder="Admin token" autocomplete="current-password"> <button onclick="go()">Open</button> <span id="lerr" class="bad"></span></div>
+<div id="app" style="display:none">
+<div id="top"></div>
+<h2 id="uploads-h">Recent uploads</h2>
+<div id="qrow"><input id="q" type="search" placeholder="Search file name, link, IP, GitHub repo, status..." autocomplete="off"><span id="qinfo" class="dim"></span></div>
+<div id="uploads"></div>
+<div id="rest"></div>
+</div>
+<script>
+const esc=s=>String(s??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
+const fmt=t=>new Date(t*1000).toLocaleString([],{dateStyle:'short',timeStyle:'short'}),nf=n=>Number(n).toLocaleString();
+const sz=n=>n>=1048576?(n/1048576).toFixed(1)+' MB':n>=1024?(n/1024).toFixed(1)+' KB':n+' B';
+let T=sessionStorage.getItem('t')||'',timer=null,Q='',qt=null;
+document.getElementById('tok').value='';
+async function api(p,body){
+ const r=await fetch(p,{method:body?'POST':'GET',headers:{Authorization:'Bearer '+T,'Content-Type':'application/json'},body:body?JSON.stringify(body):undefined});
+ if(r.status===401||r.status===429){sessionStorage.removeItem('t');throw new Error((await r.json()).detail)}
+ const j=await r.json();if(!r.ok)throw new Error(j.detail||r.status);return j;
+}
+function go(){T=document.getElementById('tok').value.trim();sessionStorage.setItem('t',T);load(true)}
+const btn=(label,cls,path,body,ask)=>`<button class="${cls}" data-p="${esc(path)}" data-b="${esc(JSON.stringify(body))}" data-ask="${esc(ask||'')}">${esc(label)}</button>`;
+document.addEventListener('click',e=>{
+ const a=e.target.closest('a.tile[href^="#"]');if(!a)return;
+ const el=document.querySelector(a.getAttribute('href'));if(!el)return;
+ e.preventDefault();el.scrollIntoView({behavior:matchMedia('(prefers-reduced-motion: reduce)').matches?'auto':'smooth',block:'start'});
+ el.classList.remove('flash');void el.offsetWidth;el.classList.add('flash');
+});
+document.addEventListener('click',async e=>{
+ const b=e.target.closest('button[data-p]');if(!b)return;
+ if(b.dataset.ask&&!confirm(b.dataset.ask))return;
+ try{await api(b.dataset.p,JSON.parse(b.dataset.b));load()}catch(err){alert(err.message)}
+});
+async function load(first){
+ try{
+ const s=await api('/api/admin/state?q='+encodeURIComponent(Q));
+ document.getElementById('login').style.display='none';document.getElementById('app').style.display='block';document.getElementById('nav').style.display='flex';
+ render(s);if(!timer)timer=setInterval(load,20000);
+ }catch(e){
+ if(first||!T){document.getElementById('lerr').textContent=e.message}
+ if(timer){clearInterval(timer);timer=null}
+ document.getElementById('login').style.display='flex';document.getElementById('app').style.display='none';document.getElementById('nav').style.display='none';
+ }
+}
+const GH='/api/admin/gh';
+let WHO=false;
+const cell=(l,v,c)=>`<td data-l="${l}"${c?` class="${c}"`:''}>${v}</td>`;
+const empty=(n,t)=>`<tr><td class="empty" colspan="${n}">${t}</td></tr>`;
+function logout(){sessionStorage.removeItem('t');T='';if(timer){clearInterval(timer);timer=null}document.getElementById('app').style.display='none';document.getElementById('nav').style.display='none';document.getElementById('login').style.display='flex'}
+function render(s){
+ const banned=s.bans,nb=Object.keys(banned).length,pend=s.gh.filter(([k,g])=>g.status==='pending'&&g.syncs>=1);
+ const tile=(n,l,c,href)=>`<${href?`a href="${href}"`:'div'} class="tile ${c||''}"><b>${n}</b><span>${l}</span></${href?'a':'div'}>`;
+ let t=`<div class="tiles">${tile(nf(s.total),'uploads saved')}${tile(nf(s.blocked),'blocked attempts',s.blocked?'amb':'')}${tile(nb,'banned IPs','','#bans')}${tile(pend.length,'awaiting approval',pend.length?'amb':'','#pending')}</div>`;
+ if(s.error)t+=`<div class="banner">${esc(s.error)}</div>`;
+ t+=`<section id="pending" class="sec${pend.length?' attn':''}"><h2>Waiting for approval</h2>`;
+ if(pend.length){
+ t+='<table class="cards"><tr><th>GitHub account</th><th>Repos</th><th>Syncs</th><th>Last sync</th><th></th></tr>';
+ for(const [k,g] of pend)t+=`<tr>${cell('Account',`<a href="https://github.com/${encodeURIComponent(g.owner)}" target="_blank" rel="noopener">${esc(g.owner)}</a>`,'name')}${cell('Repos',esc((g.repos||[]).join(', ')))}${cell('Syncs',g.syncs)}${cell('Last sync',fmt(g.last))}<td class="act">${btn('Allow','green',GH,{key:k,action:'allow'})}${btn('Ban','red',GH,{key:k,action:'ban'},'Ban this GitHub account?')}</td></tr>`;
+ t+='</table>';
+ }else t+='<p class="dim" style="margin:0">Nothing waiting. A new GitHub account shows up here after its first sync.</p>';
+ t+='</section>';
+ document.getElementById('top').innerHTML=t;
+
+ let u='<table class="cards"><tr><th>File</th><th>Time</th><th>IP</th><th>Size</th><th>Via</th><th>Status</th><th></th></tr>';
+ for(const r of s.log){
+ const st=r.status==='ok'?'ok':r.status==='deleted'?'dim':'bad',live=r.path&&r.status==='ok';
+ const acts=(r.method==='github'?'':r.ip in banned?'<span class="dim">IP banned</span>':btn('Ban IP','red','/api/admin/ban',{ip:r.ip},'Ban '+r.ip+'?'))+(live?btn('Delete','red','/api/admin/delete',{path:r.path},'Delete '+r.path+' from the CDN?'):'');
+ u+=`<tr>${cell('File',(live?`<a href="/${encodeURI(r.path)}" target="_blank" rel="noopener">${esc(r.name)}</a>`:esc(r.name))+(r.extra?`<div class="dim" style="font-weight:400">${esc(r.extra)}</div>`:''),'name')}${cell('Time',fmt(r.t),'nw')}${cell('IP',esc(r.ip),'mono')}${cell('Size',sz(r.size),'nw')}${cell('Via',esc(r.method))}${cell('Status',esc(r.status),st)}${acts?`<td class="act">${acts}</td>`:'<td class="act" style="display:none"></td>'}</tr>`;
+ }
+ if(!s.log.length)u+=empty(7,Q?'Nothing matches your search.':'No uploads recorded yet.');
+ document.getElementById('uploads').innerHTML=u+'</table>';
+ document.getElementById('qinfo').textContent=Q?`${s.matched} match${s.matched===1?'':'es'} of ${s.total}${s.matched>s.log.length?' (showing newest '+s.log.length+')':''}`:`${nf(s.total)} saved`;
+
+ let r='<h2>GitHub accounts</h2><table class="cards"><tr><th>Account</th><th>Status</th><th>Repos</th><th>Syncs</th><th>Last IP</th><th>Last sync</th><th></th></tr>';
+ for(const [k,g] of s.gh){const c=g.status==='allowed'?'ok':g.status==='banned'?'bad':'warn';
+ r+=`<tr>${cell('Account',esc(g.owner),'name')}${cell('Status',esc(g.status),c)}${cell('Repos',esc((g.repos||[]).join(', ')))}${cell('Syncs',g.syncs)}${cell('Last IP',esc(g.last_ip),'mono')}${cell('Last sync',fmt(g.last))}<td class="act">${g.status!=='allowed'?btn('Allow','green',GH,{key:k,action:'allow'}):btn('Revoke','',GH,{key:k,action:'revoke'})}${g.status!=='banned'?btn('Ban','red',GH,{key:k,action:'ban'},'Ban this GitHub account?'):''}</td></tr>`}
+ if(!s.gh.length)r+=empty(7,'No GitHub syncs yet.');
+ r+=`</table><p class="dim">Always allowed: ${esc(s.always_allow.join(', ')||'none')}</p><section id="bans" class="sec"><h2>Banned IPs</h2><table class="cards"><tr><th>IP</th><th>Since</th><th>Note</th><th></th></tr>`;
+ for(const ip in banned)r+=`<tr>${cell('IP',esc(ip),'name mono')}${cell('Since',fmt(banned[ip].t))}${cell('Note',esc(banned[ip].note||''))}<td class="act">${btn('Unban','','/api/admin/unban',{ip})}</td></tr>`;
+ if(!nb)r+=empty(4,'None.');
+ const w=s.whoami,kv=(a,b)=>`<tr><td>${a}</td><td class="mono">${esc(b==null||b===''?'-':b)}</td></tr>`;
+ r+=`</table></section><details id="who" class="who"${WHO||!w.ok?' open':''}><summary>Your IP as the server sees it: <span class="mono ${w.ok?'ok':'bad'}">${esc(w.detected)}</span> ${w.ok?'✓':'✗ looks wrong'}</summary><table class="kv">${kv('X-Forwarded-For',w.x_forwarded_for)}${kv('CF-Connecting-IP',w.cf_connecting_ip)}${kv('Socket peer',w.socket_peer)}${kv('Settings','TRUST_CF_CONNECTING_IP='+(w.trust_cf_header?1:0)+', TRUSTED_PROXY_HOPS='+w.proxy_hops)}</table><p class="dim">This must be your real public IP, or bans will hit the wrong address.</p></details>`;
+ document.getElementById('rest').innerHTML=r;
+ document.getElementById('who').addEventListener('toggle',e=>{WHO=e.target.open});
+}
+document.getElementById('q').addEventListener('input',e=>{clearTimeout(qt);qt=setTimeout(()=>{Q=e.target.value.trim();load()},250)});
+document.getElementById('tok').addEventListener('keydown',e=>{if(e.key==='Enter')go()});
+if(T)load(true);
+</script></body></html>"""
diff --git a/app/gh_oidc.py b/app/gh_oidc.py
@@ -71,6 +71,7 @@ def verify_actions_token(token: str) -> dict:
return {
"owner": owner,
+ "owner_id": str(claims.get("repository_owner_id", "")),
"repo": repo_name,
"actor": claims.get("actor", ""),
"ref": claims.get("ref", ""),
diff --git a/app/main.py b/app/main.py
@@ -30,15 +30,20 @@ from .storage import (
search_files,
write_batch_manifest,
get_batch_manifest,
- HF_REPO_ID,
+ HF_BUCKET_ID,
+ bucket_url,
+ auth_headers,
format_size,
)
from .gh_oidc import verify_actions_token
from .shortener import shorten_url, get_destination_url
-from .upload_guard import router as upload_guard_router, get_client_ip, inspect_upload, record_upload
+from .upload_guard import get_client_ip, inspect_upload, is_video_upload
+from .stats import router as stats_router
+from .admin import router as admin_router, record_upload, is_banned, gh_check, gh_record
app = FastAPI()
-app.include_router(upload_guard_router)
+app.include_router(admin_router)
+app.include_router(stats_router)
templates = Jinja2Templates(directory="app/templates")
STATIC_DIR = Path(__file__).parent / "static"
@@ -146,9 +151,9 @@ def _resolve_content_type(filename: str, upstream_content_type: str | None) -> s
return upstream_content_type
async def stream_raw(path: str, request: Request):
- hf_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/{path}"
+ hf_url = bucket_url(path)
client = httpx.AsyncClient(follow_redirects=True)
- req_headers = {}
+ req_headers = dict(auth_headers())
range_header = request.headers.get("range")
if range_header:
req_headers["Range"] = range_header
@@ -184,9 +189,9 @@ async def stream_raw(path: str, request: Request):
@app.get("/api/get/{path:path}")
async def download_file(path: str, request: Request):
- hf_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/{path}"
+ hf_url = bucket_url(path)
client = httpx.AsyncClient(follow_redirects=True)
- req_headers = {}
+ req_headers = dict(auth_headers())
range_header = request.headers.get("range")
if range_header:
req_headers["Range"] = range_header
@@ -253,6 +258,38 @@ class _UploadRateLimiter:
_upload_limiter = _UploadRateLimiter()
+
+
+class _CountRateLimiter:
+ """Sliding-window request counter per key (e.g. per client IP)."""
+
+ def __init__(self, per_minute: int, per_hour: int):
+ self.per_minute = per_minute
+ self.per_hour = per_hour
+ self._hits: dict[str, deque] = {}
+
+ def check_and_record(self, key: str):
+ now = time.time()
+ dq = self._hits.setdefault(key, deque())
+ while dq and now - dq[0] > 3600:
+ dq.popleft()
+ if sum(1 for t in dq if now - t <= 60) >= self.per_minute:
+ raise HTTPException(
+ status_code=429,
+ detail=f"Rate limit exceeded: {self.per_minute} requests/minute. Try again shortly.",
+ )
+ if len(dq) >= self.per_hour:
+ raise HTTPException(
+ status_code=429,
+ detail=f"Rate limit exceeded: {self.per_hour} requests/hour. Try again later.",
+ )
+ dq.append(now)
+ if len(self._hits) > 10000:
+ self._hits = {k: v for k, v in self._hits.items() if v and now - v[-1] <= 3600}
+
+
+_shorten_limiter = _CountRateLimiter(per_minute=10, per_hour=60)
+SHORTEN_MAX_URL_LENGTH = 2048
BATCH_MIN_FILES = 2
ALLOWED_UPLOAD_FOLDERS = {"uploads", "third-party"}
@@ -278,7 +315,7 @@ async def handle_upload(
results = []
for file in files:
- temp_path = f"/tmp/{uuid.uuid4()}-{file.filename}"
+ temp_path = f"/tmp/{uuid.uuid4()}"
with open(temp_path, "wb") as f:
shutil.copyfileobj(file.file, f)
@@ -372,6 +409,10 @@ async def gh_sync(
owner, repo = claims["owner"], claims["repo"]
client_ip = get_client_ip(request)
+ if await is_banned(client_ip):
+ raise HTTPException(status_code=403, detail="You are banned from this CDN.")
+ await gh_check(owner, claims["owner_id"])
+
contents = await archive.read()
_gh_sync_limiter.check_and_record(
f"gh-sync:{owner}/{repo}:{client_ip}",
@@ -393,12 +434,23 @@ async def gh_sync(
detail="Could not read archive (expected a .tar.gz).",
)
+ for root, _dirs, names in os.walk(tmp_extract):
+ for name in names:
+ full = os.path.join(root, name)
+ if is_video_upload(name, full):
+ raise HTTPException(
+ status_code=415,
+ detail=f"Video files are not allowed on this CDN (found {name}). Nothing was synced.",
+ )
+
dest_prefix = await upload_folder_scoped(
tmp_extract,
owner,
repo,
)
+ await gh_record(owner, claims["owner_id"], repo, claims["actor"], client_ip, len(contents), dest_prefix)
+
return {
"synced_to": dest_prefix,
"cdn_url": f"{CDN_BASE_URL}/{dest_prefix}/",
@@ -421,6 +473,15 @@ class ShortenRequest(BaseModel):
@app.post("/api/shorten")
async def api_shorten(request: Request, body: ShortenRequest):
url = body.url.strip()
+ if len(url) > SHORTEN_MAX_URL_LENGTH:
+ raise HTTPException(
+ status_code=400,
+ detail=f"URL too long (max {SHORTEN_MAX_URL_LENGTH} characters).",
+ )
+ client_ip = get_client_ip(request)
+ if await is_banned(client_ip):
+ raise HTTPException(status_code=403, detail="You are banned from this CDN.")
+ _shorten_limiter.check_and_record(client_ip)
_assert_public_url(url)
short_id = await shorten_url(url)
@@ -520,7 +581,7 @@ async def handle_upload_from_url(
or f"download-{uuid.uuid4().hex[:8]}"
)
- temp_path = f"/tmp/{uuid.uuid4()}-{filename}"
+ temp_path = f"/tmp/{uuid.uuid4()}"
downloaded = 0
async with httpx.AsyncClient(
@@ -547,7 +608,6 @@ async def handle_upload_from_url(
if "filename=" in cd:
filename = cd.split("filename=")[-1].strip('"; ') or filename
- temp_path = f"/tmp/{uuid.uuid4()}-{filename}"
with open(temp_path, "wb") as f:
async for chunk in r.aiter_bytes():
@@ -670,13 +730,10 @@ async def download_zip(path: str):
semaphore = asyncio.Semaphore(ZIP_FETCH_CONCURRENCY)
async def fetch(client: httpx.AsyncClient, f: dict):
- hf_url = (
- f"https://huggingface.co/datasets/"
- f"{HF_REPO_ID}/resolve/main/{f['path']}"
- )
+ hf_url = bucket_url(f["path"])
async with semaphore:
- r = await client.get(hf_url)
+ r = await client.get(hf_url, headers=auth_headers())
return f, r
@@ -714,7 +771,7 @@ async def download_zip(path: str):
zip_filename = (
clean_path.rstrip("/").split("/")[-1]
if clean_path
- else HF_REPO_ID.split("/")[-1]
+ else HF_BUCKET_ID.split("/")[-1]
) + ".zip"
return StreamingResponse(
@@ -818,13 +875,10 @@ async def download_zip_batch(batch_id: str):
semaphore = asyncio.Semaphore(ZIP_FETCH_CONCURRENCY)
async def fetch(client: httpx.AsyncClient, f: dict):
- hf_url = (
- f"https://huggingface.co/datasets/"
- f"{HF_REPO_ID}/resolve/main/{f['hf_path']}"
- )
+ hf_url = bucket_url(f["hf_path"])
async with semaphore:
- r = await client.get(hf_url)
+ r = await client.get(hf_url, headers=auth_headers())
return f, r
diff --git a/app/shortener.py b/app/shortener.py
@@ -1,15 +1,10 @@
-import os
-import io
import random
import string
import asyncio
import re
import httpx
-from huggingface_hub import HfApi
-HF_REPO_ID = os.getenv("HF_REPO_ID", "notamitgamer/cdn")
-HF_TOKEN = os.getenv("HF_TOKEN")
-api = HfApi(token=HF_TOKEN) if HF_TOKEN else None
+from .storage import HF_TOKEN, bucket_url, auth_headers, _do_upload_bytes
# Reusable HTTP client for fast connection pooling & HTTP keep-alive
_client = httpx.AsyncClient(follow_redirects=True, timeout=10.0)
@@ -22,8 +17,8 @@ def generate_id(length: int = 8) -> str:
return "".join(random.choices(chars, k=length))
async def shorten_url(destination_url: str) -> str:
- """Save a destination URL under an in-memory byte payload to Hugging Face."""
- if not api:
+ """Save a destination URL as a tiny object in the Hugging Face bucket."""
+ if not HF_TOKEN:
raise RuntimeError("HF_TOKEN is not configured.")
destination_url = destination_url.strip()
@@ -32,8 +27,7 @@ async def shorten_url(destination_url: str) -> str:
attempts = 0
while True:
short_id = generate_id()
- hf_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/_shortened/{short_id}"
- r = await _client.head(hf_url)
+ r = await _client.head(bucket_url(f"_shortened/{short_id}"), headers=auth_headers())
if r.status_code == 404:
break
attempts += 1
@@ -42,31 +36,22 @@ async def shorten_url(destination_url: str) -> str:
short_id = generate_id(length=12)
break
- # Upload directly from memory without writing to disk
- payload = io.BytesIO(destination_url.encode("utf-8"))
-
- def _upload():
- api.upload_file(
- path_or_fileobj=payload,
- path_in_repo=f"_shortened/{short_id}",
- repo_id=HF_REPO_ID,
- repo_type="dataset",
- token=HF_TOKEN,
- commit_message=f"shorten: {short_id}",
- )
-
- await asyncio.to_thread(_upload)
+ # Upload straight from memory without writing to disk
+ await asyncio.to_thread(
+ _do_upload_bytes,
+ destination_url.encode("utf-8"),
+ f"_shortened/{short_id}",
+ )
return short_id
async def get_destination_url(short_id: str) -> str | None:
- """Look up the destination URL directly from Hugging Face resolve endpoints."""
+ """Look up the destination URL directly from the bucket's resolve endpoint."""
short_id = short_id.strip().lower()
if not _ID_REGEX.match(short_id):
return None
- hf_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/_shortened/{short_id}"
try:
- r = await _client.get(hf_url)
+ r = await _client.get(bucket_url(f"_shortened/{short_id}"), headers=auth_headers())
if r.status_code == 200:
return r.text.strip()
except httpx.RequestError:
@@ -75,4 +60,4 @@ async def get_destination_url(short_id: str) -> str | None:
def list_all_ids() -> list[str]:
"""Stub kept for router compatibility."""
- return []
-\ No newline at end of file
+ return []
diff --git a/app/stats.py b/app/stats.py
@@ -0,0 +1,127 @@
+"""Public stats page: what is in the CDN (types, sizes, folders, growth).
+
+Everything is computed from the file list storage.py already caches, then kept
+for STATS_TTL seconds, so viewing the page never adds load on the bucket.
+Only aggregate numbers are exposed - no IPs, no upload log.
+"""
+from __future__ import annotations
+
+import asyncio
+import statistics
+import time
+from collections import Counter, defaultdict
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, Request
+from fastapi.responses import JSONResponse
+from fastapi.templating import Jinja2Templates
+
+from . import storage
+
+router = APIRouter()
+templates = Jinja2Templates(directory="app/templates")
+
+STATS_TTL = 600
+_cache: dict = {"at": 0.0, "data": None}
+_lock = asyncio.Lock()
+
+CATEGORIES = [
+ ("images", "Images", "png jpg jpeg gif webp avif svg ico bmp heic tiff"),
+ ("documents", "Documents", "pdf doc docx ppt pptx xls xlsx odt ods odp txt md rtf csv epub"),
+ ("code", "Code & data", "js mjs css html htm json xml yml yaml toml py ts tsx jsx c cpp h java go rs sh sql map wasm"),
+ ("audio", "Audio", "mp3 wav ogg flac m4a aac opus"),
+ ("video", "Video", "mp4 mkv webm mov avi m4v"),
+ ("archives", "Archives", "zip tar gz tgz 7z rar xz bz2"),
+ ("fonts", "Fonts", "woff woff2 ttf otf eot"),
+]
+_EXT_TO_CAT = {e: (k, label) for k, label, exts in CATEGORIES for e in exts.split()}
+SIZE_BUCKETS = [(10 * 1024, "< 10 KB"), (100 * 1024, "10-100 KB"), (1024 ** 2, "100 KB-1 MB"),
+ (10 * 1024 ** 2, "1-10 MB"), (100 * 1024 ** 2, "10-100 MB"), (float("inf"), "> 100 MB")]
+TOP_LEVEL_FOLDERS = {"uploads", "third-party"}
+
+
+def _ext(name: str) -> str:
+ return name.rsplit(".", 1)[-1].lower() if "." in name.strip(".") else ""
+
+
+def _folder(path: str) -> str:
+ parts = path.split("/")
+ if len(parts) == 1:
+ return "(root)"
+ return parts[0] if parts[0] in TOP_LEVEL_FOLDERS else "/".join(parts[:2]) if len(parts) > 2 else parts[0]
+
+
+def compute(items: list[dict]) -> dict:
+ files = [i for i in items if not i["is_dir"] and i["name"] != ".gitattributes"]
+ if not files:
+ return {"empty": True}
+
+ sizes = [f["size"] for f in files]
+ cats: dict[str, dict] = {}
+ exts: dict[str, list] = defaultdict(lambda: [0, 0])
+ folders: dict[str, list] = defaultdict(lambda: [0, 0])
+ buckets = Counter()
+ months: dict[str, list] = defaultdict(lambda: [0, 0])
+
+ for f in files:
+ ext = _ext(f["name"])
+ key, label = _EXT_TO_CAT.get(ext, ("other", "Other"))
+ c = cats.setdefault(key, {"key": key, "label": label, "files": 0, "bytes": 0})
+ c["files"] += 1
+ c["bytes"] += f["size"]
+ exts[ext or "(none)"][0] += 1
+ exts[ext or "(none)"][1] += f["size"]
+ fo = folders[_folder(f["path"])]
+ fo[0] += 1
+ fo[1] += f["size"]
+ buckets[next(lbl for limit, lbl in SIZE_BUCKETS if f["size"] < limit)] += 1
+ if f.get("ts"):
+ m = months[datetime.fromtimestamp(f["ts"], timezone.utc).strftime("%Y-%m")]
+ m[0] += 1
+ m[1] += f["size"]
+
+ top_ext = sorted(exts.items(), key=lambda kv: kv[1][0], reverse=True)
+ top_folders = sorted(folders.items(), key=lambda kv: kv[1][1], reverse=True)
+ largest = sorted(files, key=lambda f: f["size"], reverse=True)[:8]
+ month_rows = [{"m": k, "files": v[0], "bytes": v[1]} for k, v in sorted(months.items())][-12:]
+
+ return {
+ "empty": False,
+ "generated": int(time.time()),
+ "totals": {
+ "files": len(files), "bytes": sum(sizes), "avg": int(sum(sizes) / len(sizes)),
+ "median": int(statistics.median(sizes)), "types": len(exts),
+ },
+ "categories": sorted(cats.values(), key=lambda c: c["files"], reverse=True),
+ "extensions": [{"ext": e, "files": v[0], "bytes": v[1]} for e, v in top_ext[:10]],
+ "extensions_more": sum(v[0] for _, v in top_ext[10:]),
+ "sizes": [{"label": lbl, "files": buckets.get(lbl, 0)} for _, lbl in SIZE_BUCKETS],
+ "folders": [{"name": n, "files": v[0], "bytes": v[1]} for n, v in top_folders[:8]],
+ "largest": [{"name": f["name"], "path": f["path"], "size": f["size"]} for f in largest],
+ "months": month_rows,
+ }
+
+
+async def get_stats() -> dict:
+ now = time.time()
+ if _cache["data"] is not None and now - _cache["at"] < STATS_TTL:
+ return _cache["data"]
+ async with _lock:
+ if _cache["data"] is not None and time.time() - _cache["at"] < STATS_TTL:
+ return _cache["data"]
+ items = await storage._get_full_tree()
+ if not items: # listing failed: keep serving the last good result if we have one
+ return _cache["data"] or {"empty": True}
+ data = await asyncio.to_thread(compute, items)
+ _cache.update(at=time.time(), data=data)
+ return data
+
+
+@router.get("/stats")
+async def stats_page(request: Request):
+ return templates.TemplateResponse(request, "stats.html", {"page": "stats"})
+
+
+@router.get("/api/stats")
+async def stats_api():
+ return JSONResponse(await get_stats(), headers={"Cache-Control": "public, max-age=300"})
diff --git a/app/storage.py b/app/storage.py
@@ -1,6 +1,7 @@
import os
import re
import time
+import unicodedata
import uuid
import json
import tempfile
@@ -8,8 +9,15 @@ import asyncio
import httpx
from huggingface_hub import HfApi
-HF_REPO_ID = os.getenv("HF_REPO_ID", "notamitgamer/cdn")
+# Files live in a Hugging Face Storage Bucket (mutable, non-versioned object
+# storage) instead of a git-backed dataset repo: every write is a plain object
+# write rather than a commit, so uploads and folder syncs avoid git overhead and
+# commit rate limits. HF_REPO_ID is still honoured as a fallback so existing
+# deployments keep working until HF_BUCKET_ID is set.
+HF_BUCKET_ID = os.getenv("HF_BUCKET_ID") or os.getenv("HF_REPO_ID", "notamitgamer/cdn")
+HF_REPO_ID = HF_BUCKET_ID # backwards-compatible alias used by older imports
HF_TOKEN = os.getenv("HF_TOKEN")
+BUCKET_PRIVATE = os.getenv("HF_BUCKET_PRIVATE", "0") == "1"
CACHE_TTL = 60
MAX_CACHE_SIZE = 500
ZIP_MAX_FILES = 300
@@ -19,6 +27,29 @@ SEARCH_RESULT_LIMIT = 10
api = HfApi(token=HF_TOKEN)
+
+def bucket_url(path: str) -> str:
+ """Direct (CDN-backed) download URL for an object in the bucket."""
+ return f"https://huggingface.co/buckets/{HF_BUCKET_ID}/resolve/{path}"
+
+
+def auth_headers() -> dict:
+ """Bearer header, only needed when the bucket is private."""
+ if BUCKET_PRIVATE and HF_TOKEN:
+ return {"Authorization": f"Bearer {HF_TOKEN}"}
+ return {}
+
+
+_bucket_ready = False
+
+
+def _ensure_bucket():
+ global _bucket_ready
+ if _bucket_ready:
+ return
+ api.create_bucket(HF_BUCKET_ID, private=BUCKET_PRIVATE, exist_ok=True)
+ _bucket_ready = True
+
_cache = {}
_key_locks: dict[str, asyncio.Lock] = {}
@@ -69,9 +100,9 @@ async def get_file_info(path: str) -> dict:
if cached is not None:
return cached
- hf_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/{path}"
+ hf_url = bucket_url(path)
async with httpx.AsyncClient(follow_redirects=True) as client:
- r = await client.head(hf_url)
+ r = await client.head(hf_url, headers=auth_headers())
exists = r.status_code == 200
size = None
content_type = None
@@ -85,17 +116,29 @@ async def get_file_info(path: str) -> dict:
_set_cache(cache_key, result)
return result
+def _under(path: str, items: list) -> list:
+ """Keep only entries inside `path` (defensive: guards against prefix over-match
+ such as 'docs' also matching 'docs-old/...')."""
+ if not path:
+ return items
+ prefix = path.rstrip("/") + "/"
+ return [it for it in items if it.path.startswith(prefix)]
+
def _fetch_tree(path: str):
- return list(api.list_repo_tree(
- repo_id=HF_REPO_ID, path_in_repo=path, repo_type="dataset", expand=False
+ path = path.strip("/")
+ items = list(api.list_bucket_tree(
+ HF_BUCKET_ID, prefix=path or None, recursive=False
))
+ return _under(path, items)
def _fetch_tree_recursive(path: str = ""):
- return list(api.list_repo_tree(
- repo_id=HF_REPO_ID, path_in_repo=path, repo_type="dataset", recursive=True
+ path = path.strip("/")
+ items = list(api.list_bucket_tree(
+ HF_BUCKET_ID, prefix=path or None, recursive=True
))
+ return _under(path, items)
-_HIDDEN_PREFIXES = ("_batches", "_shortened", "_logs",)
+_HIDDEN_PREFIXES = ("_batches", "_shortened", "_logs", "_admin",)
def _is_hidden_path(path: str) -> bool:
return any(path == p or path.startswith(p + "/") for p in _HIDDEN_PREFIXES)
@@ -196,11 +239,13 @@ async def _get_full_tree():
name = item.path.split("/")[-1]
is_dir = not hasattr(item, "size")
size = getattr(item, "size", 0) or 0
+ up = getattr(item, "uploaded_at", None)
items.append({
"name": name,
"path": item.path,
"is_dir": is_dir,
"size": size,
+ "ts": int(up.timestamp()) if up else 0,
"size_str": format_size(size) if not is_dir else "-",
})
@@ -241,17 +286,54 @@ async def search_files(query: str):
]
def _do_upload(temp_path: str, hf_path: str):
- api.upload_file(
- path_or_fileobj=temp_path,
- path_in_repo=hf_path,
- repo_id=HF_REPO_ID,
- repo_type="dataset",
- token=HF_TOKEN
+ _ensure_bucket()
+ api.batch_bucket_files(
+ HF_BUCKET_ID,
+ add=[(temp_path, hf_path)],
+ )
+
+def _do_upload_bytes(data: bytes, hf_path: str):
+ _ensure_bucket()
+ api.batch_bucket_files(
+ HF_BUCKET_ID,
+ add=[(data, hf_path)],
)
+def delete_object(path: str):
+ """Delete one object, or everything under path/ (for a synced owner/repo folder)."""
+ _ensure_bucket()
+ path = path.strip("/")
+ items = [it.path for it in api.list_bucket_tree(HF_BUCKET_ID, prefix=path, recursive=True)
+ if it.type == "file" and (it.path == path or it.path.startswith(path + "/"))]
+ if items:
+ api.batch_bucket_files(HF_BUCKET_ID, delete=items)
+ _cache.clear()
+
+
+_UNSAFE_NAME_CHARS = re.compile(r"[^A-Za-z0-9._-]+")
+
+
+def sanitize_filename(filename: str, max_len: int = 100) -> str:
+ """Turn any user-supplied filename into a safe object-key segment.
+
+ Accepts spaces, accents, symbols, emoji and path fragments: directory parts
+ are dropped, accents are folded to ASCII, and anything else unsafe becomes
+ a single underscore. The extension is kept.
+ """
+ name = (filename or "").replace("\\", "/").split("/")[-1]
+ name = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode("ascii")
+ stem, dot, ext = name.rpartition(".")
+ if not dot:
+ stem, ext = name, ""
+ stem = _UNSAFE_NAME_CHARS.sub("_", stem).strip("._") or "file"
+ ext = _UNSAFE_NAME_CHARS.sub("", ext)[:10]
+ stem = stem[: max_len - len(ext) - 1]
+ return f"{stem}.{ext}" if ext else stem
+
+
async def upload_temp_file(temp_path: str, filename: str, folder: str = "uploads") -> str:
slug = str(uuid.uuid4())[:8]
- safe_filename = filename.replace(" ", "_")
+ safe_filename = sanitize_filename(filename)
hf_path = f"{folder}/{slug}-{safe_filename}"
await asyncio.to_thread(_do_upload, temp_path, hf_path)
@@ -260,17 +342,34 @@ async def upload_temp_file(temp_path: str, filename: str, folder: str = "uploads
return hf_path
def _do_upload_folder_scoped(local_dir: str, dest_prefix: str):
- # path_in_repo + delete_patterns are both scoped to dest_prefix, so this call
- # can only ever create/update/delete files under that one folder - it never
- # touches anything outside of it in the dataset repo.
- api.upload_folder(
- repo_id=HF_REPO_ID,
- repo_type="dataset",
- folder_path=local_dir,
- path_in_repo=dest_prefix,
- delete_patterns="*",
- commit_message=f"gh-sync: update {dest_prefix}",
- token=HF_TOKEN,
+ # Mirror local_dir into the bucket under dest_prefix. Every added path and every
+ # deleted path is built from dest_prefix, so this can only ever create, update
+ # or delete objects under that one folder - it never touches anything outside it.
+ _ensure_bucket()
+ dest_prefix = dest_prefix.strip("/")
+
+ local_files = {}
+ for root, _dirs, names in os.walk(local_dir):
+ for name in names:
+ full = os.path.join(root, name)
+ rel = os.path.relpath(full, local_dir).replace(os.sep, "/")
+ local_files[f"{dest_prefix}/{rel}"] = full
+
+ remote_files = {
+ it.path
+ for it in api.list_bucket_tree(HF_BUCKET_ID, prefix=dest_prefix, recursive=True)
+ if it.type == "file" and it.path.startswith(dest_prefix + "/")
+ }
+
+ stale = sorted(remote_files - local_files.keys())
+ adds = [(full, remote) for remote, full in sorted(local_files.items())]
+
+ if not adds and not stale:
+ return
+ api.batch_bucket_files(
+ HF_BUCKET_ID,
+ add=adds or None,
+ delete=stale or None,
)
async def upload_folder_scoped(local_dir: str, owner: str, repo: str) -> str:
@@ -303,9 +402,9 @@ async def get_batch_manifest(batch_id: str):
if cached is not None:
return cached
- hf_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/_batches/{batch_id}.json"
+ hf_url = bucket_url(f"_batches/{batch_id}.json")
async with httpx.AsyncClient(follow_redirects=True) as client:
- r = await client.get(hf_url)
+ r = await client.get(hf_url, headers=auth_headers())
if r.status_code != 200:
return None
try:
diff --git a/app/templates/_navbar.html b/app/templates/_navbar.html
@@ -73,6 +73,7 @@
<a href="/new" class="{{ 'active' if page == 'upload' }}"{{ ' aria-current="page"'|safe if page == 'upload' }}><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="17 8 12 3 7 8"/><line x1="12" y1="3" x2="12" y2="15"/></svg><span>Upload</span></a>
<a href="/shorten" class="{{ 'active' if page == 'shorten' }}"{{ ' aria-current="page"'|safe if page == 'shorten' }}><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"/><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"/></svg><span>Shorten</span></a>
<a href="/history" class="{{ 'active' if page == 'history' }}"{{ ' aria-current="page"'|safe if page == 'history' }}><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="10"/><polyline points="12 6 12 12 16 14"/></svg><span>History</span></a>
+ <a href="/stats" class="{{ 'active' if page == 'stats' }}"{{ ' aria-current="page"'|safe if page == 'stats' }}><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><line x1="6" y1="20" x2="6" y2="12"/><line x1="12" y1="20" x2="12" y2="4"/><line x1="18" y1="20" x2="18" y2="9"/></svg><span>Stats</span></a>
<a href="/gh-sync" class="{{ 'active' if page == 'gh-sync' }}"{{ ' aria-current="page"'|safe if page == 'gh-sync' }}><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="23 4 23 10 17 10"/><polyline points="1 20 1 14 7 14"/><path d="M3.51 9a9 9 0 0 1 14.85-3.36L23 10M1 14l4.64 4.36A9 9 0 0 0 20.49 15"/></svg><span>Sync</span></a>
<a href="/documentation" class="{{ 'active' if page == 'documentation' }}"{{ ' aria-current="page"'|safe if page == 'documentation' }}><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M4 19.5A2.5 2.5 0 0 1 6.5 17H20"/><path d="M6.5 2H20v20H6.5A2.5 2.5 0 0 1 4 19.5v-15A2.5 2.5 0 0 1 6.5 2z"/></svg><span>Docs</span></a>
</nav>
@@ -96,6 +97,8 @@
{% set ns.items = ns.items + [{'name': 'find', 'href': '/find'}] %}
{% elif page == 'history' %}
{% set ns.items = ns.items + [{'name': 'history', 'href': '/history'}] %}
+ {% elif page == 'stats' %}
+ {% set ns.items = ns.items + [{'name': 'stats', 'href': '/stats'}] %}
{% elif page == 'documentation' %}
{% set ns.items = ns.items + [{'name': 'documentation', 'href': '/documentation'}] %}
{% elif page == 'gh-sync' %}
diff --git a/app/templates/documentation.html b/app/templates/documentation.html
@@ -179,7 +179,7 @@
<h3>How it works</h3>
<p>Each file is streamed to a local temp path, checked against the rate limiter (50 MB/minute, 300 MB/hour
per client IP), then uploaded to Hugging Face at <code><folder>/<8-char-uuid>-<original-filename></code>
- (spaces in the filename become underscores). The random prefix means two uploads can never collide, and the
+ (any characters other than letters, digits, <code>.</code>, <code>_</code> and <code>-</code> become underscores, and accents are simplified). The random prefix means two uploads can never collide, and the
filename portion is still readable. If 2 or more files were uploaded together, a <em>batch</em> is also created
(see <a href="#batches">Batches</a> below) so the whole set can be shared/downloaded as one link.</p>
<h3>Response <code>200</code></h3>
diff --git a/app/templates/gh_sync_docs.html b/app/templates/gh_sync_docs.html
@@ -190,7 +190,10 @@ jobs:
<h2>Notes</h2>
<ul>
- <li>Any file type is accepted, and there's currently no limit on file size or on how many files a sync can push — send the whole repo as-is.</li>
+ <li>Any file type except video is accepted (a sync containing a video is rejected as a whole), and there's currently no limit on file size or on how many files a sync can push — send the whole repo as-is.</li>
+ <li><strong>First sync is free, later ones need approval.</strong> The first sync from a new GitHub account
+ goes through immediately. After that, further syncs from that account are refused until the CDN owner
+ approves it. Accounts that upload abusive content are banned.</li>
<li>Each sync fully replaces the contents of <em>your own</em> <code>username/repo</code> folder only
(delete-then-upload), so removed files disappear from the CDN too. No other folder is ever touched.</li>
<li>Change <code>branches: [ main ]</code> to whichever branch you want mirrored.</li>
diff --git a/app/templates/index.html b/app/templates/index.html
@@ -1134,14 +1134,10 @@
const allFiles = Array.from(fileList);
const files = [];
const rejectedSize = [];
- const rejectedName = [];
- const allowedNameRegex = /^[a-zA-Z0-9_.\-]+$/;
allFiles.forEach(f => {
if (f.size > MAX_FILE_SIZE_BYTES) {
rejectedSize.push(f.name);
- } else if (!allowedNameRegex.test(f.name)) {
- rejectedName.push(f.name);
} else {
files.push(f);
}
@@ -1151,9 +1147,6 @@
if (rejectedSize.length) {
errorMessages.push(`Skipped ${rejectedSize.length} file(s) over ${formatBytes(MAX_FILE_SIZE_BYTES)}: ${rejectedSize.join(', ')}`);
}
- if (rejectedName.length) {
- errorMessages.push(`Skipped ${rejectedName.length} file(s) with invalid characters (only a-z, A-Z, 0-9, _, -, . allowed): ${rejectedName.join(', ')}`);
- }
if (errorMessages.length) {
showError(errorMessages.join(' | '));
diff --git a/app/templates/stats.html b/app/templates/stats.html
@@ -0,0 +1,152 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Stats · amit/cdn</title>
+<meta name="description" content="What's on the CDN: file types, sizes, folders and growth.">
+<link rel="icon" href="/favicon.ico" sizes="any">
+<link rel="icon" type="image/svg+xml" href="/static/icons/logo.svg">
+<link rel="apple-touch-icon" sizes="180x180" href="/static/icons/apple-touch-icon.png">
+<meta name="theme-color" content="#ffffff" media="(prefers-color-scheme: light)">
+<meta name="theme-color" content="#0d1117" media="(prefers-color-scheme: dark)">
+<meta name="color-scheme" content="light dark">
+<style>
+ *, *::before, *::after { box-sizing: border-box; -webkit-tap-highlight-color: transparent; }
+ body { background: var(--ui-bg); color: var(--ui-text); margin: 0; }
+ .wrap { max-width: var(--ui-width); margin: 0 auto; padding: 28px 16px 72px; }
+ h1 { margin: 0 0 4px; }
+ h2 { font-size: 1rem; margin: 0; font-weight: 600; }
+ .muted { color: var(--ui-muted); font-size: .88rem; }
+ .tiles { display: grid; grid-template-columns: repeat(2, 1fr); gap: 10px; margin: 20px 0 14px; }
+ .tile, .card { background: var(--ui-surface); border: 1px solid var(--ui-border); border-radius: 10px; }
+ .tile { padding: 12px 14px; }
+ .tile b { display: block; font-size: 1.45rem; line-height: 1.2; font-variant-numeric: tabular-nums; }
+ .tile span { color: var(--ui-muted); font-size: .8rem; }
+ .grid { display: grid; grid-template-columns: 1fr; gap: 14px; }
+ .card { padding: 14px 16px 16px; min-width: 0; }
+ .card > header { display: flex; align-items: center; justify-content: space-between; gap: 10px; margin-bottom: 12px; flex-wrap: wrap; }
+ .seg { display: inline-flex; border: 1px solid var(--ui-border); border-radius: 8px; overflow: hidden; }
+ .seg button { font: inherit; font-size: .8rem; padding: 6px 12px; min-height: 32px; border: 0; background: transparent; color: var(--ui-muted); cursor: pointer; }
+ .seg button[aria-pressed="true"] { background: var(--ui-accent-solid); color: #fff; }
+
+ .donut-wrap { display: flex; flex-wrap: wrap; align-items: center; gap: 16px 22px; }
+ .donut { width: 150px; height: 150px; flex: none; margin: 0 auto; position: relative; }
+ .donut svg { width: 100%; height: 100%; transform: rotate(-90deg); }
+ .donut circle { fill: none; stroke-width: 7; }
+ .donut .hole { position: absolute; inset: 0; display: grid; place-content: center; text-align: center; line-height: 1.2; }
+ .donut .hole b { font-size: 1.15rem; font-variant-numeric: tabular-nums; }
+ .legend { list-style: none; margin: 0; padding: 0; flex: 1 1 200px; display: grid; gap: 7px; font-size: .88rem; }
+ .legend li { display: flex; align-items: center; gap: 8px; }
+ .legend i { width: 11px; height: 11px; border-radius: 3px; flex: none; }
+ .legend .n { margin-left: auto; color: var(--ui-muted); font-variant-numeric: tabular-nums; white-space: nowrap; }
+
+ .bars { display: grid; gap: 11px; }
+ .bar .top { display: flex; justify-content: space-between; gap: 10px; font-size: .86rem; }
+ .bar .top span:first-child { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; }
+ .bar .top span:last-child { color: var(--ui-muted); white-space: nowrap; font-variant-numeric: tabular-nums; }
+ .track { height: 8px; border-radius: 4px; background: var(--ui-border); margin-top: 4px; overflow: hidden; }
+ .track i { display: block; height: 100%; border-radius: 4px; background: var(--ui-accent); min-width: 2px; }
+
+ .cols { display: flex; align-items: flex-end; gap: 6px; height: 170px; padding-top: 18px; }
+ .col { flex: 1; min-width: 0; height: 100%; display: flex; flex-direction: column; justify-content: flex-end; align-items: center; }
+ .col .v { font-size: .68rem; color: var(--ui-muted); font-variant-numeric: tabular-nums; }
+ .col i { width: 100%; max-width: 44px; border-radius: 4px 4px 0 0; background: var(--ui-accent); min-height: 2px; }
+ .labels { display: flex; gap: 6px; margin-top: 6px; }
+ .labels span { flex: 1; min-width: 0; text-align: center; font-size: .66rem; color: var(--ui-muted); line-height: 1.25; overflow-wrap: anywhere; }
+
+ .files { list-style: none; margin: 0; padding: 0; }
+ .files li { display: flex; gap: 12px; justify-content: space-between; padding: 9px 0; border-top: 1px solid var(--ui-border); font-size: .88rem; }
+ .files li:first-child { border-top: 0; padding-top: 0; }
+ .files a { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; color: var(--ui-accent); text-decoration: none; }
+ .files span { color: var(--ui-muted); white-space: nowrap; font-variant-numeric: tabular-nums; }
+ .note { padding: 40px 0; text-align: center; color: var(--ui-muted); }
+ .note button { font: inherit; margin-top: 10px; padding: 8px 16px; border-radius: 8px; border: 1px solid var(--ui-border); background: var(--ui-surface); color: var(--ui-text); cursor: pointer; }
+ footer { max-width: var(--ui-width); margin: 0 auto; padding: 0 16px 32px; text-align: center; }
+
+ @media (min-width: 760px) {
+ .wrap { padding: 44px 20px 96px; }
+ .tiles { grid-template-columns: repeat(4, 1fr); }
+ .grid { grid-template-columns: 1fr 1fr; }
+ .span2 { grid-column: 1 / -1; }
+ .col .v { font-size: .75rem; } .labels span { font-size: .74rem; }
+ }
+ :root { --c0: #4c8dff; --c1: #3fb950; --c2: #d29922; --c3: #f85149; --c4: #a371f7; --c5: #39c5cf; --c6: #db61a2; --c7: #8b949e; }
+</style>
+</head>
+<body>
+{% include "_navbar.html" %}
+<main class="wrap">
+ <h1>CDN stats</h1>
+ <p class="muted" id="sub">Loading…</p>
+ <div id="out"></div>
+</main>
+<footer><p class="muted">© 2025 - 2026 notamitgamer. All rights reserved.</p></footer>
+<script>
+(function () {
+ var C = ['--c0','--c1','--c2','--c3','--c4','--c5','--c6'], OTHER = 'var(--c7)';
+ var $ = function (id) { return document.getElementById(id); };
+ var esc = function (s) { return String(s == null ? '' : s).replace(/[&<>"']/g, function (c) { return {'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]; }); };
+ var nf = new Intl.NumberFormat();
+ function B(n) { var u = ['B','KB','MB','GB','TB'], i = 0; while (n >= 1024 && i < 4) { n /= 1024; i++; } return (i ? n.toFixed(n < 10 ? 1 : 0) : n) + ' ' + u[i]; }
+ function pct(a, b) { return b ? Math.round(a / b * 100) : 0; }
+ function color(i, key) { return key === 'other' ? OTHER : 'var(' + C[i % C.length] + ')'; }
+
+ function donut(cats, mode, t) {
+ var total = mode === 'size' ? t.bytes : t.files, acc = 0, segs = '';
+ cats.forEach(function (c, i) {
+ var v = mode === 'size' ? c.bytes : c.files, p = total ? v / total * 100 : 0;
+ if (p > 0) segs += '<circle cx="21" cy="21" r="15.9155" stroke="' + color(i, c.key) + '" stroke-dasharray="' + p.toFixed(3) + ' ' + (100 - p).toFixed(3) + '" stroke-dashoffset="' + (-acc).toFixed(3) + '"/>';
+ acc += p;
+ });
+ var legend = cats.map(function (c, i) {
+ var v = mode === 'size' ? c.bytes : c.files;
+ return '<li><i style="background:' + color(i, c.key) + '"></i>' + esc(c.label) + '<span class="n">' + (mode === 'size' ? B(v) : nf.format(v)) + ' · ' + pct(v, total) + '%</span></li>';
+ }).join('');
+ return '<div class="donut-wrap"><div class="donut" role="img" aria-label="File types by ' + (mode === 'size' ? 'size' : 'count') + '"><svg viewBox="0 0 42 42">' +
+ '<circle cx="21" cy="21" r="15.9155" stroke="var(--ui-border)"/>' + segs + '</svg><div class="hole"><b>' + (mode === 'size' ? B(total) : nf.format(total)) +
+ '</b><span class="muted">' + (mode === 'size' ? 'total' : 'files') + '</span></div></div><ul class="legend">' + legend + '</ul></div>';
+ }
+ function bars(rows, label, val, num, sub) {
+ var max = Math.max.apply(null, rows.map(num).concat(1));
+ return '<div class="bars">' + rows.map(function (r) {
+ return '<div class="bar"><div class="top"><span>' + esc(label(r)) + '</span><span>' + val(r) + '</span></div><div class="track"><i style="width:' + Math.max(1, num(r) / max * 100).toFixed(1) + '%"></i></div></div>';
+ }).join('') + '</div>';
+ }
+ function cols(rows, label, num, fmt, tip) {
+ var max = Math.max.apply(null, rows.map(num).concat(1));
+ return '<div class="cols">' + rows.map(function (r) {
+ return '<div class="col" title="' + esc(tip(r)) + '"><span class="v">' + fmt(num(r)) + '</span><i style="height:' + Math.max(1, num(r) / max * 100).toFixed(1) + '%"></i></div>';
+ }).join('') + '</div><div class="labels">' + rows.map(function (r) { return '<span>' + esc(label(r)) + '</span>'; }).join('') + '</div>';
+ }
+ function short(n) { return n >= 10000 ? (n / 1000).toFixed(0) + 'k' : n >= 1000 ? (n / 1000).toFixed(1) + 'k' : String(n); }
+ function mon(m) { var p = m.split('-'); return new Date(Date.UTC(+p[0], +p[1] - 1, 1)).toLocaleString(undefined, { month: 'short', timeZone: 'UTC' }) + (p[1] === '01' ? ' ’' + p[0].slice(2) : ''); }
+
+ function render(d) {
+ if (d.empty) { $('sub').textContent = ''; $('out').innerHTML = '<div class="note">No stats yet - the file list could not be read, or the CDN is empty.<br><button onclick="location.reload()">Try again</button></div>'; return; }
+ var t = d.totals, mode = 'files';
+ $('sub').textContent = 'Updated ' + new Date(d.generated * 1000).toLocaleString([], { dateStyle: 'medium', timeStyle: 'short' }) + ' · refreshed every few minutes';
+ var h = '<div class="tiles">' +
+ [[nf.format(t.files), 'files'], [B(t.bytes), 'total size'], [B(t.median), 'median file'], [nf.format(t.types), 'file types']].map(function (x) { return '<div class="tile"><b>' + x[0] + '</b><span>' + x[1] + '</span></div>'; }).join('') +
+ '</div><div class="grid">' +
+ '<section class="card span2"><header><h2>What\'s on the CDN</h2><div class="seg" id="seg"><button aria-pressed="true" data-m="files">Files</button><button aria-pressed="false" data-m="size">Size</button></div></header><div id="donut"></div></section>' +
+ '<section class="card"><header><h2>Top file types</h2></header>' + bars(d.extensions, function (r) { return r.ext === '(none)' ? '(no extension)' : '.' + r.ext; }, function (r) { return nf.format(r.files) + ' · ' + B(r.bytes); }, function (r) { return r.files; }) +
+ (d.extensions_more ? '<p class="muted" style="margin:12px 0 0">+ ' + nf.format(d.extensions_more) + ' more files in rarer types</p>' : '') + '</section>' +
+ '<section class="card"><header><h2>File sizes</h2></header>' + cols(d.sizes, function (r) { return r.label; }, function (r) { return r.files; }, short, function (r) { return r.label + ': ' + nf.format(r.files) + ' files'; }) + '</section>' +
+ '<section class="card"><header><h2>Biggest folders</h2></header>' + bars(d.folders, function (r) { return r.name; }, function (r) { return B(r.bytes) + ' · ' + nf.format(r.files) + ' files'; }, function (r) { return r.bytes; }) + '</section>' +
+ (d.months.length > 1 ? '<section class="card"><header><h2>Files added per month</h2></header>' + cols(d.months, function (r) { return mon(r.m); }, function (r) { return r.files; }, short, function (r) { return r.m + ': ' + nf.format(r.files) + ' files, ' + B(r.bytes); }) + '</section>' : '') +
+ '<section class="card span2"><header><h2>Largest files</h2></header><ul class="files">' + d.largest.map(function (f) { return '<li><a href="/' + encodeURI(f.path) + '" title="' + esc(f.path) + '">' + esc(f.name) + '</a><span>' + B(f.size) + '</span></li>'; }).join('') + '</ul></section></div>';
+ $('out').innerHTML = h;
+ function draw() { $('donut').innerHTML = donut(d.categories, mode, t); }
+ draw();
+ $('seg').addEventListener('click', function (e) {
+ var b = e.target.closest('button'); if (!b) return; mode = b.dataset.m;
+ [].forEach.call($('seg').children, function (x) { x.setAttribute('aria-pressed', x === b); }); draw();
+ });
+ }
+ fetch('/api/stats').then(function (r) { if (!r.ok) throw 0; return r.json(); }).then(render)
+ .catch(function () { $('sub').textContent = ''; $('out').innerHTML = '<div class="note">Could not load the stats.<br><button onclick="location.reload()">Retry</button></div>'; });
+})();
+</script>
+</body>
+</html>
diff --git a/app/upload_guard.py b/app/upload_guard.py
@@ -1,30 +1,21 @@
"""Upload safety checks.
-Every upload endpoint calls inspect_upload() before the file is stored and
-record_upload() after it succeeds. Keeping the logic in this module means the
-rules can grow without touching the upload handlers in main.py.
+Every upload endpoint calls inspect_upload() before the file is stored.
+Keeping the logic in this module means the rules can grow without touching
+the upload handlers in main.py.
"""
from __future__ import annotations
-import asyncio
-import hashlib
import httpx
-import hmac
import ipaddress
-import json
import os
-import tempfile
-import threading
import time
from dataclasses import dataclass
-from cryptography.fernet import Fernet
-from fastapi import APIRouter, HTTPException, Request
-from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi import HTTPException, Request
-from .storage import _do_upload, HF_REPO_ID, HF_TOKEN
+from .admin import is_banned, record_blocked
-router = APIRouter()
# Video containers that are refused outright, by extension.
BLOCKED_VIDEO_EXTENSIONS = {
@@ -40,21 +31,6 @@ _NON_VIDEO_FTYP_BRANDS = {
b"jp2 ", b"jpx ", b"jpm ",
}
-LOG_PATH = os.getenv("UPLOAD_LOG_PATH", "/tmp/upload_log.jsonl")
-LOG_SALT = os.getenv("UPLOAD_LOG_SALT", "change-me")
-ADMIN_TOKEN = os.getenv("ADMIN_TOKEN", "")
-_log_lock = threading.Lock()
-# Persistence to the Hugging Face dataset. The dataset is publicly readable, so
-# IPs are Fernet-encrypted before upload; without LOG_ENC_KEY nothing is uploaded.
-# Generate a key with: python -c "from cryptography.fernet import Fernet;print(Fernet.generate_key().decode())"
-LOG_ENC_KEY = os.getenv("LOG_ENC_KEY", "")
-REMOTE_LOG_PATH = "_logs/uploads.jsonl"
-SYNC_INTERVAL = int(os.getenv("LOG_SYNC_SECONDS", "60"))
-MAX_LOG_ROWS = 50000
-_dirty = False
-_ready = False
-_ready_lock = asyncio.Lock()
-_sync_task = None
VPN_BLOCK = os.getenv("VPN_BLOCK", "1") == "1"
VPN_FAIL_OPEN = os.getenv("VPN_FAIL_OPEN", "1") == "1" # allow upload if lookup fails
PROXYCHECK_KEY = os.getenv("PROXYCHECK_KEY", "")
@@ -74,6 +50,32 @@ def _valid_ip(value: str | None) -> str | None:
return None
+# Cloudflare's published edge ranges (https://www.cloudflare.com/ips/). A request that
+# really arrived through Cloudflare carries the visitor's address in CF-Connecting-IP.
+_CLOUDFLARE_NETS = [ipaddress.ip_network(n) for n in (
+ "103.21.244.0/22", "103.22.200.0/22", "103.31.4.0/22", "104.16.0.0/13", "104.24.0.0/14",
+ "108.162.192.0/18", "131.0.72.0/22", "141.101.64.0/18", "162.158.0.0/15", "172.64.0.0/13",
+ "173.245.48.0/20", "188.114.96.0/20", "190.93.240.0/20", "197.234.240.0/22", "198.41.128.0/17",
+ "2400:cb00::/32", "2606:4700::/32", "2803:f800::/32", "2405:b500::/32", "2405:8100::/32",
+ "2a06:98c0::/29", "2c0f:f248::/32",
+)]
+
+
+def is_cloudflare_ip(value: str) -> bool:
+ try:
+ ip = ipaddress.ip_address(value)
+ except ValueError:
+ return False
+ return any(ip.version == n.version and ip in n for n in _CLOUDFLARE_NETS)
+
+
+def is_public_ip(value: str) -> bool:
+ try:
+ return ipaddress.ip_address(value).is_global
+ except ValueError:
+ return False
+
+
def get_client_ip(request: Request) -> str:
"""Best-effort real client IP behind Cloudflare / Render's proxy.
@@ -89,6 +91,19 @@ def get_client_ip(request: Request) -> str:
xff = request.headers.get("x-forwarded-for")
if xff:
parts = [p.strip() for p in xff.split(",") if p.strip()]
+ # Hosts like Render append their own internal (10.x etc.) proxy addresses to
+ # the right-hand end. Those are never the visitor, so drop them before counting hops.
+ public_parts = list(parts)
+ while public_parts and not is_public_ip(public_parts[-1]):
+ public_parts.pop()
+ parts = public_parts or parts
+ # The last remaining entry is the address that actually connected to the platform
+ # (appended by it, so the visitor can't forge it). If that is a Cloudflare edge, the
+ # request really came through Cloudflare and CF-Connecting-IP is genuine.
+ if parts and is_cloudflare_ip(parts[-1]):
+ ip = _valid_ip(request.headers.get("cf-connecting-ip"))
+ if ip:
+ return ip
if parts:
idx = len(parts) - TRUSTED_PROXY_HOPS
ip = _valid_ip(parts[idx] if idx >= 0 else parts[0])
@@ -148,7 +163,6 @@ async def inspect_upload(
method: str = "file",
) -> UploadContext:
"""Run all upload checks. Raises HTTPException when the upload is refused."""
- await _ensure_ready()
ctx = UploadContext(
ip=get_client_ip(request),
filename=filename or "",
@@ -158,14 +172,18 @@ async def inspect_upload(
user_agent=request.headers.get("user-agent", "")[:300],
)
+ if await is_banned(ctx.ip):
+ await record_blocked(ctx, "banned")
+ raise HTTPException(status_code=403, detail="You are banned from uploading to this CDN.")
+
if VPN_BLOCK and ctx.ip not in VPN_ALLOWLIST:
v = await is_vpn(ctx.ip)
if v or (v is None and not VPN_FAIL_OPEN):
- _log(ctx, None, "blocked_vpn")
+ await record_blocked(ctx, "blocked_vpn")
raise HTTPException(status_code=403, detail="Uploads from VPNs/proxies are not allowed.")
if is_video_upload(ctx.filename, temp_path):
- _log(ctx, None, "blocked_video")
+ await record_blocked(ctx, "blocked_video")
raise HTTPException(
status_code=415,
detail="Video uploads (including .mp4) are not allowed on this CDN.",
@@ -197,234 +215,3 @@ async def is_vpn(ip: str) -> bool | None:
_vpn_cache.clear()
_vpn_cache[ip] = (time.time() + (_VPN_TTL if verdict is not None else 300), verdict)
return verdict
-
-
-def _fernet():
- if not LOG_ENC_KEY:
- return None
- try:
- return Fernet(LOG_ENC_KEY.encode())
- except ValueError:
- return None
-
-
-async def _fetch_remote() -> str | None:
- url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/{REMOTE_LOG_PATH}"
- headers = {"Authorization": f"Bearer {HF_TOKEN}"} if HF_TOKEN else {}
- try:
- async with httpx.AsyncClient(follow_redirects=True, timeout=20.0) as c:
- r = await c.get(url, headers=headers)
- return r.text if r.status_code == 200 else None
- except httpx.HTTPError:
- return None
-
-
-async def _restore() -> None:
- """After a restart the local file is gone: rebuild it from the HF copy."""
- f = _fernet()
- if not f or os.path.exists(LOG_PATH):
- return
- text = await _fetch_remote()
- if not text:
- return
- out = []
- for line in text.splitlines():
- try:
- row = json.loads(line)
- if row.get("ip"):
- row["ip"] = f.decrypt(row["ip"].encode()).decode()
- out.append(json.dumps(row))
- except Exception:
- continue
- if out:
- with _log_lock, open(LOG_PATH, "a") as fh:
- fh.write("\n".join(out) + "\n")
-
-
-async def _push() -> None:
- global _dirty
- f = _fernet()
- if not f:
- return
- _dirty = False
- tmp = None
- try:
- with _log_lock, open(LOG_PATH) as fh:
- lines = fh.readlines()[-MAX_LOG_ROWS:]
- out = []
- for l in lines:
- if not l.strip():
- continue
- row = json.loads(l)
- if row.get("ip"):
- row["ip"] = f.encrypt(row["ip"].encode()).decode()
- out.append(json.dumps(row))
- fd, tmp = tempfile.mkstemp(suffix=".jsonl")
- with os.fdopen(fd, "w") as fh:
- fh.write("\n".join(out) + "\n")
- await asyncio.to_thread(_do_upload, tmp, REMOTE_LOG_PATH)
- except Exception:
- _dirty = True # retry on the next tick
- finally:
- if tmp and os.path.exists(tmp):
- os.remove(tmp)
-
-
-async def _sync_loop() -> None:
- while True:
- await asyncio.sleep(SYNC_INTERVAL)
- if _dirty:
- await _push()
-
-
-async def _ensure_ready() -> None:
- global _ready, _sync_task
- if _ready:
- return
- async with _ready_lock:
- if _ready:
- return
- await _restore()
- if _fernet():
- _sync_task = asyncio.create_task(_sync_loop())
- _ready = True
-
-
-def _anon(ip: str) -> str:
- return hmac.new(LOG_SALT.encode(), ip.encode(), hashlib.sha256).hexdigest()[:10]
-
-
-def _log(ctx: UploadContext, hf_path: str | None, status: str) -> None:
- global _dirty
- row = {
- "time": int(time.time()),
- "status": status,
- "filename": ctx.filename,
- "path": hf_path,
- "size": ctx.size,
- "folder": ctx.folder,
- "method": ctx.method,
- "uploader": _anon(ctx.ip),
- "ip": ctx.ip,
- "user_agent": ctx.user_agent,
- }
- try:
- with _log_lock, open(LOG_PATH, "a") as f:
- f.write(json.dumps(row) + "\n")
- _dirty = True
- except OSError:
- pass
-
-
-async def record_upload(ctx: UploadContext, hf_path: str) -> None:
- await _ensure_ready()
- _log(ctx, hf_path, "ok")
-
-
-def _read_log(limit: int) -> list[dict]:
- try:
- with open(LOG_PATH) as f:
- lines = f.readlines()[-limit:]
- except OSError:
- return []
- return [json.loads(l) for l in reversed(lines) if l.strip()]
-
-
-@router.get("/api/uploads")
-async def public_upload_log(limit: int = 100):
- """Public log. The uploader is an anonymous ID, never the raw IP."""
- await _ensure_ready()
- rows = _read_log(max(1, min(limit, 500)))
- public = ("time", "status", "filename", "path", "size", "folder", "method", "uploader")
- return JSONResponse({"uploads": [{k: r.get(k) for k in public} for r in rows]})
-
-
-@router.get("/api/admin/uploads")
-async def admin_upload_log(request: Request, limit: int = 200):
- """Full log with raw IPs. Needs: Authorization: Bearer $ADMIN_TOKEN."""
- auth = request.headers.get("authorization", "")
- if not ADMIN_TOKEN or not hmac.compare_digest(auth, f"Bearer {ADMIN_TOKEN}"):
- raise HTTPException(status_code=401, detail="Unauthorized")
- await _ensure_ready()
- return JSONResponse({"uploads": _read_log(max(1, min(limit, 2000)))})
-
-
-@router.get("/api/admin/ip")
-async def admin_ip_debug(request: Request):
- """Shows which IP the server detects for YOU and the raw headers it used."""
- auth = request.headers.get("authorization", "")
- if not ADMIN_TOKEN or not hmac.compare_digest(auth, f"Bearer {ADMIN_TOKEN}"):
- raise HTTPException(status_code=401, detail="Unauthorized")
- h = request.headers
- return {
- "detected_ip": get_client_ip(request),
- "cf_connecting_ip": h.get("cf-connecting-ip"),
- "x_forwarded_for": h.get("x-forwarded-for"),
- "socket_peer": request.client.host if request.client else None,
- "trust_cf_header": TRUST_CF_CONNECTING_IP,
- "trusted_proxy_hops": TRUSTED_PROXY_HOPS,
- }
-
-_ADMIN_HTML = """<!doctype html><html lang="en"><head><meta charset="utf-8">
-<meta name="viewport" content="width=device-width,initial-scale=1"><meta name="robots" content="noindex"><title>Upload log</title>
-<style>
-*{box-sizing:border-box}
-body{font:14px/1.45 system-ui,sans-serif;margin:0;padding:16px;background:#0d1117;color:#e6edf3;max-width:1200px;margin-inline:auto}
-h1{font-size:18px;margin:0 0 12px}
-form{display:flex;gap:8px;flex-wrap:wrap}
-input,select,button{font:inherit;padding:10px 12px;background:#161b22;color:inherit;border:1px solid #30363d;border-radius:8px;min-height:42px}
-input{flex:1 1 220px}select{flex:1 1 110px}button{flex:0 0 auto;background:#238636;border-color:#238636;font-weight:600}
-#msg{color:#f85149;margin:8px 0 0;min-height:1.4em}#count{color:#8b949e;margin:4px 0 8px;font-size:13px}
-a{color:#58a6ff}
-table{border-collapse:collapse;width:100%}
-th,td{border-bottom:1px solid #30363d;padding:8px;text-align:left;vertical-align:top;overflow-wrap:anywhere}
-th{color:#8b949e;font-weight:600;font-size:12px;text-transform:uppercase}
-.st{justify-self:start;display:inline-block;padding:1px 8px;border-radius:99px;font-size:12px;font-weight:600;background:#23863633;color:#3fb950}
-.st.blocked_video,.st.blocked_vpn{background:#f8514933;color:#f85149}
-@media(max-width:700px){
- thead{display:none}
- table,tbody,tr,td{display:block;width:100%}
- tr{background:#161b22;border:1px solid #30363d;border-radius:10px;margin:0 0 10px;padding:6px 12px}
- td{border:0;padding:4px 0;display:grid;grid-template-columns:78px 1fr;gap:8px}
- td::before{content:attr(data-l);color:#8b949e;font-size:12px;padding-top:2px}
- td.ua{font-size:12px;color:#8b949e}
-}
-</style></head><body>
-<h1>Upload log</h1>
-<form id="f"><input id="t" type="password" placeholder="Admin token" autocomplete="off">
-<select id="s"><option value="">All</option><option value="ok">OK</option><option value="blocked_video">Blocked video</option><option value="blocked_vpn">Blocked VPN</option></select>
-<button>Load</button></form>
-<div id="msg"></div><div id="count"></div>
-<table><thead><tr><th>Time</th><th>Status</th><th>File</th><th>Size</th><th>IP</th><th>Uploader</th><th>Via</th><th>User agent</th></tr></thead><tbody id="b"></tbody></table>
-<script>
-const $=id=>document.getElementById(id);
-const H=["Time","Status","File","Size","IP","Uploader","Via","Agent"];
-$("t").value=sessionStorage.getItem("tk")||"";
-async function load(){
- $("msg").textContent="";
- const r=await fetch("/api/admin/uploads?limit=500",{headers:{Authorization:"Bearer "+$("t").value}});
- if(!r.ok){$("msg").textContent=r.status==401?"Wrong token":"Error "+r.status;return}
- sessionStorage.setItem("tk",$("t").value);
- const f=$("s").value,rows=(await r.json()).uploads.filter(x=>!f||x.status==f),b=$("b");b.textContent="";
- $("count").textContent=rows.length+(rows.length==1?" entry":" entries")+", newest first";
- for(const x of rows){
- const tr=b.insertRow();
- const v=[new Date(x.time*1000).toLocaleString(),x.status,x.path||x.filename,(x.size/1048576).toFixed(2)+" MB",x.ip,x.uploader,x.method,x.user_agent];
- v.forEach((t,i)=>{
- const td=tr.insertCell();td.dataset.l=H[i];
- if(i==1){const s=document.createElement("span");s.className="st "+x.status;s.textContent=x.status.replace("_"," ");td.append(s)}
- else if(i==2&&x.path){const a=document.createElement("a");a.href="/"+x.path;a.textContent=t;td.append(a)}
- else td.textContent=t;
- if(i==7)td.className="ua";
- });
- }
-}
-$("f").onsubmit=e=>{e.preventDefault();load()};
-if($("t").value)load();
-</script></body></html>"""
-
-
-@router.get("/admin", response_class=HTMLResponse)
-async def admin_page():
- """Admin UI. The page itself holds no data; it asks for the token and calls the API."""
- return HTMLResponse(_ADMIN_HTML, headers={"Cache-Control": "no-store", "X-Robots-Tag": "noindex"})
diff --git a/render.yaml b/render.yaml
@@ -4,7 +4,7 @@ services:
env: docker
plan: free
envVars:
- - key: HF_REPO_ID
+ - key: HF_BUCKET_ID
value: notamitgamer/cdn
- key: HF_TOKEN
@@ -14,4 +14,13 @@ services:
value: raw.cdn.amit.is-a.dev
- key: RAW_BASE_URL
- value: https://raw.cdn.amit.is-a.dev
-\ No newline at end of file
+ value: https://raw.cdn.amit.is-a.dev
+
+ # Admin dashboard at /admin. Leave unset to switch the dashboard, bans and
+ # the GitHub allowlist off. ADMIN_DATA_KEY encrypts the saved state in the
+ # bucket; set it separately so rotating ADMIN_TOKEN doesn't lock you out of it.
+ - key: ADMIN_TOKEN
+ sync: false
+
+ - key: ADMIN_DATA_KEY
+ sync: false
diff --git a/requirements.txt b/requirements.txt
@@ -1,8 +1,8 @@
fastapi
uvicorn
-huggingface_hub
+huggingface_hub>=1.5
httpx
python-multipart
jinja2
pyjwt[crypto]
-yt-dlp
-\ No newline at end of file
+yt-dlp
