commit 36b3d0ff05af28e3238803be7310506298247986
parent e93ad33e1df77d90d69ecbad41d345d8148b4bc1
Author: Amit Dutta <mail@amit.is-a.dev>
Date: Fri, 9 Oct 2026 06:34:55 +0530
Merge pull request #32 from notamitgamer/docs-update-sync
Rate limits: new limits, a dedicated docs section and page notices; docs and sync page refresh
Diffstat:
6 files changed, 237 insertions(+), 67 deletions(-)
diff --git a/app/admin.py b/app/admin.py
@@ -437,6 +437,10 @@ function render(s){
t+='</table>';
}else t+='<p class="dim" style="margin:0">Nothing waiting. A new GitHub account shows up here after its first sync.</p>';
t+='</section>';
+ t+='<section id="bans" class="sec"><h2>Banned IPs</h2><table class="cards"><tr><th>IP</th><th>Since</th><th>Note</th><th></th></tr>';
+ for(const ip in banned)t+=`<tr>${cell('IP',esc(ip),'name mono')}${cell('Since',fmt(banned[ip].t))}${cell('Note',esc(banned[ip].note||''))}<td class="act">${btn('Unban','','/api/admin/unban',{ip})}</td></tr>`;
+ if(!nb)t+=empty(4,'None.');
+ t+='</table></section>';
document.getElementById('top').innerHTML=t;
let u='<table class="cards"><tr><th>File</th><th>Time</th><th>IP</th><th>Size</th><th>Via</th><th>Status</th><th></th></tr>';
@@ -453,11 +457,9 @@ function render(s){
for(const [k,g] of s.gh){const c=g.status==='allowed'?'ok':g.status==='banned'?'bad':'warn';
r+=`<tr>${cell('Account',esc(g.owner),'name')}${cell('Status',esc(g.status),c)}${cell('Repos',esc((g.repos||[]).join(', ')))}${cell('Syncs',g.syncs)}${cell('Last IP',esc(g.last_ip),'mono')}${cell('Last sync',fmt(g.last))}<td class="act">${g.status!=='allowed'?btn('Allow','green',GH,{key:k,action:'allow'}):btn('Revoke','',GH,{key:k,action:'revoke'})}${g.status!=='banned'?btn('Ban','red',GH,{key:k,action:'ban'},'Ban this GitHub account?'):''}</td></tr>`}
if(!s.gh.length)r+=empty(7,'No GitHub syncs yet.');
- r+=`</table><p class="dim">Always allowed: ${esc(s.always_allow.join(', ')||'none')}</p><section id="bans" class="sec"><h2>Banned IPs</h2><table class="cards"><tr><th>IP</th><th>Since</th><th>Note</th><th></th></tr>`;
- for(const ip in banned)r+=`<tr>${cell('IP',esc(ip),'name mono')}${cell('Since',fmt(banned[ip].t))}${cell('Note',esc(banned[ip].note||''))}<td class="act">${btn('Unban','','/api/admin/unban',{ip})}</td></tr>`;
- if(!nb)r+=empty(4,'None.');
+ r+=`</table><p class="dim">Always allowed: ${esc(s.always_allow.join(', ')||'none')}</p>`;
const w=s.whoami,kv=(a,b)=>`<tr><td>${a}</td><td class="mono">${esc(b==null||b===''?'-':b)}</td></tr>`;
- r+=`</table></section><details id="who" class="who"${WHO||!w.ok?' open':''}><summary>Your IP as the server sees it: <span class="mono ${w.ok?'ok':'bad'}">${esc(w.detected)}</span> ${w.ok?'✓':'✗ looks wrong'}</summary><table class="kv">${kv('X-Forwarded-For',w.x_forwarded_for)}${kv('CF-Connecting-IP',w.cf_connecting_ip)}${kv('Socket peer',w.socket_peer)}${kv('Settings','TRUST_CF_CONNECTING_IP='+(w.trust_cf_header?1:0)+', TRUSTED_PROXY_HOPS='+w.proxy_hops)}</table><p class="dim">This must be your real public IP, or bans will hit the wrong address.</p></details>`;
+ r+=`<details id="who" class="who"${WHO||!w.ok?' open':''}><summary>Your IP as the server sees it: <span class="mono ${w.ok?'ok':'bad'}">${esc(w.detected)}</span> ${w.ok?'✓':'✗ looks wrong'}</summary><table class="kv">${kv('X-Forwarded-For',w.x_forwarded_for)}${kv('CF-Connecting-IP',w.cf_connecting_ip)}${kv('Socket peer',w.socket_peer)}${kv('Settings','TRUST_CF_CONNECTING_IP='+(w.trust_cf_header?1:0)+', TRUSTED_PROXY_HOPS='+w.proxy_hops)}</table><p class="dim">This must be your real public IP, or bans will hit the wrong address.</p></details>`;
document.getElementById('rest').innerHTML=r;
document.getElementById('who').addEventListener('toggle',e=>{WHO=e.target.open});
}
diff --git a/app/main.py b/app/main.py
@@ -220,44 +220,40 @@ async def download_file(path: str, request: Request):
)
-UPLOAD_LIMIT_PER_MINUTE = 50 * 1024 * 1024
-UPLOAD_LIMIT_PER_HOUR = 300 * 1024 * 1024
+UPLOAD_LIMIT_PER_HOUR = 500 * 1024 * 1024 # file uploads (/api/put), per IP
+URL_UPLOAD_LIMIT_PER_HOUR = 50 * 1024 * 1024 # uploads from a link (/api/put-url), per IP
class _UploadRateLimiter:
- def __init__(self):
- self._usage: dict[str, deque] = {}
+ """Sliding one-hour window of uploaded bytes, per key (the client IP). No per-minute limit."""
- def _prune(self, ip: str, now: float):
- dq = self._usage.get(ip)
- if not dq:
- return
- while dq and now - dq[0][0] > 3600:
- dq.popleft()
+ def __init__(self, per_hour: int, name: str = "Upload"):
+ self.per_hour = per_hour
+ self.name = name
+ self._usage: dict[str, deque] = {}
- def check_and_record(self, ip: str, size: int):
+ def check_and_record(self, key: str, size: int):
now = time.time()
- dq = self._usage.setdefault(ip, deque())
- self._prune(ip, now)
- minute_used = sum(s for t, s in dq if now - t <= 60)
- hour_used = sum(s for t, s in dq)
-
- if minute_used + size > UPLOAD_LIMIT_PER_MINUTE:
+ if size > self.per_hour: # can never fit, so don't say "try again later"
raise HTTPException(
- status_code=429,
- detail=f"Upload rate limit exceeded: {format_size(UPLOAD_LIMIT_PER_MINUTE)}/minute. Try again shortly.",
+ status_code=413,
+ detail=f"File too large (limit {format_size(self.per_hour)} per file).",
)
-
- if hour_used + size > UPLOAD_LIMIT_PER_HOUR:
+ dq = self._usage.setdefault(key, deque())
+ while dq and now - dq[0][0] > 3600:
+ dq.popleft()
+ if sum(s for _, s in dq) + size > self.per_hour:
raise HTTPException(
status_code=429,
- detail=f"Upload rate limit exceeded: {format_size(UPLOAD_LIMIT_PER_HOUR)}/hour. Try again later.",
+ detail=f"{self.name} rate limit exceeded: {format_size(self.per_hour)}/hour. Try again later.",
)
-
dq.append((now, size))
+ if len(self._usage) > 10000: # drop idle clients so the table can't grow forever
+ self._usage = {k: v for k, v in self._usage.items() if v and now - v[-1][0] <= 3600}
-_upload_limiter = _UploadRateLimiter()
+_upload_limiter = _UploadRateLimiter(UPLOAD_LIMIT_PER_HOUR, "Upload")
+_url_upload_limiter = _UploadRateLimiter(URL_UPLOAD_LIMIT_PER_HOUR, "URL upload")
class _CountRateLimiter:
@@ -288,7 +284,7 @@ class _CountRateLimiter:
self._hits = {k: v for k, v in self._hits.items() if v and now - v[-1] <= 3600}
-_shorten_limiter = _CountRateLimiter(per_minute=10, per_hour=60)
+_shorten_limiter = _CountRateLimiter(per_minute=10, per_hour=50)
SHORTEN_MAX_URL_LENGTH = 2048
BATCH_MIN_FILES = 2
ALLOWED_UPLOAD_FOLDERS = {"uploads", "third-party"}
@@ -356,10 +352,6 @@ async def handle_upload(
return response
-GH_SYNC_LIMIT_PER_HOUR = 2 * 1024 * 1024 * 1024
-_gh_sync_limiter = _UploadRateLimiter()
-
-
def _safe_extract_tar(tar: tarfile.TarFile, dest: str):
dest_real = os.path.realpath(dest)
@@ -413,11 +405,7 @@ async def gh_sync(
raise HTTPException(status_code=403, detail="You are banned from this CDN.")
await gh_check(owner, claims["owner_id"])
- contents = await archive.read()
- _gh_sync_limiter.check_and_record(
- f"gh-sync:{owner}/{repo}:{client_ip}",
- len(contents),
- )
+ contents = await archive.read() # no size/rate limit on GitHub sync (identity, approval, bans and the video rule apply)
with tempfile.TemporaryDirectory() as tmp_upload, tempfile.TemporaryDirectory() as tmp_extract:
archive_path = os.path.join(tmp_upload, "payload.tar.gz")
@@ -508,7 +496,7 @@ async def mask_redirect(request: Request, short_id: str):
)
-_MAX_URL_UPLOAD_BYTES = UPLOAD_LIMIT_PER_HOUR
+_MAX_URL_UPLOAD_BYTES = URL_UPLOAD_LIMIT_PER_HOUR
def _assert_public_url(url: str):
@@ -644,7 +632,7 @@ async def handle_upload_from_url(
folder,
"url",
)
- _upload_limiter.check_and_record(
+ _url_upload_limiter.check_and_record(
client_ip,
downloaded,
)
diff --git a/app/templates/documentation.html b/app/templates/documentation.html
@@ -67,9 +67,12 @@
<li><a href="#browsing">Browsing & viewing</a></li>
<li><a href="#gh-sync">GitHub Actions sync</a></li>
<li><a href="#uploading">Uploading files</a></li>
+ <li><a href="#shortener">URL shortener</a></li>
+ <li><a href="#rate-limits">Rate limits</a></li>
<li><a href="#downloading">Downloading & zips</a></li>
<li><a href="#batches">Multi-file batches</a></li>
<li><a href="#search">Search</a></li>
+ <li><a href="#stats">Stats</a></li>
<li><a href="#raw-domain">The raw domain</a></li>
<li><a href="#utility">Utility</a></li>
<li><a href="#architecture">How storage works</a></li>
@@ -91,9 +94,9 @@
<tr><td>nonexistent</td><td class="desc">404 page.</td></tr>
</table>
<h3>How it decides</h3>
- <p>It calls <code>get_file_info(path)</code> first (a HEAD request against the Hugging Face dataset repo's
- <code>resolve/main/</code> URL for that path). If that says the path exists as a file, it renders the file
- page. Otherwise it calls <code>list_directory(path)</code> (a tree listing from the HF API); a non-empty
+ <p>It calls <code>get_file_info(path)</code> first (a HEAD request against the Hugging Face bucket's
+ <code>resolve</code> URL for that path). If that says the path exists as a file, it renders the file
+ page. Otherwise it calls <code>list_directory(path)</code> (a tree listing from the HF bucket API); a non-empty
result renders the folder listing, an empty result on a non-root path is a 404.</p>
<h3>Errors</h3>
<ul class="err-list"><li><span class="err-code">404</span> Path doesn't exist as a file or a non-empty folder.</li></ul>
@@ -134,10 +137,12 @@
<h3>How it works, step by step</h3>
<ol style="color:var(--muted)">
<li>Verifies the token's signature against GitHub's own public keys (<code>token.actions.githubusercontent.com/.well-known/jwks</code>), checks it isn't expired, and checks the <code>aud</code> claim matches this service.</li>
- <li>Reads <code>repository_owner</code> and <code>repository</code> straight out of the <em>verified</em> token claims — this is the destination folder, and it is never taken from anything else in the request. A workflow file cannot forge these claims because GitHub itself stamps them for the run.</li>
- <li>Runs a per-repo, per-IP rate check (2 GB/hour) as an abuse guard — this is not a content-size cap, just throttling.</li>
+ <li>Reads <code>repository_owner</code>, <code>repository</code> and the numeric <code>repository_owner_id</code> straight out of the <em>verified</em> token claims — this is the destination folder and the identity of the sender, and it is never taken from anything else in the request. A workflow file cannot forge these claims because GitHub itself stamps them for the run.</li>
+ <li>Refuses banned visitors and banned GitHub accounts, then applies the <strong>approval rule</strong>: the <em>first</em> sync from a GitHub account is accepted automatically; after that, the account must have been approved by the CDN owner or the sync is refused with <code>403</code>. Accounts are tracked by GitHub's numeric owner ID, so renaming an account or re-registering an old username does not bypass this. The CDN owner's own account is always allowed.</li>
+
<li>Extracts the tar archive into a temp directory, rejecting anything that tries to write outside that directory (path traversal) or that is a symlink.</li>
- <li>Uploads that folder to the underlying Hugging Face dataset repo with <code>path_in_repo</code> <strong>and</strong> <code>delete_patterns</code> both scoped to exactly <code><owner>/<repo>/</code> — so the call can create, update, or delete files only inside that one folder, never anywhere else. Deleted files in the source repo are removed from the CDN folder too (this is a mirror, not an accumulate-forever upload).</li>
+ <li>Rejects the <strong>whole sync</strong> if any file is a video, detected by extension <em>and</em> by file signature (so renaming a video does not help). Nothing is uploaded in that case.</li>
+ <li>Uploads that folder to the Hugging Face storage bucket with the destination and the delete scope both limited to exactly <code><owner>/<repo>/</code> — so the call can create, update, or delete files only inside that one folder, never anywhere else. Files deleted from the source repo are removed from the CDN folder too (this is a mirror, not an accumulate-forever upload). The bucket keeps no version history, so those removals are permanent.</li>
</ol>
<h3>Response <code>200</code></h3>
@@ -153,12 +158,14 @@
<ul class="err-list">
<li><span class="err-code">401</span> Token missing, expired, badly signed, wrong audience, or its owner/repository claims don't line up (malformed token).</li>
<li><span class="err-code">400</span> Archive isn't a readable <code>.tar.gz</code>, or it contains an unsafe path / a symlink.</li>
- <li><span class="err-code">429</span> Per-repo/IP hourly byte limit (2 GB) exceeded.</li>
+ <li><span class="err-code">403</span> The sender's IP or GitHub account is banned, <em>or</em> this account's first sync was already accepted and further syncs need approval from the CDN owner. The message says which.</li>
+ <li><span class="err-code">415</span> The archive contains a video file. The message names the file and nothing is synced.</li>
+
</ul>
<h3>Limits, currently</h3>
- <p>No file-type restriction and no per-file or total size cap beyond the 2 GB/hour abuse-rate guard above.
- This is intentional for now, not an oversight.</p>
+ <p>Any file type except video. <strong>GitHub sync has no rate limit</strong> and no limit on the size or number of files:
+ what protects it instead is the sign-in identity, the approval rule, bans and the video rule above. See <a href="#rate-limits">Rate limits</a>.</p>
</div>
</section>
@@ -177,10 +184,13 @@
<tr><td><code>folder</code></td><td class="desc">Optional. Must be exactly <code>uploads</code> or <code>third-party</code> — these are the only two folders this endpoint is allowed to write to. Defaults to <code>uploads</code>.</td></tr>
</table>
<h3>How it works</h3>
- <p>Each file is streamed to a local temp path, checked against the rate limiter (50 MB/minute, 300 MB/hour
- per client IP), then uploaded to Hugging Face at <code><folder>/<8-char-uuid>-<original-filename></code>
- (any characters other than letters, digits, <code>.</code>, <code>_</code> and <code>-</code> become underscores, and accents are simplified). The random prefix means two uploads can never collide, and the
- filename portion is still readable. If 2 or more files were uploaded together, a <em>batch</em> is also created
+ <p>Each file is streamed to a temporary file named only by a random ID (your filename is never used in a filesystem path),
+ then checked in this order: banned IP, VPN/proxy address, video (by extension or file signature), and the
+ upload limit (500 MB per hour per client IP). It is then stored in the bucket at
+ <code><folder>/<8-char-uuid>-<filename></code>. Any filename is accepted: directory parts are dropped,
+ accents are simplified, and anything other than letters, digits, <code>.</code>, <code>_</code> and <code>-</code>
+ becomes an underscore (<code>my file (1).png</code> is stored as <code>ab12cd34-my_file_1.png</code>).
+ The random prefix means two uploads can never collide, and the filename portion is still readable. If 2 or more files were uploaded together, a <em>batch</em> is also created
(see <a href="#batches">Batches</a> below) so the whole set can be shared/downloaded as one link.</p>
<h3>Response <code>200</code></h3>
<pre>{
@@ -194,14 +204,19 @@
<h3>Errors</h3>
<ul class="err-list">
<li><span class="err-code">400</span> <code>folder</code> isn't <code>uploads</code> or <code>third-party</code>.</li>
- <li><span class="err-code">429</span> Per-IP rate limit exceeded (50 MB/min or 300 MB/hour).</li>
+ <li><span class="err-code">403</span> Your IP is banned, or the upload comes from a VPN/proxy address.</li>
+ <li><span class="err-code">415</span> The file is a video (<code>.mp4</code>, <code>.mov</code>, <code>.webm</code>, <code>.mkv</code> and similar, or recognised by its contents). Videos are not accepted on this CDN.</li>
+ <li><span class="err-code">413</span> A single file is larger than 500 MB (waiting won't help; split it).</li>
+ <li><span class="err-code">429</span> You have used your 500 MB for the hour. See <a href="#rate-limits">Rate limits</a>.</li>
</ul>
</div>
<div class="endpoint">
<div class="route-row"><span class="method post">POST</span><span class="route-path">/api/put-url</span></div>
<p>Uploads a file the server fetches from a URL you give it, instead of one you send directly. Same destination
- rules and rate limiter as <code>/api/put</code>.</p>
+ rules and checks (ban, VPN/proxy, video) as <code>/api/put</code>, but with its own smaller allowance: <strong>50 MB per hour per IP</strong>,
+ separate from your file-upload allowance. The file name is taken from the URL or the
+ response's <code>Content-Disposition</code> and sanitised the same way.</p>
<h3>Request</h3>
<p>JSON body:</p>
<pre>{ "url": "https://example.com/file.png", "folder": "uploads" }</pre>
@@ -217,12 +232,103 @@
<h3>Errors</h3>
<ul class="err-list">
<li><span class="err-code">400</span> Not <code>https://</code>, hostname doesn't resolve, resolves to a private/internal address, or the source returned a non-200.</li>
- <li><span class="err-code">413</span> File exceeds the fetch size limit (currently the same 300 MB/hour figure used as the upload cap).</li>
- <li><span class="err-code">429</span> Per-IP rate limit exceeded.</li>
+ <li><span class="err-code">403</span> Your IP is banned, or the request comes from a VPN/proxy address.</li>
+ <li><span class="err-code">413</span> The file is larger than 50 MB, the most that can be fetched from a link. The download is stopped as soon as it passes the limit.</li>
+ <li><span class="err-code">415</span> The downloaded file is a video.</li>
+ <li><span class="err-code">429</span> You have used your 50 MB for the hour for uploads from a link. See <a href="#rate-limits">Rate limits</a>.</li>
</ul>
</div>
</section>
+ <section id="shortener">
+ <h2>URL shortener</h2>
+
+ <div class="endpoint">
+ <div class="route-row"><span class="method get">GET</span><span class="route-path">/shorten</span></div>
+ <p>The page for creating short links. It calls the API below.</p>
+ </div>
+
+ <div class="endpoint">
+ <div class="route-row"><span class="method post">POST</span><span class="route-path">/api/shorten</span></div>
+ <p>Turns a long <code>https://</code> URL into a short <code>/mask/<id></code> link on this domain.</p>
+ <h3>Request</h3>
+ <pre>{ "url": "https://example.com/some/long/page" }</pre>
+ <h3>How it works / safety</h3>
+ <p>Only <code>https://</code> URLs of at most 2048 characters are accepted. The hostname is resolved and every
+ address is checked — private, loopback, link-local and other internal addresses are rejected, the same SSRF
+ guard used by <code>/api/put-url</code>. Each client IP is limited to <strong>10 requests per minute and 50 per hour</strong>;
+ the limit is counted before the hostname lookup, so rejected attempts count too. The destination is stored as a small
+ object in the bucket under a random 8-character ID.</p>
+ <h3>Response <code>200</code></h3>
+ <pre>{
+ "id": "abc123XY",
+ "masked_url": "https://cdn.amit.is-a.dev/mask/abc123XY"
+}</pre>
+ <h3>Errors</h3>
+ <ul class="err-list">
+ <li><span class="err-code">400</span> Not <code>https://</code>, longer than 2048 characters, invalid, host doesn't resolve, or it points to a private/internal address.</li>
+ <li><span class="err-code">403</span> Your IP is banned.</li>
+ <li><span class="err-code">429</span> More than 10 requests in a minute or 50 in an hour from your IP. See <a href="#rate-limits">Rate limits</a>.</li>
+ </ul>
+ </div>
+
+ <div class="endpoint">
+ <div class="route-row"><span class="method get">GET</span><span class="route-path">/mask/{short_id}</span></div>
+ <p>Redirects to the stored destination with a <code>307</code> and a <code>Referrer-Policy: no-referrer</code> header, so the
+ destination site doesn't see which page the visitor came from.</p>
+ <h3>Errors</h3>
+ <ul class="err-list">
+ <li><span class="err-code">404</span> No short link with that ID.</li>
+ </ul>
+ </div>
+ </section>
+
+ <section id="rate-limits">
+ <h2>Rate limits</h2>
+ <p class="lede">Limits keep the storage free for everyone and stop one visitor from filling it. They apply only to things that
+ <em>write</em>: uploading files, uploading from a link, and shortening a link. Browsing, search, viewing, downloads, zips and
+ GitHub sync are not rate-limited by this app.</p>
+
+ <h3>At a glance</h3>
+ <table>
+ <tr><th>What</th><th class="desc">Limit</th></tr>
+ <tr><td style="white-space:nowrap"><code>/api/put</code></td><td class="desc"><strong>500 MB per hour</strong>, counted in bytes, per IP. No per-minute limit. A single file can be at most 500 MB.</td></tr>
+ <tr><td style="white-space:nowrap"><code>/api/put-url</code></td><td class="desc"><strong>50 MB per hour</strong>, counted in bytes, per IP. This is a separate allowance from file uploads. A single file can be at most 50 MB.</td></tr>
+ <tr><td style="white-space:nowrap"><code>/api/shorten</code></td><td class="desc"><strong>10 per minute</strong> and <strong>50 per hour</strong>, counted in requests, per IP.</td></tr>
+ <tr><td style="white-space:nowrap"><code>/api/gh-sync</code></td><td class="desc"><strong>No rate limit.</strong></td></tr>
+ </table>
+
+ <h3>How the limits work</h3>
+ <ul style="color:var(--muted)">
+ <li><strong>Sliding window.</strong> Your usage is totalled over the <em>last 60 minutes</em> (the shortener also looks at the last 60 seconds). There is no clock-time reset: old uploads age out one by one, so your allowance returns gradually.</li>
+ <li><strong>Per IP.</strong> The limit follows the address your request really comes from. People on the same network (office, school, mobile carrier) share one allowance. Uploads from VPN or proxy addresses are refused outright, whatever your usage.</li>
+ <li><strong>Bytes, not files.</strong> Ten 1 MB files use the same allowance as one 10 MB file. Each file is checked when it arrives.</li>
+ <li><strong>Two separate allowances.</strong> Uploading from a link never uses up your 500 MB of file uploads, and file uploads never use up your 50 MB for links.</li>
+ <li><strong>Biggest single file.</strong> 500 MB for file uploads and 50 MB from a link. A file over that is refused with <code>413</code> rather than <code>429</code>, because waiting would not help. From a link, the download stops as soon as it passes 50 MB.</li>
+ <li><strong>Batches stop at the limit.</strong> If you upload several files in one request and the limit is reached partway, the request fails with <code>429</code>. Files already stored earlier in that request stay stored.</li>
+ <li><strong>What does not count.</strong> Uploads refused before the limit check (banned IP, VPN/proxy address, video) do not use your allowance. Shortener requests <em>do</em> count even when the URL turns out to be invalid, because the limit is checked before the link is examined.</li>
+ <li><strong>GitHub sync is not rate-limited.</strong> It is protected by the sign-in identity, the approval rule, bans and the video rule instead (see <a href="#gh-sync">GitHub Actions sync</a>).</li>
+ <li><strong>In memory.</strong> Counters are kept in the server's memory, so a restart or redeploy clears them.</li>
+
+ </ul>
+
+ <h3>When you hit a limit</h3>
+ <p>The response is <code>429</code> (or <code>413</code> for a file that is too big) with a JSON message. There is no <code>Retry-After</code> header, so use the message to see which limit you hit:</p>
+ <pre>{ "detail": "Upload rate limit exceeded: 500.00 MB/hour. Try again later." }
+{ "detail": "URL upload rate limit exceeded: 50.00 MB/hour. Try again later." }
+{ "detail": "Rate limit exceeded: 10 requests/minute. Try again shortly." }
+{ "detail": "Rate limit exceeded: 50 requests/hour. Try again later." }
+{ "detail": "File too large (limit 500.00 MB per file)." }</pre>
+ <p>After a <em>minute</em> message, wait up to a minute. After an <em>hour</em> message, wait until earlier uploads are an hour old. For <em>file too large</em>, waiting doesn't help: make the file smaller or split it.</p>
+
+ <h3>Staying under the limits</h3>
+ <ul style="color:var(--muted)">
+ <li>Upload big batches in smaller groups, and leave time between them if you are close to 500 MB in an hour.</li>
+ <li>Compress or resize large files before uploading.</li>
+ <li>Have a whole project to move? <a href="#gh-sync">GitHub sync</a> has no rate limit.</li>
+ </ul>
+ </section>
+
<section id="downloading">
<h2>Downloading & zips</h2>
@@ -287,12 +393,44 @@
</div>
</section>
+ <section id="stats">
+ <h2>Stats</h2>
+
+ <div class="endpoint">
+ <div class="route-row"><span class="method get">GET</span><span class="route-path">/stats</span></div>
+ <p>A page of charts about what is on the CDN: file types (by count or by size), the most common extensions, a
+ file-size histogram, the biggest folders, files added per month, and the largest files. It loads its data from
+ <code>/api/stats</code> and draws everything with plain CSS and inline SVG, so the page stays tiny.</p>
+ </div>
+
+ <div class="endpoint">
+ <div class="route-row"><span class="method get">GET</span><span class="route-path">/api/stats</span></div>
+ <p>The JSON behind the page. It contains aggregate numbers only — no IP addresses and no upload history. It is
+ computed from the same cached file listing used for browsing and search and recomputed at most every 10 minutes, so
+ calling it never adds load on the bucket. The response is sent with <code>Cache-Control: public, max-age=300</code>.</p>
+ <h3>Response <code>200</code></h3>
+ <pre>{
+ "empty": false,
+ "generated": 1791270000, // unix time
+ "totals": { "files": 900, "bytes": 1060758477, "avg": 1178620, "median": 171280, "types": 14 },
+ "categories": [ { "key": "images", "label": "Images", "files": 421, "bytes": 123456 } ],
+ "extensions": [ { "ext": "png", "files": 236, "bytes": 49283712 } ], // top 10 by count
+ "extensions_more": 72, // files in rarer types
+ "sizes": [ { "label": "< 10 KB", "files": 129 } ],
+ "folders": [ { "name": "uploads", "files": 375, "bytes": 564133888 } ], // top 8 by size
+ "largest": [ { "name": "a.zip", "path": "uploads/ab12cd34-a.zip", "size": 45088768 } ],
+ "months": [ { "m": "2026-09", "files": 71, "bytes": 1234567 } ] // last 12 months
+}</pre>
+ <p>If the file listing can't be read, it returns the last good result, or <code>{ "empty": true }</code> if there is none yet.</p>
+ </div>
+ </section>
+
<section id="raw-domain">
<h2>The raw domain</h2>
<div class="endpoint">
<div class="route-row"><span class="method get">GET</span><span class="route-path">/{path}</span> <span class="domain-tag">raw.cdn.amit.is-a.dev</span></div>
<p>A separate hostname, not a path prefix on the main domain. Requests here are proxied straight through to
- the underlying Hugging Face file (<code>resolve/main/<path></code>), with no HTML wrapper — this is
+ the underlying file in the Hugging Face bucket (its <code>resolve</code> URL), with no HTML wrapper — this is
meant for direct file links (<code><img src></code>, downloads, embeds), not browsing.</p>
<h3>If the path is a folder, not a file</h3>
<p>The direct file fetch 404s (Hugging Face's <code>resolve</code> only serves blobs). Before giving up, this
@@ -327,12 +465,15 @@
<h2>How storage actually works</h2>
<p class="lede">Everything above talks about "the CDN" as one thing, but there's no custom database or file
server behind it — every file this service serves, lists, or accepts lives in a single
- <strong>Hugging Face dataset repository</strong>. This app is a FastAPI backend in front of that repo:</p>
+ <strong>Hugging Face Storage Bucket</strong>. A bucket is plain object storage, not a git repository, so there is no
+ commit history that grows as files pile up. This app is a FastAPI backend in front of that bucket:</p>
<ul style="color:var(--muted)">
- <li>File listings (folders, search, file-exists checks) come from Hugging Face's repo-tree API, with a short-lived in-memory cache on this server's side so repeated browsing doesn't re-hit that API constantly.</li>
- <li>Serving an actual file's bytes (viewing, raw links, downloads, zips) streams from the dataset repo's public <code>resolve/main/<path></code> URL — this server proxies that stream rather than storing a separate copy.</li>
- <li>Uploads (<code>/api/put</code>, <code>/api/put-url</code>, <code>/api/gh-sync</code>) all ultimately call the Hugging Face Hub API to commit files into that same dataset repo, each scoped to whatever folder/path rules that endpoint enforces.</li>
- <li>There is one HF write token, held only by this server - never handed to uploaders, contributors, or GitHub Actions runs. Everything that writes to storage goes through this backend's own validation first (folder allow-lists, OIDC identity checks, rate limits) precisely because the write credential itself has no per-user restriction built in - the backend <em>is</em> the access control.</li>
+ <li>File listings (folders, search, file-exists checks, stats) come from the bucket's tree API, with a short-lived in-memory cache on this server's side so repeated browsing doesn't re-hit that API constantly.</li>
+ <li>Serving an actual file's bytes (viewing, raw links, downloads, zips) streams from the bucket's public <code>resolve</code> URL — this server proxies that stream rather than storing a separate copy.</li>
+ <li>Uploads (<code>/api/put</code>, <code>/api/put-url</code>, <code>/api/gh-sync</code>) all ultimately call the Hugging Face bucket API to write objects into that same bucket, each scoped to whatever folder/path rules that endpoint enforces.</li>
+ <li>There is one HF write token, held only by this server - never handed to uploaders, contributors, or GitHub Actions runs. Everything that writes to storage goes through this backend's own validation first (folder allow-lists, OIDC identity checks, bans, rate limits) precisely because the write credential itself has no per-user restriction built in - the backend <em>is</em> the access control.</li>
+ <li><strong>No version history.</strong> Overwritten or deleted files can't be restored from history, and a GitHub sync that removes files from a folder removes them for good.</li>
+ <li>A few internal folders live in the same bucket (batch records, short links and the site's own moderation data). They are hidden from listings and search, and the moderation data is encrypted.</li>
</ul>
</section>
diff --git a/app/templates/gh_sync_docs.html b/app/templates/gh_sync_docs.html
@@ -137,6 +137,9 @@
The CDN backend verifies that token's signature, reads the owner/repo straight from it, and uses
<em>only that</em> to decide where your files go. Nothing in your request body or headers is trusted
for the destination path.</p>
+ <p>The same token also says <em>who</em> you are, by GitHub's permanent numeric account ID — renaming
+ your account, or someone registering a username you dropped, doesn't change that identity.
+ That's what the approval step below is based on.</p>
<h2>1. Add the workflow</h2>
<p>Create <code>.github/workflows/cdn-sync.yml</code> in the repo you want mirrored. The token is
@@ -177,7 +180,7 @@ jobs:
env:
OIDC_TOKEN: ${{ steps.oidc.outputs.token }}
run: |
- curl -sf -X POST https://cdn.amit.is-a.dev/api/gh-sync \
+ curl -sS --fail-with-body -X POST https://cdn.amit.is-a.dev/api/gh-sync \
-F "token=$OIDC_TOKEN" \
-F "archive=@/tmp/payload.tar.gz"
{% endraw %}</pre>
@@ -190,17 +193,42 @@ jobs:
<h2>Notes</h2>
<ul>
- <li>Any file type except video is accepted (a sync containing a video is rejected as a whole), and there's currently no limit on file size or on how many files a sync can push — send the whole repo as-is.</li>
<li><strong>First sync is free, later ones need approval.</strong> The first sync from a new GitHub account
goes through immediately. After that, further syncs from that account are refused until the CDN owner
- approves it. Accounts that upload abusive content are banned.</li>
+ approves it, so after your first push, ask for approval. Approved accounts sync freely from then on.
+ Accounts that upload abusive content are banned.</li>
+ <li><strong>No videos.</strong> Any file type is accepted <em>except</em> video, which is detected by file
+ extension and by file contents (renaming won't help). If the repo contains one, the whole sync is
+ rejected and nothing is uploaded. Leave videos out of the archive, for example by adding
+ <code>--exclude='*.mp4'</code> to the <code>tar</code> command.</li>
+ <li><strong>No rate limit.</strong> There's no limit on file count, archive size or how often you sync. What protects it instead is your
+ sign-in identity, the approval rule, bans and the no-video rule. A smaller archive syncs faster, so leave build output out, for example
+ <code>--exclude='node_modules'</code> in the <code>tar</code> command.
+ <a href="/documentation#rate-limits">How rate limits work</a>.</li>
<li>Each sync fully replaces the contents of <em>your own</em> <code>username/repo</code> folder only
- (delete-then-upload), so removed files disappear from the CDN too. No other folder is ever touched.</li>
+ (delete-then-upload), so removed files disappear from the CDN too. No other folder is ever touched.
+ The storage keeps no version history, so a removed or overwritten file can't be brought back — keep the
+ original in your repo.</li>
<li>Change <code>branches: [ main ]</code> to whichever branch you want mirrored.</li>
<li>Want it on every branch/tag push instead? Adjust the <code>on:</code> block — the destination
folder is still locked to your username/repo either way.</li>
</ul>
+ <h2>If a sync is refused</h2>
+ <p>The step now shows the server's reason in the Actions log (that's what <code>--fail-with-body</code> is for).</p>
+ <ul>
+ <li><code>403</code> <em>"First sync from '<you>' was accepted; further syncs need approval…"</em> — your account
+ has already synced once and is waiting to be approved. Nothing is wrong with your workflow; ask the CDN owner to
+ approve the account, then re-run the job.</li>
+ <li><code>403</code> <em>"…banned…"</em> — the account (or the network the job ran from) has been blocked.</li>
+ <li><code>415</code> <em>"Video files are not allowed…"</em> — the message names the file; remove it from the repo
+ or exclude it from the archive.</li>
+ <li><code>401</code> — the identity token was missing, expired or not for this service. Check that
+ <code>permissions: id-token: write</code> is set and that the audience is <code>cdn.amit.is-a.dev</code>.</li>
+
+ </ul>
+ <p>The full request/response details are in the <a href="/documentation#gh-sync">documentation</a>.</p>
+
<footer>
<p>© 2025 - 2026 notamitgamer. All rights reserved.</p>
</footer>
diff --git a/app/templates/index.html b/app/templates/index.html
@@ -605,6 +605,9 @@
font-weight: 600;
}
+ .rate-note { display:flex; gap:.65rem; align-items:flex-start; margin:0 0 1.25rem; padding:.75rem 1rem; border:1px solid rgba(245,158,11,.45); background:rgba(245,158,11,.1); border-radius:var(--radius-md); font-size:.88rem; line-height:1.5; color:var(--text-main); }
+ .rate-note svg { flex:none; margin-top:.15rem; color:#d97706; }
+ .rate-note a { color:var(--accent-primary); font-weight:500; white-space:nowrap; }
#dropzone {
border: 2px dashed var(--border-hover);
border-radius: var(--radius-lg);
@@ -1044,6 +1047,8 @@
<button type="button" class="folder-option" data-folder="third-party">third-party</button>
</div>
+ <div class="rate-note" role="note"><svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="10"/><polyline points="12 6 12 12 16 14"/></svg><span><strong>Rate limits apply.</strong> Each IP can upload <strong>500 MB per hour</strong> (a single file can be at most <strong>500 MB</strong>). Uploading from a link is limited to <strong>50 MB per hour</strong>. Going over returns an error until your allowance comes back. <a href="/documentation#rate-limits">How rate limits work</a></span></div>
+
<div style="display:flex; gap:0.75rem; margin-bottom: 1.5rem;">
<input type="text" id="url-input" class="url-input" placeholder="Paste a link to fetch and upload…">
<button type="button" class="icon-btn" id="url-upload-btn" title="Upload from URL" aria-label="Upload from URL">
diff --git a/app/templates/url_shortener.html b/app/templates/url_shortener.html
@@ -225,6 +225,10 @@
.container { width: 100%; padding: 0 1.25rem; }
.result-url { flex-direction: column; align-items: flex-start; }
}
+
+ .rate-note { display:flex; gap:.65rem; align-items:flex-start; margin:0 0 1.25rem; padding:.75rem 1rem; border:1px solid rgba(245,158,11,.45); background:rgba(245,158,11,.1); border-radius:var(--radius-md); font-size:.88rem; line-height:1.5; color:var(--text-main); }
+ .rate-note svg { flex:none; margin-top:.15rem; color:#d97706; }
+ .rate-note a { color:var(--accent-primary); font-weight:500; white-space:nowrap; }
</style>
</head>
<body>
@@ -235,6 +239,8 @@
Paste any public <code>https://</code> URL and get back a masked, shareable link.
The destination is never exposed until someone visits the masked URL.
</blockquote>
+
+ <div class="rate-note" role="note"><svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="10"/><polyline points="12 6 12 12 16 14"/></svg><span><strong>Rate limit:</strong> each IP can create <strong>10 links per minute</strong> and <strong>50 per hour</strong>. Failed attempts count too. Only <code>https://</code> URLs up to 2048 characters are accepted. <a href="/documentation#rate-limits">How rate limits work</a></span></div>
<div class="input-group" id="shorten-group">
<input type="url" id="url-input" class="url-input" placeholder="Paste a URL to shorten (e.g. https://example.com)" autocomplete="off" spellcheck="false" required>
