fast_tree.py - cdnV2 -

fast_tree.py (5156B)
      1 """Data for instant folder navigation (static/fasttree.js).
      2 
      3 The browser keeps two things between visits:
      4 
      5 * the **tree**: every non-hidden file as ``[path, size]`` (``/api/tree``), small enough that the whole
      6   bucket's structure fits in a few kilobytes gzipped, and
      7 * the **shell**: a server-rendered folder page with *slots* where the folder-specific parts go, plus
      8   the HTML of each of those parts with ``@@placeholders@@`` (``/api/shell``).
      9 
     10 With both in hand a folder page can be assembled locally, with no request, and handed to Turbo. The
     11 markup comes from the same Jinja macros (templates/_listing.html) the server uses for ordinary
     12 renders, so the client never carries a second copy of it; only the data is filled in. See
     13 tests/test_fast_tree.py, which checks that a locally assembled page equals the server's.
     14 """
     15 
     16 import gzip
     17 import hashlib
     18 import json
     19 import os
     20 
     21 from markupsafe import Markup
     22 from starlette.requests import Request
     23 from starlette.responses import Response
     24 
     25 # Above this many files the tree is not shipped and the browser falls back to normal navigation.
     26 MAX_FILES = int(os.getenv("FAST_TREE_MAX_FILES", "20000"))
     27 
     28 # Markers left in the shell where the folder-specific HTML goes. Each must appear exactly once.
     29 SLOTS = (
     30     "<!--@root-header-->",
     31     "<!--@breadcrumb-->",
     32     "<!--@toolbar-->",
     33     "<!--@list-->",
     34     "<!--@stats-->",
     35     "/*@visit*/",
     36 )
     37 
     38 _tree_memo: tuple | None = None   # (items list, payload, etag): the items list is cached by storage, so identity is a cheap "unchanged" test
     39 _shell_memo: dict = {}
     40 
     41 
     42 def _digest(*chunks: str) -> str:
     43     h = hashlib.sha1()
     44     for c in chunks:
     45         h.update(c.encode("utf-8"))
     46     return h.hexdigest()[:16]
     47 
     48 
     49 def tree_payload(items: list) -> tuple[dict, str]:
     50     """(payload, etag) for /api/tree, built from storage's full tree listing."""
     51     global _tree_memo
     52     if _tree_memo is not None and _tree_memo[0] is items:
     53         return _tree_memo[1], _tree_memo[2]
     54 
     55     files = sorted([it["path"], int(it["size"] or 0)] for it in items if not it["is_dir"])
     56     etag = _digest(json.dumps(files, separators=(",", ":"), ensure_ascii=False))
     57     if len(files) > MAX_FILES:
     58         payload = {"v": etag, "n": len(files), "too_large": True}
     59     else:
     60         payload = {"v": etag, "n": len(files), "files": files}
     61     _tree_memo = (items, payload, etag)
     62     return payload, etag
     63 
     64 
     65 def shell_payload(env, ctx: dict) -> tuple[dict, str]:
     66     """(payload, etag) for /api/shell: the skeleton page and the fill-in fragments."""
     67     key = (ctx.get("cdn_base_url"), ctx.get("raw_base_url"))
     68     if key in _shell_memo:
     69         return _shell_memo[key]
     70 
     71     html = env.get_template("index.html").render(
     72         {**ctx, "page": "listing", "path": "", "items": [], "skeleton": True}
     73     )
     74     for slot in SLOTS:
     75         if html.count(slot) != 1:
     76             raise RuntimeError(f"shell slot {slot!r} should appear exactly once, found {html.count(slot)}")
     77 
     78     m = env.get_template("_listing.html").module
     79     parts = {
     80         "root_header": m.root_header(),
     81         "count": m.count("@@n@@", "@@word@@", "@@s@@"),
     82         "toolbar": m.toolbar("@@counts@@", "@@path@@"),
     83         "go_up": m.go_up("@@parent_path@@"),
     84         "row_dir": m.row_dir("@@name@@", "@@path@@"),
     85         "row_file": m.row_file("@@name@@", "@@path@@", "@@size_str@@"),
     86         "empty": m.empty_state(),
     87         "stats": m.footer_stats("@@n@@", "@@s@@", "@@size_str@@"),
     88         "crumb_container": m.crumb_container("@@inner@@", ctx["cdn_base_url"], "@@tail@@"),
     89         "crumb_link": m.crumb_link("@@href@@", "@@name@@"),
     90         "crumb_collapsed": m.crumb_collapsed("@@menu@@"),
     91         "copy_btn": m.copy_btn("@@path@@"),
     92         "visit": m.visit_script(Markup(
     93             'type: "folder",\n path: @@path_json@@,\n cdn_url: @@cdn_url_json@@,'
     94         )),
     95     }
     96     parts = {k: str(v) for k, v in parts.items()}
     97     payload = {"html": html, "parts": parts, "cdn_base": ctx["cdn_base_url"]}
     98     etag = _digest(html, json.dumps(parts, sort_keys=True), ctx["cdn_base_url"])
     99     payload["v"] = etag
    100     _shell_memo[key] = (payload, etag)
    101     return payload, etag
    102 
    103 
    104 def _etag_matches(header: str, etag: str) -> bool:
    105     # Proxies may turn a strong ETag into a weak one (W/"...") when they compress; accept either.
    106     for token in header.split(","):
    107         token = token.strip()
    108         if token.startswith("W/"):
    109             token = token[2:]
    110         if token == f'"{etag}"':
    111             return True
    112     return False
    113 
    114 
    115 def json_response(request: Request, payload: dict, etag: str) -> Response:
    116     """JSON with an ETag (so an unchanged tree costs a tiny 304) and gzip."""
    117     headers = {"ETag": f'"{etag}"', "Cache-Control": "no-cache", "Vary": "Accept-Encoding"}
    118     if _etag_matches(request.headers.get("if-none-match", ""), etag):
    119         return Response(status_code=304, headers=headers)
    120     body = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
    121     if "gzip" in request.headers.get("accept-encoding", "").lower():
    122         body = gzip.compress(body, 6)
    123         headers["Content-Encoding"] = "gzip"
    124     return Response(body, media_type="application/json", headers=headers)