test_fast_tree.py

test_fast_tree.py (8867B)
      1 """Offline checks for instant folder navigation (app/fast_tree.py + app/static/fasttree.js).
      2 
      3 Run from the repo root:  HF_BUCKET_ID=ci/x python tests/test_fast_tree.py     (needs `node`)
      4 
      5 The important one: a folder page assembled in the browser from /api/tree + /api/shell must be the same
      6 HTML the server renders for that folder. If someone edits the listing markup in a way the client
      7 can't reproduce, this fails with a diff pointing at the first difference.
      8 """
      9 
     10 import json
     11 import os
     12 import re
     13 import subprocess
     14 import sys
     15 import tempfile
     16 import urllib.parse
     17 
     18 os.environ.setdefault("HF_BUCKET_ID", "ci/placeholder")
     19 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     20 sys.path.insert(0, ROOT)
     21 sys.path.insert(0, os.path.join(ROOT, "tests"))
     22 
     23 from fastapi.testclient import TestClient  # noqa: E402
     24 
     25 import app.main as m  # noqa: E402
     26 from app import fast_tree, storage  # noqa: E402
     27 import fake_bucket  # noqa: E402
     28 
     29 fake_bucket.install(storage, m)
     30 client = TestClient(m.app)
     31 
     32 
     33 def check(cond, msg):
     34     if not cond:
     35         raise AssertionError(msg)
     36     print("ok  -", msg)
     37 
     38 
     39 def node(*args):
     40     out = subprocess.run(["node", *args], capture_output=True, text=True, cwd=ROOT)
     41     if out.returncode != 0:
     42         raise AssertionError(f"node failed: {out.stderr}")
     43     return out.stdout
     44 
     45 
     46 def norm(html):
     47     return re.sub(r"\s+", " ", html).strip()
     48 
     49 
     50 # --- /api/tree ---------------------------------------------------------------------------------------
     51 
     52 gz = {"Accept-Encoding": "gzip"}
     53 r = client.get("/api/tree", headers=gz)
     54 tree = r.json()
     55 paths = {p for p, _ in tree["files"]}
     56 check(r.status_code == 200 and r.headers.get("content-encoding") == "gzip", "/api/tree is served gzipped")
     57 check(tree["n"] == len(tree["files"]) == len(paths), "...with one entry per file")
     58 check("README.md" in paths and ".gitattributes" in paths and "bsc/semester_1/notes.txt" in paths, "...containing the files")
     59 check(not any(p.startswith(("_logs", "_batches")) for p in paths), "...and none of the hidden internal prefixes")
     60 check(r.headers["etag"] == f'"{tree["v"]}"' and r.headers["cache-control"] == "no-cache", "...with an ETag that is revalidated each time")
     61 
     62 r2 = client.get("/api/tree", headers={"If-None-Match": r.headers["etag"]})
     63 check(r2.status_code == 304 and not r2.content, "an unchanged tree costs an empty 304")
     64 check(client.get("/api/tree", headers={"If-None-Match": "W/" + r.headers["etag"]}).status_code == 304, "...even when a proxy weakened the ETag")
     65 check(client.get("/api/tree", headers={"If-None-Match": '"stale"'}).status_code == 200, "a different ETag gets the full tree")
     66 
     67 fake_bucket.FILES["brand/new.txt"] = 5
     68 storage._cache.clear()
     69 check(client.get("/api/tree").json()["v"] != tree["v"], "a new file changes the tree's version")
     70 del fake_bucket.FILES["brand/new.txt"]
     71 storage._cache.clear()
     72 
     73 fast_tree.MAX_FILES, saved_max = 3, fast_tree.MAX_FILES
     74 fast_tree._tree_memo = None
     75 big = client.get("/api/tree").json()
     76 check(big.get("too_large") is True and "files" not in big, "a tree over FAST_TREE_MAX_FILES is not shipped")
     77 fast_tree.MAX_FILES = saved_max
     78 fast_tree._tree_memo = None
     79 
     80 saved_files = dict(fake_bucket.FILES)
     81 fake_bucket.FILES.clear()
     82 storage._cache.clear()
     83 check(client.get("/api/tree").status_code == 503, "an empty/failed listing is a 503, never an empty tree")
     84 fake_bucket.FILES.update(saved_files)
     85 storage._cache.clear()
     86 
     87 # --- /api/shell --------------------------------------------------------------------------------------
     88 
     89 r = client.get("/api/shell", headers=gz)
     90 shell = r.json()
     91 check(r.status_code == 200 and r.headers.get("content-encoding") == "gzip", "/api/shell is served gzipped")
     92 check(all(shell["html"].count(s) == 1 for s in fast_tree.SLOTS), "every slot appears exactly once in the shell")
     93 check("skeleton" not in shell["html"] and "{{" not in shell["html"], "the shell is fully rendered HTML")
     94 check(shell["cdn_base"] == m.CDN_BASE_URL, "...and carries the CDN base URL for the visited-history entry")
     95 check(client.get("/api/shell", headers={"If-None-Match": r.headers["etag"]}).status_code == 304, "the shell revalidates with a 304 too")
     96 check(client.get("/static/fasttree.js").status_code == 200, "/static/fasttree.js is served")
     97 
     98 # --- the page built in the browser == the page the server renders ------------------------------------
     99 
    100 tmp = tempfile.mkdtemp()
    101 tree = client.get("/api/tree").json()
    102 json.dump(tree, open(f"{tmp}/tree.json", "w"))
    103 json.dump(shell, open(f"{tmp}/shell.json", "w"))
    104 
    105 folders = [
    106     "",                                   # root: header, no breadcrumb, no "go up"
    107     "bsc", "bsc/semester_1",
    108     "bsc/semester_1/deep/er/est",         # 6 crumbs: collapsed breadcrumb
    109     "bsc/semester_1/deep/er/est/more",
    110     "a&b", "a&b/it's here",               # HTML-special characters in folder names
    111     "uploads",                            # a file name with a space
    112     "bsc/ünï 😀",                          # non-ASCII + astral characters (escaping in the history script)
    113     "solo",                               # 1152 B: exact tie, Python rounds half to even ("1.12 KB")
    114     "solo2",                              # singular counts: "1 file"
    115 ]
    116 built = json.loads(node("tests/compose.js", f"{tmp}/tree.json", f"{tmp}/shell.json", *folders))
    117 for folder in folders:
    118     url = "/" + urllib.parse.quote(folder, safe="/")
    119     server = client.get(url, headers={"Accept-Encoding": "identity"})
    120     assert server.status_code == 200, (url, server.status_code)
    121     a, b = norm(server.text), norm(built[folder] or "")
    122     if a != b:
    123         i = next((k for k in range(min(len(a), len(b))) if a[k] != b[k]), min(len(a), len(b)))
    124         raise AssertionError(
    125             f"locally built page differs from the server's for {folder!r} at char {i}\n"
    126             f"  server: ...{a[max(0, i - 120):i + 160]}\n  client: ...{b[max(0, i - 120):i + 160]}"
    127         )
    128     print(f"ok  - folder {folder or '(root)'!r}: built page == server page")
    129 
    130 missing = json.loads(node("tests/compose.js", f"{tmp}/tree.json", f"{tmp}/shell.json", "no/such/folder"))
    131 check(missing["no/such/folder"] is None, "a folder that isn't in the tree is not built locally (the server decides)")
    132 
    133 # --- text helpers match Python/Jinja byte for byte ---------------------------------------------------
    134 
    135 sizes = [0, 1, 1023, 1024, 1025, 1152, 1536, 2047, 3000, 999999, 1048575, 1048576, 2500000, 10**9,
    136          5 * 2**30 + 12345, 2**40, 3 * 2**40 + 7, 2**50, 1000 * 2**40]
    137 js = json.loads(node("-e", "const {formatSize}=require('./app/static/fasttree.js');"
    138                            "console.log(JSON.stringify(JSON.parse(process.argv[1]).map(formatSize)))", json.dumps(sizes)))
    139 py = [storage.format_size(s) for s in sizes]
    140 check(js == py, f"format_size agrees with Python for {len(sizes)} sizes, including exact rounding ties")
    141 
    142 tricky = ["plain", "a&b", "it's", "<script>alert(1)</script>", "ünï 😀", "tab\there", 'q"uote', "back\\slash",
    143           "\x7f", "\u2028", "line\nbreak", "$&", "@@name@@", "</script>"]
    144 render_tojson = m.templates.env.from_string("{{ x|tojson }}").render   # the real filter, as the templates use it
    145 js = json.loads(node("-e", "const {jsonForScript}=require('./app/static/fasttree.js');"
    146                            "console.log(JSON.stringify(JSON.parse(process.argv[1]).map(jsonForScript)))", json.dumps(tricky)))
    147 check(js == [render_tojson(x=t) for t in tricky], "jsonForScript agrees with Jinja's |tojson")
    148 
    149 from markupsafe import escape  # noqa: E402
    150 js = json.loads(node("-e", "const {escapeHtml}=require('./app/static/fasttree.js');"
    151                            "console.log(JSON.stringify(JSON.parse(process.argv[1]).map(escapeHtml)))", json.dumps(tricky)))
    152 check(js == [str(escape(t)) for t in tricky], "escapeHtml agrees with markupsafe.escape")
    153 
    154 # --- real Turbo + real pages, in jsdom ---------------------------------------------------------------
    155 
    156 e2e_dir = tempfile.mkdtemp()
    157 json.dump(tree, open(f"{e2e_dir}/tree.json", "w"))
    158 json.dump(shell, open(f"{e2e_dir}/shell.json", "w"))
    159 for name, url in {"from": "/bsc", "to": "/bsc/semester_1", "root": "/"}.items():
    160     open(f"{e2e_dir}/{name}.html", "w", encoding="utf-8").write(client.get(url, headers={"Accept-Encoding": "identity"}).text)
    161 
    162 have_deps = all(os.path.isdir(os.path.join(ROOT, "node_modules", d)) for d in ("jsdom", "@hotwired/turbo"))
    163 if have_deps:
    164     out = subprocess.run(["node", "tests/turbo_e2e.js", e2e_dir], capture_output=True, text=True, cwd=ROOT)
    165     print(out.stdout.rstrip())
    166     check(out.returncode == 0, "end-to-end: real Turbo renders the locally built page, history and Back work, no page request")
    167 elif os.getenv("REQUIRE_E2E"):
    168     raise AssertionError("jsdom / @hotwired/turbo are not installed but REQUIRE_E2E is set")
    169 else:
    170     print("skip - end-to-end test (npm install --no-save --no-package-lock jsdom @hotwired/turbo@8.0.4 to enable)")
    171 
    172 print("\nall fast-tree checks passed")