feat(tooling): port the doc/lint/grammar checks to Ludic (no Python)
Replace the Python doc/lint/vocabulary guards with Ludic equivalents that run through the `x` task runner, so the checks need no Python interpreter: x check-docs every ```ludic doc fence parses (or is marked) x check-impl every implemented feature has a docs/language page x check-vocabulary vocabulary in sync across grammar / lexer / header / parser x lint-asset <file> validate one editor .json / .xml asset New fragments: tools/x/json.ludic (a small JSON reader — objects/arrays/ strings with \uXXXX + surrogates/numbers/literals, used by the vocabulary check's grammar navigation and the asset validator) and tools/x/checks.ludic (the checks + string helpers + a minimal XML well-formedness validator). `x test-tools` now runs the vocabulary + docs-coverage checks and the JSON/XML asset validation through Ludic instead of python3; ci.yml's docs-coverage step calls `x check-impl` / `x check-vocabulary`. Each port was verified against its former Python script for exact verdict parity on the clean tree and on injected drift (a removed keyword, a broken grammar alternation, an undocumented method). Deletes the superseded scripts: tools/check-vocabulary.py, tools/check-docs.py, tools/docgen/check-impl.py, tools/docgen/validate.py. The docgen site generator (gen.py/check.py/palette.py) and the LSP protocol driver (test-lsp.py) remain and are tracked separately. Toolchain unchanged (seed byte-identical); `x test` (56) and `x test-tools` (29) stay green. Part of #31 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
c040fff8c8
commit
e65e862244
11 changed files with 922 additions and 486 deletions
|
|
@ -64,7 +64,7 @@ hovering any token shows a summary card from the real API data.
|
|||
```bash
|
||||
python3 tools/docgen/gen.py --out build/pages # generate the whole site
|
||||
python3 tools/docgen/check.py build/pages # coverage + duplicate-token + link guard
|
||||
python3 tools/docgen/validate.py # compile every ```ludic example with bin/ludicc
|
||||
bin/x check-docs # parse every ```ludic doc fence (Ludic; no Python)
|
||||
```
|
||||
|
||||
`check.py` fails CI if any symbol in `inventory.json` lacks a page, if a token is
|
||||
|
|
|
|||
|
|
@ -1,163 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Docs coverage, anchored to the implementation — not a hand-kept list.
|
||||
|
||||
The failure mode this prevents: you add a feature to the language and forget to
|
||||
document it. Nobody notices until someone goes looking for the docs that were
|
||||
never written. So the set of things that MUST have a doc page is read straight
|
||||
from the implementation, and every one is required to have a page (and every
|
||||
namespace-method page is required to correspond to something real):
|
||||
|
||||
* namespace methods selfhost/backend/emit_call.ludic `emit_ns_call` + the
|
||||
`is_<ns>_ns` predicates it delegates to (Math/Text/List)
|
||||
* keywords tools/ludic-tools/ludic_syntax.h LUDIC_KW_* (minus
|
||||
LUDIC_KW_RESERVED, which is not-yet-implemented)
|
||||
* primitive types ludic_syntax.h LUDIC_TYPES
|
||||
* phases ludic_syntax.h LUDIC_PHASES
|
||||
|
||||
Coverage means: the symbol appears in the `tokens:` of some docs/language page.
|
||||
For namespace methods that page is docs/language/<ns>/<ns>-<method>.md with
|
||||
token `<Ns>.<method>`. Run: python3 tools/docgen/check-impl.py
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
LANG = os.path.join(ROOT, "docs", "language")
|
||||
problems = []
|
||||
|
||||
|
||||
def read(*parts):
|
||||
with open(os.path.join(ROOT, *parts), encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
def fn_body(text, name):
|
||||
"""The source of `function <name>` up to the next top-level `function ` (or EOF)."""
|
||||
m = re.search(r"^function %s\b" % re.escape(name), text, re.M)
|
||||
if not m:
|
||||
return ""
|
||||
rest = text[m.end():]
|
||||
nxt = re.search(r"^function ", rest, re.M)
|
||||
return text[m.start(): m.end() + (nxt.start() if nxt else len(rest))]
|
||||
|
||||
|
||||
def all_selfhost_source():
|
||||
src = ""
|
||||
d = os.path.join(ROOT, "selfhost")
|
||||
files = []
|
||||
for dirpath, _dirs, names in os.walk(d):
|
||||
for fn in names:
|
||||
if fn.endswith(".ludic"):
|
||||
files.append(os.path.join(dirpath, fn))
|
||||
for path in sorted(files):
|
||||
src += open(path, encoding="utf-8").read() + "\n"
|
||||
return src
|
||||
|
||||
|
||||
def compiler_ns_methods():
|
||||
"""{Namespace: set(method)} the compiler actually dispatches on."""
|
||||
allsrc = all_selfhost_source()
|
||||
body = fn_body(allsrc, "emit_ns_call")
|
||||
if not body:
|
||||
problems.append("emit_ns_call not found in selfhost/ sources")
|
||||
return {}
|
||||
result = {}
|
||||
# each `if (ns == "X")` opens a block that runs to the next such marker
|
||||
chunks = re.split(r'if \(ns == "', body)
|
||||
for chunk in chunks[1:]:
|
||||
mn = re.match(r'(\w+)"', chunk)
|
||||
if not mn:
|
||||
continue
|
||||
ns = mn.group(1)
|
||||
methods = set(re.findall(r'meth == "([A-Za-z_][A-Za-z0-9_]*)"', chunk))
|
||||
if not methods:
|
||||
# delegated form: `if is_<ns>_ns(meth) { return emit_<ns>_ns(...) }`
|
||||
pm = re.search(r"is_(\w+)_ns\(meth\)", chunk)
|
||||
if pm:
|
||||
pred = fn_body(allsrc, "is_%s_ns" % pm.group(1))
|
||||
methods = set(re.findall(r'meth == "([A-Za-z_][A-Za-z0-9_]*)"', pred))
|
||||
if methods:
|
||||
result.setdefault(ns, set()).update(methods)
|
||||
return result
|
||||
|
||||
|
||||
def documented():
|
||||
"""(tokens set, {(ns, member): id} for namespace-method pages)."""
|
||||
tokens = set()
|
||||
nsmethods = {}
|
||||
for cat in sorted(os.listdir(LANG)):
|
||||
cdir = os.path.join(LANG, cat)
|
||||
if not os.path.isdir(cdir):
|
||||
continue
|
||||
for fn in os.listdir(cdir):
|
||||
if not fn.endswith(".md") or fn == "_section.md":
|
||||
continue
|
||||
meta = {}
|
||||
for line in read("docs", "language", cat, fn).split("\n"):
|
||||
m = re.match(r"^(\w+):\s*(.*)$", line)
|
||||
if m:
|
||||
meta[m.group(1)] = m.group(2).strip()
|
||||
if line.strip() == "---" and meta:
|
||||
break
|
||||
tokens.update(meta.get("tokens", "").split())
|
||||
if meta.get("kind") == "namespace-method" and meta.get("ns") and meta.get("member"):
|
||||
nsmethods[(meta["ns"], meta["member"])] = meta.get("id", fn[:-3])
|
||||
return tokens, nsmethods
|
||||
|
||||
|
||||
def vocab_sets():
|
||||
h = read("tools", "ludic-tools", "ludic_syntax.h")
|
||||
|
||||
def clist(name):
|
||||
m = re.search(r"\b" + name + r"\s*\[\s*\]\s*=\s*\{(.*?)\}\s*;", h, re.S)
|
||||
if not m:
|
||||
problems.append("ludic_syntax.h: %s not found" % name)
|
||||
return set()
|
||||
return set(re.findall(r'"([^"]+)"', m.group(1)))
|
||||
|
||||
reserved = clist("LUDIC_KW_RESERVED")
|
||||
keywords = (clist("LUDIC_KW_DECL") | clist("LUDIC_KW_CLAUSE")
|
||||
| clist("LUDIC_KW_STMT")) - reserved
|
||||
return keywords, clist("LUDIC_TYPES"), clist("LUDIC_PHASES")
|
||||
|
||||
|
||||
def main():
|
||||
ns_impl = compiler_ns_methods()
|
||||
tokens, ns_doc = documented()
|
||||
|
||||
# 1) every implemented namespace method has a page with the right token
|
||||
for ns, methods in sorted(ns_impl.items()):
|
||||
for meth in sorted(methods):
|
||||
page = os.path.join(LANG, ns.lower(), "%s-%s.md" % (ns.lower(), meth))
|
||||
if (ns, meth) not in ns_doc:
|
||||
problems.append("undocumented %s.%s — add %s (tokens: %s.%s)"
|
||||
% (ns, meth, os.path.relpath(page, ROOT), ns, meth))
|
||||
elif "%s.%s" % (ns, meth) not in tokens:
|
||||
problems.append("%s.%s documented but its page lacks that `tokens:` entry" % (ns, meth))
|
||||
|
||||
# 2) every documented namespace method corresponds to real dispatch
|
||||
for (ns, meth), sid in sorted(ns_doc.items()):
|
||||
if meth not in ns_impl.get(ns, set()):
|
||||
problems.append("stale doc %s: %s.%s is not dispatched by the compiler" % (sid, ns, meth))
|
||||
|
||||
# 3) every implemented keyword / type / phase is documented
|
||||
keywords, types, phases = vocab_sets()
|
||||
for label, names in (("keyword", keywords), ("type", types), ("phase", phases)):
|
||||
for n in sorted(names):
|
||||
if n not in tokens:
|
||||
problems.append("undocumented %s: %s (no docs/language page lists it in `tokens:`)" % (label, n))
|
||||
|
||||
n = sum(len(v) for v in ns_impl.values())
|
||||
if problems:
|
||||
print("docs-vs-implementation drift:", file=sys.stderr)
|
||||
for p in problems:
|
||||
print(" -", p, file=sys.stderr)
|
||||
return 1
|
||||
print("docs cover the implementation: %d namespace methods, %d keywords, %d types, %d phases"
|
||||
% (n, len(keywords), len(types), len(phases)))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Compile-validate every ```ludic example in the docs against the real ludicc.
|
||||
Full programs (start with program/module) compile directly; fragments are wrapped
|
||||
(as declarations, then as statements) like tools/check-docs.py. Reports failures."""
|
||||
import os, re, subprocess, tempfile, sys
|
||||
REPO=os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); LC=os.path.join(REPO,"bin","ludicc")
|
||||
FENCE=re.compile(r"```ludic\n(.*?)\n```", re.S)
|
||||
DECL=("program","property","model","handler","enum","function","extern","const","var","ui","event","module","import","scene","machine")
|
||||
|
||||
def first_kw(body):
|
||||
for l in body.split("\n"):
|
||||
s=l.strip()
|
||||
if s and not s.startswith("#"): return s.split("(")[0].split()[0].split("{")[0]
|
||||
return ""
|
||||
|
||||
def candidates(body):
|
||||
kw=first_kw(body)
|
||||
if kw in ("program","module"): return [body]
|
||||
d="program DocCheck {\n"+body+"\n}\n"
|
||||
s="program DocCheck {\n handler DocS phase Start {\n"+body+"\n }\n}\n"
|
||||
return [d,s] if kw in DECL else [s,d]
|
||||
|
||||
def compiles(src, full):
|
||||
with tempfile.NamedTemporaryFile("w",suffix=".ludic",delete=False) as f:
|
||||
f.write(src); tmp=f.name
|
||||
flag=["--emit-llvm","-o",tmp+".ll"] if full else ["--fmt"]
|
||||
r=subprocess.run([LC,tmp]+flag,capture_output=True)
|
||||
os.unlink(tmp)
|
||||
try: os.unlink(tmp+".ll")
|
||||
except OSError: pass
|
||||
return r.returncode==0, (r.stderr or b"").decode("utf-8","replace").strip().split("\n")[0]
|
||||
|
||||
def files():
|
||||
for base in ("docs/language","docs/site/snippets"):
|
||||
for root,_,fs in os.walk(os.path.join(REPO,base)):
|
||||
for fn in fs:
|
||||
if fn.endswith(".md") or fn.endswith(".ludic"): yield os.path.join(root,fn)
|
||||
|
||||
ok=fail=skip=0; fails=[]
|
||||
for p in files():
|
||||
t=open(p).read()
|
||||
blocks=FENCE.findall(t) if p.endswith(".md") else [t]
|
||||
for b in blocks:
|
||||
if 'import "' in b: skip+=1; continue
|
||||
good=False; err=""
|
||||
full = first_kw(b) in ("program","module")
|
||||
for cand in candidates(b):
|
||||
c,e=compiles(cand, full)
|
||||
if c: good=True; break
|
||||
if not err: err=e
|
||||
if good: ok+=1
|
||||
else: fail+=1; fails.append((os.path.relpath(p,REPO),err))
|
||||
print(f"examples: {ok} compile, {fail} fail, {skip} skipped (import)")
|
||||
for p,e in fails: print(f" FAIL {p}: {e[:100]}")
|
||||
sys.exit(1 if fail else 0)
|
||||
Loading…
Add table
Add a link
Reference in a new issue