From 2ad6edaae3084b85c58eb3fcb94cfdbc1bc4e9e5 Mon Sep 17 00:00:00 2001 From: Orkuncakilkaya Date: Tue, 29 Sep 2026 23:59:30 +0300 Subject: [PATCH] ludic syntax, and ludic-dev syntax: every grammar written from the compiler's vocabulary and checked against it `ludic syntax [--json] [-o FILE]` prints what `ludicc --emit-syntax` does (a line per entry, or the JSON). `ludic-dev syntax` writes, between "ludic-dev syntax: begin" / "end" lines, the keyword, type, phase and attribute tables of ludic_syntax.h, LudicVocabulary's sets (JetBrains), ludic-mode.el's lists and the language server's word tests (is_keyword_word and the rest; is_contextual_word is every word the parser does not reserve, and every declaring or modifying one), and every TextMate pattern marked "comment": "ludic-dev syntax: " (shared and the VS Code copy). The grammars gain module uses port bind action reducer dispatch registry def open component prop view alias friend unsafe numbers of as from mut system; import and extern colour as declarations; the phase clause knows Overlay; the bitwise pattern matches | and ^ on their own again. `ludic-dev syntax --check` - and check-vocabulary, whose old parser comparison it replaces, and the regression suite (syntax_cases, one line per file) - fails when a written list is behind, when a grammar lacks a keyword, type or phase, when docs/language has no page for a keyword, type, phase or attribute, or when the parser (a scan of selfhost/frontend: is_id / text == words, a == / ann == attributes) tests a word or reads an attribute vocab.ludic lacks, or the reverse. Co-Authored-By: Claude Opus 5.5 --- LANGUAGE.md | 11 + tools/editors/README.md | 24 +- tools/editors/emacs/ludic-mode.el | 30 +- .../main/kotlin/io/ludic/ide/LudicTokens.kt | 33 +- tools/editors/shared/ludic.tmLanguage.json | 21 +- .../vscode/syntaxes/ludic.tmLanguage.json | 21 +- tools/ludic-cli/checks.ludic | 55 +- tools/ludic-cli/dev.ludic | 3 + tools/ludic-cli/main.ludic | 3 + tools/ludic-cli/syntax_cli.ludic | 86 +++ tools/ludic-cli/syntax_gen.ludic | 508 ++++++++++++++++++ tools/ludic-cli/test.ludic | 2 + tools/ludic-tools/lsp.ludic | 41 +- tools/ludic-tools/lsp/types.ludic | 18 +- tools/ludic-tools/ludic_syntax.h | 46 +- 15 files changed, 784 insertions(+), 118 deletions(-) create mode 100644 tools/ludic-cli/syntax_cli.ludic create mode 100644 tools/ludic-cli/syntax_gen.ludic diff --git a/LANGUAGE.md b/LANGUAGE.md index c9568b84..ef6ed06f 100644 --- a/LANGUAGE.md +++ b/LANGUAGE.md @@ -2246,6 +2246,7 @@ ludic test packages/ludic.base # the test programs under a directory (a ludic build --check # every check a build makes, nothing written ludic build --check --diagnostics=json # the same, every error as a JSON array on stdout ludic schema -o build/schema.json # records, registries, entries, consts, components and natives, for editors +ludic syntax --json # the language's vocabulary: keywords, declarations, types, phases, attributes, operators ludic deps # how tangled the modules are, as the compiler resolved them ludic deps --check tests/deps-baseline.txt # fail when a number rose (--baseline FILE writes them) @@ -2361,6 +2362,16 @@ Plugins for VS Code and JetBrains IDEs, plus configuration for Neovim, Helix, Emacs, Sublime and Zed, are in `tools/editors/` — see [tools/editors/README.md](tools/editors/README.md). +The vocabulary every editor highlights with is the compiler's: `ludic syntax +--json` (`ludicc --emit-syntax`) prints every keyword with its role and whether +the parser reserves it, every declaration's form, the built-in types and phases, +every attribute with what it goes on, its arguments and what it means, the +operators and the literal forms. The grammars in `tools/editors/`, the language +server's word lists and `ludic_syntax.h` are written from it between marked lines +(`bin/ludic-dev syntax`), and `bin/ludic-dev syntax --check` fails when one falls +behind, when docs/language lacks a page for an entry, or when the parser tests a +word the vocabulary does not have. + ## Working programs - `examples/games/chronorift.ludic` — a co-op JRPG (overworld, dungeon, boss, shop, diff --git a/tools/editors/README.md b/tools/editors/README.md index 9f59b52c..0f4a332e 100644 --- a/tools/editors/README.md +++ b/tools/editors/README.md @@ -146,11 +146,21 @@ output. The formatter cannot change what a program means. ## Keeping it honest -The vocabulary is written down in five places that cannot include each other — -the compiler's two tables, `ludic_syntax.h`, the TextMate grammar (JSON), and the -JetBrains lexer (Kotlin). Adding a builtin and forgetting the rest is silent -failure, so `bin/ludic-dev check-vocabulary` (written in Ludic) compares all five, and -`bin/ludic-dev test-tools` runs it. +The language's words come from one place: the compiler's vocabulary table +(`selfhost/frontend/vocab.ludic`), printed by `ludic syntax --json` (`ludicc +--emit-syntax`) and held to the parser's own recognisers. The keyword, type and +phase lists in `ludic_syntax.h`, the TextMate grammar (every pattern marked +`"comment": "ludic-dev syntax: "`), the JetBrains lexer's +`LudicVocabulary`, `ludic-mode.el` and the language server are written from it +between `ludic-dev syntax: begin` / `end` lines by `bin/ludic-dev syntax`; do not +edit inside them. -When you add a keyword or builtin: put it in `ludic_syntax.h`, then run -`bin/ludic-dev test-tools` and let it tell you which copies still need it. +`bin/ludic-dev syntax --check` (and `check-vocabulary`, which `test-tools` runs, +and the regression suite) fails when a generated list is behind, when a grammar +lacks a word, when docs/language has no page for a keyword, type, phase or +attribute, or when the parser (`selfhost/frontend`) tests a word or reads an +attribute the vocabulary lacks. Builtins are still listed in `ludic_syntax.h`, +and `check-vocabulary` compares them with the grammar and the Kotlin lexer. + +When you add a keyword or an attribute: add its row to `vocab.ludic`, rebuild the +compiler, run `bin/ludic-dev syntax`, and write its docs/language page. diff --git a/tools/editors/emacs/ludic-mode.el b/tools/editors/emacs/ludic-mode.el index 08d9d0c7..63d75cb6 100644 --- a/tools/editors/emacs/ludic-mode.el +++ b/tools/editors/emacs/ludic-mode.el @@ -10,23 +10,27 @@ ;;; Code: -;; The vocabulary mirrors tools/ludic-tools/ludic_syntax.h; that header is the reference. +;; The vocabulary is written by `ludic-dev syntax` from the compiler's own table +;; (`ludicc --emit-syntax`) between the marked lines. +;; ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit (defconst ludic--declaration-keywords - '("program" "import" "property" "model" "enum" "ui" "namespace" - "const" "var" "function" "extern" "handler" "entry" "event" "scene" "test" "prefab")) - + '("program" "module" "import" "property" "state" "event" "action" "reducer" "enum" "model" + "prefab" "scene" "handler" "ui" "namespace" "alias" "port" "bind" "registry" "def" + "component" "view" "prop" "const" "var" "function" "fn" "extern" "entry" "test")) (defconst ludic--clause-keywords - '("phase" "query" "on" "cancellable" "public" "layer" "start" "shows" "lasts" "loads" "then" "export" "internal")) - + '("export" "internal" "friend" "uses" "open" "of" "as" "from" "numbers" "unsafe" "mut" "phase" + "query" "on" "cancellable" "public" "layer" "start" "shows" "lasts" "then" "loads" + "system")) (defconst ludic--statement-keywords - '("let" "return" "if" "else" "while" "for" "in" "spawn" "despawn" - "enable" "disable" "attach" "detach" "match" "machine" "state" "become" "where" - "and" "or" "not" "break" "continue" "new" "emit" "cancel" "try")) - + '("let" "if" "else" "while" "for" "in" "where" "match" "machine" "become" "return" "break" + "continue" "new" "spawn" "despawn" "enable" "disable" "attach" "detach" "emit" "cancel" + "dispatch" "try" "and" "or" "not")) (defconst ludic--types - '("int" "long" "fixed" "countdown" "bool" "entity" "string" "pointer" "byte" - "words" "fixeds" "pointers" "Vector" "IVec2" "Rect" "void")) -(defconst ludic--phases '("Start" "Input" "FixedUpdate" "Update" "LateUpdate" "Render" "Overlay")) + '("int" "long" "float" "double" "fixed" "bool" "byte" "string" "pointer" "void" "entity" + "countdown" "words" "floats" "doubles" "fixeds" "pointers" "Vector" "IVec2" "Rect")) +(defconst ludic--phases + '("Start" "Input" "FixedUpdate" "Update" "LateUpdate" "Render" "Overlay")) +;; ludic-dev syntax: end (defconst ludic-font-lock-keywords (list diff --git a/tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt b/tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt index f03c560a..380232f3 100644 --- a/tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt +++ b/tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt @@ -55,24 +55,37 @@ object LudicTokens { } /** - * The vocabulary, mirroring tools/ludic-tools/ludic_syntax.h. When the language - * grows a keyword both sides change together; that header is the reference. + * The vocabulary. The keyword, type and phase sets are written by `ludic-dev + * syntax` from the compiler's own table (`ludicc --emit-syntax`); the rest + * mirrors tools/ludic-tools/ludic_syntax.h. */ object LudicVocabulary { + // ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit val DECL = setOf( - "program", "import", "property", "model", "enum", "ui", "namespace", - "const", "var", "function", "extern", "handler", "entry", "event", "scene", "test" + "program", "module", "import", "property", "state", "event", "action", "reducer", "enum", + "model", "prefab", "scene", "handler", "ui", "namespace", "alias", "port", "bind", + "registry", "def", "component", "view", "prop", "const", "var", "function", "fn", "extern", + "entry", "test" ) val CLAUSE = setOf( - "phase", "query", "on", "cancellable", "public", "layer", "start" + "export", "internal", "friend", "uses", "open", "of", "as", "from", "numbers", "unsafe", + "mut", "phase", "query", "on", "cancellable", "public", "layer", "start", "shows", "lasts", + "then", "loads", "system" ) val STMT = setOf( - "let", "return", "if", "else", "while", "for", "in", "spawn", "despawn", - "enable", "disable", "match", "machine", "prefab", "state", "become", "where", - "and", "or", "not", "break", "continue", "new", "emit", "cancel", "try" + "let", "if", "else", "while", "for", "in", "where", "match", "machine", "become", "return", + "break", "continue", "new", "spawn", "despawn", "enable", "disable", "attach", "detach", + "emit", "cancel", "dispatch", "try", "and", "or", "not" ) - val PRIMITIVES = setOf("int", "long", "fixed", "float", "double", "countdown", "bool", "entity", "string", "pointer", "byte", "words", "fixeds", "floats", "doubles", "pointers", "Vector", "IVec2", "Rect", "void") - val PHASES = setOf("Start", "Input", "FixedUpdate", "Update", "LateUpdate", "Render", "Overlay") + val PRIMITIVES = setOf( + "int", "long", "float", "double", "fixed", "bool", "byte", "string", "pointer", "void", + "entity", "countdown", "words", "floats", "doubles", "fixeds", "pointers", "Vector", + "IVec2", "Rect" + ) + val PHASES = setOf( + "Start", "Input", "FixedUpdate", "Update", "LateUpdate", "Render", "Overlay" + ) + // ludic-dev syntax: end val WIDGETS = setOf("panel", "col", "row", "label", "button", "image", "spacer") val BUILTINS = setOf( diff --git a/tools/editors/shared/ludic.tmLanguage.json b/tools/editors/shared/ludic.tmLanguage.json index 4480bf7f..9931756e 100644 --- a/tools/editors/shared/ludic.tmLanguage.json +++ b/tools/editors/shared/ludic.tmLanguage.json @@ -3,7 +3,7 @@ "name": "Ludic", "scopeName": "source.ludic", "fileTypes": ["ludic"], - "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. Generated shapes are kept in sync with tools/ludic-tools/ludic_syntax.h.", + "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. The keyword, type and phase alternations are written by `ludic-dev syntax` from the compiler's own vocabulary (`ludicc --emit-syntax`): every pattern marked \"ludic-dev syntax: \".", "patterns": [ { "include": "#comment" }, { "include": "#declaration" }, @@ -126,7 +126,7 @@ "captures": { "1": { "name": "keyword.control.import.ludic" } } }, { - "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render)\\b", + "comment": "ludic-dev syntax: phase-clause", "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b", "captures": { "1": { "name": "keyword.other.clause.ludic" }, "2": { "name": "constant.language.phase.ludic" } @@ -180,14 +180,13 @@ }, "keyword": { "patterns": [ - { "name": "keyword.control.ludic", "match": "\\b(if|else|while|for|in|match|machine|become|return|spawn|despawn|enable|disable|attach|detach|emit|cancel|where|break|continue|new|try)\\b" }, - { "name": "keyword.operator.logical.ludic", "match": "\\b(and|or|not)\\b" }, - { "name": "keyword.other.clause.ludic", "match": "\\b(phase|query|on|cancellable|public|layer|start|shows|lasts|loads|then|export|internal)\\b" }, - { "name": "keyword.other.ludic", "match": "\\b(import|extern)\\b" }, - { "name": "storage.type.ludic", "match": "\\b(program|property|model|prefab|namespace|enum|ui|const|var|let|function|fn|handler|entry|event|scene|state|test)\\b" }, - { "name": "support.type.primitive.ludic", "match": "\\b(int|long|fixed|float|double|countdown|bool|entity|string|pointer|byte|words|fixeds|floats|doubles|pointers|Vector|IVec2|Rect|void)\\b" }, - { "name": "constant.language.boolean.ludic", "match": "\\b(true|false|null)\\b" }, - { "name": "constant.language.phase.ludic", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" } + { "name": "keyword.control.ludic", "comment": "ludic-dev syntax: statement", "match": "\\b(let|if|else|while|for|in|where|match|machine|become|return|break|continue|new|spawn|despawn|enable|disable|attach|detach|emit|cancel|dispatch|try)\\b" }, + { "name": "keyword.operator.logical.ludic", "comment": "ludic-dev syntax: operator", "match": "\\b(and|or|not)\\b" }, + { "name": "keyword.other.clause.ludic", "comment": "ludic-dev syntax: modifier", "match": "\\b(export|internal|friend|uses|open|of|as|from|numbers|unsafe|mut|phase|query|on|cancellable|public|layer|start|shows|lasts|then|loads|system)\\b" }, + { "name": "storage.type.ludic", "comment": "ludic-dev syntax: declaration", "match": "\\b(program|module|import|property|state|event|action|reducer|enum|model|prefab|scene|handler|ui|namespace|alias|port|bind|registry|def|component|view|prop|const|var|function|fn|extern|entry|test)\\b" }, + { "name": "support.type.primitive.ludic", "comment": "ludic-dev syntax: types", "match": "\\b(int|long|float|double|fixed|bool|byte|string|pointer|void|entity|countdown|words|floats|doubles|fixeds|pointers|Vector|IVec2|Rect)\\b" }, + { "name": "constant.language.boolean.ludic", "comment": "ludic-dev syntax: constant", "match": "\\b(true|false|null)\\b" }, + { "name": "constant.language.phase.ludic", "comment": "ludic-dev syntax: phases", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" } ] }, "widget": { @@ -241,7 +240,7 @@ "patterns": [ { "name": "keyword.operator.arrow.ludic", "match": "->|=>" }, { "name": "keyword.operator.range.ludic", "match": "\\.\\." }, - { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\|\\^|~" }, + { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\||\\^|~" }, { "name": "keyword.operator.comparison.ludic", "match": "==|!=|<=|>=|<|>" }, { "name": "keyword.operator.assignment.ludic", "match": "\\+=|-=|\\*=|/=|=" }, { "name": "keyword.operator.arithmetic.ludic", "match": "[+\\-*/%]" } diff --git a/tools/editors/vscode/syntaxes/ludic.tmLanguage.json b/tools/editors/vscode/syntaxes/ludic.tmLanguage.json index 4480bf7f..9931756e 100644 --- a/tools/editors/vscode/syntaxes/ludic.tmLanguage.json +++ b/tools/editors/vscode/syntaxes/ludic.tmLanguage.json @@ -3,7 +3,7 @@ "name": "Ludic", "scopeName": "source.ludic", "fileTypes": ["ludic"], - "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. Generated shapes are kept in sync with tools/ludic-tools/ludic_syntax.h.", + "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. The keyword, type and phase alternations are written by `ludic-dev syntax` from the compiler's own vocabulary (`ludicc --emit-syntax`): every pattern marked \"ludic-dev syntax: \".", "patterns": [ { "include": "#comment" }, { "include": "#declaration" }, @@ -126,7 +126,7 @@ "captures": { "1": { "name": "keyword.control.import.ludic" } } }, { - "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render)\\b", + "comment": "ludic-dev syntax: phase-clause", "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b", "captures": { "1": { "name": "keyword.other.clause.ludic" }, "2": { "name": "constant.language.phase.ludic" } @@ -180,14 +180,13 @@ }, "keyword": { "patterns": [ - { "name": "keyword.control.ludic", "match": "\\b(if|else|while|for|in|match|machine|become|return|spawn|despawn|enable|disable|attach|detach|emit|cancel|where|break|continue|new|try)\\b" }, - { "name": "keyword.operator.logical.ludic", "match": "\\b(and|or|not)\\b" }, - { "name": "keyword.other.clause.ludic", "match": "\\b(phase|query|on|cancellable|public|layer|start|shows|lasts|loads|then|export|internal)\\b" }, - { "name": "keyword.other.ludic", "match": "\\b(import|extern)\\b" }, - { "name": "storage.type.ludic", "match": "\\b(program|property|model|prefab|namespace|enum|ui|const|var|let|function|fn|handler|entry|event|scene|state|test)\\b" }, - { "name": "support.type.primitive.ludic", "match": "\\b(int|long|fixed|float|double|countdown|bool|entity|string|pointer|byte|words|fixeds|floats|doubles|pointers|Vector|IVec2|Rect|void)\\b" }, - { "name": "constant.language.boolean.ludic", "match": "\\b(true|false|null)\\b" }, - { "name": "constant.language.phase.ludic", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" } + { "name": "keyword.control.ludic", "comment": "ludic-dev syntax: statement", "match": "\\b(let|if|else|while|for|in|where|match|machine|become|return|break|continue|new|spawn|despawn|enable|disable|attach|detach|emit|cancel|dispatch|try)\\b" }, + { "name": "keyword.operator.logical.ludic", "comment": "ludic-dev syntax: operator", "match": "\\b(and|or|not)\\b" }, + { "name": "keyword.other.clause.ludic", "comment": "ludic-dev syntax: modifier", "match": "\\b(export|internal|friend|uses|open|of|as|from|numbers|unsafe|mut|phase|query|on|cancellable|public|layer|start|shows|lasts|then|loads|system)\\b" }, + { "name": "storage.type.ludic", "comment": "ludic-dev syntax: declaration", "match": "\\b(program|module|import|property|state|event|action|reducer|enum|model|prefab|scene|handler|ui|namespace|alias|port|bind|registry|def|component|view|prop|const|var|function|fn|extern|entry|test)\\b" }, + { "name": "support.type.primitive.ludic", "comment": "ludic-dev syntax: types", "match": "\\b(int|long|float|double|fixed|bool|byte|string|pointer|void|entity|countdown|words|floats|doubles|fixeds|pointers|Vector|IVec2|Rect)\\b" }, + { "name": "constant.language.boolean.ludic", "comment": "ludic-dev syntax: constant", "match": "\\b(true|false|null)\\b" }, + { "name": "constant.language.phase.ludic", "comment": "ludic-dev syntax: phases", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" } ] }, "widget": { @@ -241,7 +240,7 @@ "patterns": [ { "name": "keyword.operator.arrow.ludic", "match": "->|=>" }, { "name": "keyword.operator.range.ludic", "match": "\\.\\." }, - { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\|\\^|~" }, + { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\||\\^|~" }, { "name": "keyword.operator.comparison.ludic", "match": "==|!=|<=|>=|<|>" }, { "name": "keyword.operator.assignment.ludic", "match": "\\+=|-=|\\*=|/=|=" }, { "name": "keyword.operator.arithmetic.ludic", "match": "[+\\-*/%]" } diff --git a/tools/ludic-cli/checks.ludic b/tools/ludic-cli/checks.ludic index d5e95642..c3629eca 100644 --- a/tools/ludic-cli/checks.ludic +++ b/tools/ludic-cli/checks.ludic @@ -484,19 +484,7 @@ function grammar_alt(root: JVal, node: pointer, marker: pointer) -> []pointer { return out } -# keywords the self-host parser dispatches on: is_id("x") + streq(t.text, "x") -function parser_keywords() -> []pointer { - let out = new []pointer - add_parser_kw(out, "selfhost/frontend/parse.ludic") - add_parser_kw(out, "selfhost/frontend/parse_game.ludic") - return out -} -function add_parser_kw(out: []pointer, path: pointer) -> void { - let t = read_file(path) - if t == null { return } - add_lower(out, collect_after(t, "is_id(" + dq())) - add_lower(out, collect_after(t, "streq(t.text, " + dq())) -} +# the lower-case names among `src` into `out` (a set) function add_lower(out: []pointer, src: []pointer) -> void { var i = 0 while i < len(src) { if all_lower(src[i]) { set_add(out, src[i]) }; i += 1 } @@ -537,7 +525,6 @@ function cmd_check_vocab() -> int { let h_widgets = table_set(h, "LUDIC_WIDGETS") let h_builtins = c_table_names(h, "LUDIC_BUILTINS") let h_intrinsics = c_table_names(h, "LUDIC_INTRINSICS") - let h_reserved = table_set(h, "LUDIC_KW_RESERVED") # --- against the JetBrains lexer --- let kt = read_file("tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt") @@ -561,16 +548,26 @@ function cmd_check_vocab() -> int { cmp_sets("primitive types", h_types, grammar_alt(g, "keyword", "fixed"), "ludic.tmLanguage.json") } - # --- against the self-host parser --- - let pkw = parser_keywords() - if len(pkw) == 0 { - cv_problem("could not extract any keywords from selfhost/frontend/parse*.ludic") - } else { - cmp_unparsed("declaration keywords", h_decl, pkw, h_reserved) - cmp_unparsed("clause keywords", h_clause, pkw, h_reserved) - # reserved words the parser now accepts should be promoted - var i = 0 - while i < len(h_reserved) { if set_has(pkw, h_reserved[i]) { cv_problem("LUDIC_KW_RESERVED lists a keyword the parser now accepts — promote it: " + h_reserved[i]) }; i += 1 } + # --- the keyword, type and phase tables, the docs and the parser, against the compiler's own + # vocabulary (ludicc --emit-syntax; syntax_gen.ludic) --- + if sx_vocab() == null { cv_problem("bin/ludicc has no --emit-syntax: rebuild it (ludic-dev build)") } + else { + g_sx_prob = new []pointer + let ts = sx_targets() + var ti = 0 + while ti < len(ts) { + let text = read_file(ts[ti]) + if text != null { + let want = sx_regenerate(ts[ti], text) + if want != null and not (want == text) { cv_problem(ts[ti] + " is behind the vocabulary - run ludic-dev syntax") } + } + sx_carries(ts[ti]) + ti += 1 + } + sx_docs() + sx_parser() + var pi2 = 0 + while pi2 < len(g_sx_prob) { cv_problem(g_sx_prob[pi2]); pi2 += 1 } } if CV_N > 0 { @@ -581,16 +578,6 @@ function cmd_check_vocab() -> int { return 0 } -# keywords in `kws` the parser never dispatches on (minus reserved) are a problem -function cmp_unparsed(label: pointer, kws: []pointer, pkw: []pointer, reserved: []pointer) -> void { - var i = 0 - while i < len(kws) { - if not set_has(pkw, kws[i]) { - if not set_has(reserved, kws[i]) { cv_problem("ludic_syntax.h lists a " + label + " the selfhost parser never dispatches on: " + kws[i]) } - } - i += 1 - } -} # ============================================================================ # json/xml asset validation — replaces the python3 json.load / xml.dom checks diff --git a/tools/ludic-cli/dev.ludic b/tools/ludic-cli/dev.ludic index 18867334..c65ef467 100644 --- a/tools/ludic-cli/dev.ludic +++ b/tools/ludic-cli/dev.ludic @@ -30,6 +30,7 @@ program LudicDev { import "lsp_test.ludic" import "forgejo.ludic" import "checks.ludic" + import "syntax_gen.ludic" import "docgen.ludic" import "docgen_gen.ludic" import "docgen_check.ludic" @@ -67,6 +68,7 @@ program LudicDev { print(" check-docs every ```ludic doc fence parses (or is marked skip/expect-error)") print(" check-impl every implemented feature has a docs/language page") print(" check-vocabulary the vocabulary is in sync across grammar / lexer / header / parser") + print(" syntax [--check] write every grammar's keyword, type and phase lists from ludicc --emit-syntax") print(" lint-asset validate one editor .json / .xml asset") print(" docs-gen [--out DIR] generate the documentation site (default build/pages)") print(" docs-check [DIR] coverage/integrity guard over a generated docs site") @@ -108,6 +110,7 @@ program LudicDev { if (cmd == "check-docs") { return cmd_check_docs() } if (cmd == "check-impl") { return cmd_check_impl() } if (cmd == "check-vocabulary") { return cmd_check_vocab() } + if (cmd == "syntax") { return cmd_syntax_gen() } if (cmd == "lint-asset") { return cmd_lint_asset() } if (cmd == "docs-palette") { return cmd_docs_palette() } if (cmd == "glgen") { return cmd_glgen() } diff --git a/tools/ludic-cli/main.ludic b/tools/ludic-cli/main.ludic index 77514736..5e82c029 100644 --- a/tools/ludic-cli/main.ludic +++ b/tools/ludic-cli/main.ludic @@ -32,6 +32,7 @@ program Ludic { import "scripts.ludic" import "deps.ludic" import "schema.ludic" + import "syntax_cli.ludic" import "ui_preview.ludic" import "testpar.ludic" import "migrate.ludic" @@ -53,6 +54,7 @@ program Ludic { print(" deps [file] [--graph|--dot|--writes|--uses MOD|--check F|--baseline F]") print(" the module graph as the compiler sees it, and how tangled it is") print(" schema [file] [-o FILE] records, registries and their entries, consts, as JSON (for editors)") + print(" syntax [--json] [-o FILE] the language's vocabulary: keywords, declarations, types, phases, attributes, operators") print(" migrate state [file] [--prune] [--runtime] module-level vars into states, passed as parameters (0.S); --prune drops states nothing uses, --tighten also mut nothing writes") print(" clean remove build/") print("") @@ -108,6 +110,7 @@ program Ludic { if (cmd == "test") { return cmd_test() } if (cmd == "deps") { return cmd_deps() } if (cmd == "schema") { return cmd_schema() } + if (cmd == "syntax") { return cmd_syntax() } if (cmd == "migrate") { return cmd_migrate() } if (cmd == "clean") { return cmd_clean() } if (cmd == "fmt") { return cmd_fmt() } diff --git a/tools/ludic-cli/syntax_cli.ludic b/tools/ludic-cli/syntax_cli.ludic new file mode 100644 index 00000000..7dfc4eb7 --- /dev/null +++ b/tools/ludic-cli/syntax_cli.ludic @@ -0,0 +1,86 @@ +# ---- ludic syntax ------------------------------------------------------------- +# The language's vocabulary as the compiler holds it (`ludicc --emit-syntax`, +# selfhost/frontend/vocab.ludic): every keyword with its role and whether it is +# reserved, every declaration's form, the built-in types, the phases, every +# attribute with where it goes, what it takes and what it means, the operators and +# the literal forms. An editor's grammar is written from it (`ludic-dev syntax`). +# +# ludic syntax one line per entry: kind, name, role +# ludic syntax --json the JSON, as the compiler writes it +# ludic syntax ... -o FILE into FILE + +# the vocabulary's JSON from the compiler, or null when it cannot say +function syntax_json_text() -> pointer { + ensure_ludicc() + let js = tmp_path("syntax.json") + if not shq(`{ludicc()} --emit-syntax > {sh_single(js)} 2> {tmp_path("syntax.err")}`) { return null } + let s = read_file(js) + if s == null or slen(s) == 0 or s[0] != '{' { return null } + return s +} + +function cmd_syntax() -> int { + var json = false + var dest = "" + var ai = 2 + while ai < arg_count() { + let a = arg(ai) + if a == "--json" { json = true } + else if a == "-o" or a == "--out" { + if ai + 1 >= arg_count() { err(`ludic syntax: {a} needs a file\n`); return 2 } + ai += 1 + dest = arg(ai) + } + else { + err(`ludic syntax: unknown argument {a}\n`) + err(" usage: ludic syntax [--json] [-o FILE]\n") + return 2 + } + ai += 1 + } + let s = syntax_json_text() + if s == null { err("ludic syntax: the compiler wrote no vocabulary (is it older than ludic syntax?)\n"); return 1 } + var text = s + if not json { text = syntax_lines(s) } + if dest == "" { out(text); return 0 } + if not write_file(dest, text) { err(`ludic syntax: cannot write {dest}\n`); return 1 } + return 0 +} + +# the plain listing, `keyword module declaration`, one entry a line - read off the JSON's own +# layout (a list opens on ` "name": [`, one entry per line after it) rather than parsed +function syntax_lines(s: pointer) -> pointer { + var o = "" + var kind = "" + let n = slen(s) + var i = 0 + while i < n { + let ln = line_at(s, i) + i = i + slen(ln) + 1 + if s_starts(ln, " \"") and s_contains(ln, "[") { + kind = sslice(ln, 3, s_index(ln, "\"", 3) - 1) # "keywords" -> keyword + continue + } + if not s_starts(ln, " {\"") { continue } + var name = syntax_field(ln, s_index(ln, "\": ", 0) + 3) + if kind == "attribute" { name = "@" + name } + var role = "" + let r = s_index(ln, "\"role\": ", 0) + if r >= 0 { role = syntax_field(ln, r + 8) } + let f = s_index(ln, "\"form\": ", 0) + if f >= 0 { role = syntax_field(ln, f + 8) } + o = o + kind + "\t" + name + "\t" + role + "\n" + } + return o +} +# the JSON string starting at ln[q] (its opening quote), unescaped +function syntax_field(ln: pointer, q: int) -> pointer { + var o = "" + var i = q + 1 + while ln[i] != 0 and ln[i] != '"' { + if ln[i] == '\\' { i += 1 } + o = o + sslice(ln, i, i + 1) + i += 1 + } + return o +} diff --git a/tools/ludic-cli/syntax_gen.ludic b/tools/ludic-cli/syntax_gen.ludic new file mode 100644 index 00000000..b6fc6e69 --- /dev/null +++ b/tools/ludic-cli/syntax_gen.ludic @@ -0,0 +1,508 @@ +# syntax_gen.ludic — one vocabulary, written into every editor's grammar and checked everywhere +# else (`ludic-dev syntax [--check]`). +# +# The vocabulary is what `ludicc --emit-syntax` prints (selfhost/frontend/vocab.ludic, held to the +# parser's own recognisers). From it, between marked lines, this writes: +# +# tools/ludic-tools/ludic_syntax.h LUDIC_KW_DECL / _CLAUSE / _STMT, LUDIC_TYPES, +# LUDIC_PHASES, LUDIC_ATTRIBUTES +# tools/ludic-tools/lsp.ludic, lsp/types.ludic the server's word tests (keyword, type, phase, +# the words that may also be names) +# tools/editors/shared/ludic.tmLanguage.json every pattern marked "ludic-dev syntax: " +# (and its copy in tools/editors/vscode/syntaxes/) +# tools/editors/jetbrains/.../LudicTokens.kt LudicVocabulary's keyword, type and phase sets +# tools/editors/emacs/ludic-mode.el the keyword, type and phase lists +# +# and it checks what cannot be written: that docs/language has a page for every keyword, type, +# phase and attribute, and that the parser tests no word and reads no attribute the vocabulary +# lacks (a scan of selfhost/frontend) - nor the vocabulary lists one the parser never tests. +# +# ludic-dev syntax rewrite the marked regions from the vocabulary +# ludic-dev syntax --check fail, naming each, on a region behind the vocabulary or a gap + +var g_sx: JVal = null +var g_sx_prob: []pointer = new []pointer + +# the vocabulary, from bin/ludicc (null when that compiler has no --emit-syntax) +function sx_vocab() -> JVal { + if g_sx != null { return g_sx } + ensure_ludicc() + let js = tmp_path("syntax.json") + if not shq(`bin/ludicc --emit-syntax > {js} 2> {tmp_path("syntax.err")}`) { return null } + let s = read_file(js) + if s == null or slen(s) == 0 or s[0] != '{' { return null } + g_sx = json_parse(s) + return g_sx +} + +# the `key` of every entry of `list` whose role is `role` ("" for all) +function sx_words(list: pointer, key: pointer, role: pointer) -> []pointer { + let out = new []pointer + let xs = j_get(g_sx, list) + var i = 0 + while i < len(xs.kids) { + let e = xs.kids[i] + if slen(role) == 0 or j_get(e, "role").s == role { push(out, j_get(e, key).s) } + i += 1 + } + return out +} +function sx_kw(role: pointer) -> []pointer { return sx_words("keywords", "word", role) } +# a statement is written like one, and and/or/not highlight with them +function sx_stmt() -> []pointer { return set_union(sx_kw("statement"), sx_kw("operator")) } +# every keyword that is not a literal (true, false and null lex as booleans) +function sx_all_kw() -> []pointer { return set_union(set_union(sx_kw("declaration"), sx_kw("modifier")), sx_stmt()) } +# the words the language server lets stand as a name where one is written (`var view = ...`, +# `a.model`): every word the parser does not reserve, and every declaring or modifying word +function sx_contextual() -> []pointer { + let out = new []pointer + let xs = j_get(g_sx, "keywords") + var i = 0 + while i < len(xs.kids) { + let e = xs.kids[i] + let r = j_get(e, "role").s + if (j_get(e, "reserved").b == 0 and r != "constant" and r != "operator") or r == "declaration" or r == "modifier" { push(out, j_get(e, "word").s) } + i += 1 + } + return out +} +function sx_attr_names() -> []pointer { + let out = new []pointer + let xs = sx_words("attributes", "name", "") + var i = 0 + while i < len(xs) { push(out, "@" + xs[i]); i += 1 } + return out +} + +# ---- writing a list in each file's own spelling ---- +# words quoted with `q`, joined by `sep`, after `lead` on the first line and under `indent` on the +# rest, wrapped before column `width`; `last` follows the final word (a C table's terminating 0) +function sx_wrap(words: []pointer, q: pointer, sep: pointer, lead: pointer, indent: pointer, width: int, last: pointer) -> pointer { + var out = lead + var col = slen(lead) + var fresh = true + var i = 0 + while i < len(words) { + var item = q + words[i] + q + if i + 1 < len(words) { item = item + s_trim(sep) } else { item = item + last } + var gap = "" + if not fresh { gap = sep_gap(sep) } + if not fresh and col + slen(gap) + slen(item) > width { + out = out + "\n" + indent + col = slen(indent) + gap = "" + } + out = out + gap + item + col = col + slen(gap) + slen(item) + fresh = false + i += 1 + } + return out +} +# the spaces a separator puts between two items (", " -> " ", "," -> "") +function sep_gap(sep: pointer) -> pointer { + if s_ends(sep, " ") { return " " } + return "" +} +function sx_c_table(name: pointer, words: []pointer) -> pointer { + return "static const char* " + name + "[] = {\n" + sx_wrap(words, "\"", ",", " ", " ", 100, ", 0") + "\n};\n" +} +function sx_kt_set(name: pointer, words: []pointer) -> pointer { + return " val " + name + " = setOf(\n" + sx_wrap(words, "\"", ", ", " ", " ", 100, "") + "\n )\n" +} +function sx_el_list(name: pointer, words: []pointer) -> pointer { + return "(defconst " + name + "\n" + sx_wrap(words, "\"", " ", " '(", " ", 96, "))") + "\n" +} +# a Ludic word test: `if (w == "a") or ... { return true }`, six to a line +function sx_ludic_test(fname: pointer, words: []pointer, ind: pointer) -> pointer { + var out = ind + "function " + fname + "(w: pointer) -> bool {\n" + var i = 0 + while i < len(words) { + var line = ind + " if " + var k = 0 + while k < 6 and i < len(words) { + if k > 0 { line = line + " or " } + line = line + "(w == \"" + words[i] + "\")" + k += 1 + i += 1 + } + out = out + line + " { return true }\n" + } + return out + ind + " return false\n" + ind + "}\n" +} +# a TextMate alternation, as it is written inside a JSON string: \\b(a|b|c)\\b +function sx_tm_alt(words: []pointer) -> pointer { + var out = "" + var i = 0 + while i < len(words) { + if i > 0 { out = out + "|" } + out = out + words[i] + i += 1 + } + return "\\\\b(" + out + ")\\\\b" +} + +# ---- the regions ---- +function sx_begin_note() -> pointer { return "ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit" } +function sx_header_region() -> pointer { + var o = "/* " + sx_begin_note() + " */\n" + o = o + sx_c_table("LUDIC_KW_DECL", sx_kw("declaration")) + o = o + sx_c_table("LUDIC_KW_CLAUSE", sx_kw("modifier")) + o = o + sx_c_table("LUDIC_KW_STMT", sx_stmt()) + o = o + sx_c_table("LUDIC_TYPES", sx_words("types", "name", "")) + o = o + sx_c_table("LUDIC_PHASES", sx_words("phases", "name", "")) + o = o + sx_c_table("LUDIC_ATTRIBUTES", sx_words("attributes", "name", "")) + return o + "/* ludic-dev syntax: end */\n" +} +function sx_kotlin_region() -> pointer { + var o = " // " + sx_begin_note() + "\n" + o = o + sx_kt_set("DECL", sx_kw("declaration")) + o = o + sx_kt_set("CLAUSE", sx_kw("modifier")) + o = o + sx_kt_set("STMT", sx_stmt()) + o = o + sx_kt_set("PRIMITIVES", sx_words("types", "name", "")) + o = o + sx_kt_set("PHASES", sx_words("phases", "name", "")) + return o + " // ludic-dev syntax: end\n" +} +function sx_emacs_region() -> pointer { + var o = ";; " + sx_begin_note() + "\n" + o = o + sx_el_list("ludic--declaration-keywords", sx_kw("declaration")) + o = o + sx_el_list("ludic--clause-keywords", sx_kw("modifier")) + o = o + sx_el_list("ludic--statement-keywords", sx_stmt()) + o = o + sx_el_list("ludic--types", sx_words("types", "name", "")) + o = o + sx_el_list("ludic--phases", sx_words("phases", "name", "")) + return o + ";; ludic-dev syntax: end\n" +} +function sx_lsp_region() -> pointer { + var o = " # " + sx_begin_note() + "\n" + o = o + sx_ludic_test("is_type_word", sx_words("types", "name", ""), " ") + o = o + sx_ludic_test("is_phase_word", sx_words("phases", "name", ""), " ") + o = o + sx_ludic_test("is_keyword_word", sx_all_kw(), " ") + return o + " # ludic-dev syntax: end\n" +} +function sx_lsp_types_region() -> pointer { + return "# " + sx_begin_note() + "\n" + sx_ludic_test("is_contextual_word", sx_contextual(), "") + "# ludic-dev syntax: end\n" +} + +# `text` with the lines from the one holding "ludic-dev syntax: begin" to the one holding +# "ludic-dev syntax: end" replaced by `region`; null when the markers are not there +function sx_splice(text: pointer, region: pointer) -> pointer { + let b = s_index(text, "ludic-dev syntax: begin", 0) + if b < 0 { return null } + let e = s_index(text, "ludic-dev syntax: end", b) + if e < 0 { return null } + var ls = b + while ls > 0 and text[ls - 1] != '\n' { ls -= 1 } + var le = e + while text[le] != 0 and text[le] != '\n' { le += 1 } + if text[le] == '\n' { le += 1 } + return sslice(text, 0, ls) + region + sslice(text, le, slen(text)) +} + +# the TextMate grammar: each pattern whose line says "comment": "ludic-dev syntax: " has its +# "match" on that line rewritten +function sx_tm_group(group: pointer) -> pointer { + if group == "declaration" { return sx_tm_alt(sx_kw("declaration")) } + if group == "modifier" { return sx_tm_alt(sx_kw("modifier")) } + if group == "statement" { return sx_tm_alt(sx_kw("statement")) } + if group == "operator" { return sx_tm_alt(sx_kw("operator")) } + if group == "constant" { return sx_tm_alt(sx_kw("constant")) } + if group == "types" { return sx_tm_alt(sx_words("types", "name", "")) } + if group == "phases" { return sx_tm_alt(sx_words("phases", "name", "")) } + if group == "phase-clause" { + let alt = sx_tm_alt(sx_words("phases", "name", "")) + return "\\\\b(phase)\\\\s+" + sslice(alt, 3, slen(alt)) + } + return null +} +function sx_tm_rewrite(text: pointer) -> pointer { + let marker = "\"comment\": \"ludic-dev syntax: " + var out = "" + var i = 0 + let n = slen(text) + while i < n { + let ln = line_at(text, i) + i = i + slen(ln) + 1 + var line = ln + let m = s_index(ln, marker, 0) + if m >= 0 { + let g0 = m + slen(marker) + let group = sslice(ln, g0, s_index(ln, "\"", g0)) + let alt = sx_tm_group(group) + let mq = s_index(ln, "\"match\": \"", 0) + if alt == null or mq < 0 { push(g_sx_prob, `ludic.tmLanguage.json: an unknown group "{group}" or no "match" beside it`) } + else { + let a = mq + 10 + var e = a + while ln[e] != 0 and ln[e] != '"' { if ln[e] == '\\' { e += 1 }; e += 1 } + line = sslice(ln, 0, a) + alt + sslice(ln, e, slen(ln)) + } + } + out = out + line + if i <= n { out = out + "\n" } + } + return out +} + +# the files and what each should hold; kind: "region" (between the markers) or "tm" +function sx_targets() -> []pointer { + let t = new []pointer + push(t, "tools/ludic-tools/ludic_syntax.h") + push(t, "tools/ludic-tools/lsp.ludic") + push(t, "tools/ludic-tools/lsp/types.ludic") + push(t, "tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt") + push(t, "tools/editors/emacs/ludic-mode.el") + push(t, "tools/editors/shared/ludic.tmLanguage.json") + push(t, "tools/editors/vscode/syntaxes/ludic.tmLanguage.json") + return t +} +# what `path` should read, from what it reads now; null (and a problem) when it cannot be written +function sx_regenerate(path: pointer, text: pointer) -> pointer { + if s_ends(path, ".tmLanguage.json") { return sx_tm_rewrite(text) } + var region: pointer = null + if s_ends(path, "ludic_syntax.h") { region = sx_header_region() } + if s_ends(path, "ludic-tools/lsp.ludic") { region = sx_lsp_region() } + if s_ends(path, "lsp/types.ludic") { region = sx_lsp_types_region() } + if s_ends(path, "LudicTokens.kt") { region = sx_kotlin_region() } + if s_ends(path, "ludic-mode.el") { region = sx_emacs_region() } + let out = sx_splice(text, region) + if out == null { push(g_sx_prob, `{path}: no "ludic-dev syntax: begin" ... "end" lines to write between`) } + return out +} + +# ---- what cannot be generated, checked ---- +# the words a file's text quotes (every "..." in it), for "does it carry every keyword" +function sx_quoted(text: pointer) -> []pointer { + let out = new []pointer + var i = 0 + let n = slen(text) + while i < n { + if text[i] == '"' { + var j = i + 1 + while j < n and text[j] != '"' and text[j] != '\n' { j += 1 } + set_add(out, sslice(text, i + 1, j)) + i = j + 1 + } else { i += 1 } + } + return out +} +# the alternations of a TextMate grammar: every a|b|c word inside \b( )\b +function sx_tm_words(text: pointer) -> []pointer { + let out = new []pointer + let open = bx3('\\', 'b', '(') + var i = 0 + while true { + let p = s_index(text, open, i) + if p < 0 { break } + let e = s_index(text, ")", p) # an alternation holds no ')'; the \\b after it is escaped twice here + if e < 0 { break } + let alts = split_pipe(sslice(text, p + 3, e)) + var k = 0 + while k < len(alts) { set_add(out, alts[k]); k += 1 } + i = e + 1 + } + return out +} +# every word of `want` missing from `have` is a problem naming the file +function sx_need(file: pointer, what: pointer, want: []pointer, have: []pointer) -> int { + var missing = 0 + var i = 0 + while i < len(want) { + if not set_has(have, want[i]) { + push(g_sx_prob, `{file} is missing the {what} {want[i]}`) + missing += 1 + } + i += 1 + } + return missing +} +# does `file` carry every keyword, type and phase of the vocabulary? (the number missing) +function sx_carries(file: pointer) -> int { + let t = read_file(file) + if t == null { push(g_sx_prob, `{file}: cannot read it`); return 1 } + var have: []pointer = null + if s_ends(file, ".tmLanguage.json") { have = sx_tm_words(t) } else { have = sx_quoted(t) } + # the server's other file holds only the words that may stand as names + if s_ends(file, "lsp/types.ludic") { return sx_need(file, "keyword that may be a name", sx_contextual(), have) } + var n = sx_need(file, "keyword", sx_all_kw(), have) + n += sx_need(file, "type", sx_words("types", "name", ""), have) + n += sx_need(file, "phase", sx_words("phases", "name", ""), have) + return n +} + +# docs/language: every keyword, type, phase and attribute is some page's token +function sx_docs() -> int { + let tokens = new []pointer + let pairs = new []pointer + collect_docs(tokens, pairs) + var n = sx_need("docs/language", "page for the keyword", set_union(sx_all_kw(), sx_kw("constant")), tokens) + n += sx_need("docs/language", "page for the type", sx_words("types", "name", ""), tokens) + n += sx_need("docs/language", "page for the phase", sx_words("phases", "name", ""), tokens) + n += sx_need("docs/language", "page for the attribute", sx_attr_names(), tokens) + return n +} + +# the parser, scanned: the lower-case words it tests (is_id("w"), text == "w") and the attributes +# it reads (a == "Name", ann == "Name") across selfhost/frontend +function sx_parser_scan(words: []pointer, attrs: []pointer) -> void { + let list = capture("ls selfhost/frontend/*.ludic") + var i = 0 + let n = slen(list) + while i < n { + let path = s_trim(line_at(list, i)) + i = i + slen(line_at(list, i)) + 1 + if slen(path) == 0 or s_ends(path, "/vocab.ludic") { continue } + let t = read_file(path) + if t == null { continue } + add_lower(words, collect_after(t, "is_id(" + dq())) + add_lower(words, collect_after(t, "is_kw(" + dq())) + add_lower(words, collect_after(t, "text == " + dq())) + sx_add_attrs(attrs, collect_after(t, " a == " + dq())) + sx_add_attrs(attrs, collect_after(t, "(a == " + dq())) + sx_add_attrs(attrs, collect_after(t, " ann == " + dq())) + sx_add_attrs(attrs, collect_after(t, "(ann == " + dq())) + } +} +function sx_add_attrs(out: []pointer, src: []pointer) -> void { + var i = 0 + while i < len(src) { + let w = src[i] + if slen(w) > 0 and sx_ident(w) and not (w == "gltf") { set_add(out, w) } # "gltf": @Asset's argument + i += 1 + } +} +function sx_ident(w: pointer) -> bool { + var i = 0 + while w[i] != 0 { + let c = w[i] + if not ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9') or c == '_') { return false } + i += 1 + } + return true +} +# both directions: nothing the parser tests is missing from the vocabulary, and nothing the +# vocabulary lists as read is something the parser never tests +function sx_parser() -> int { + let words = new []pointer + let attrs = new []pointer + sx_parser_scan(words, attrs) + if len(words) == 0 { push(g_sx_prob, "could not read any word the parser tests from selfhost/frontend"); return 1 } + # words the parser tests that are not the language's words: a builtin it notes, a type after `numbers` + let known = set_union(set_union(sx_all_kw(), sx_kw("constant")), sx_words("types", "name", "")) + let vattrs = sx_words("attributes", "name", "") + var n = 0 + var i = 0 + while i < len(words) { + let w = words[i] + if not set_has(known, w) and not set_has(vattrs, w) and not (w == "text_of") { + push(g_sx_prob, `the parser tests the word {w}, which vocab.ludic lacks - add its row and run ludic-dev syntax`) + n += 1 + } + i += 1 + } + i = 0 + while i < len(attrs) { + if not set_has(vattrs, attrs[i]) { + push(g_sx_prob, `the parser reads @{attrs[i]}, which vocab.ludic lacks - add its row and run ludic-dev syntax`) + n += 1 + } + i += 1 + } + let kws = set_union(sx_all_kw(), sx_kw("constant")) + i = 0 + while i < len(kws) { + if not set_has(words, kws[i]) { push(g_sx_prob, `vocab.ludic lists the keyword {kws[i]}, which the parser never tests`); n += 1 } + i += 1 + } + let checked = sx_words("attributes", "name", "checked") + i = 0 + while i < len(checked) { + if not set_has(attrs, checked[i]) { push(g_sx_prob, `vocab.ludic lists @{checked[i]} as read, and the parser never reads it`); n += 1 } + i += 1 + } + return n +} + +# ---- the command ---- +function sx_report(what: pointer) -> int { + if len(g_sx_prob) == 0 { return 0 } + err(`{what}:\n`) + var i = 0 + while i < len(g_sx_prob) { err(` - {g_sx_prob[i]}\n`); i += 1 } + return 1 +} +# usage: ludic-dev syntax [--check] +function cmd_syntax_gen() -> int { + let check = argn(2, "") == "--check" + if sx_vocab() == null { err("ludic-dev syntax: bin/ludicc has no --emit-syntax (run ludic-dev build)\n"); return 2 } + g_sx_prob = new []pointer + let ts = sx_targets() + var changed = 0 + var i = 0 + while i < len(ts) { + let path = ts[i] + i += 1 + let text = read_file(path) + if text == null { push(g_sx_prob, `{path}: cannot read it`); continue } + let want = sx_regenerate(path, text) + if want == null or want == text { continue } + if check { push(g_sx_prob, `{path} is behind the vocabulary - run ludic-dev syntax`) } + else { + write_file(path, want) + print(` wrote {path}`) + changed += 1 + } + } + if check { + i = 0 + while i < len(ts) { sx_carries(ts[i]); i += 1 } + sx_docs() + sx_parser() + return sx_report("the vocabulary and what is written from it disagree") + } + if changed == 0 { print(" every grammar already says what ludicc --emit-syntax does") } + return sx_report("ludic-dev syntax") +} + +# ---- the regression suite's cases (test.ludic): one line each ---- +function syntax_cases() -> void { + print("== one vocabulary: ludic syntax --json, and every grammar, the server and the docs against it ==") + if not shq("bin/ludic syntax --json > /dev/null 2>&1") or sx_vocab() == null { bad("ludic syntax --json printed no vocabulary"); return } + # the words editor tooling had lost track of (the scripting proposal's R8) + let r8 = ["module", "uses", "port", "bind", "action", "reducer", "dispatch", "registry", "def", "open", "component", "prop", "view", "alias", "friend", "unsafe", "numbers", "of", "as", "from", "attach", "detach"] + let kws = sx_all_kw() + var gone = "" + var i = 0 + while i < len(r8) { if not set_has(kws, r8[i]) { gone = gone + " " + r8[i] }; i += 1 } + if slen(gone) == 0 { ok("ludic syntax --json: every keyword, among them module, uses, port, bind, action, reducer, dispatch, registry, def, component, view") } + else { bad2("ludic syntax --json lacks keywords", gone) } + let ats = sx_words("attributes", "name", "") + let r8a = ["Ref", "OneOf", "Range", "Unit", "Asset", "Color", "Node", "Clip", "Material", "Tint", "Derived", "Text", "Multiline", "Key", "AppendOnly", "ByKey", "PerMap", "Chunked", "max", "owns", "frame", "Sync", "Computed", "alloc_ok"] + gone = "" + i = 0 + while i < len(r8a) { if not set_has(ats, r8a[i]) { gone = gone + " @" + r8a[i] }; i += 1 } + if slen(gone) == 0 { ok("ludic syntax --json: every attribute, with where it goes, its arguments and its doc") } + else { bad2("ludic syntax --json lacks attributes", gone) } + let ts = sx_targets() + i = 0 + while i < len(ts) { + g_sx_prob = new []pointer + let path = ts[i] + i += 1 + let text = read_file(path) + var behind = false + if text != null { + let want = sx_regenerate(path, text) + behind = want == null or not (want == text) + } + let missing = sx_carries(path) + if missing == 0 and not behind { ok(`{path} carries the vocabulary's words (as ludic-dev syntax writes them)`) } + else if missing > 0 { bad2(`{path} is missing words`, g_sx_prob[0]) } + else { bad2(`{path} is behind the vocabulary`, "run ludic-dev syntax") } + } + g_sx_prob = new []pointer + if sx_docs() == 0 { ok("docs/language has a page for every keyword, type, phase and attribute") } + else { bad2(`docs/language lacks {string(len(g_sx_prob))} page(s)`, g_sx_prob[0]) } + g_sx_prob = new []pointer + if sx_parser() == 0 { ok("the parser tests no word and reads no attribute the vocabulary lacks, and the reverse") } + else { bad2("the parser and vocab.ludic disagree", g_sx_prob[0]) } +} diff --git a/tools/ludic-cli/test.ludic b/tools/ludic-cli/test.ludic index 32da04b3..87627f5f 100644 --- a/tools/ludic-cli/test.ludic +++ b/tools/ludic-cli/test.ludic @@ -1437,6 +1437,8 @@ function cmd_dev_test() -> int { smoke("library/cursor_capture") # #89 Input.cursor_mode compiles (no-op headless; windowed links cocoa.ll) smoke("rendering/gl_triangle") # Gl.* (OpenGL 4.1 core) compiles headless; the run needs a GPU context + syntax_cases() + print("== the compiler and the CLI (ludicc / ludic) ==") # ludicc comes out of the IR seed with clang alone; the CLI is then compiled # by it, from Ludic. diff --git a/tools/ludic-tools/lsp.ludic b/tools/ludic-tools/lsp.ludic index b030c9a4..c68ff219 100644 --- a/tools/ludic-tools/lsp.ludic +++ b/tools/ludic-tools/lsp.ludic @@ -226,21 +226,38 @@ program LudicLsp { if c == ',' or c == ':' or c == '.' or c == '!' or c == '@' or c == ';' { return true } # , : . ! @ ; return c == '&' or c == '|' or c == '^' or c == '~' # & | ^ ~ } + # the vocabulary, written by `ludic-dev syntax` from `ludicc --emit-syntax` + # ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit function is_type_word(w: pointer) -> bool { - return (w == "int") or (w == "long") or (w == "fixed") or (w == "float") or (w == "double") or (w == "floats") or (w == "doubles") or (w == "countdown") or (w == "bool") or (w == "entity") or (w == "string") or (w == "pointer") or (w == "byte") or (w == "words") or (w == "fixeds") or (w == "pointers") or (w == "Vector") or (w == "IVec2") or (w == "Rect") or (w == "void") - } - function is_phase_word(w: pointer) -> bool { - return (w == "Start") or (w == "Input") or (w == "FixedUpdate") or (w == "Update") or (w == "LateUpdate") or (w == "Render") - } - function is_keyword_word(w: pointer) -> bool { - if (w == "program") or (w == "import") or (w == "property") or (w == "model") or (w == "enum") or (w == "ui") { return true } - if (w == "const") or (w == "var") or (w == "function") or (w == "extern") or (w == "handler") or (w == "entry") or (w == "event") or (w == "scene") or (w == "mut") { return true } - if (w == "phase") or (w == "query") or (w == "on") or (w == "cancellable") or (w == "public") or (w == "layer") or (w == "start") { return true } - if (w == "let") or (w == "return") or (w == "if") or (w == "else") or (w == "while") or (w == "for") or (w == "in") or (w == "spawn") or (w == "despawn") { return true } - if (w == "enable") or (w == "disable") or (w == "match") or (w == "machine") or (w == "state") or (w == "become") or (w == "where") or (w == "prefab") { return true } - if (w == "and") or (w == "or") or (w == "not") or (w == "break") or (w == "continue") or (w == "new") or (w == "emit") or (w == "cancel") { return true } + if (w == "int") or (w == "long") or (w == "float") or (w == "double") or (w == "fixed") or (w == "bool") { return true } + if (w == "byte") or (w == "string") or (w == "pointer") or (w == "void") or (w == "entity") or (w == "countdown") { return true } + if (w == "words") or (w == "floats") or (w == "doubles") or (w == "fixeds") or (w == "pointers") or (w == "Vector") { return true } + if (w == "IVec2") or (w == "Rect") { return true } return false } + function is_phase_word(w: pointer) -> bool { + if (w == "Start") or (w == "Input") or (w == "FixedUpdate") or (w == "Update") or (w == "LateUpdate") or (w == "Render") { return true } + if (w == "Overlay") { return true } + return false + } + function is_keyword_word(w: pointer) -> bool { + if (w == "program") or (w == "module") or (w == "import") or (w == "property") or (w == "state") or (w == "event") { return true } + if (w == "action") or (w == "reducer") or (w == "enum") or (w == "model") or (w == "prefab") or (w == "scene") { return true } + if (w == "handler") or (w == "ui") or (w == "namespace") or (w == "alias") or (w == "port") or (w == "bind") { return true } + if (w == "registry") or (w == "def") or (w == "component") or (w == "view") or (w == "prop") or (w == "const") { return true } + if (w == "var") or (w == "function") or (w == "fn") or (w == "extern") or (w == "entry") or (w == "test") { return true } + if (w == "export") or (w == "internal") or (w == "friend") or (w == "uses") or (w == "open") or (w == "of") { return true } + if (w == "as") or (w == "from") or (w == "numbers") or (w == "unsafe") or (w == "mut") or (w == "phase") { return true } + if (w == "query") or (w == "on") or (w == "cancellable") or (w == "public") or (w == "layer") or (w == "start") { return true } + if (w == "shows") or (w == "lasts") or (w == "then") or (w == "loads") or (w == "system") or (w == "let") { return true } + if (w == "if") or (w == "else") or (w == "while") or (w == "for") or (w == "in") or (w == "where") { return true } + if (w == "match") or (w == "machine") or (w == "become") or (w == "return") or (w == "break") or (w == "continue") { return true } + if (w == "new") or (w == "spawn") or (w == "despawn") or (w == "enable") or (w == "disable") or (w == "attach") { return true } + if (w == "detach") or (w == "emit") or (w == "cancel") or (w == "dispatch") or (w == "try") or (w == "and") { return true } + if (w == "or") or (w == "not") { return true } + return false + } + # ludic-dev syntax: end function is_widget_word(w: pointer) -> bool { return (w == "panel") or (w == "col") or (w == "row") or (w == "label") or (w == "button") or (w == "image") or (w == "spacer") } diff --git a/tools/ludic-tools/lsp/types.ludic b/tools/ludic-tools/lsp/types.ludic index 464c6e21..a95dd9bd 100644 --- a/tools/ludic-tools/lsp/types.ludic +++ b/tools/ludic-tools/lsp/types.ludic @@ -224,13 +224,23 @@ function on_type_definition(msg: JVal, id: JVal) -> void { # `model: Model` and `a.model` are all ordinary names. A word is a keyword only # where it could start or continue a declaration. +# a keyword that may stand as a name where one is written (written by `ludic-dev syntax`) +# ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit function is_contextual_word(w: pointer) -> bool { - if (w == "program") or (w == "import") or (w == "property") or (w == "model") or (w == "enum") or (w == "ui") { return true } - if (w == "function") or (w == "extern") or (w == "handler") or (w == "entry") or (w == "event") or (w == "scene") { return true } - if (w == "phase") or (w == "query") or (w == "on") or (w == "cancellable") or (w == "public") or (w == "layer") or (w == "start") { return true } - if (w == "state") or (w == "machine") or (w == "prefab") or (w == "namespace") or (w == "test") { return true } + if (w == "program") or (w == "module") or (w == "import") or (w == "property") or (w == "state") or (w == "event") { return true } + if (w == "action") or (w == "reducer") or (w == "enum") or (w == "model") or (w == "prefab") or (w == "scene") { return true } + if (w == "handler") or (w == "ui") or (w == "namespace") or (w == "alias") or (w == "port") or (w == "bind") { return true } + if (w == "registry") or (w == "def") or (w == "component") or (w == "view") or (w == "prop") or (w == "const") { return true } + if (w == "var") or (w == "function") or (w == "fn") or (w == "extern") or (w == "entry") or (w == "test") { return true } + if (w == "export") or (w == "internal") or (w == "friend") or (w == "uses") or (w == "open") or (w == "of") { return true } + if (w == "as") or (w == "from") or (w == "numbers") or (w == "unsafe") or (w == "mut") or (w == "phase") { return true } + if (w == "query") or (w == "on") or (w == "cancellable") or (w == "public") or (w == "layer") or (w == "start") { return true } + if (w == "shows") or (w == "lasts") or (w == "then") or (w == "loads") or (w == "system") or (w == "in") { return true } + if (w == "where") or (w == "enable") or (w == "disable") or (w == "attach") or (w == "detach") or (w == "emit") { return true } + if (w == "dispatch") or (w == "try") { return true } return false } +# ludic-dev syntax: end function demote_contextual_keywords(D: Doc) -> void { let N = ntok(D) diff --git a/tools/ludic-tools/ludic_syntax.h b/tools/ludic-tools/ludic_syntax.h index 6ec194ac..d6a7cf73 100644 --- a/tools/ludic-tools/ludic_syntax.h +++ b/tools/ludic-tools/ludic_syntax.h @@ -48,34 +48,48 @@ typedef struct { } LLex; /* ---------- the vocabulary ------------------------------------------------*/ -/* Reserved words, grouped so editors can colour them differently. The groups - * mirror the compiler's parser: anything parse_decl() dispatches on is a - * declaration keyword, anything stmt() dispatches on is a statement keyword. */ -static const char* LUDIC_KW_DECL[] = { - "program","import","property","model","enum","ui","namespace", - "const","var","function","extern","handler","entry","event","scene","test", 0 -}; -static const char* LUDIC_KW_CLAUSE[] = { - "phase","query","on","cancellable","public","layer","start", 0 -}; +/* Keywords by role (declaring, modifying, statement), the built-in types, the + * phases and the attribute names: one vocabulary, printed by the compiler + * (`ludicc --emit-syntax`, selfhost/frontend/vocab.ludic, which is held to the + * parser's own recognisers) and written between the marked lines below by + * `ludic-dev syntax`; `ludic-dev syntax --check` fails when they fall behind. */ /* Documented design targets the self-hosted parser does not accept yet. Kept * out of the highlighted vocabulary (they would read as working keywords) until - * they are implemented; check-vocabulary.py verifies the lists above are a - * subset of what selfhost/parse*.ludic actually dispatches on. */ + * they are implemented. */ static const char* LUDIC_KW_RESERVED[] = { 0 }; +/* ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit */ +static const char* LUDIC_KW_DECL[] = { + "program","module","import","property","state","event","action","reducer","enum","model", + "prefab","scene","handler","ui","namespace","alias","port","bind","registry","def","component", + "view","prop","const","var","function","fn","extern","entry","test", 0 +}; +static const char* LUDIC_KW_CLAUSE[] = { + "export","internal","friend","uses","open","of","as","from","numbers","unsafe","mut","phase", + "query","on","cancellable","public","layer","start","shows","lasts","then","loads","system", 0 +}; static const char* LUDIC_KW_STMT[] = { - "let","return","if","else","while","for","in","spawn","despawn", - "enable","disable","match","machine","state","become","where","prefab", - "and","or","not","break","continue","new","emit","cancel","try", 0 + "let","if","else","while","for","in","where","match","machine","become","return","break", + "continue","new","spawn","despawn","enable","disable","attach","detach","emit","cancel", + "dispatch","try","and","or","not", 0 }; static const char* LUDIC_TYPES[] = { - "int","long","fixed","float","double","countdown","bool","entity","string","pointer","byte","words","fixeds","floats","doubles","pointers","Vector","IVec2","Rect","void", 0 + "int","long","float","double","fixed","bool","byte","string","pointer","void","entity", + "countdown","words","floats","doubles","fixeds","pointers","Vector","IVec2","Rect", 0 }; static const char* LUDIC_PHASES[] = { "Start","Input","FixedUpdate","Update","LateUpdate","Render","Overlay", 0 }; +static const char* LUDIC_ATTRIBUTES[] = { + "Ref","OneOf","Range","Unit","Asset","Color","Node","Clip","Material","Tint","Derived","Text", + "Multiline","Key","Computed","Sync","frame","max","owns","AppendOnly","ByKey","PerMap", + "Chunked","export","Public","On","Queries","OnSpawn","OnDespawn","OnAttach","OnDetach", + "OnEnable","OnDisable","OnStart","OnQuit","Owned","Server","Predicted","ToServer","ToClients", + "System","EngineSystem","Namespace","deterministic","ClearColor","alloc_ok","creates", + "releases","Handles","Reads","Writes", 0 +}; +/* ludic-dev syntax: end */ static const char* LUDIC_WIDGETS[] = { "panel","col","row","label","button","image","spacer", 0 };