diff --git a/LANGUAGE.md b/LANGUAGE.md index c9568b84..ef6ed06f 100644 --- a/LANGUAGE.md +++ b/LANGUAGE.md @@ -2246,6 +2246,7 @@ ludic test packages/ludic.base # the test programs under a directory (a ludic build --check # every check a build makes, nothing written ludic build --check --diagnostics=json # the same, every error as a JSON array on stdout ludic schema -o build/schema.json # records, registries, entries, consts, components and natives, for editors +ludic syntax --json # the language's vocabulary: keywords, declarations, types, phases, attributes, operators ludic deps # how tangled the modules are, as the compiler resolved them ludic deps --check tests/deps-baseline.txt # fail when a number rose (--baseline FILE writes them) @@ -2361,6 +2362,16 @@ Plugins for VS Code and JetBrains IDEs, plus configuration for Neovim, Helix, Emacs, Sublime and Zed, are in `tools/editors/` — see [tools/editors/README.md](tools/editors/README.md). +The vocabulary every editor highlights with is the compiler's: `ludic syntax +--json` (`ludicc --emit-syntax`) prints every keyword with its role and whether +the parser reserves it, every declaration's form, the built-in types and phases, +every attribute with what it goes on, its arguments and what it means, the +operators and the literal forms. The grammars in `tools/editors/`, the language +server's word lists and `ludic_syntax.h` are written from it between marked lines +(`bin/ludic-dev syntax`), and `bin/ludic-dev syntax --check` fails when one falls +behind, when docs/language lacks a page for an entry, or when the parser tests a +word the vocabulary does not have. + ## Working programs - `examples/games/chronorift.ludic` — a co-op JRPG (overworld, dungeon, boss, shop, diff --git a/tools/editors/README.md b/tools/editors/README.md index 9f59b52c..0f4a332e 100644 --- a/tools/editors/README.md +++ b/tools/editors/README.md @@ -146,11 +146,21 @@ output. The formatter cannot change what a program means. ## Keeping it honest -The vocabulary is written down in five places that cannot include each other — -the compiler's two tables, `ludic_syntax.h`, the TextMate grammar (JSON), and the -JetBrains lexer (Kotlin). Adding a builtin and forgetting the rest is silent -failure, so `bin/ludic-dev check-vocabulary` (written in Ludic) compares all five, and -`bin/ludic-dev test-tools` runs it. +The language's words come from one place: the compiler's vocabulary table +(`selfhost/frontend/vocab.ludic`), printed by `ludic syntax --json` (`ludicc +--emit-syntax`) and held to the parser's own recognisers. The keyword, type and +phase lists in `ludic_syntax.h`, the TextMate grammar (every pattern marked +`"comment": "ludic-dev syntax: "`), the JetBrains lexer's +`LudicVocabulary`, `ludic-mode.el` and the language server are written from it +between `ludic-dev syntax: begin` / `end` lines by `bin/ludic-dev syntax`; do not +edit inside them. -When you add a keyword or builtin: put it in `ludic_syntax.h`, then run -`bin/ludic-dev test-tools` and let it tell you which copies still need it. +`bin/ludic-dev syntax --check` (and `check-vocabulary`, which `test-tools` runs, +and the regression suite) fails when a generated list is behind, when a grammar +lacks a word, when docs/language has no page for a keyword, type, phase or +attribute, or when the parser (`selfhost/frontend`) tests a word or reads an +attribute the vocabulary lacks. Builtins are still listed in `ludic_syntax.h`, +and `check-vocabulary` compares them with the grammar and the Kotlin lexer. + +When you add a keyword or an attribute: add its row to `vocab.ludic`, rebuild the +compiler, run `bin/ludic-dev syntax`, and write its docs/language page. diff --git a/tools/editors/emacs/ludic-mode.el b/tools/editors/emacs/ludic-mode.el index 08d9d0c7..63d75cb6 100644 --- a/tools/editors/emacs/ludic-mode.el +++ b/tools/editors/emacs/ludic-mode.el @@ -10,23 +10,27 @@ ;;; Code: -;; The vocabulary mirrors tools/ludic-tools/ludic_syntax.h; that header is the reference. +;; The vocabulary is written by `ludic-dev syntax` from the compiler's own table +;; (`ludicc --emit-syntax`) between the marked lines. +;; ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit (defconst ludic--declaration-keywords - '("program" "import" "property" "model" "enum" "ui" "namespace" - "const" "var" "function" "extern" "handler" "entry" "event" "scene" "test" "prefab")) - + '("program" "module" "import" "property" "state" "event" "action" "reducer" "enum" "model" + "prefab" "scene" "handler" "ui" "namespace" "alias" "port" "bind" "registry" "def" + "component" "view" "prop" "const" "var" "function" "fn" "extern" "entry" "test")) (defconst ludic--clause-keywords - '("phase" "query" "on" "cancellable" "public" "layer" "start" "shows" "lasts" "loads" "then" "export" "internal")) - + '("export" "internal" "friend" "uses" "open" "of" "as" "from" "numbers" "unsafe" "mut" "phase" + "query" "on" "cancellable" "public" "layer" "start" "shows" "lasts" "then" "loads" + "system")) (defconst ludic--statement-keywords - '("let" "return" "if" "else" "while" "for" "in" "spawn" "despawn" - "enable" "disable" "attach" "detach" "match" "machine" "state" "become" "where" - "and" "or" "not" "break" "continue" "new" "emit" "cancel" "try")) - + '("let" "if" "else" "while" "for" "in" "where" "match" "machine" "become" "return" "break" + "continue" "new" "spawn" "despawn" "enable" "disable" "attach" "detach" "emit" "cancel" + "dispatch" "try" "and" "or" "not")) (defconst ludic--types - '("int" "long" "fixed" "countdown" "bool" "entity" "string" "pointer" "byte" - "words" "fixeds" "pointers" "Vector" "IVec2" "Rect" "void")) -(defconst ludic--phases '("Start" "Input" "FixedUpdate" "Update" "LateUpdate" "Render" "Overlay")) + '("int" "long" "float" "double" "fixed" "bool" "byte" "string" "pointer" "void" "entity" + "countdown" "words" "floats" "doubles" "fixeds" "pointers" "Vector" "IVec2" "Rect")) +(defconst ludic--phases + '("Start" "Input" "FixedUpdate" "Update" "LateUpdate" "Render" "Overlay")) +;; ludic-dev syntax: end (defconst ludic-font-lock-keywords (list diff --git a/tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt b/tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt index f03c560a..380232f3 100644 --- a/tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt +++ b/tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt @@ -55,24 +55,37 @@ object LudicTokens { } /** - * The vocabulary, mirroring tools/ludic-tools/ludic_syntax.h. When the language - * grows a keyword both sides change together; that header is the reference. + * The vocabulary. The keyword, type and phase sets are written by `ludic-dev + * syntax` from the compiler's own table (`ludicc --emit-syntax`); the rest + * mirrors tools/ludic-tools/ludic_syntax.h. */ object LudicVocabulary { + // ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit val DECL = setOf( - "program", "import", "property", "model", "enum", "ui", "namespace", - "const", "var", "function", "extern", "handler", "entry", "event", "scene", "test" + "program", "module", "import", "property", "state", "event", "action", "reducer", "enum", + "model", "prefab", "scene", "handler", "ui", "namespace", "alias", "port", "bind", + "registry", "def", "component", "view", "prop", "const", "var", "function", "fn", "extern", + "entry", "test" ) val CLAUSE = setOf( - "phase", "query", "on", "cancellable", "public", "layer", "start" + "export", "internal", "friend", "uses", "open", "of", "as", "from", "numbers", "unsafe", + "mut", "phase", "query", "on", "cancellable", "public", "layer", "start", "shows", "lasts", + "then", "loads", "system" ) val STMT = setOf( - "let", "return", "if", "else", "while", "for", "in", "spawn", "despawn", - "enable", "disable", "match", "machine", "prefab", "state", "become", "where", - "and", "or", "not", "break", "continue", "new", "emit", "cancel", "try" + "let", "if", "else", "while", "for", "in", "where", "match", "machine", "become", "return", + "break", "continue", "new", "spawn", "despawn", "enable", "disable", "attach", "detach", + "emit", "cancel", "dispatch", "try", "and", "or", "not" ) - val PRIMITIVES = setOf("int", "long", "fixed", "float", "double", "countdown", "bool", "entity", "string", "pointer", "byte", "words", "fixeds", "floats", "doubles", "pointers", "Vector", "IVec2", "Rect", "void") - val PHASES = setOf("Start", "Input", "FixedUpdate", "Update", "LateUpdate", "Render", "Overlay") + val PRIMITIVES = setOf( + "int", "long", "float", "double", "fixed", "bool", "byte", "string", "pointer", "void", + "entity", "countdown", "words", "floats", "doubles", "fixeds", "pointers", "Vector", + "IVec2", "Rect" + ) + val PHASES = setOf( + "Start", "Input", "FixedUpdate", "Update", "LateUpdate", "Render", "Overlay" + ) + // ludic-dev syntax: end val WIDGETS = setOf("panel", "col", "row", "label", "button", "image", "spacer") val BUILTINS = setOf( diff --git a/tools/editors/shared/ludic.tmLanguage.json b/tools/editors/shared/ludic.tmLanguage.json index 4480bf7f..9931756e 100644 --- a/tools/editors/shared/ludic.tmLanguage.json +++ b/tools/editors/shared/ludic.tmLanguage.json @@ -3,7 +3,7 @@ "name": "Ludic", "scopeName": "source.ludic", "fileTypes": ["ludic"], - "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. Generated shapes are kept in sync with tools/ludic-tools/ludic_syntax.h.", + "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. The keyword, type and phase alternations are written by `ludic-dev syntax` from the compiler's own vocabulary (`ludicc --emit-syntax`): every pattern marked \"ludic-dev syntax: \".", "patterns": [ { "include": "#comment" }, { "include": "#declaration" }, @@ -126,7 +126,7 @@ "captures": { "1": { "name": "keyword.control.import.ludic" } } }, { - "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render)\\b", + "comment": "ludic-dev syntax: phase-clause", "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b", "captures": { "1": { "name": "keyword.other.clause.ludic" }, "2": { "name": "constant.language.phase.ludic" } @@ -180,14 +180,13 @@ }, "keyword": { "patterns": [ - { "name": "keyword.control.ludic", "match": "\\b(if|else|while|for|in|match|machine|become|return|spawn|despawn|enable|disable|attach|detach|emit|cancel|where|break|continue|new|try)\\b" }, - { "name": "keyword.operator.logical.ludic", "match": "\\b(and|or|not)\\b" }, - { "name": "keyword.other.clause.ludic", "match": "\\b(phase|query|on|cancellable|public|layer|start|shows|lasts|loads|then|export|internal)\\b" }, - { "name": "keyword.other.ludic", "match": "\\b(import|extern)\\b" }, - { "name": "storage.type.ludic", "match": "\\b(program|property|model|prefab|namespace|enum|ui|const|var|let|function|fn|handler|entry|event|scene|state|test)\\b" }, - { "name": "support.type.primitive.ludic", "match": "\\b(int|long|fixed|float|double|countdown|bool|entity|string|pointer|byte|words|fixeds|floats|doubles|pointers|Vector|IVec2|Rect|void)\\b" }, - { "name": "constant.language.boolean.ludic", "match": "\\b(true|false|null)\\b" }, - { "name": "constant.language.phase.ludic", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" } + { "name": "keyword.control.ludic", "comment": "ludic-dev syntax: statement", "match": "\\b(let|if|else|while|for|in|where|match|machine|become|return|break|continue|new|spawn|despawn|enable|disable|attach|detach|emit|cancel|dispatch|try)\\b" }, + { "name": "keyword.operator.logical.ludic", "comment": "ludic-dev syntax: operator", "match": "\\b(and|or|not)\\b" }, + { "name": "keyword.other.clause.ludic", "comment": "ludic-dev syntax: modifier", "match": "\\b(export|internal|friend|uses|open|of|as|from|numbers|unsafe|mut|phase|query|on|cancellable|public|layer|start|shows|lasts|then|loads|system)\\b" }, + { "name": "storage.type.ludic", "comment": "ludic-dev syntax: declaration", "match": "\\b(program|module|import|property|state|event|action|reducer|enum|model|prefab|scene|handler|ui|namespace|alias|port|bind|registry|def|component|view|prop|const|var|function|fn|extern|entry|test)\\b" }, + { "name": "support.type.primitive.ludic", "comment": "ludic-dev syntax: types", "match": "\\b(int|long|float|double|fixed|bool|byte|string|pointer|void|entity|countdown|words|floats|doubles|fixeds|pointers|Vector|IVec2|Rect)\\b" }, + { "name": "constant.language.boolean.ludic", "comment": "ludic-dev syntax: constant", "match": "\\b(true|false|null)\\b" }, + { "name": "constant.language.phase.ludic", "comment": "ludic-dev syntax: phases", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" } ] }, "widget": { @@ -241,7 +240,7 @@ "patterns": [ { "name": "keyword.operator.arrow.ludic", "match": "->|=>" }, { "name": "keyword.operator.range.ludic", "match": "\\.\\." }, - { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\|\\^|~" }, + { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\||\\^|~" }, { "name": "keyword.operator.comparison.ludic", "match": "==|!=|<=|>=|<|>" }, { "name": "keyword.operator.assignment.ludic", "match": "\\+=|-=|\\*=|/=|=" }, { "name": "keyword.operator.arithmetic.ludic", "match": "[+\\-*/%]" } diff --git a/tools/editors/vscode/syntaxes/ludic.tmLanguage.json b/tools/editors/vscode/syntaxes/ludic.tmLanguage.json index 4480bf7f..9931756e 100644 --- a/tools/editors/vscode/syntaxes/ludic.tmLanguage.json +++ b/tools/editors/vscode/syntaxes/ludic.tmLanguage.json @@ -3,7 +3,7 @@ "name": "Ludic", "scopeName": "source.ludic", "fileTypes": ["ludic"], - "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. Generated shapes are kept in sync with tools/ludic-tools/ludic_syntax.h.", + "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. The keyword, type and phase alternations are written by `ludic-dev syntax` from the compiler's own vocabulary (`ludicc --emit-syntax`): every pattern marked \"ludic-dev syntax: \".", "patterns": [ { "include": "#comment" }, { "include": "#declaration" }, @@ -126,7 +126,7 @@ "captures": { "1": { "name": "keyword.control.import.ludic" } } }, { - "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render)\\b", + "comment": "ludic-dev syntax: phase-clause", "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b", "captures": { "1": { "name": "keyword.other.clause.ludic" }, "2": { "name": "constant.language.phase.ludic" } @@ -180,14 +180,13 @@ }, "keyword": { "patterns": [ - { "name": "keyword.control.ludic", "match": "\\b(if|else|while|for|in|match|machine|become|return|spawn|despawn|enable|disable|attach|detach|emit|cancel|where|break|continue|new|try)\\b" }, - { "name": "keyword.operator.logical.ludic", "match": "\\b(and|or|not)\\b" }, - { "name": "keyword.other.clause.ludic", "match": "\\b(phase|query|on|cancellable|public|layer|start|shows|lasts|loads|then|export|internal)\\b" }, - { "name": "keyword.other.ludic", "match": "\\b(import|extern)\\b" }, - { "name": "storage.type.ludic", "match": "\\b(program|property|model|prefab|namespace|enum|ui|const|var|let|function|fn|handler|entry|event|scene|state|test)\\b" }, - { "name": "support.type.primitive.ludic", "match": "\\b(int|long|fixed|float|double|countdown|bool|entity|string|pointer|byte|words|fixeds|floats|doubles|pointers|Vector|IVec2|Rect|void)\\b" }, - { "name": "constant.language.boolean.ludic", "match": "\\b(true|false|null)\\b" }, - { "name": "constant.language.phase.ludic", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" } + { "name": "keyword.control.ludic", "comment": "ludic-dev syntax: statement", "match": "\\b(let|if|else|while|for|in|where|match|machine|become|return|break|continue|new|spawn|despawn|enable|disable|attach|detach|emit|cancel|dispatch|try)\\b" }, + { "name": "keyword.operator.logical.ludic", "comment": "ludic-dev syntax: operator", "match": "\\b(and|or|not)\\b" }, + { "name": "keyword.other.clause.ludic", "comment": "ludic-dev syntax: modifier", "match": "\\b(export|internal|friend|uses|open|of|as|from|numbers|unsafe|mut|phase|query|on|cancellable|public|layer|start|shows|lasts|then|loads|system)\\b" }, + { "name": "storage.type.ludic", "comment": "ludic-dev syntax: declaration", "match": "\\b(program|module|import|property|state|event|action|reducer|enum|model|prefab|scene|handler|ui|namespace|alias|port|bind|registry|def|component|view|prop|const|var|function|fn|extern|entry|test)\\b" }, + { "name": "support.type.primitive.ludic", "comment": "ludic-dev syntax: types", "match": "\\b(int|long|float|double|fixed|bool|byte|string|pointer|void|entity|countdown|words|floats|doubles|fixeds|pointers|Vector|IVec2|Rect)\\b" }, + { "name": "constant.language.boolean.ludic", "comment": "ludic-dev syntax: constant", "match": "\\b(true|false|null)\\b" }, + { "name": "constant.language.phase.ludic", "comment": "ludic-dev syntax: phases", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" } ] }, "widget": { @@ -241,7 +240,7 @@ "patterns": [ { "name": "keyword.operator.arrow.ludic", "match": "->|=>" }, { "name": "keyword.operator.range.ludic", "match": "\\.\\." }, - { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\|\\^|~" }, + { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\||\\^|~" }, { "name": "keyword.operator.comparison.ludic", "match": "==|!=|<=|>=|<|>" }, { "name": "keyword.operator.assignment.ludic", "match": "\\+=|-=|\\*=|/=|=" }, { "name": "keyword.operator.arithmetic.ludic", "match": "[+\\-*/%]" } diff --git a/tools/ludic-cli/checks.ludic b/tools/ludic-cli/checks.ludic index d5e95642..c3629eca 100644 --- a/tools/ludic-cli/checks.ludic +++ b/tools/ludic-cli/checks.ludic @@ -484,19 +484,7 @@ function grammar_alt(root: JVal, node: pointer, marker: pointer) -> []pointer { return out } -# keywords the self-host parser dispatches on: is_id("x") + streq(t.text, "x") -function parser_keywords() -> []pointer { - let out = new []pointer - add_parser_kw(out, "selfhost/frontend/parse.ludic") - add_parser_kw(out, "selfhost/frontend/parse_game.ludic") - return out -} -function add_parser_kw(out: []pointer, path: pointer) -> void { - let t = read_file(path) - if t == null { return } - add_lower(out, collect_after(t, "is_id(" + dq())) - add_lower(out, collect_after(t, "streq(t.text, " + dq())) -} +# the lower-case names among `src` into `out` (a set) function add_lower(out: []pointer, src: []pointer) -> void { var i = 0 while i < len(src) { if all_lower(src[i]) { set_add(out, src[i]) }; i += 1 } @@ -537,7 +525,6 @@ function cmd_check_vocab() -> int { let h_widgets = table_set(h, "LUDIC_WIDGETS") let h_builtins = c_table_names(h, "LUDIC_BUILTINS") let h_intrinsics = c_table_names(h, "LUDIC_INTRINSICS") - let h_reserved = table_set(h, "LUDIC_KW_RESERVED") # --- against the JetBrains lexer --- let kt = read_file("tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt") @@ -561,16 +548,26 @@ function cmd_check_vocab() -> int { cmp_sets("primitive types", h_types, grammar_alt(g, "keyword", "fixed"), "ludic.tmLanguage.json") } - # --- against the self-host parser --- - let pkw = parser_keywords() - if len(pkw) == 0 { - cv_problem("could not extract any keywords from selfhost/frontend/parse*.ludic") - } else { - cmp_unparsed("declaration keywords", h_decl, pkw, h_reserved) - cmp_unparsed("clause keywords", h_clause, pkw, h_reserved) - # reserved words the parser now accepts should be promoted - var i = 0 - while i < len(h_reserved) { if set_has(pkw, h_reserved[i]) { cv_problem("LUDIC_KW_RESERVED lists a keyword the parser now accepts — promote it: " + h_reserved[i]) }; i += 1 } + # --- the keyword, type and phase tables, the docs and the parser, against the compiler's own + # vocabulary (ludicc --emit-syntax; syntax_gen.ludic) --- + if sx_vocab() == null { cv_problem("bin/ludicc has no --emit-syntax: rebuild it (ludic-dev build)") } + else { + g_sx_prob = new []pointer + let ts = sx_targets() + var ti = 0 + while ti < len(ts) { + let text = read_file(ts[ti]) + if text != null { + let want = sx_regenerate(ts[ti], text) + if want != null and not (want == text) { cv_problem(ts[ti] + " is behind the vocabulary - run ludic-dev syntax") } + } + sx_carries(ts[ti]) + ti += 1 + } + sx_docs() + sx_parser() + var pi2 = 0 + while pi2 < len(g_sx_prob) { cv_problem(g_sx_prob[pi2]); pi2 += 1 } } if CV_N > 0 { @@ -581,16 +578,6 @@ function cmd_check_vocab() -> int { return 0 } -# keywords in `kws` the parser never dispatches on (minus reserved) are a problem -function cmp_unparsed(label: pointer, kws: []pointer, pkw: []pointer, reserved: []pointer) -> void { - var i = 0 - while i < len(kws) { - if not set_has(pkw, kws[i]) { - if not set_has(reserved, kws[i]) { cv_problem("ludic_syntax.h lists a " + label + " the selfhost parser never dispatches on: " + kws[i]) } - } - i += 1 - } -} # ============================================================================ # json/xml asset validation — replaces the python3 json.load / xml.dom checks diff --git a/tools/ludic-cli/dev.ludic b/tools/ludic-cli/dev.ludic index 18867334..c65ef467 100644 --- a/tools/ludic-cli/dev.ludic +++ b/tools/ludic-cli/dev.ludic @@ -30,6 +30,7 @@ program LudicDev { import "lsp_test.ludic" import "forgejo.ludic" import "checks.ludic" + import "syntax_gen.ludic" import "docgen.ludic" import "docgen_gen.ludic" import "docgen_check.ludic" @@ -67,6 +68,7 @@ program LudicDev { print(" check-docs every ```ludic doc fence parses (or is marked skip/expect-error)") print(" check-impl every implemented feature has a docs/language page") print(" check-vocabulary the vocabulary is in sync across grammar / lexer / header / parser") + print(" syntax [--check] write every grammar's keyword, type and phase lists from ludicc --emit-syntax") print(" lint-asset validate one editor .json / .xml asset") print(" docs-gen [--out DIR] generate the documentation site (default build/pages)") print(" docs-check [DIR] coverage/integrity guard over a generated docs site") @@ -108,6 +110,7 @@ program LudicDev { if (cmd == "check-docs") { return cmd_check_docs() } if (cmd == "check-impl") { return cmd_check_impl() } if (cmd == "check-vocabulary") { return cmd_check_vocab() } + if (cmd == "syntax") { return cmd_syntax_gen() } if (cmd == "lint-asset") { return cmd_lint_asset() } if (cmd == "docs-palette") { return cmd_docs_palette() } if (cmd == "glgen") { return cmd_glgen() } diff --git a/tools/ludic-cli/main.ludic b/tools/ludic-cli/main.ludic index 77514736..5e82c029 100644 --- a/tools/ludic-cli/main.ludic +++ b/tools/ludic-cli/main.ludic @@ -32,6 +32,7 @@ program Ludic { import "scripts.ludic" import "deps.ludic" import "schema.ludic" + import "syntax_cli.ludic" import "ui_preview.ludic" import "testpar.ludic" import "migrate.ludic" @@ -53,6 +54,7 @@ program Ludic { print(" deps [file] [--graph|--dot|--writes|--uses MOD|--check F|--baseline F]") print(" the module graph as the compiler sees it, and how tangled it is") print(" schema [file] [-o FILE] records, registries and their entries, consts, as JSON (for editors)") + print(" syntax [--json] [-o FILE] the language's vocabulary: keywords, declarations, types, phases, attributes, operators") print(" migrate state [file] [--prune] [--runtime] module-level vars into states, passed as parameters (0.S); --prune drops states nothing uses, --tighten also mut nothing writes") print(" clean remove build/") print("") @@ -108,6 +110,7 @@ program Ludic { if (cmd == "test") { return cmd_test() } if (cmd == "deps") { return cmd_deps() } if (cmd == "schema") { return cmd_schema() } + if (cmd == "syntax") { return cmd_syntax() } if (cmd == "migrate") { return cmd_migrate() } if (cmd == "clean") { return cmd_clean() } if (cmd == "fmt") { return cmd_fmt() } diff --git a/tools/ludic-cli/syntax_cli.ludic b/tools/ludic-cli/syntax_cli.ludic new file mode 100644 index 00000000..7dfc4eb7 --- /dev/null +++ b/tools/ludic-cli/syntax_cli.ludic @@ -0,0 +1,86 @@ +# ---- ludic syntax ------------------------------------------------------------- +# The language's vocabulary as the compiler holds it (`ludicc --emit-syntax`, +# selfhost/frontend/vocab.ludic): every keyword with its role and whether it is +# reserved, every declaration's form, the built-in types, the phases, every +# attribute with where it goes, what it takes and what it means, the operators and +# the literal forms. An editor's grammar is written from it (`ludic-dev syntax`). +# +# ludic syntax one line per entry: kind, name, role +# ludic syntax --json the JSON, as the compiler writes it +# ludic syntax ... -o FILE into FILE + +# the vocabulary's JSON from the compiler, or null when it cannot say +function syntax_json_text() -> pointer { + ensure_ludicc() + let js = tmp_path("syntax.json") + if not shq(`{ludicc()} --emit-syntax > {sh_single(js)} 2> {tmp_path("syntax.err")}`) { return null } + let s = read_file(js) + if s == null or slen(s) == 0 or s[0] != '{' { return null } + return s +} + +function cmd_syntax() -> int { + var json = false + var dest = "" + var ai = 2 + while ai < arg_count() { + let a = arg(ai) + if a == "--json" { json = true } + else if a == "-o" or a == "--out" { + if ai + 1 >= arg_count() { err(`ludic syntax: {a} needs a file\n`); return 2 } + ai += 1 + dest = arg(ai) + } + else { + err(`ludic syntax: unknown argument {a}\n`) + err(" usage: ludic syntax [--json] [-o FILE]\n") + return 2 + } + ai += 1 + } + let s = syntax_json_text() + if s == null { err("ludic syntax: the compiler wrote no vocabulary (is it older than ludic syntax?)\n"); return 1 } + var text = s + if not json { text = syntax_lines(s) } + if dest == "" { out(text); return 0 } + if not write_file(dest, text) { err(`ludic syntax: cannot write {dest}\n`); return 1 } + return 0 +} + +# the plain listing, `keyword module declaration`, one entry a line - read off the JSON's own +# layout (a list opens on ` "name": [`, one entry per line after it) rather than parsed +function syntax_lines(s: pointer) -> pointer { + var o = "" + var kind = "" + let n = slen(s) + var i = 0 + while i < n { + let ln = line_at(s, i) + i = i + slen(ln) + 1 + if s_starts(ln, " \"") and s_contains(ln, "[") { + kind = sslice(ln, 3, s_index(ln, "\"", 3) - 1) # "keywords" -> keyword + continue + } + if not s_starts(ln, " {\"") { continue } + var name = syntax_field(ln, s_index(ln, "\": ", 0) + 3) + if kind == "attribute" { name = "@" + name } + var role = "" + let r = s_index(ln, "\"role\": ", 0) + if r >= 0 { role = syntax_field(ln, r + 8) } + let f = s_index(ln, "\"form\": ", 0) + if f >= 0 { role = syntax_field(ln, f + 8) } + o = o + kind + "\t" + name + "\t" + role + "\n" + } + return o +} +# the JSON string starting at ln[q] (its opening quote), unescaped +function syntax_field(ln: pointer, q: int) -> pointer { + var o = "" + var i = q + 1 + while ln[i] != 0 and ln[i] != '"' { + if ln[i] == '\\' { i += 1 } + o = o + sslice(ln, i, i + 1) + i += 1 + } + return o +} diff --git a/tools/ludic-cli/syntax_gen.ludic b/tools/ludic-cli/syntax_gen.ludic new file mode 100644 index 00000000..b6fc6e69 --- /dev/null +++ b/tools/ludic-cli/syntax_gen.ludic @@ -0,0 +1,508 @@ +# syntax_gen.ludic — one vocabulary, written into every editor's grammar and checked everywhere +# else (`ludic-dev syntax [--check]`). +# +# The vocabulary is what `ludicc --emit-syntax` prints (selfhost/frontend/vocab.ludic, held to the +# parser's own recognisers). From it, between marked lines, this writes: +# +# tools/ludic-tools/ludic_syntax.h LUDIC_KW_DECL / _CLAUSE / _STMT, LUDIC_TYPES, +# LUDIC_PHASES, LUDIC_ATTRIBUTES +# tools/ludic-tools/lsp.ludic, lsp/types.ludic the server's word tests (keyword, type, phase, +# the words that may also be names) +# tools/editors/shared/ludic.tmLanguage.json every pattern marked "ludic-dev syntax: " +# (and its copy in tools/editors/vscode/syntaxes/) +# tools/editors/jetbrains/.../LudicTokens.kt LudicVocabulary's keyword, type and phase sets +# tools/editors/emacs/ludic-mode.el the keyword, type and phase lists +# +# and it checks what cannot be written: that docs/language has a page for every keyword, type, +# phase and attribute, and that the parser tests no word and reads no attribute the vocabulary +# lacks (a scan of selfhost/frontend) - nor the vocabulary lists one the parser never tests. +# +# ludic-dev syntax rewrite the marked regions from the vocabulary +# ludic-dev syntax --check fail, naming each, on a region behind the vocabulary or a gap + +var g_sx: JVal = null +var g_sx_prob: []pointer = new []pointer + +# the vocabulary, from bin/ludicc (null when that compiler has no --emit-syntax) +function sx_vocab() -> JVal { + if g_sx != null { return g_sx } + ensure_ludicc() + let js = tmp_path("syntax.json") + if not shq(`bin/ludicc --emit-syntax > {js} 2> {tmp_path("syntax.err")}`) { return null } + let s = read_file(js) + if s == null or slen(s) == 0 or s[0] != '{' { return null } + g_sx = json_parse(s) + return g_sx +} + +# the `key` of every entry of `list` whose role is `role` ("" for all) +function sx_words(list: pointer, key: pointer, role: pointer) -> []pointer { + let out = new []pointer + let xs = j_get(g_sx, list) + var i = 0 + while i < len(xs.kids) { + let e = xs.kids[i] + if slen(role) == 0 or j_get(e, "role").s == role { push(out, j_get(e, key).s) } + i += 1 + } + return out +} +function sx_kw(role: pointer) -> []pointer { return sx_words("keywords", "word", role) } +# a statement is written like one, and and/or/not highlight with them +function sx_stmt() -> []pointer { return set_union(sx_kw("statement"), sx_kw("operator")) } +# every keyword that is not a literal (true, false and null lex as booleans) +function sx_all_kw() -> []pointer { return set_union(set_union(sx_kw("declaration"), sx_kw("modifier")), sx_stmt()) } +# the words the language server lets stand as a name where one is written (`var view = ...`, +# `a.model`): every word the parser does not reserve, and every declaring or modifying word +function sx_contextual() -> []pointer { + let out = new []pointer + let xs = j_get(g_sx, "keywords") + var i = 0 + while i < len(xs.kids) { + let e = xs.kids[i] + let r = j_get(e, "role").s + if (j_get(e, "reserved").b == 0 and r != "constant" and r != "operator") or r == "declaration" or r == "modifier" { push(out, j_get(e, "word").s) } + i += 1 + } + return out +} +function sx_attr_names() -> []pointer { + let out = new []pointer + let xs = sx_words("attributes", "name", "") + var i = 0 + while i < len(xs) { push(out, "@" + xs[i]); i += 1 } + return out +} + +# ---- writing a list in each file's own spelling ---- +# words quoted with `q`, joined by `sep`, after `lead` on the first line and under `indent` on the +# rest, wrapped before column `width`; `last` follows the final word (a C table's terminating 0) +function sx_wrap(words: []pointer, q: pointer, sep: pointer, lead: pointer, indent: pointer, width: int, last: pointer) -> pointer { + var out = lead + var col = slen(lead) + var fresh = true + var i = 0 + while i < len(words) { + var item = q + words[i] + q + if i + 1 < len(words) { item = item + s_trim(sep) } else { item = item + last } + var gap = "" + if not fresh { gap = sep_gap(sep) } + if not fresh and col + slen(gap) + slen(item) > width { + out = out + "\n" + indent + col = slen(indent) + gap = "" + } + out = out + gap + item + col = col + slen(gap) + slen(item) + fresh = false + i += 1 + } + return out +} +# the spaces a separator puts between two items (", " -> " ", "," -> "") +function sep_gap(sep: pointer) -> pointer { + if s_ends(sep, " ") { return " " } + return "" +} +function sx_c_table(name: pointer, words: []pointer) -> pointer { + return "static const char* " + name + "[] = {\n" + sx_wrap(words, "\"", ",", " ", " ", 100, ", 0") + "\n};\n" +} +function sx_kt_set(name: pointer, words: []pointer) -> pointer { + return " val " + name + " = setOf(\n" + sx_wrap(words, "\"", ", ", " ", " ", 100, "") + "\n )\n" +} +function sx_el_list(name: pointer, words: []pointer) -> pointer { + return "(defconst " + name + "\n" + sx_wrap(words, "\"", " ", " '(", " ", 96, "))") + "\n" +} +# a Ludic word test: `if (w == "a") or ... { return true }`, six to a line +function sx_ludic_test(fname: pointer, words: []pointer, ind: pointer) -> pointer { + var out = ind + "function " + fname + "(w: pointer) -> bool {\n" + var i = 0 + while i < len(words) { + var line = ind + " if " + var k = 0 + while k < 6 and i < len(words) { + if k > 0 { line = line + " or " } + line = line + "(w == \"" + words[i] + "\")" + k += 1 + i += 1 + } + out = out + line + " { return true }\n" + } + return out + ind + " return false\n" + ind + "}\n" +} +# a TextMate alternation, as it is written inside a JSON string: \\b(a|b|c)\\b +function sx_tm_alt(words: []pointer) -> pointer { + var out = "" + var i = 0 + while i < len(words) { + if i > 0 { out = out + "|" } + out = out + words[i] + i += 1 + } + return "\\\\b(" + out + ")\\\\b" +} + +# ---- the regions ---- +function sx_begin_note() -> pointer { return "ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit" } +function sx_header_region() -> pointer { + var o = "/* " + sx_begin_note() + " */\n" + o = o + sx_c_table("LUDIC_KW_DECL", sx_kw("declaration")) + o = o + sx_c_table("LUDIC_KW_CLAUSE", sx_kw("modifier")) + o = o + sx_c_table("LUDIC_KW_STMT", sx_stmt()) + o = o + sx_c_table("LUDIC_TYPES", sx_words("types", "name", "")) + o = o + sx_c_table("LUDIC_PHASES", sx_words("phases", "name", "")) + o = o + sx_c_table("LUDIC_ATTRIBUTES", sx_words("attributes", "name", "")) + return o + "/* ludic-dev syntax: end */\n" +} +function sx_kotlin_region() -> pointer { + var o = " // " + sx_begin_note() + "\n" + o = o + sx_kt_set("DECL", sx_kw("declaration")) + o = o + sx_kt_set("CLAUSE", sx_kw("modifier")) + o = o + sx_kt_set("STMT", sx_stmt()) + o = o + sx_kt_set("PRIMITIVES", sx_words("types", "name", "")) + o = o + sx_kt_set("PHASES", sx_words("phases", "name", "")) + return o + " // ludic-dev syntax: end\n" +} +function sx_emacs_region() -> pointer { + var o = ";; " + sx_begin_note() + "\n" + o = o + sx_el_list("ludic--declaration-keywords", sx_kw("declaration")) + o = o + sx_el_list("ludic--clause-keywords", sx_kw("modifier")) + o = o + sx_el_list("ludic--statement-keywords", sx_stmt()) + o = o + sx_el_list("ludic--types", sx_words("types", "name", "")) + o = o + sx_el_list("ludic--phases", sx_words("phases", "name", "")) + return o + ";; ludic-dev syntax: end\n" +} +function sx_lsp_region() -> pointer { + var o = " # " + sx_begin_note() + "\n" + o = o + sx_ludic_test("is_type_word", sx_words("types", "name", ""), " ") + o = o + sx_ludic_test("is_phase_word", sx_words("phases", "name", ""), " ") + o = o + sx_ludic_test("is_keyword_word", sx_all_kw(), " ") + return o + " # ludic-dev syntax: end\n" +} +function sx_lsp_types_region() -> pointer { + return "# " + sx_begin_note() + "\n" + sx_ludic_test("is_contextual_word", sx_contextual(), "") + "# ludic-dev syntax: end\n" +} + +# `text` with the lines from the one holding "ludic-dev syntax: begin" to the one holding +# "ludic-dev syntax: end" replaced by `region`; null when the markers are not there +function sx_splice(text: pointer, region: pointer) -> pointer { + let b = s_index(text, "ludic-dev syntax: begin", 0) + if b < 0 { return null } + let e = s_index(text, "ludic-dev syntax: end", b) + if e < 0 { return null } + var ls = b + while ls > 0 and text[ls - 1] != '\n' { ls -= 1 } + var le = e + while text[le] != 0 and text[le] != '\n' { le += 1 } + if text[le] == '\n' { le += 1 } + return sslice(text, 0, ls) + region + sslice(text, le, slen(text)) +} + +# the TextMate grammar: each pattern whose line says "comment": "ludic-dev syntax: " has its +# "match" on that line rewritten +function sx_tm_group(group: pointer) -> pointer { + if group == "declaration" { return sx_tm_alt(sx_kw("declaration")) } + if group == "modifier" { return sx_tm_alt(sx_kw("modifier")) } + if group == "statement" { return sx_tm_alt(sx_kw("statement")) } + if group == "operator" { return sx_tm_alt(sx_kw("operator")) } + if group == "constant" { return sx_tm_alt(sx_kw("constant")) } + if group == "types" { return sx_tm_alt(sx_words("types", "name", "")) } + if group == "phases" { return sx_tm_alt(sx_words("phases", "name", "")) } + if group == "phase-clause" { + let alt = sx_tm_alt(sx_words("phases", "name", "")) + return "\\\\b(phase)\\\\s+" + sslice(alt, 3, slen(alt)) + } + return null +} +function sx_tm_rewrite(text: pointer) -> pointer { + let marker = "\"comment\": \"ludic-dev syntax: " + var out = "" + var i = 0 + let n = slen(text) + while i < n { + let ln = line_at(text, i) + i = i + slen(ln) + 1 + var line = ln + let m = s_index(ln, marker, 0) + if m >= 0 { + let g0 = m + slen(marker) + let group = sslice(ln, g0, s_index(ln, "\"", g0)) + let alt = sx_tm_group(group) + let mq = s_index(ln, "\"match\": \"", 0) + if alt == null or mq < 0 { push(g_sx_prob, `ludic.tmLanguage.json: an unknown group "{group}" or no "match" beside it`) } + else { + let a = mq + 10 + var e = a + while ln[e] != 0 and ln[e] != '"' { if ln[e] == '\\' { e += 1 }; e += 1 } + line = sslice(ln, 0, a) + alt + sslice(ln, e, slen(ln)) + } + } + out = out + line + if i <= n { out = out + "\n" } + } + return out +} + +# the files and what each should hold; kind: "region" (between the markers) or "tm" +function sx_targets() -> []pointer { + let t = new []pointer + push(t, "tools/ludic-tools/ludic_syntax.h") + push(t, "tools/ludic-tools/lsp.ludic") + push(t, "tools/ludic-tools/lsp/types.ludic") + push(t, "tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt") + push(t, "tools/editors/emacs/ludic-mode.el") + push(t, "tools/editors/shared/ludic.tmLanguage.json") + push(t, "tools/editors/vscode/syntaxes/ludic.tmLanguage.json") + return t +} +# what `path` should read, from what it reads now; null (and a problem) when it cannot be written +function sx_regenerate(path: pointer, text: pointer) -> pointer { + if s_ends(path, ".tmLanguage.json") { return sx_tm_rewrite(text) } + var region: pointer = null + if s_ends(path, "ludic_syntax.h") { region = sx_header_region() } + if s_ends(path, "ludic-tools/lsp.ludic") { region = sx_lsp_region() } + if s_ends(path, "lsp/types.ludic") { region = sx_lsp_types_region() } + if s_ends(path, "LudicTokens.kt") { region = sx_kotlin_region() } + if s_ends(path, "ludic-mode.el") { region = sx_emacs_region() } + let out = sx_splice(text, region) + if out == null { push(g_sx_prob, `{path}: no "ludic-dev syntax: begin" ... "end" lines to write between`) } + return out +} + +# ---- what cannot be generated, checked ---- +# the words a file's text quotes (every "..." in it), for "does it carry every keyword" +function sx_quoted(text: pointer) -> []pointer { + let out = new []pointer + var i = 0 + let n = slen(text) + while i < n { + if text[i] == '"' { + var j = i + 1 + while j < n and text[j] != '"' and text[j] != '\n' { j += 1 } + set_add(out, sslice(text, i + 1, j)) + i = j + 1 + } else { i += 1 } + } + return out +} +# the alternations of a TextMate grammar: every a|b|c word inside \b( )\b +function sx_tm_words(text: pointer) -> []pointer { + let out = new []pointer + let open = bx3('\\', 'b', '(') + var i = 0 + while true { + let p = s_index(text, open, i) + if p < 0 { break } + let e = s_index(text, ")", p) # an alternation holds no ')'; the \\b after it is escaped twice here + if e < 0 { break } + let alts = split_pipe(sslice(text, p + 3, e)) + var k = 0 + while k < len(alts) { set_add(out, alts[k]); k += 1 } + i = e + 1 + } + return out +} +# every word of `want` missing from `have` is a problem naming the file +function sx_need(file: pointer, what: pointer, want: []pointer, have: []pointer) -> int { + var missing = 0 + var i = 0 + while i < len(want) { + if not set_has(have, want[i]) { + push(g_sx_prob, `{file} is missing the {what} {want[i]}`) + missing += 1 + } + i += 1 + } + return missing +} +# does `file` carry every keyword, type and phase of the vocabulary? (the number missing) +function sx_carries(file: pointer) -> int { + let t = read_file(file) + if t == null { push(g_sx_prob, `{file}: cannot read it`); return 1 } + var have: []pointer = null + if s_ends(file, ".tmLanguage.json") { have = sx_tm_words(t) } else { have = sx_quoted(t) } + # the server's other file holds only the words that may stand as names + if s_ends(file, "lsp/types.ludic") { return sx_need(file, "keyword that may be a name", sx_contextual(), have) } + var n = sx_need(file, "keyword", sx_all_kw(), have) + n += sx_need(file, "type", sx_words("types", "name", ""), have) + n += sx_need(file, "phase", sx_words("phases", "name", ""), have) + return n +} + +# docs/language: every keyword, type, phase and attribute is some page's token +function sx_docs() -> int { + let tokens = new []pointer + let pairs = new []pointer + collect_docs(tokens, pairs) + var n = sx_need("docs/language", "page for the keyword", set_union(sx_all_kw(), sx_kw("constant")), tokens) + n += sx_need("docs/language", "page for the type", sx_words("types", "name", ""), tokens) + n += sx_need("docs/language", "page for the phase", sx_words("phases", "name", ""), tokens) + n += sx_need("docs/language", "page for the attribute", sx_attr_names(), tokens) + return n +} + +# the parser, scanned: the lower-case words it tests (is_id("w"), text == "w") and the attributes +# it reads (a == "Name", ann == "Name") across selfhost/frontend +function sx_parser_scan(words: []pointer, attrs: []pointer) -> void { + let list = capture("ls selfhost/frontend/*.ludic") + var i = 0 + let n = slen(list) + while i < n { + let path = s_trim(line_at(list, i)) + i = i + slen(line_at(list, i)) + 1 + if slen(path) == 0 or s_ends(path, "/vocab.ludic") { continue } + let t = read_file(path) + if t == null { continue } + add_lower(words, collect_after(t, "is_id(" + dq())) + add_lower(words, collect_after(t, "is_kw(" + dq())) + add_lower(words, collect_after(t, "text == " + dq())) + sx_add_attrs(attrs, collect_after(t, " a == " + dq())) + sx_add_attrs(attrs, collect_after(t, "(a == " + dq())) + sx_add_attrs(attrs, collect_after(t, " ann == " + dq())) + sx_add_attrs(attrs, collect_after(t, "(ann == " + dq())) + } +} +function sx_add_attrs(out: []pointer, src: []pointer) -> void { + var i = 0 + while i < len(src) { + let w = src[i] + if slen(w) > 0 and sx_ident(w) and not (w == "gltf") { set_add(out, w) } # "gltf": @Asset's argument + i += 1 + } +} +function sx_ident(w: pointer) -> bool { + var i = 0 + while w[i] != 0 { + let c = w[i] + if not ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9') or c == '_') { return false } + i += 1 + } + return true +} +# both directions: nothing the parser tests is missing from the vocabulary, and nothing the +# vocabulary lists as read is something the parser never tests +function sx_parser() -> int { + let words = new []pointer + let attrs = new []pointer + sx_parser_scan(words, attrs) + if len(words) == 0 { push(g_sx_prob, "could not read any word the parser tests from selfhost/frontend"); return 1 } + # words the parser tests that are not the language's words: a builtin it notes, a type after `numbers` + let known = set_union(set_union(sx_all_kw(), sx_kw("constant")), sx_words("types", "name", "")) + let vattrs = sx_words("attributes", "name", "") + var n = 0 + var i = 0 + while i < len(words) { + let w = words[i] + if not set_has(known, w) and not set_has(vattrs, w) and not (w == "text_of") { + push(g_sx_prob, `the parser tests the word {w}, which vocab.ludic lacks - add its row and run ludic-dev syntax`) + n += 1 + } + i += 1 + } + i = 0 + while i < len(attrs) { + if not set_has(vattrs, attrs[i]) { + push(g_sx_prob, `the parser reads @{attrs[i]}, which vocab.ludic lacks - add its row and run ludic-dev syntax`) + n += 1 + } + i += 1 + } + let kws = set_union(sx_all_kw(), sx_kw("constant")) + i = 0 + while i < len(kws) { + if not set_has(words, kws[i]) { push(g_sx_prob, `vocab.ludic lists the keyword {kws[i]}, which the parser never tests`); n += 1 } + i += 1 + } + let checked = sx_words("attributes", "name", "checked") + i = 0 + while i < len(checked) { + if not set_has(attrs, checked[i]) { push(g_sx_prob, `vocab.ludic lists @{checked[i]} as read, and the parser never reads it`); n += 1 } + i += 1 + } + return n +} + +# ---- the command ---- +function sx_report(what: pointer) -> int { + if len(g_sx_prob) == 0 { return 0 } + err(`{what}:\n`) + var i = 0 + while i < len(g_sx_prob) { err(` - {g_sx_prob[i]}\n`); i += 1 } + return 1 +} +# usage: ludic-dev syntax [--check] +function cmd_syntax_gen() -> int { + let check = argn(2, "") == "--check" + if sx_vocab() == null { err("ludic-dev syntax: bin/ludicc has no --emit-syntax (run ludic-dev build)\n"); return 2 } + g_sx_prob = new []pointer + let ts = sx_targets() + var changed = 0 + var i = 0 + while i < len(ts) { + let path = ts[i] + i += 1 + let text = read_file(path) + if text == null { push(g_sx_prob, `{path}: cannot read it`); continue } + let want = sx_regenerate(path, text) + if want == null or want == text { continue } + if check { push(g_sx_prob, `{path} is behind the vocabulary - run ludic-dev syntax`) } + else { + write_file(path, want) + print(` wrote {path}`) + changed += 1 + } + } + if check { + i = 0 + while i < len(ts) { sx_carries(ts[i]); i += 1 } + sx_docs() + sx_parser() + return sx_report("the vocabulary and what is written from it disagree") + } + if changed == 0 { print(" every grammar already says what ludicc --emit-syntax does") } + return sx_report("ludic-dev syntax") +} + +# ---- the regression suite's cases (test.ludic): one line each ---- +function syntax_cases() -> void { + print("== one vocabulary: ludic syntax --json, and every grammar, the server and the docs against it ==") + if not shq("bin/ludic syntax --json > /dev/null 2>&1") or sx_vocab() == null { bad("ludic syntax --json printed no vocabulary"); return } + # the words editor tooling had lost track of (the scripting proposal's R8) + let r8 = ["module", "uses", "port", "bind", "action", "reducer", "dispatch", "registry", "def", "open", "component", "prop", "view", "alias", "friend", "unsafe", "numbers", "of", "as", "from", "attach", "detach"] + let kws = sx_all_kw() + var gone = "" + var i = 0 + while i < len(r8) { if not set_has(kws, r8[i]) { gone = gone + " " + r8[i] }; i += 1 } + if slen(gone) == 0 { ok("ludic syntax --json: every keyword, among them module, uses, port, bind, action, reducer, dispatch, registry, def, component, view") } + else { bad2("ludic syntax --json lacks keywords", gone) } + let ats = sx_words("attributes", "name", "") + let r8a = ["Ref", "OneOf", "Range", "Unit", "Asset", "Color", "Node", "Clip", "Material", "Tint", "Derived", "Text", "Multiline", "Key", "AppendOnly", "ByKey", "PerMap", "Chunked", "max", "owns", "frame", "Sync", "Computed", "alloc_ok"] + gone = "" + i = 0 + while i < len(r8a) { if not set_has(ats, r8a[i]) { gone = gone + " @" + r8a[i] }; i += 1 } + if slen(gone) == 0 { ok("ludic syntax --json: every attribute, with where it goes, its arguments and its doc") } + else { bad2("ludic syntax --json lacks attributes", gone) } + let ts = sx_targets() + i = 0 + while i < len(ts) { + g_sx_prob = new []pointer + let path = ts[i] + i += 1 + let text = read_file(path) + var behind = false + if text != null { + let want = sx_regenerate(path, text) + behind = want == null or not (want == text) + } + let missing = sx_carries(path) + if missing == 0 and not behind { ok(`{path} carries the vocabulary's words (as ludic-dev syntax writes them)`) } + else if missing > 0 { bad2(`{path} is missing words`, g_sx_prob[0]) } + else { bad2(`{path} is behind the vocabulary`, "run ludic-dev syntax") } + } + g_sx_prob = new []pointer + if sx_docs() == 0 { ok("docs/language has a page for every keyword, type, phase and attribute") } + else { bad2(`docs/language lacks {string(len(g_sx_prob))} page(s)`, g_sx_prob[0]) } + g_sx_prob = new []pointer + if sx_parser() == 0 { ok("the parser tests no word and reads no attribute the vocabulary lacks, and the reverse") } + else { bad2("the parser and vocab.ludic disagree", g_sx_prob[0]) } +} diff --git a/tools/ludic-cli/test.ludic b/tools/ludic-cli/test.ludic index 32da04b3..87627f5f 100644 --- a/tools/ludic-cli/test.ludic +++ b/tools/ludic-cli/test.ludic @@ -1437,6 +1437,8 @@ function cmd_dev_test() -> int { smoke("library/cursor_capture") # #89 Input.cursor_mode compiles (no-op headless; windowed links cocoa.ll) smoke("rendering/gl_triangle") # Gl.* (OpenGL 4.1 core) compiles headless; the run needs a GPU context + syntax_cases() + print("== the compiler and the CLI (ludicc / ludic) ==") # ludicc comes out of the IR seed with clang alone; the CLI is then compiled # by it, from Ludic. diff --git a/tools/ludic-tools/lsp.ludic b/tools/ludic-tools/lsp.ludic index b030c9a4..c68ff219 100644 --- a/tools/ludic-tools/lsp.ludic +++ b/tools/ludic-tools/lsp.ludic @@ -226,21 +226,38 @@ program LudicLsp { if c == ',' or c == ':' or c == '.' or c == '!' or c == '@' or c == ';' { return true } # , : . ! @ ; return c == '&' or c == '|' or c == '^' or c == '~' # & | ^ ~ } + # the vocabulary, written by `ludic-dev syntax` from `ludicc --emit-syntax` + # ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit function is_type_word(w: pointer) -> bool { - return (w == "int") or (w == "long") or (w == "fixed") or (w == "float") or (w == "double") or (w == "floats") or (w == "doubles") or (w == "countdown") or (w == "bool") or (w == "entity") or (w == "string") or (w == "pointer") or (w == "byte") or (w == "words") or (w == "fixeds") or (w == "pointers") or (w == "Vector") or (w == "IVec2") or (w == "Rect") or (w == "void") - } - function is_phase_word(w: pointer) -> bool { - return (w == "Start") or (w == "Input") or (w == "FixedUpdate") or (w == "Update") or (w == "LateUpdate") or (w == "Render") - } - function is_keyword_word(w: pointer) -> bool { - if (w == "program") or (w == "import") or (w == "property") or (w == "model") or (w == "enum") or (w == "ui") { return true } - if (w == "const") or (w == "var") or (w == "function") or (w == "extern") or (w == "handler") or (w == "entry") or (w == "event") or (w == "scene") or (w == "mut") { return true } - if (w == "phase") or (w == "query") or (w == "on") or (w == "cancellable") or (w == "public") or (w == "layer") or (w == "start") { return true } - if (w == "let") or (w == "return") or (w == "if") or (w == "else") or (w == "while") or (w == "for") or (w == "in") or (w == "spawn") or (w == "despawn") { return true } - if (w == "enable") or (w == "disable") or (w == "match") or (w == "machine") or (w == "state") or (w == "become") or (w == "where") or (w == "prefab") { return true } - if (w == "and") or (w == "or") or (w == "not") or (w == "break") or (w == "continue") or (w == "new") or (w == "emit") or (w == "cancel") { return true } + if (w == "int") or (w == "long") or (w == "float") or (w == "double") or (w == "fixed") or (w == "bool") { return true } + if (w == "byte") or (w == "string") or (w == "pointer") or (w == "void") or (w == "entity") or (w == "countdown") { return true } + if (w == "words") or (w == "floats") or (w == "doubles") or (w == "fixeds") or (w == "pointers") or (w == "Vector") { return true } + if (w == "IVec2") or (w == "Rect") { return true } return false } + function is_phase_word(w: pointer) -> bool { + if (w == "Start") or (w == "Input") or (w == "FixedUpdate") or (w == "Update") or (w == "LateUpdate") or (w == "Render") { return true } + if (w == "Overlay") { return true } + return false + } + function is_keyword_word(w: pointer) -> bool { + if (w == "program") or (w == "module") or (w == "import") or (w == "property") or (w == "state") or (w == "event") { return true } + if (w == "action") or (w == "reducer") or (w == "enum") or (w == "model") or (w == "prefab") or (w == "scene") { return true } + if (w == "handler") or (w == "ui") or (w == "namespace") or (w == "alias") or (w == "port") or (w == "bind") { return true } + if (w == "registry") or (w == "def") or (w == "component") or (w == "view") or (w == "prop") or (w == "const") { return true } + if (w == "var") or (w == "function") or (w == "fn") or (w == "extern") or (w == "entry") or (w == "test") { return true } + if (w == "export") or (w == "internal") or (w == "friend") or (w == "uses") or (w == "open") or (w == "of") { return true } + if (w == "as") or (w == "from") or (w == "numbers") or (w == "unsafe") or (w == "mut") or (w == "phase") { return true } + if (w == "query") or (w == "on") or (w == "cancellable") or (w == "public") or (w == "layer") or (w == "start") { return true } + if (w == "shows") or (w == "lasts") or (w == "then") or (w == "loads") or (w == "system") or (w == "let") { return true } + if (w == "if") or (w == "else") or (w == "while") or (w == "for") or (w == "in") or (w == "where") { return true } + if (w == "match") or (w == "machine") or (w == "become") or (w == "return") or (w == "break") or (w == "continue") { return true } + if (w == "new") or (w == "spawn") or (w == "despawn") or (w == "enable") or (w == "disable") or (w == "attach") { return true } + if (w == "detach") or (w == "emit") or (w == "cancel") or (w == "dispatch") or (w == "try") or (w == "and") { return true } + if (w == "or") or (w == "not") { return true } + return false + } + # ludic-dev syntax: end function is_widget_word(w: pointer) -> bool { return (w == "panel") or (w == "col") or (w == "row") or (w == "label") or (w == "button") or (w == "image") or (w == "spacer") } diff --git a/tools/ludic-tools/lsp/types.ludic b/tools/ludic-tools/lsp/types.ludic index 464c6e21..a95dd9bd 100644 --- a/tools/ludic-tools/lsp/types.ludic +++ b/tools/ludic-tools/lsp/types.ludic @@ -224,13 +224,23 @@ function on_type_definition(msg: JVal, id: JVal) -> void { # `model: Model` and `a.model` are all ordinary names. A word is a keyword only # where it could start or continue a declaration. +# a keyword that may stand as a name where one is written (written by `ludic-dev syntax`) +# ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit function is_contextual_word(w: pointer) -> bool { - if (w == "program") or (w == "import") or (w == "property") or (w == "model") or (w == "enum") or (w == "ui") { return true } - if (w == "function") or (w == "extern") or (w == "handler") or (w == "entry") or (w == "event") or (w == "scene") { return true } - if (w == "phase") or (w == "query") or (w == "on") or (w == "cancellable") or (w == "public") or (w == "layer") or (w == "start") { return true } - if (w == "state") or (w == "machine") or (w == "prefab") or (w == "namespace") or (w == "test") { return true } + if (w == "program") or (w == "module") or (w == "import") or (w == "property") or (w == "state") or (w == "event") { return true } + if (w == "action") or (w == "reducer") or (w == "enum") or (w == "model") or (w == "prefab") or (w == "scene") { return true } + if (w == "handler") or (w == "ui") or (w == "namespace") or (w == "alias") or (w == "port") or (w == "bind") { return true } + if (w == "registry") or (w == "def") or (w == "component") or (w == "view") or (w == "prop") or (w == "const") { return true } + if (w == "var") or (w == "function") or (w == "fn") or (w == "extern") or (w == "entry") or (w == "test") { return true } + if (w == "export") or (w == "internal") or (w == "friend") or (w == "uses") or (w == "open") or (w == "of") { return true } + if (w == "as") or (w == "from") or (w == "numbers") or (w == "unsafe") or (w == "mut") or (w == "phase") { return true } + if (w == "query") or (w == "on") or (w == "cancellable") or (w == "public") or (w == "layer") or (w == "start") { return true } + if (w == "shows") or (w == "lasts") or (w == "then") or (w == "loads") or (w == "system") or (w == "in") { return true } + if (w == "where") or (w == "enable") or (w == "disable") or (w == "attach") or (w == "detach") or (w == "emit") { return true } + if (w == "dispatch") or (w == "try") { return true } return false } +# ludic-dev syntax: end function demote_contextual_keywords(D: Doc) -> void { let N = ntok(D) diff --git a/tools/ludic-tools/ludic_syntax.h b/tools/ludic-tools/ludic_syntax.h index 6ec194ac..d6a7cf73 100644 --- a/tools/ludic-tools/ludic_syntax.h +++ b/tools/ludic-tools/ludic_syntax.h @@ -48,34 +48,48 @@ typedef struct { } LLex; /* ---------- the vocabulary ------------------------------------------------*/ -/* Reserved words, grouped so editors can colour them differently. The groups - * mirror the compiler's parser: anything parse_decl() dispatches on is a - * declaration keyword, anything stmt() dispatches on is a statement keyword. */ -static const char* LUDIC_KW_DECL[] = { - "program","import","property","model","enum","ui","namespace", - "const","var","function","extern","handler","entry","event","scene","test", 0 -}; -static const char* LUDIC_KW_CLAUSE[] = { - "phase","query","on","cancellable","public","layer","start", 0 -}; +/* Keywords by role (declaring, modifying, statement), the built-in types, the + * phases and the attribute names: one vocabulary, printed by the compiler + * (`ludicc --emit-syntax`, selfhost/frontend/vocab.ludic, which is held to the + * parser's own recognisers) and written between the marked lines below by + * `ludic-dev syntax`; `ludic-dev syntax --check` fails when they fall behind. */ /* Documented design targets the self-hosted parser does not accept yet. Kept * out of the highlighted vocabulary (they would read as working keywords) until - * they are implemented; check-vocabulary.py verifies the lists above are a - * subset of what selfhost/parse*.ludic actually dispatches on. */ + * they are implemented. */ static const char* LUDIC_KW_RESERVED[] = { 0 }; +/* ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit */ +static const char* LUDIC_KW_DECL[] = { + "program","module","import","property","state","event","action","reducer","enum","model", + "prefab","scene","handler","ui","namespace","alias","port","bind","registry","def","component", + "view","prop","const","var","function","fn","extern","entry","test", 0 +}; +static const char* LUDIC_KW_CLAUSE[] = { + "export","internal","friend","uses","open","of","as","from","numbers","unsafe","mut","phase", + "query","on","cancellable","public","layer","start","shows","lasts","then","loads","system", 0 +}; static const char* LUDIC_KW_STMT[] = { - "let","return","if","else","while","for","in","spawn","despawn", - "enable","disable","match","machine","state","become","where","prefab", - "and","or","not","break","continue","new","emit","cancel","try", 0 + "let","if","else","while","for","in","where","match","machine","become","return","break", + "continue","new","spawn","despawn","enable","disable","attach","detach","emit","cancel", + "dispatch","try","and","or","not", 0 }; static const char* LUDIC_TYPES[] = { - "int","long","fixed","float","double","countdown","bool","entity","string","pointer","byte","words","fixeds","floats","doubles","pointers","Vector","IVec2","Rect","void", 0 + "int","long","float","double","fixed","bool","byte","string","pointer","void","entity", + "countdown","words","floats","doubles","fixeds","pointers","Vector","IVec2","Rect", 0 }; static const char* LUDIC_PHASES[] = { "Start","Input","FixedUpdate","Update","LateUpdate","Render","Overlay", 0 }; +static const char* LUDIC_ATTRIBUTES[] = { + "Ref","OneOf","Range","Unit","Asset","Color","Node","Clip","Material","Tint","Derived","Text", + "Multiline","Key","Computed","Sync","frame","max","owns","AppendOnly","ByKey","PerMap", + "Chunked","export","Public","On","Queries","OnSpawn","OnDespawn","OnAttach","OnDetach", + "OnEnable","OnDisable","OnStart","OnQuit","Owned","Server","Predicted","ToServer","ToClients", + "System","EngineSystem","Namespace","deterministic","ClearColor","alloc_ok","creates", + "releases","Handles","Reads","Writes", 0 +}; +/* ludic-dev syntax: end */ static const char* LUDIC_WIDGETS[] = { "panel","col","row","label","button","image","spacer", 0 };