ludic syntax, and ludic-dev syntax: every grammar written from the compiler's vocabulary and checked against it

`ludic syntax [--json] [-o FILE]` prints what `ludicc --emit-syntax` does (a line
per entry, or the JSON). `ludic-dev syntax` writes, between "ludic-dev syntax:
begin" / "end" lines, the keyword, type, phase and attribute tables of
ludic_syntax.h, LudicVocabulary's sets (JetBrains), ludic-mode.el's lists and the
language server's word tests (is_keyword_word and the rest; is_contextual_word is
every word the parser does not reserve, and every declaring or modifying one),
and every TextMate pattern marked "comment": "ludic-dev syntax: <group>" (shared
and the VS Code copy). The grammars gain module uses port bind action reducer
dispatch registry def open component prop view alias friend unsafe numbers of as
from mut system; import and extern colour as declarations; the phase clause
knows Overlay; the bitwise pattern matches | and ^ on their own again.

`ludic-dev syntax --check` - and check-vocabulary, whose old parser comparison it
replaces, and the regression suite (syntax_cases, one line per file) - fails when
a written list is behind, when a grammar lacks a keyword, type or phase, when
docs/language has no page for a keyword, type, phase or attribute, or when the
parser (a scan of selfhost/frontend: is_id / text == words, a == / ann ==
attributes) tests a word or reads an attribute vocab.ludic lacks, or the reverse.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tools/editors/README.md

@@ -146,11 +146,21 @@ output. The formatter cannot change what a program means.
 
 ## Keeping it honest
 
-The vocabulary is written down in five places that cannot include each other —
-the compiler's two tables, `ludic_syntax.h`, the TextMate grammar (JSON), and the
-JetBrains lexer (Kotlin). Adding a builtin and forgetting the rest is silent
-failure, so `bin/ludic-dev check-vocabulary` (written in Ludic) compares all five, and
-`bin/ludic-dev test-tools` runs it.
+The language's words come from one place: the compiler's vocabulary table
+(`selfhost/frontend/vocab.ludic`), printed by `ludic syntax --json` (`ludicc
+--emit-syntax`) and held to the parser's own recognisers. The keyword, type and
+phase lists in `ludic_syntax.h`, the TextMate grammar (every pattern marked
+`"comment": "ludic-dev syntax: <group>"`), the JetBrains lexer's
+`LudicVocabulary`, `ludic-mode.el` and the language server are written from it
+between `ludic-dev syntax: begin` / `end` lines by `bin/ludic-dev syntax`; do not
+edit inside them.
 
-When you add a keyword or builtin: put it in `ludic_syntax.h`, then run
-`bin/ludic-dev test-tools` and let it tell you which copies still need it.
+`bin/ludic-dev syntax --check` (and `check-vocabulary`, which `test-tools` runs,
+and the regression suite) fails when a generated list is behind, when a grammar
+lacks a word, when docs/language has no page for a keyword, type, phase or
+attribute, or when the parser (`selfhost/frontend`) tests a word or reads an
+attribute the vocabulary lacks. Builtins are still listed in `ludic_syntax.h`,
+and `check-vocabulary` compares them with the grammar and the Kotlin lexer.
+
+When you add a keyword or an attribute: add its row to `vocab.ludic`, rebuild the
+compiler, run `bin/ludic-dev syntax`, and write its docs/language page.

tools/editors/emacs/ludic-mode.el

@@ -10,23 +10,27 @@
 
 ;;; Code:
 
-;; The vocabulary mirrors tools/ludic-tools/ludic_syntax.h; that header is the reference.
+;; The vocabulary is written by `ludic-dev syntax` from the compiler's own table
+;; (`ludicc --emit-syntax`) between the marked lines.
+;; ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit
 (defconst ludic--declaration-keywords
-  '("program" "import" "property" "model" "enum" "ui" "namespace"
-    "const" "var" "function" "extern" "handler" "entry" "event" "scene" "test" "prefab"))
-
+  '("program" "module" "import" "property" "state" "event" "action" "reducer" "enum" "model"
+    "prefab" "scene" "handler" "ui" "namespace" "alias" "port" "bind" "registry" "def"
+    "component" "view" "prop" "const" "var" "function" "fn" "extern" "entry" "test"))
 (defconst ludic--clause-keywords
-  '("phase" "query" "on" "cancellable" "public" "layer" "start" "shows" "lasts" "loads" "then" "export" "internal"))
-
+  '("export" "internal" "friend" "uses" "open" "of" "as" "from" "numbers" "unsafe" "mut" "phase"
+    "query" "on" "cancellable" "public" "layer" "start" "shows" "lasts" "then" "loads"
+    "system"))
 (defconst ludic--statement-keywords
-  '("let" "return" "if" "else" "while" "for" "in" "spawn" "despawn"
-    "enable" "disable" "attach" "detach" "match" "machine" "state" "become" "where"
-    "and" "or" "not" "break" "continue" "new" "emit" "cancel" "try"))
-
+  '("let" "if" "else" "while" "for" "in" "where" "match" "machine" "become" "return" "break"
+    "continue" "new" "spawn" "despawn" "enable" "disable" "attach" "detach" "emit" "cancel"
+    "dispatch" "try" "and" "or" "not"))
 (defconst ludic--types
-  '("int" "long" "fixed" "countdown" "bool" "entity" "string" "pointer" "byte"
-    "words" "fixeds" "pointers" "Vector" "IVec2" "Rect" "void"))
-(defconst ludic--phases '("Start" "Input" "FixedUpdate" "Update" "LateUpdate" "Render" "Overlay"))
+  '("int" "long" "float" "double" "fixed" "bool" "byte" "string" "pointer" "void" "entity"
+    "countdown" "words" "floats" "doubles" "fixeds" "pointers" "Vector" "IVec2" "Rect"))
+(defconst ludic--phases
+  '("Start" "Input" "FixedUpdate" "Update" "LateUpdate" "Render" "Overlay"))
+;; ludic-dev syntax: end
 
 (defconst ludic-font-lock-keywords
   (list

tools/editors/jetbrains/src/main/kotlin/io/ludic/ide/LudicTokens.kt

@@ -55,24 +55,37 @@ object LudicTokens {
 }
 
 /**
- * The vocabulary, mirroring tools/ludic-tools/ludic_syntax.h. When the language
- * grows a keyword both sides change together; that header is the reference.
+ * The vocabulary. The keyword, type and phase sets are written by `ludic-dev
+ * syntax` from the compiler's own table (`ludicc --emit-syntax`); the rest
+ * mirrors tools/ludic-tools/ludic_syntax.h.
  */
 object LudicVocabulary {
+    // ludic-dev syntax: begin - generated from `ludicc --emit-syntax` (selfhost/frontend/vocab.ludic); run `ludic-dev syntax`, do not edit
     val DECL = setOf(
-        "program", "import", "property", "model", "enum", "ui", "namespace",
-        "const", "var", "function", "extern", "handler", "entry", "event", "scene", "test"
+        "program", "module", "import", "property", "state", "event", "action", "reducer", "enum",
+        "model", "prefab", "scene", "handler", "ui", "namespace", "alias", "port", "bind",
+        "registry", "def", "component", "view", "prop", "const", "var", "function", "fn", "extern",
+        "entry", "test"
     )
     val CLAUSE = setOf(
-        "phase", "query", "on", "cancellable", "public", "layer", "start"
+        "export", "internal", "friend", "uses", "open", "of", "as", "from", "numbers", "unsafe",
+        "mut", "phase", "query", "on", "cancellable", "public", "layer", "start", "shows", "lasts",
+        "then", "loads", "system"
     )
     val STMT = setOf(
-        "let", "return", "if", "else", "while", "for", "in", "spawn", "despawn",
-        "enable", "disable", "match", "machine", "prefab", "state", "become", "where",
-        "and", "or", "not", "break", "continue", "new", "emit", "cancel", "try"
+        "let", "if", "else", "while", "for", "in", "where", "match", "machine", "become", "return",
+        "break", "continue", "new", "spawn", "despawn", "enable", "disable", "attach", "detach",
+        "emit", "cancel", "dispatch", "try", "and", "or", "not"
     )
-    val PRIMITIVES = setOf("int", "long", "fixed", "float", "double", "countdown", "bool", "entity", "string", "pointer", "byte", "words", "fixeds", "floats", "doubles", "pointers", "Vector", "IVec2", "Rect", "void")
-    val PHASES = setOf("Start", "Input", "FixedUpdate", "Update", "LateUpdate", "Render", "Overlay")
+    val PRIMITIVES = setOf(
+        "int", "long", "float", "double", "fixed", "bool", "byte", "string", "pointer", "void",
+        "entity", "countdown", "words", "floats", "doubles", "fixeds", "pointers", "Vector",
+        "IVec2", "Rect"
+    )
+    val PHASES = setOf(
+        "Start", "Input", "FixedUpdate", "Update", "LateUpdate", "Render", "Overlay"
+    )
+    // ludic-dev syntax: end
     val WIDGETS = setOf("panel", "col", "row", "label", "button", "image", "spacer")
 
     val BUILTINS = setOf(

tools/editors/shared/ludic.tmLanguage.json

@@ -3,7 +3,7 @@
   "name": "Ludic",
   "scopeName": "source.ludic",
   "fileTypes": ["ludic"],
-  "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. Generated shapes are kept in sync with tools/ludic-tools/ludic_syntax.h.",
+  "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. The keyword, type and phase alternations are written by `ludic-dev syntax` from the compiler's own vocabulary (`ludicc --emit-syntax`): every pattern marked \"ludic-dev syntax: <group>\".",
   "patterns": [
     { "include": "#comment" },
     { "include": "#declaration" },
@@ -126,7 +126,7 @@
           "captures": { "1": { "name": "keyword.control.import.ludic" } }
         },
         {
-          "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render)\\b",
+          "comment": "ludic-dev syntax: phase-clause", "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b",
           "captures": {
             "1": { "name": "keyword.other.clause.ludic" },
             "2": { "name": "constant.language.phase.ludic" }
@@ -180,14 +180,13 @@
     },
     "keyword": {
       "patterns": [
-        { "name": "keyword.control.ludic", "match": "\\b(if|else|while|for|in|match|machine|become|return|spawn|despawn|enable|disable|attach|detach|emit|cancel|where|break|continue|new|try)\\b" },
-        { "name": "keyword.operator.logical.ludic", "match": "\\b(and|or|not)\\b" },
-        { "name": "keyword.other.clause.ludic", "match": "\\b(phase|query|on|cancellable|public|layer|start|shows|lasts|loads|then|export|internal)\\b" },
-        { "name": "keyword.other.ludic", "match": "\\b(import|extern)\\b" },
-        { "name": "storage.type.ludic", "match": "\\b(program|property|model|prefab|namespace|enum|ui|const|var|let|function|fn|handler|entry|event|scene|state|test)\\b" },
-        { "name": "support.type.primitive.ludic", "match": "\\b(int|long|fixed|float|double|countdown|bool|entity|string|pointer|byte|words|fixeds|floats|doubles|pointers|Vector|IVec2|Rect|void)\\b" },
-        { "name": "constant.language.boolean.ludic", "match": "\\b(true|false|null)\\b" },
-        { "name": "constant.language.phase.ludic", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" }
+        { "name": "keyword.control.ludic", "comment": "ludic-dev syntax: statement", "match": "\\b(let|if|else|while|for|in|where|match|machine|become|return|break|continue|new|spawn|despawn|enable|disable|attach|detach|emit|cancel|dispatch|try)\\b" },
+        { "name": "keyword.operator.logical.ludic", "comment": "ludic-dev syntax: operator", "match": "\\b(and|or|not)\\b" },
+        { "name": "keyword.other.clause.ludic", "comment": "ludic-dev syntax: modifier", "match": "\\b(export|internal|friend|uses|open|of|as|from|numbers|unsafe|mut|phase|query|on|cancellable|public|layer|start|shows|lasts|then|loads|system)\\b" },
+        { "name": "storage.type.ludic", "comment": "ludic-dev syntax: declaration", "match": "\\b(program|module|import|property|state|event|action|reducer|enum|model|prefab|scene|handler|ui|namespace|alias|port|bind|registry|def|component|view|prop|const|var|function|fn|extern|entry|test)\\b" },
+        { "name": "support.type.primitive.ludic", "comment": "ludic-dev syntax: types", "match": "\\b(int|long|float|double|fixed|bool|byte|string|pointer|void|entity|countdown|words|floats|doubles|fixeds|pointers|Vector|IVec2|Rect)\\b" },
+        { "name": "constant.language.boolean.ludic", "comment": "ludic-dev syntax: constant", "match": "\\b(true|false|null)\\b" },
+        { "name": "constant.language.phase.ludic", "comment": "ludic-dev syntax: phases", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" }
       ]
     },
     "widget": {
@@ -241,7 +240,7 @@
       "patterns": [
         { "name": "keyword.operator.arrow.ludic", "match": "->|=>" },
         { "name": "keyword.operator.range.ludic", "match": "\\.\\." },
-        { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\|\\^|~" },
+        { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\||\\^|~" },
         { "name": "keyword.operator.comparison.ludic", "match": "==|!=|<=|>=|<|>" },
         { "name": "keyword.operator.assignment.ludic", "match": "\\+=|-=|\\*=|/=|=" },
         { "name": "keyword.operator.arithmetic.ludic", "match": "[+\\-*/%]" }

tools/editors/vscode/syntaxes/ludic.tmLanguage.json

@@ -3,7 +3,7 @@
   "name": "Ludic",
   "scopeName": "source.ludic",
   "fileTypes": ["ludic"],
-  "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. Generated shapes are kept in sync with tools/ludic-tools/ludic_syntax.h.",
+  "comment": "Static grammar for Ludic. It is the fallback: when ludic-lsp is running, its semantic tokens refine everything here with real name resolution. The keyword, type and phase alternations are written by `ludic-dev syntax` from the compiler's own vocabulary (`ludicc --emit-syntax`): every pattern marked \"ludic-dev syntax: <group>\".",
   "patterns": [
     { "include": "#comment" },
     { "include": "#declaration" },
@@ -126,7 +126,7 @@
           "captures": { "1": { "name": "keyword.control.import.ludic" } }
         },
         {
-          "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render)\\b",
+          "comment": "ludic-dev syntax: phase-clause", "match": "\\b(phase)\\s+(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b",
           "captures": {
             "1": { "name": "keyword.other.clause.ludic" },
             "2": { "name": "constant.language.phase.ludic" }
@@ -180,14 +180,13 @@
     },
     "keyword": {
       "patterns": [
-        { "name": "keyword.control.ludic", "match": "\\b(if|else|while|for|in|match|machine|become|return|spawn|despawn|enable|disable|attach|detach|emit|cancel|where|break|continue|new|try)\\b" },
-        { "name": "keyword.operator.logical.ludic", "match": "\\b(and|or|not)\\b" },
-        { "name": "keyword.other.clause.ludic", "match": "\\b(phase|query|on|cancellable|public|layer|start|shows|lasts|loads|then|export|internal)\\b" },
-        { "name": "keyword.other.ludic", "match": "\\b(import|extern)\\b" },
-        { "name": "storage.type.ludic", "match": "\\b(program|property|model|prefab|namespace|enum|ui|const|var|let|function|fn|handler|entry|event|scene|state|test)\\b" },
-        { "name": "support.type.primitive.ludic", "match": "\\b(int|long|fixed|float|double|countdown|bool|entity|string|pointer|byte|words|fixeds|floats|doubles|pointers|Vector|IVec2|Rect|void)\\b" },
-        { "name": "constant.language.boolean.ludic", "match": "\\b(true|false|null)\\b" },
-        { "name": "constant.language.phase.ludic", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" }
+        { "name": "keyword.control.ludic", "comment": "ludic-dev syntax: statement", "match": "\\b(let|if|else|while|for|in|where|match|machine|become|return|break|continue|new|spawn|despawn|enable|disable|attach|detach|emit|cancel|dispatch|try)\\b" },
+        { "name": "keyword.operator.logical.ludic", "comment": "ludic-dev syntax: operator", "match": "\\b(and|or|not)\\b" },
+        { "name": "keyword.other.clause.ludic", "comment": "ludic-dev syntax: modifier", "match": "\\b(export|internal|friend|uses|open|of|as|from|numbers|unsafe|mut|phase|query|on|cancellable|public|layer|start|shows|lasts|then|loads|system)\\b" },
+        { "name": "storage.type.ludic", "comment": "ludic-dev syntax: declaration", "match": "\\b(program|module|import|property|state|event|action|reducer|enum|model|prefab|scene|handler|ui|namespace|alias|port|bind|registry|def|component|view|prop|const|var|function|fn|extern|entry|test)\\b" },
+        { "name": "support.type.primitive.ludic", "comment": "ludic-dev syntax: types", "match": "\\b(int|long|float|double|fixed|bool|byte|string|pointer|void|entity|countdown|words|floats|doubles|fixeds|pointers|Vector|IVec2|Rect)\\b" },
+        { "name": "constant.language.boolean.ludic", "comment": "ludic-dev syntax: constant", "match": "\\b(true|false|null)\\b" },
+        { "name": "constant.language.phase.ludic", "comment": "ludic-dev syntax: phases", "match": "\\b(Start|Input|FixedUpdate|Update|LateUpdate|Render|Overlay)\\b" }
       ]
     },
     "widget": {
@@ -241,7 +240,7 @@
       "patterns": [
         { "name": "keyword.operator.arrow.ludic", "match": "->|=>" },
         { "name": "keyword.operator.range.ludic", "match": "\\.\\." },
-        { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\|\\^|~" },
+        { "name": "keyword.operator.bitwise.ludic", "match": "<<|>>|&|\\||\\^|~" },
         { "name": "keyword.operator.comparison.ludic", "match": "==|!=|<=|>=|<|>" },
         { "name": "keyword.operator.assignment.ludic", "match": "\\+=|-=|\\*=|/=|=" },
         { "name": "keyword.operator.arithmetic.ludic", "match": "[+\\-*/%]" }