feat(test): ludic test <dir>, each test block in a process of its own

A directory stands for its *_test.ludic files and the files straight
inside any tests/ under it. The runner answers --list and runs one test
by name, and ludic test runs every block in a child process, so tests
no longer share globals. A failed expect names the file it is written
in (the path the compiler was given) and its line. Reseed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

LANGUAGE.md

@@ -1775,11 +1775,27 @@ ludic run                                # compile src/main.ludic and run it
 ludic build --headless                   # headless build (renders out.ppm; reads stdin)
 ludic test                               # compile and run the project's `test` blocks
 ludic test tests/math.ludic --test adds  # just the test named "adds" (-v: every result line)
+ludic test packages/ludic.base          # the test programs under a directory (a package's)
 
 ludicc app.ludic -o build/app            # the compiler directly: a native binary
 ludicc app.ludic --emit-llvm -o app.ll   # stop at LLVM IR
 ```
 
+A test program is a file of `test "name" { ... }` blocks with `expect(cond)`, `expect_eq(a, b)` and
+`expect_near(a, b, tol)` in them. `ludic test` finds `tests/*.ludic` and `src/**/*_test.ludic`;
+given a directory, it runs every `*_test.ludic` under it and every file straight inside a `tests/`
+directory under it. **Each test block runs in a process of its own**, so a global one test changes
+is back to its initial value in the next - no test depends on another having run, or not. A
+failed assertion names its file, as the compiler was given it, and its line:
+
+```
+pk/src/sums_test.ludic:6: expect_eq failed (got 4, want 5)
+FAIL - wrong
+```
+
+A test program's runner takes a test's name as its one argument, and `--list` to name them all -
+which is how `ludic test` runs them one at a time.
+
 `ludic` is the CLI (`ludic help`); `ludicc` is the compiler it drives, built from
 the IR seed by `bin/ludic-dev build-cli`. **[COMPILING.md](COMPILING.md) is the
 authoritative CLI reference** — the full flag set (`-o`, `--windowed`,

changes/test-per-process.md

@@ -0,0 +1,7 @@
+bump: minor
+type: feature
+**`ludic test <dir>`, and every test block in a fresh process.** A directory stands for the
+`*_test.ludic` files under it and the files straight inside any `tests/` directory under it. Each
+`test` block now runs in a child process of its own (the runner answers `--list` and runs one test
+by name), so tests no longer share globals; a file's line says how many tests it ran. A failed
+`expect` names the file it is written in, not only the base name of the program's root.

selfhost/backend/emit_call.ludic

@@ -23,6 +23,12 @@ function emit_expect_fail(cond1: pointer, msgsym: pointer, got: pointer, want: p
   emit(`{lok}:\n`)
 }
 
+# an assertion names the file it is written in, as the compiler was given it, and its line
+function expect_where(e: Node) -> pointer {
+  if e.file != null { return `{e.file}:{itoa(e.line)}` }
+  return `{g_src_name}:{itoa(e.line)}`
+}
+
 # lowercase an ASCII identifier (for #62 package-namespace aliasing: Foo -> foo)
 function ns_lower(s: pointer) -> pointer {
   let n = cstr_len(s)
@@ -673,7 +679,7 @@ function emit_call(e: Node) -> Val {
     g_uses_expect = true
     let a = emit_expr(e.kids[0])
     let c = emit_bind(`icmp ne i32 {a.code}, 0`)
-    let msg = emit_str_const(`{g_src_name}:{itoa(e.line)}: expect failed`)
+    let msg = emit_str_const(`{expect_where(e)}: expect failed`)
     emit_expect_fail(c, msg, "", "")
     return val("0", "void")
   }
@@ -681,7 +687,7 @@ function emit_call(e: Node) -> Val {
     g_uses_expect = true
     let a = emit_expr(e.kids[0]);  let b = emit_expr(e.kids[1])
     let c = emit_bind(`icmp eq i32 {a.code}, {b.code}`)
-    let msg = emit_str_const(`{g_src_name}:{itoa(e.line)}: expect_eq failed`)
+    let msg = emit_str_const(`{expect_where(e)}: expect_eq failed`)
     emit_expect_fail(c, msg, a.code, b.code)
     return val("0", "void")
   }
@@ -693,7 +699,7 @@ function emit_call(e: Node) -> Val {
     let isneg = emit_bind(`icmp slt i32 {d}, 0`)
     let ad = emit_bind(`select i1 {isneg}, i32 {neg}, i32 {d}`)
     let c = emit_bind(`icmp sle i32 {ad}, {tol.code}`)
-    let msg = emit_str_const(`{g_src_name}:{itoa(e.line)}: expect_near failed`)
+    let msg = emit_str_const(`{expect_where(e)}: expect_near failed`)
     emit_expect_fail(c, msg, a.code, b.code)
     return val("0", "void")
   }

selfhost/backend/emit_decl.ludic

@@ -125,6 +125,26 @@ function emit_test_runner() -> void {
   emit(`  store ptr {fval}, ptr %filter\n`)
   emit(`  br label %{lstart}\n`)
   emit(`{lstart}:\n`)
+  # `--list` names every test, one a line: `ludic test` runs each in a process of its own
+  let llist = lbl("tlist");  let lrun0 = lbl("trun0")
+  let fl0 = emit_bind("load ptr, ptr %filter")
+  let hasf0 = emit_bind(`icmp ne ptr {fl0}, null`)
+  let lcmp0 = lbl("tlcmp")
+  emit(`  br i1 {hasf0}, label %{lcmp0}, label %{lrun0}\n`)
+  emit(`{lcmp0}:\n`)
+  let listflag = emit_str_const("--list")
+  let lc = emit_bind(`call i32 @strcmp(ptr {fl0}, ptr {listflag})`)
+  let islist = emit_bind(`icmp eq i32 {lc}, 0`)
+  emit(`  br i1 {islist}, label %{llist}, label %{lrun0}\n`)
+  emit(`{llist}:\n`)
+  var li = 0
+  while li < len(g_tests) {
+    let nm = emit_str_const(g_tests[li].s)
+    emit(`  call i32 (ptr, ...) @printf(ptr @.fmt_line, ptr {nm})\n`)
+    li += 1
+  }
+  emit("  ret i32 0\n")
+  emit(`{lrun0}:\n`)
   if has_ecs() { emit("  call void @rt_init()\n") }
   emit("  call void @L_init_globals()\n")
   i = 0

tools/ludic-cli/project.ludic

@@ -350,10 +350,17 @@ function test_files() -> []pointer {
   return files
 }
 
-# ludic test [file...] [--verbose] [--test NAME] — compile each test program
+# the test programs under a directory - a package's: every *_test.ludic, and every file straight
-# headlessly and run it. A test program's own runner prints ok/FAIL per test
+# inside a tests/ directory, anywhere under it
-# block and exits non-zero if any failed, so this reports one line per file and
+function test_files_in(dir: pointer) -> []pointer {
-# forwards the failure.
+  let q = sh_single(dir)
+  return split_lines(capture(`{{ find {q} -name '*_test.ludic' -type f; find {q} -path '*/tests/*.ludic' -type f | grep -v '/tests/.*/'; }} 2>/dev/null | sort -u`))
+}
+
+# ludic test [file|dir...] [--verbose] [--test NAME] — compile each test program
+# headlessly and run every test block in a process of its own, so no test sees
+# what another left in a global. A directory stands for the test programs under
+# it. One line per file; a failure prints what its tests printed.
 #
 # --verbose streams every runner line, framed by `RUN <file>` ... `PASS|FAIL
 # <file>`: the shape an IDE test view parses. --test NAME runs only the test
@@ -373,9 +380,15 @@ function cmd_test() -> int {
     }
     else if a[0] == '-' {
       err(`ludic test: unknown option {a}\n`)
-      err("  usage: ludic test [file...] [--verbose] [--test NAME]\n")
+      err("  usage: ludic test [file|dir...] [--verbose] [--test NAME]\n")
       return 1
     }
+    else if shq(`test -d {sh_single(a)}`) {
+      let found = test_files_in(a)
+      if len(found) == 0 { err(`ludic test: no tests under {a} (expected *_test.ludic or tests/*.ludic)\n`);  return 1 }
+      var k = 0
+      while k < len(found) { push(files, found[k]);  k += 1 }
+    }
     else { push(files, a) }
     ai += 1
   }
@@ -390,9 +403,8 @@ function cmd_test() -> int {
     mi += 1
   }
   shell("mkdir -p build")
-  var filter = ""
-  if only != "" { filter = ` {sh_single(only)}` }
   var failed = 0
+  var ntests = 0
   var i = 0
   while i < len(files) {
     let f = files[i]
@@ -402,20 +414,21 @@ function cmd_test() -> int {
       let line = `  {c_red()}FAIL{c_reset()} {f}  (did not compile)`
       if verbose { say(line) } else { print(line) }
       failed += 1
-    } else if verbose {
-      let rc = sh(`{bin}{filter} < /dev/null 2>&1`)
-      if rc == 0 { say(`  {c_green()}PASS{c_reset()} {f}`) }
-      else { say(`  {c_red()}FAIL{c_reset()} {f}  (exit {string(rc)})`);  failed += 1 }
     } else {
-      # a test program prints one line per test block; that belongs on screen
+      let r = test_run_file(bin, only, verbose)
-      # when something failed and nowhere when everything passed.
+      ntests += g_tr_ran
-      let log = `{bin}.out`
+      var what = `({string(g_tr_ran)} tests)`
-      let rc = sh(`{bin}{filter} < /dev/null > {log} 2>&1`)
+      if g_tr_ran == 1 { what = "(1 test)" }
-      if rc == 0 { print(`  {c_green()}PASS{c_reset()} {f}`) }
+      if r == 0 {
-      else {
+        let line = `  {c_green()}PASS{c_reset()} {f}  {what}`
-        print(`  {c_red()}FAIL{c_reset()} {f}  (exit {string(rc)})`)
+        if verbose { say(line) } else { print(line) }
-        let txt = read_file(log)
+      } else {
-        if txt != null { out(txt) }
+        var line = `  {c_red()}FAIL{c_reset()} {f}  ({string(g_tr_bad)} of {string(g_tr_ran)} tests failed)`
+        if g_tr_ran == 0 { line = `  {c_red()}FAIL{c_reset()} {f}  (no test named "{only}")` }
+        if verbose { say(line) } else {
+          print(line)
+          out(g_tr_log)
+        }
         failed += 1
       }
     }
@@ -423,13 +436,56 @@ function cmd_test() -> int {
   }
   print("")
   if failed == 0 {
-    print(`== {string(len(files))} test files passed ==`)
+    print(`== {string(len(files))} test files passed ({string(ntests)} tests) ==`)
     return 0
   }
   print(`== {string(failed)} of {string(len(files))} test files failed ==`)
   return 1
 }
 
+var g_tr_ran: int = 0
+var g_tr_bad: int = 0
+var g_tr_log: pointer = ""
+# every test block of one compiled test program, each in a child process of its own: the runner
+# names them with --list and runs just one when given its name. Non-zero when any failed (or
+# none matched --test).
+function test_run_file(bin: pointer, only: pointer, verbose: bool) -> int {
+  g_tr_ran = 0
+  g_tr_bad = 0
+  g_tr_log = ""
+  let names = split_lines(capture(`{bin} --list < /dev/null`))
+  var n = 0
+  while n < len(names) {
+    let nm = names[n]
+    n += 1
+    if nm == "" { continue }
+    if only != "" and not (nm == only) { continue }
+    g_tr_ran += 1
+    let log = `{bin}.out`
+    let rc = sh(`{bin} {sh_single(nm)} < /dev/null > {log} 2>&1`)
+    # the child's own summary line counts one test; the file's line says it better
+    let txt = capture(`grep -v '^== ' {log}`)
+    if verbose { shell(`grep -v '^== ' {log}`) }
+    if rc != 0 {
+      g_tr_bad += 1
+      g_tr_log = g_tr_log + txt
+      var ends = false
+      if len(txt) > 0 { ends = txt[len(txt) - 1] == 10 }
+      if not ends { g_tr_log = g_tr_log + "\n" }
+      if not s_contains(txt, "FAIL - ") {
+        g_tr_log = g_tr_log + `FAIL - {nm}  (exit {string(rc)})\n`
+      }
+    }
+  }
+  if only != "" and g_tr_ran == 0 {
+    g_tr_log = `no test named "{only}"\n`
+    if verbose { say(`no test named "{only}"`) }
+    return 1
+  }
+  if g_tr_bad > 0 { return 1 }
+  return 0
+}
+
 # print a line through the shell. The runner's own stdout is buffered, so a
 # print() made between two child processes would surface after both of them;
 # under --verbose the framing has to interleave with the runner output exactly.

tools/ludic-cli/test.ludic

@@ -435,6 +435,28 @@ function scaffold_names_case() -> void {
   ok(lbl)
 }
 
+# `ludic test <dir>`: the test programs under a package, each test block in a process of its own
+# (a global one test changes is fresh in the next), and a failed expect named by file:line
+function test_dir_case() -> void {
+  let lbl = "ludic test <dir>: *_test.ludic and tests/*.ludic, a process per test, expect at file:line"
+  let work = `{tmp_dir()}/testdir`
+  let root = capture_line("pwd")
+  shell(`rm -rf {work} && mkdir -p {work}/pk/tests {work}/pk/src/deep`)
+  var a = "program State {\n  var counter: int = 0\n  var seen: []int = new []int\n"
+  a = a + "  test \"first\" {\n    counter += 1\n    push(seen, 1)\n    expect_eq(counter, 1)\n    expect_eq(len(seen), 1)\n  }\n"
+  a = a + "  test \"second\" {\n    counter += 1\n    push(seen, 2)\n    expect_eq(counter, 1)\n    expect_eq(len(seen), 1)\n  }\n}\n"
+  write_file(`{work}/pk/tests/state.ludic`, a)
+  write_file(`{work}/pk/src/deep/sums_test.ludic`, "program Sums {\n  test \"adds\" {\n    expect_eq(2 + 2, 4)\n  }\n  test \"wrong\" {\n    expect_eq(2 + 2, 5)\n  }\n}\n")
+  write_file(`{work}/pk/src/helper.ludic`, "program NotATest {\n  entry { exit(3) }\n}\n")
+  if shq(`cd {work} && {root}/bin/ludic test pk > out.txt 2>&1`) { bad2(lbl, "a failing test passed");  return }
+  let got = capture(`cat {work}/out.txt`)
+  if not s_contains(got, "pk/src/deep/sums_test.ludic:6: expect_eq failed (got 4, want 5)") { bad2(lbl, `no file:line for the failure in [{s_trim(got)}]`);  return }
+  if not s_contains(got, "pk/tests/state.ludic  (2 tests)") { bad2(lbl, `the two tests shared a global: [{s_trim(got)}]`);  return }
+  if s_contains(got, "helper.ludic") { bad2(lbl, "a file that is not a test was run");  return }
+  if not shq(`cd {work} && {root}/bin/ludic test pk/tests > out2.txt 2>&1`) { bad2(lbl, capture_line(`tail -1 {work}/out2.txt`));  return }
+  ok(lbl)
+}
+
 # package.ludic drives the CLI: `entry` (or the one program under src/) is what
 # build compiles, scripts run by name, and hooks wrap commands — a failing
 # `before` stops the command, `after` runs only on success.
@@ -957,6 +979,7 @@ function cmd_dev_test() -> int {
   install_layout_case()
   scaffold_names_case()
   package_scripts_case()
+  test_dir_case()
   pack_roundtrip_case()
   packignore_case()
   os_dirs_case()