diff --git a/LANGUAGE.md b/LANGUAGE.md index ca4ffae8..d08b473b 100644 --- a/LANGUAGE.md +++ b/LANGUAGE.md @@ -1827,6 +1827,7 @@ ludic run # compile src/main.ludic and run it ludic build --headless # headless build (renders out.ppm; reads stdin) ludic test # compile and run the project's `test` blocks ludic test tests/math.ludic --test adds # just the test named "adds" (-v: every result line) +ludic test -j 4 # four tests at once (default: one per CPU) ludic test packages/ludic.base # the test programs under a directory (a package's) ludic deps # how tangled the modules are, as the compiler resolved them ludic deps --check tests/deps-baseline.txt # fail when a number rose (--baseline FILE writes them) @@ -1861,7 +1862,10 @@ FAIL - wrong ``` A test program's runner takes a test's name as its one argument, and `--list` to name them all - -which is how `ludic test` runs them one at a time. +which is how `ludic test` runs them one at a time. Because every test is its own process they run side +by side: `ludic test -j N` runs N at once (the machine's CPU count by default; `-j 1` one after the +other), each with a `TMPDIR` - and so an `Os.temp_dir()` - of its own, and the report comes out in file +order as a sequential run's does. `ludic` is the CLI (`ludic help`); `ludicc` is the compiler it drives, built from the IR seed by `bin/ludic-dev build-cli`. **[COMPILING.md](COMPILING.md) is the diff --git a/changes/test-parallel.md b/changes/test-parallel.md new file mode 100644 index 00000000..0db3faa1 --- /dev/null +++ b/changes/test-parallel.md @@ -0,0 +1,5 @@ +bump: minor +type: feature +**`ludic test -j N` runs tests side by side.** Every file is compiled, linked and every test block +run as up to N jobs at once (default: the machine's CPU count), each test with a `TMPDIR` of its own, +and the report comes out in file order. `ludic test packages` takes about half the time. diff --git a/tools/ludic-cli/deps.ludic b/tools/ludic-cli/deps.ludic index 3d938784..1550619e 100644 --- a/tools/ludic-cli/deps.ludic +++ b/tools/ludic-cli/deps.ludic @@ -87,15 +87,6 @@ function dp_load(path: pointer) -> bool { } return true } -function s_to_int(s: pointer) -> int { - var v = 0 - var i = 0 - while s[i] >= 48 and s[i] <= 57 { - v = v * 10 + (s[i] - 48) - i += 1 - } - return v -} function dp_own(k: int) -> bool { return dp_pkg[k] == 0 } # an edge the numbers count: between two of the program's own modules function dp_counted(e: int) -> bool { return dp_own(dp_ef[e]) and dp_own(dp_et[e]) } diff --git a/tools/ludic-cli/main.ludic b/tools/ludic-cli/main.ludic index 3bb91ed3..c067f4a5 100644 --- a/tools/ludic-cli/main.ludic +++ b/tools/ludic-cli/main.ludic @@ -30,6 +30,7 @@ program Ludic { import "bundle.ludic" import "scripts.ludic" import "deps.ludic" + import "testpar.ludic" function usage() -> void { print("ludic — the toolchain for the Ludic language") @@ -41,7 +42,7 @@ program Ludic { print(" run [file] [--headless] build the project and run it") print(" build [file] [--headless] [-o out]") print(" compile to build/ (a native binary, nothing to ship beside it)") - print(" test [file...] [-v] [--test NAME]") + print(" test [file|dir...] [-j N] [-v] [--test NAME]") print(" compile and run the project's tests (-v: every test's line)") print(" deps [file] [--graph|--dot|--writes|--uses MOD|--check F|--baseline F]") print(" the module graph as the compiler sees it, and how tangled it is") diff --git a/tools/ludic-cli/prelude.ludic b/tools/ludic-cli/prelude.ludic index c0223792..51c22f47 100644 --- a/tools/ludic-cli/prelude.ludic +++ b/tools/ludic-cli/prelude.ludic @@ -458,3 +458,13 @@ function cc() -> pointer { } return "clang -Wno-override-module" } +# the leading decimal digits of `s` as an int (0 for none) +function s_to_int(s: pointer) -> int { + var v = 0 + var i = 0 + while s[i] >= 48 and s[i] <= 57 { + v = v * 10 + (s[i] - 48) + i += 1 + } + return v +} diff --git a/tools/ludic-cli/project.ludic b/tools/ludic-cli/project.ludic index 76e6fc12..e946f91a 100644 --- a/tools/ludic-cli/project.ludic +++ b/tools/ludic-cli/project.ludic @@ -373,6 +373,16 @@ function cmd_test() -> int { while ai < arg_count() { let a = arg(ai) if (a == "--verbose") or (a == "-v") { verbose = true } + else if a == "-j" or a == "--jobs" { + if ai + 1 >= arg_count() { err("ludic test: -j needs a number\n"); return 1 } + ai += 1 + g_test_jobs = s_to_int(arg(ai)) + if g_test_jobs < 1 { g_test_jobs = 1 } + } + else if s_starts(a, "-j") and slen(a) > 2 { + g_test_jobs = s_to_int(sslice(a, 2, slen(a))) + if g_test_jobs < 1 { g_test_jobs = 1 } + } else if a == "--test" { if ai + 1 >= arg_count() { err("ludic test: --test needs a test name\n"); return 1 } ai += 1 @@ -380,7 +390,7 @@ function cmd_test() -> int { } else if a[0] == '-' { err(`ludic test: unknown option {a}\n`) - err(" usage: ludic test [file|dir...] [--verbose] [--test NAME]\n") + err(" usage: ludic test [file|dir...] [-j N] [--verbose] [--test NAME]\n") return 1 } else if shq(`test -d {sh_single(a)}`) { @@ -403,6 +413,7 @@ function cmd_test() -> int { mi += 1 } shell("mkdir -p build") + if not host_windows() { return test_parallel(files, only, verbose) } var failed = 0 var ntests = 0 var i = 0 diff --git a/tools/ludic-cli/test.ludic b/tools/ludic-cli/test.ludic index dd95c93c..5a65117b 100644 --- a/tools/ludic-cli/test.ludic +++ b/tools/ludic-cli/test.ludic @@ -515,6 +515,15 @@ function test_dir_case() -> void { if s_contains(got, "helper.ludic") { bad2(lbl, "a file that is not a test was run"); return } if not shq(`cd {work} && {root}/bin/ludic test pk/tests > out2.txt 2>&1`) { bad2(lbl, capture_line(`tail -1 {work}/out2.txt`)); return } ok(lbl) + # -j: two tests at once, each writing one fixed scratch path - each has a TMPDIR of its own + let lbl2 = "ludic test -j 2: tests run side by side, each with its own Os.temp_dir()" + var t = "program Scratch {\n" + t = t + " function hold(v: string) -> bool {\n let p = Os.temp_dir() + \"/scratch.txt\"\n Fs.write_text(p, v)\n for i in 0 .. 3000 {\n if Fs.read_text(p) != v { return false }\n }\n return true\n }\n" + t = t + " test \"one\" {\n expect(hold(\"one\"))\n }\n test \"two\" {\n expect(hold(\"two\"))\n }\n}\n" + write_file(`{work}/pk/tests/scratch.ludic`, t) + if not shq(`cd {work} && {root}/bin/ludic test pk/tests -j 2 > out3.txt 2>&1`) { bad2(lbl2, capture(`cat {work}/out3.txt`)); return } + if not shq(`grep -q 'pk/tests/scratch.ludic (2 tests)' {work}/out3.txt`) { bad2(lbl2, capture(`cat {work}/out3.txt`)); return } + ok(lbl2) } # package.ludic drives the CLI: `entry` (or the one program under src/) is what diff --git a/tools/ludic-cli/testpar.ludic b/tools/ludic-cli/testpar.ludic new file mode 100644 index 00000000..3885f6c1 --- /dev/null +++ b/tools/ludic-cli/testpar.ludic @@ -0,0 +1,119 @@ +# ---- ludic test in parallel --------------------------------------------------- +# Each test block is a process of its own, so nothing stops them running side by side: every file +# is compiled, then linked, then every test of every file run, each stage as up to N shell jobs at +# once (xargs -P N; -j N, the machine's CPUs by default). What each wrote and how it exited are +# read back in file order, so the report is the one a sequential run gives. Each test gets a TMPDIR +# of its own, so Os.temp_dir() is never shared between two tests running at once. +var g_test_jobs: int = 0 +function test_jobs() -> int { + if g_test_jobs > 0 { return g_test_jobs } + var n = s_to_int(capture_line("(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null)")) + if n < 1 { n = 1 } + return n +} +# run every
*.sh under `dir`, up to N at once
+function test_stage(dir: pointer, pre: pointer) -> void {
+  shell(`ls {dir}/{pre}*.sh 2>/dev/null | xargs -P {string(test_jobs())} -n 1 sh`)
+}
+function test_rc(path: pointer) -> int {
+  let t = read_file(path)
+  if t == null { return 127 }
+  return s_to_int(s_trim(t))
+}
+function test_parallel(files: []pointer, only: pointer, verbose: bool) -> int {
+  let jd = `{tmp_dir()}/testjobs`
+  shell(`rm -rf {jd} && mkdir -p {jd}`)
+  ensure_ludicc()
+  let pbf = prebuilt_link_flags()
+  let bins = new []pointer
+  for i in 0 .. len(files) {
+    let bin = `{jd}/b{string(i)}_{flat(strip_ext(files[i]))}`
+    push(bins, bin)
+    write_file(`{jd}/c{string(i)}.sh`, `{ludicc()} --headless{title_flag()}{unsafe_flag()} {sh_single(files[i])} --emit-llvm -o {bin}.ll > {bin}.log 2>&1; echo $? > {bin}.rc1\n`)
+  }
+  test_stage(jd, "c")
+  for i in 0 .. len(files) {
+    let bin = bins[i]
+    let ll = `{bin}.ll`
+    if test_rc(`{bin}.rc1`) == 0 {
+      let flags = `{gl_link_flags(ll)}{vk_link_flags(ll)}{http_link_flags(ll)}{udp_link_flags(ll)}{process_link_flags(ll)}{threads_link_flags(ll)}{pbf}`
+      write_file(`{jd}/l{string(i)}.sh`, `{cc()} -O2 {ll}{flags} -o {bin} >> {bin}.log 2>&1; echo $? > {bin}.rc2\n`)
+    }
+  }
+  test_stage(jd, "l")
+  # the tests of every file that built, each its own job
+  let names = new [][]pointer
+  var k = 0
+  for i in 0 .. len(files) {
+    let bin = bins[i]
+    let mine = new []pointer
+    if test_rc(`{bin}.rc1`) == 0 and test_rc(`{bin}.rc2`) == 0 {
+      let listed = split_lines(capture(`{bin} --list < /dev/null`))
+      for t in 0 .. len(listed) {
+        let nm = listed[t]
+        if nm != "" and (only == "" or nm == only) {
+          let j = string(len(mine))
+          push(mine, nm)
+          # its own temporary directory too: two tests of one file may use one fixed scratch path
+          let td = `{bin}.t{j}.tmp`
+          write_file(`{jd}/t{string(k)}.sh`, `mkdir -p {td}; TMPDIR={td} {bin} {sh_single(nm)} < /dev/null > {bin}.t{j}.out 2>&1; echo $? > {bin}.t{j}.rc\n`)
+          k += 1
+        }
+      }
+    }
+    push(names, mine)
+  }
+  test_stage(jd, "t")
+  return test_report(files, bins, names, only, verbose)
+}
+# the report, file by file, as the sequential run gives it
+function test_report(files: []pointer, bins: []pointer, names: [][]pointer, only: pointer, verbose: bool) -> int {
+  var failed = 0
+  var ntests = 0
+  for i in 0 .. len(files) {
+    let f = files[i]
+    let bin = bins[i]
+    if verbose { say(`  RUN  {f}`) }
+    if test_rc(`{bin}.rc1`) != 0 or test_rc(`{bin}.rc2`) != 0 {
+      let line = `  {c_red()}FAIL{c_reset()} {f}  (did not compile)`
+      if verbose { say(line) } else { print(line) }
+      let why = capture(`grep -i error {bin}.log | head -3`)
+      if len(why) > 0 { out(why) }
+      failed += 1
+      continue
+    }
+    let mine = names[i]
+    var bad = 0
+    var log = ""
+    for t in 0 .. len(mine) {
+      let o = `{bin}.t{string(t)}.out`
+      let rc = test_rc(`{bin}.t{string(t)}.rc`)
+      let txt = capture(`grep -v '^== ' {o}`)
+      if verbose { shell(`grep -v '^== ' {o}`) }
+      if rc != 0 {
+        bad += 1
+        log = log + txt
+        if not s_contains(txt, "FAIL - ") { log = log + `FAIL - {mine[t]}  (exit {string(rc)})\n` }
+      }
+    }
+    ntests += len(mine)
+    var line = ""
+    if len(mine) == 0 and only != "" { line = `  {c_red()}FAIL{c_reset()} {f}  (no test named "{only}")` }
+    else if bad > 0 { line = `  {c_red()}FAIL{c_reset()} {f}  ({string(bad)} of {string(len(mine))} tests failed)` }
+    else if len(mine) == 1 { line = `  {c_green()}PASS{c_reset()} {f}  (1 test)` }
+    else { line = `  {c_green()}PASS{c_reset()} {f}  ({string(len(mine))} tests)` }
+    if line == "" { continue }
+    if verbose { say(line) } else {
+      print(line)
+      if bad > 0 { out(log) }
+    }
+    if bad > 0 or (len(mine) == 0 and only != "") { failed += 1 }
+  }
+  print("")
+  if failed == 0 {
+    print(`== {string(len(files))} test files passed ({string(ntests)} tests) ==`)
+    return 0
+  }
+  print(`== {string(failed)} of {string(len(files))} test files failed ==`)
+  return 1
+}