feat(render3d): every shader variant as Vulkan SPIR-V, built ahead of time

Vulkan cannot compile GLSL when the game starts, so the programs render3d builds are listed
(shaders/variants.list, 45 of them, collected with R3D_PROGRAMS_LOG across the self-tests, the
screens, the viewpoints and the debug switches) and `ludic-dev shaders` compiles each into
shaders/spv/<id>.vert.spv and .frag.spv with a manifest of what the backend needs: each
stage's uniform block and member offsets, the samplers' bindings, the vertex inputs.

The GLSL is the renderer's own, assembled as programs.ludic assembles it, through glslang's
relaxed Vulkan mode, so gpu_uniform / u_* can write the same uniforms into a block on Vulkan.
Bindings are assigned by the tool (glslang's own numbering put several samplers of one stage
on binding 0): the vertex block is 0, the fragment block 1, samplers from 2 in name order,
shared across both stages. Every stage passes spirv-val; `ludic-dev test` rebuilds and compares
wherever the Vulkan SDK is installed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tools/ludic-cli/dev.ludic

@@ -34,6 +34,7 @@ program LudicDev {
   import "docgen_check.ludic"
   import "glgen.ludic"
   import "vkgen.ludic"
+  import "shaders.ludic"
   import "assets.ludic"
   import "release.ludic"
   import "pkg.ludic"
@@ -71,6 +72,7 @@ program LudicDev {
     print("  docs-palette [--check]        regenerate emit_color.ludic + palette.json from the palette table")
     print("  glgen [--check]               regenerate gl_api.ludic + gl_thunks.ll from the platform gl3.h")
     print("  vkgen [--xml f] [--check]     regenerate vk_api.ludic + vk_thunks.ll from the Vulkan registry")
+    print("  shaders [--check]             build render3d's shader variants to SPIR-V + manifest (Vulkan SDK)")
     print("")
     print("release:")
     print("  release [major|minor|patch] [--dry-run] [--publish]")
@@ -109,6 +111,7 @@ program LudicDev {
     if (cmd == "docs-palette") { return cmd_docs_palette() }
     if (cmd == "glgen") { return cmd_glgen() }
     if (cmd == "vkgen") { return cmd_vkgen() }
+    if (cmd == "shaders") { return cmd_shaders() }
     if (cmd == "fetch-assets") { return cmd_fetch_assets() }
     if (cmd == "docs-gen") { return cmd_docs_gen() }
     if (cmd == "docs-check") { return cmd_docs_check() }

tools/ludic-cli/shaders.ludic

@@ -0,0 +1,223 @@
+# shaders.ludic — render3d's shaders as Vulkan SPIR-V, built ahead of time.
+#
+#   ludic-dev shaders [--check]
+#       read packages/ludic.render3d/shaders/variants.list and write, per variant,
+#         packages/ludic.render3d/shaders/spv/<id>.vert.spv and <id>.frag.spv
+#       and one packages/ludic.render3d/shaders/spv/manifest.txt the renderer reads:
+#         P <id> <vertex> <fragment> <defines>      a program (defines with ';' for newlines)
+#         B <id> <stage> <binding> <size>           a stage's uniform block
+#         U <id> <stage> <name> <offset> <type> <count> <stride>
+#         T <id> <name> <binding> <sampler type>    a sampler, the same binding in both stages
+#         I <id> <name> <location> <type>           a vertex input
+#
+# OpenGL compiles GLSL when the game starts; Vulkan cannot, so every program the renderer
+# builds (variants.list, from R3D_PROGRAMS_LOG) is compiled here. The GLSL is the renderer's
+# own, assembled exactly as programs.ludic assembles it, through glslang's relaxed Vulkan mode:
+# the loose uniforms of each stage land in one uniform block whose offsets come from the
+# reflection, which is what gpu_uniform / u_* write through on Vulkan.
+#
+# Bindings are assigned here, not by glslang: its automatic numbering gave several samplers of
+# one stage the same binding. The vertex stage's block is 0, the fragment stage's is 1, and the
+# samplers of both stages are numbered from 2 in name order, so a sampler both stages use has
+# one binding and one pipeline layout covers the program.
+#
+# Needs the Vulkan SDK (glslangValidator, spirv-cross, spirv-val): $VULKAN_SDK/bin, else the
+# newest ~/VulkanSDK/*/macOS/bin, else C:/VulkanSDK/*/Bin. The outputs are tracked, so building a
+# game never runs this.
+
+var shd_bin: pointer = null
+
+function shd_find_sdk() -> bool {
+  let sdk = getenv_or("VULKAN_SDK", "")
+  if sdk != "" and file_exists(`{sdk}/bin/glslangValidator`) { shd_bin = `{sdk}/bin`;  return true }
+  if sdk != "" and file_exists(`{sdk}/Bin/glslangValidator.exe`) { shd_bin = `{sdk}/Bin`;  return true }
+  let found = Text.trim(capture("ls -d $HOME/VulkanSDK/*/macOS/bin /c/VulkanSDK/*/Bin 2>/dev/null | tail -1"))
+  if found != "" { shd_bin = found;  return true }
+  return false
+}
+
+# FNV-1a over the variant line, as eight hex digits: the id is stable across runs and machines
+function shd_id(line: pointer) -> pointer {
+  var h = 0x811C9DC5
+  let n = slen(line)
+  var i = 0
+  while i < n { h = (h ^ line[i]) * 16777619;  i += 1 }
+  let hex = "0123456789abcdef"
+  let b = sb_new()
+  var k = 0
+  while k < 8 { sb_putc(b, hex[(h >> (28 - k * 4)) & 15]);  k += 1 }
+  return sb_str(b)
+}
+
+function shd_is_ident(c: int) -> bool { return (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9') or c == '_' }
+
+# the sampler declarations of a source: names (in order met) into shd_names, deduplicated
+var shd_names: []pointer = null
+function shd_note_name(n: pointer) -> void {
+  var i = 0
+  while i < len(shd_names) { if shd_names[i] == n { return };  i += 1 }
+  push(shd_names, n)
+}
+
+# Rewrite every `uniform <[iu]sampler...> <name>[...];` anywhere in the text (several share a line
+# in terrain.frag) as `layout(set = 0, binding = N) uniform ...`. With `collect`, only record names.
+function shd_bind_samplers(src: pointer, collect: bool) -> pointer {
+  let b = sb_new()
+  let n = slen(src)
+  var i = 0
+  while i < n {
+    let at = s_index(src, "uniform", i)
+    if at < 0 { sb_puts(b, str_sub(src, i, n));  break }
+    sb_puts(b, str_sub(src, i, at))
+    # a whole word "uniform", then whitespace, then a sampler type
+    var ok = (at == 0 or not shd_is_ident(src[at - 1])) and at + 7 < n and str_space(src[at + 7])
+    var p = at + 7
+    while ok and p < n and str_space(src[p]) { p += 1 }
+    let ts = p
+    while ok and p < n and shd_is_ident(src[p]) { p += 1 }
+    let ty = str_sub(src, ts, p)
+    if ok { ok = s_starts(ty, "sampler") or s_starts(ty, "isampler") or s_starts(ty, "usampler") }
+    while ok and p < n and str_space(src[p]) { p += 1 }
+    let ns = p
+    while ok and p < n and shd_is_ident(src[p]) { p += 1 }
+    let name = str_sub(src, ns, p)
+    if ok { ok = slen(name) > 0 }
+    if not ok { sb_puts(b, "uniform");  i = at + 7;  continue }
+    if collect { shd_note_name(name) }
+    var bind = 0
+    var k = 0
+    while k < len(shd_names) { if shd_names[k] == name { bind = k + 2 };  k += 1 }
+    sb_puts(b, `layout(set = 0, binding = {string(bind)}) uniform {ty} {name}`)
+    i = p
+  }
+  return sb_str(b)
+}
+
+var shd_noise: pointer = null
+var shd_lighting: pointer = null
+function shd_file(name: pointer) -> pointer {
+  let s = read_file(`packages/ludic.render3d/shaders/{name}`)
+  if s == null { err(`shaders: missing packages/ludic.render3d/shaders/{name}\n`);  return "" }
+  return s
+}
+
+function shd_q(p: pointer) -> pointer { return `"{p}"` }
+
+function shd_reflect_stage(mf: pointer, id: pointer, stage: pointer, spv: pointer, tmp: pointer) -> bool {
+  let js = `{tmp}/{id}.{stage}.json`
+  if not shq(`{shd_q(shd_bin + "/spirv-cross")} {shd_q(spv)} --reflect > {shd_q(js)} 2>/dev/null`) { err(`shaders: spirv-cross failed on {spv}\n`);  return false }
+  let d = json_parse(read_file(js))
+  let ubos = j_get(d, "ubos")
+  if ubos.t == JV_ARR and len(ubos.kids) > 0 {
+    let u = ubos.kids[0]
+    sb_puts(mf, `B {id} {stage} {string(j_get(u, "binding").num)} {string(j_get(u, "block_size").num)}\n`)
+    let ty = j_get(j_get(d, "types"), j_get(u, "type").s)
+    let mem = j_get(ty, "members")
+    var i = 0
+    while i < len(mem.kids) {
+      let m = mem.kids[i]
+      var count = 1
+      let arr = j_get(m, "array")
+      if arr.t == JV_ARR and len(arr.kids) > 0 { count = arr.kids[0].num }
+      var stride = 0
+      let st = j_get(m, "array_stride")
+      if st.t == JV_NUM { stride = st.num }
+      sb_puts(mf, `U {id} {stage} {j_get(m, "name").s} {string(j_get(m, "offset").num)} {j_get(m, "type").s} {string(count)} {string(stride)}\n`)
+      i += 1
+    }
+  }
+  if stage == "vert" {
+    let ins = j_get(d, "inputs")
+    var i = 0
+    while i < len(ins.kids) {
+      let v = ins.kids[i]
+      sb_puts(mf, `I {id} {j_get(v, "name").s} {string(j_get(v, "location").num)} {j_get(v, "type").s}\n`)
+      i += 1
+    }
+  }
+  return true
+}
+
+function cmd_shaders() -> int {
+  var check = false
+  var ai = 2
+  while ai < arg_count() { if arg(ai) == "--check" { check = true };  ai += 1 }
+  if not shd_find_sdk() { err("shaders: no Vulkan SDK (glslangValidator); set VULKAN_SDK\n");  return 1 }
+  let list = read_file("packages/ludic.render3d/shaders/variants.list")
+  if list == null { err("shaders: no packages/ludic.render3d/shaders/variants.list\n");  return 1 }
+  shd_noise = shd_file("noise.glsl")
+  shd_lighting = shd_file("lighting.glsl")
+  var outdir = "packages/ludic.render3d/shaders/spv"
+  let tmp = tmp_path("shaders")
+  if check { outdir = tmp_path("shaders-check") }
+  shell(`rm -rf {shd_q(tmp)} && mkdir -p {shd_q(tmp)} {shd_q(outdir)}`)
+  if not check { shell(`rm -f {shd_q(outdir)}/*.spv`) }
+  let mf = sb_new()
+  sb_puts(mf, "# manifest.txt - GENERATED by `ludic-dev shaders` from variants.list. Do not edit.\n")
+  let lines = Text.split(list, "\n")
+  var made = 0
+  var li = 0
+  while li < len(lines) {
+    let line = Text.trim(lines[li])
+    li += 1
+    if slen(line) == 0 or line[0] == '#' { continue }
+    let parts = Text.split(line, "|")
+    if len(parts) < 2 { err(`shaders: bad variant line: {line}\n`);  return 1 }
+    var defs = ""
+    if len(parts) > 2 { defs = parts[2] }
+    let id = shd_id(line)
+    let defines = Text.replace(defs, ";", "\n")
+    let vsrc = "#version 460\n" + defines + shd_file(parts[0])
+    let fsrc = "#version 460\n" + defines + shd_noise + shd_lighting + shd_file(parts[1])
+    # one binding table for both stages
+    shd_names = new []pointer
+    shd_bind_samplers(vsrc, true)
+    shd_bind_samplers(fsrc, true)
+    # name order, so the table does not depend on which stage met a sampler first
+    var x = 0
+    while x < len(shd_names) {
+      var y = x + 1
+      while y < len(shd_names) {
+        if str_cmp(shd_names[y], shd_names[x]) < 0 { let t = shd_names[x];  shd_names[x] = shd_names[y];  shd_names[y] = t }
+        y += 1
+      }
+      x += 1
+    }
+    sb_puts(mf, `P {id} {parts[0]} {parts[1]} {defs}\n`)
+    let stages = Text.split("vert frag", " ")
+    var si = 0
+    while si < 2 {
+      let stage = stages[si]
+      var src = vsrc
+      if stage == "frag" { src = fsrc }
+      let glsl = `{tmp}/{id}.{stage}.glsl`
+      if not write_file(glsl, shd_bind_samplers(src, false)) { err(`shaders: cannot write {glsl}\n`);  return 1 }
+      let spv = `{outdir}/{id}.{stage}.spv`
+      var extra = ""
+      if stage == "frag" { extra = " --shift-UBO-binding frag 1" }
+      let logf = `{tmp}/{id}.{stage}.log`
+      if not shq(`{shd_q(shd_bin + "/glslangValidator")} -V -R --auto-map-bindings --auto-map-locations{extra} -S {stage} {shd_q(glsl)} -o {shd_q(spv)} > {shd_q(logf)} 2>&1`) {
+        err(`shaders: {parts[0]} + {parts[1]} [{defs}] ({stage}) did not compile:\n{capture("grep ERROR " + shd_q(logf) + " | head -5")}\n`)
+        return 1
+      }
+      if not shq(`{shd_q(shd_bin + "/spirv-val")} --target-env vulkan1.3 {shd_q(spv)} > /dev/null 2>&1`) { err(`shaders: {spv} fails spirv-val\n`);  return 1 }
+      if not shd_reflect_stage(mf, id, stage, spv, tmp) { return 1 }
+      si += 1
+    }
+    var t = 0
+    while t < len(shd_names) {
+      sb_puts(mf, `T {id} {shd_names[t]} {string(t + 2)}\n`)
+      t += 1
+    }
+    made += 1
+  }
+  if not write_file(`{outdir}/manifest.txt`, sb_str(mf)) { err("shaders: cannot write manifest.txt\n");  return 1 }
+  if check {
+    let real = "packages/ludic.render3d/shaders/spv"
+    if not shq(`diff -r {shd_q(outdir)} {shd_q(real)} > /dev/null 2>&1`) { err("shaders: spv/ drifted from variants.list and the GLSL (run: ludic-dev shaders)\n");  return 1 }
+  }
+  var note = ""
+  if check { note = " (unchanged)" }
+  print(`OK {string(made)} programs, {string(made * 2)} SPIR-V stages{note}`)
+  return 0
+}

tools/ludic-cli/test.ludic

@@ -817,5 +817,13 @@ function cmd_dev_test() -> int {
     } else { bad2("ludic-dev vkgen --check", capture_line(`tail -1 {tmp_dir()}/vkgen.out`)) }
   } else { skip("ludic-dev vkgen --check (needs the Vulkan SDK's vk.xml)") }
 
+  # render3d's shader variants as SPIR-V: tracked, so a change to a shader or to variants.list
+  # that nobody rebuilt fails here, wherever the Vulkan SDK (glslangValidator) is installed
+  if shq("ls $VULKAN_SDK/bin/glslangValidator $HOME/VulkanSDK/*/macOS/bin/glslangValidator 2>/dev/null | grep -q glslangValidator") {
+    if shq(`bin/ludic-dev shaders --check > {tmp_dir()}/shaders.out 2>&1`) {
+      ok("ludic-dev shaders rebuilds render3d's SPIR-V byte-identically")
+    } else { bad2("ludic-dev shaders --check", capture_line(`tail -1 {tmp_dir()}/shaders.out`)) }
+  } else { skip("ludic-dev shaders --check (needs the Vulkan SDK's glslangValidator)") }
+
   return report()
 }