Merge branch 'lang/foundations' into lang/nav

changes/model-release.md

@@ -0,0 +1,10 @@
+bump: minor
+type: feature
+**A loaded model can be let go of: `model_release(model)`.** Its meshes are freed at once, and
+each texture once no other model uses it - the glTF loader shares textures between models (one
+path loaded once, a LOD chain borrowing its LOD0's material), so every cached texture now counts
+the primitives using it, and the last one frees it and forgets it, with any remembered material
+that named it. A game can unload a species' kit or a map's models when nothing of them is left,
+and loading one again afterwards is a fresh load. A scatter layer takes room for what it holds
+instead of its whole capacity up front (it doubles as it fills, to its cap), which in Maroon Lake
+was 0.4 GB of never-filled instance arrays. `examples/rendering/release.ludic` is the check.

examples/rendering/release.ludic

@@ -0,0 +1,69 @@
+# release.ludic - a loaded model let go of (model_release): its meshes go at once, a texture it
+# shares with another model stays until that one goes too, and a model loaded again afterwards is
+# whole. Prints RELEASE OK.
+#
+#   bin/ludic build examples/rendering/release.ludic --headless && ./build/release_headless
+program Release {
+  numbers float
+  import "ludic.render3d/r3d.ludic"
+
+  property Marker { on: int = 1 }
+  model Anchor { Marker }
+
+  function scene_draw(render3d_st: mut Render3dState) -> void { }
+  function scene_draw_casters(render3d_st: mut Render3dState, light_vp: floats) -> void { }
+  function stream_fill(s: Stream, cx: int, cz: int, band: int) -> void { }
+
+  function check(what: string, cond: bool) -> bool {
+    if not cond { print(`release: FAILED - {what}`) }
+    return cond
+  }
+  # whether handle `tex` is still a texture the renderer knows (its recorded kind is set)
+  function tex_live(render3d_st: mut Render3dState, tex: int) -> bool {
+    let o = gpu_tx_at(render3d_st, tex)
+    return tex != 0 and o >= 0 and render3d_st.gpu_tx[o] != 0
+  }
+
+  handler Boot(render3d_st: mut Render3dState) phase Start {
+    spawn Anchor {}
+    r3d_on_draw(render3d_st, fn scene_draw)
+    r3d_on_casters(render3d_st, fn scene_draw_casters)
+    r3d_on_stream_fill(render3d_st, fn stream_fill)
+    r3d_plate_mode(render3d_st, true)
+    let dir = "packages/ludic.lab/plate"
+    render3d_st.r3d_sky_path = `{dir}/sky.hdr`
+    if not r3d_init(render3d_st, 320, 180, "Release") {
+      quit()
+      return
+    }
+    var ok = true
+    # two models of the same file share its two textures
+    let a = gltf_load(render3d_st, dir, "plate.gltf", "plate")
+    let b = gltf_load(render3d_st, dir, "plate.gltf", "plate")
+    ok = check("both plates load", a != null and b != null and len(a.prims) > 0) and ok
+    let diff = a.prims[0].diff
+    ok = check("the plates share their textures", diff != 0 and b.prims[0].diff == diff and render3d_st.gltf_tex_n == 2) and ok
+    model_release(render3d_st, a)
+    ok = check("the first plate is emptied", len(a.prims) == 0) and ok
+    ok = check("what the second still uses stays", tex_live(render3d_st, diff) and render3d_st.gltf_tex_n == 2) and ok
+    model_release(render3d_st, b)
+    ok = check("the last user frees the textures", not tex_live(render3d_st, diff) and render3d_st.gltf_tex_n == 0) and ok
+    ok = check("and its material is forgotten", render3d_st.gltf_mat_n == 0) and ok
+    # one model whose two materials share a texture
+    let p = gltf_load(render3d_st, dir, "probe.gltf", "probe")
+    ok = check("the probe loads its three textures", p != null and render3d_st.gltf_tex_n == 3) and ok
+    model_release(render3d_st, p)
+    ok = check("the probe gives all three back", render3d_st.gltf_tex_n == 0 and render3d_st.gltf_mat_n == 0) and ok
+    # and loading again after a release is a fresh load, whole
+    let c = gltf_load(render3d_st, dir, "plate.gltf", "plate")
+    ok = check("a plate loaded again is whole", c != null and len(c.prims) > 0 and tex_live(render3d_st, c.prims[0].diff) and render3d_st.gltf_tex_n == 2) and ok
+    model_release(render3d_st, c)
+    ok = check("and goes again", render3d_st.gltf_tex_n == 0) and ok
+    if ok { print("RELEASE OK") } else { print("RELEASE FAILED") }
+    quit()
+  }
+
+  handler Present(render3d_st: mut Render3dState) phase Render {
+    r3d_present(render3d_st)
+  }
+}

packages/ludic.render3d/env.ludic

@@ -99,6 +99,7 @@ export state Render3dState {
   gltf_comps: int = 0
   gltf_tex_paths: []pointer = null
   gltf_tex_ids: words = null
+  gltf_tex_refs: words = null        # how many primitives use each cached texture (model_release)
   gltf_tex_n: int = 0
   gltf_white: int = 0
   gltf_flat: int = 0

packages/ludic.render3d/gltf.ludic

@@ -50,17 +50,17 @@ function gltf_mat_remember(render3d_st: mut Render3dState, name: string, diff: i
 }
 
 function gltf_texture(render3d_st: mut Render3dState, uri: string, srgb: bool) -> int {
-  if render3d_st.gltf_tex_paths == null { render3d_st.gltf_tex_paths = new []pointer;  render3d_st.gltf_tex_ids = words(256) }
+  if render3d_st.gltf_tex_paths == null { render3d_st.gltf_tex_paths = new []pointer;  render3d_st.gltf_tex_ids = words(256);  render3d_st.gltf_tex_refs = words(256) }
   var png: string = uri
   let n = len(uri)
   if n > 4 and uri[n - 4] == '.' and uri[n - 3] == 'j' { png = uri[0 .. n - 4] + ".png" }
   let path = render3d_st.gltf_dir + "/" + png
   var i = 0
-  while i < render3d_st.gltf_tex_n { if render3d_st.gltf_tex_paths[i] == path { return render3d_st.gltf_tex_ids[i] };  i += 1 }
+  while i < render3d_st.gltf_tex_n { if render3d_st.gltf_tex_paths[i] == path { render3d_st.gltf_tex_refs[i] += 1;  return render3d_st.gltf_tex_ids[i] };  i += 1 }
   var dil = 0
   if render3d_st.gltf_cutout { dil = 24 }
   let id = tex_load_ex(render3d_st, path, srgb, dil)
-  if render3d_st.gltf_tex_n < 256 { push(render3d_st.gltf_tex_paths, path);  render3d_st.gltf_tex_ids[render3d_st.gltf_tex_n] = id;  render3d_st.gltf_tex_n += 1 }
+  if render3d_st.gltf_tex_n < 256 { push(render3d_st.gltf_tex_paths, path);  render3d_st.gltf_tex_ids[render3d_st.gltf_tex_n] = id;  render3d_st.gltf_tex_refs[render3d_st.gltf_tex_n] = 1;  render3d_st.gltf_tex_n += 1 }
   return id
 }
 
@@ -150,7 +150,10 @@ function gltf_prim(render3d_st: mut Render3dState, p: Val) -> Prim {
       if ud != null { gltf_mat_remember(render3d_st, mname, pr.diff, pr.nrm, pr.arm) }
       else {
         let k = gltf_mat_find(render3d_st, mname)
-        if k >= 0 { pr.diff = render3d_st.gltf_mat_diff[k];  pr.nrm = render3d_st.gltf_mat_nrm[k];  pr.arm = render3d_st.gltf_mat_arm[k] }
+        if k >= 0 {
+          pr.diff = render3d_st.gltf_mat_diff[k];  pr.nrm = render3d_st.gltf_mat_nrm[k];  pr.arm = render3d_st.gltf_mat_arm[k]
+          gltf_tex_ref(render3d_st, pr.diff);  gltf_tex_ref(render3d_st, pr.nrm);  gltf_tex_ref(render3d_st, pr.arm)
+        }
         else { print(`gltf: material {mname} has no textures and none were loaded before it`) }
       }
     }

packages/ludic.render3d/gltf_release.ludic

@@ -0,0 +1,70 @@
+# gltf_release.ludic - a loaded model let go of: its meshes, and each texture once nothing else uses
+# it. The glTF loader shares textures between models (a path loaded once, a LOD chain borrowing its
+# LOD0's material), so each cached texture carries a count of the primitives using it.
+
+# one more primitive uses the cached texture `id` (0, or a texture the cache did not load: nothing)
+function gltf_tex_ref(render3d_st: mut Render3dState, id: int) -> void {
+  if id == 0 or render3d_st.gltf_tex_refs == null { return }
+  for i in 0 .. render3d_st.gltf_tex_n { if render3d_st.gltf_tex_ids[i] == id { render3d_st.gltf_tex_refs[i] += 1;  return } }
+}
+
+# one primitive fewer: the last one frees the texture and forgets it, with every material naming it
+function gltf_tex_unref(render3d_st: mut Render3dState, id: int) -> void {
+  if id == 0 or render3d_st.gltf_tex_refs == null { return }
+  for i in 0 .. render3d_st.gltf_tex_n {
+    if render3d_st.gltf_tex_ids[i] == id {
+      render3d_st.gltf_tex_refs[i] -= 1
+      if render3d_st.gltf_tex_refs[i] > 0 { return }
+      gpu_tex_free(render3d_st, id)
+      gltf_tex_forget(render3d_st, i)
+      gltf_mats_forget(render3d_st, id)
+      return
+    }
+  }
+}
+
+# the cache entry i goes; the last entry takes its place
+function gltf_tex_forget(render3d_st: mut Render3dState, i: int) -> void {
+  let last = render3d_st.gltf_tex_n - 1
+  let paths = render3d_st.gltf_tex_paths
+  paths[i] = paths[last]
+  render3d_st.gltf_tex_ids[i] = render3d_st.gltf_tex_ids[last]
+  render3d_st.gltf_tex_refs[i] = render3d_st.gltf_tex_refs[last]
+  List.pop(paths)
+  render3d_st.gltf_tex_n = last
+}
+
+# a remembered material whose texture was freed is forgotten, so no later load borrows it
+function gltf_mats_forget(render3d_st: mut Render3dState, id: int) -> void {
+  if render3d_st.gltf_mat_names == null { return }
+  var k = 0
+  while k < render3d_st.gltf_mat_n {
+    if render3d_st.gltf_mat_diff[k] == id or render3d_st.gltf_mat_nrm[k] == id or render3d_st.gltf_mat_arm[k] == id {
+      let last = render3d_st.gltf_mat_n - 1
+      let names = render3d_st.gltf_mat_names
+      names[k] = names[last]
+      render3d_st.gltf_mat_diff[k] = render3d_st.gltf_mat_diff[last]
+      render3d_st.gltf_mat_nrm[k] = render3d_st.gltf_mat_nrm[last]
+      render3d_st.gltf_mat_arm[k] = render3d_st.gltf_mat_arm[last]
+      List.pop(names)
+      render3d_st.gltf_mat_n = last
+    } else {
+      k += 1
+    }
+  }
+}
+
+# Let a model go: every primitive's mesh, and its textures once no other model uses them. The
+# caller says when - no actor may draw the model afterwards (a released model's prims are emptied,
+# so a stale draw of it draws nothing).
+function model_release(render3d_st: mut Render3dState, m: Model) -> void {
+  if m == null or m.prims == null { return }
+  for i in 0 .. len(m.prims) {
+    let pr = m.prims[i]
+    if pr.mesh != null { gpu_mesh_free(render3d_st, pr.mesh);  pr.mesh = null }
+    gltf_tex_unref(render3d_st, pr.diff);  gltf_tex_unref(render3d_st, pr.nrm);  gltf_tex_unref(render3d_st, pr.arm)
+    pr.diff = 0;  pr.nrm = 0;  pr.arm = 0
+  }
+  let prims = m.prims
+  while len(prims) > 0 { List.pop(prims) }
+}

packages/ludic.render3d/gpu_vk_draw.ludic

@@ -1903,8 +1903,9 @@ function gvk_pipeline_fast(render3d_st: mut Render3dState, p: int, m: Mesh, st:
       return render3d_st.gvk_pc_pipe[k]
     }
   }
+  # a pipeline the driver refused is remembered as 0 too: retried on every draw, each try built its
+  # key and its create structs again and gave none of them back
   let pipe = gvk_pipeline(render3d_st, p, m, st, n_color, color_fmt, depth_fmt, samples)
-  if pipe == 0 { return pipe }
   push(render3d_st.gvk_pc_prog, p);  push(render3d_st.gvk_pc_layout, layout);  push(render3d_st.gvk_pc_state, state)
   push(render3d_st.gvk_pc_bias, bias);  push(render3d_st.gvk_pc_pass, pass);  push(render3d_st.gvk_pc_pipe, pipe)
   if p < 4096 { render3d_st.gvk_pc_last[p] = len(render3d_st.gvk_pc_prog) - 1 }

packages/ludic.render3d/gpu_vk_res.ludic

@@ -409,7 +409,7 @@ function gvk_tex_read(render3d_st: mut Render3dState, tex: int, ifmt: int, w: in
   let depth = gvk_is_depth(ifmt)
   let cb = gvk_once_begin(render3d_st)
   gvk_barrier(render3d_st, cb, image, depth, 0, 1, 1, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL)
-  let bic = bytes(VkBufferImageCopy_sizeof)
+  let bic = gvk_tmp(render3d_st, VkBufferImageCopy_sizeof)
   Vk.zero(bic, VkBufferImageCopy_sizeof)
   if depth { Vk.put_i32(bic, VkBufferImageCopy_imageSubresource + VkImageSubresourceLayers_aspectMask, VK_IMAGE_ASPECT_DEPTH_BIT) }
   else { Vk.put_i32(bic, VkBufferImageCopy_imageSubresource + VkImageSubresourceLayers_aspectMask, VK_IMAGE_ASPECT_COLOR_BIT) }

packages/ludic.render3d/r3d.ludic

@@ -26,6 +26,7 @@ import "shadow.ludic"
 import "post.ludic"
 import "quat.ludic"
 import "gltf.ludic"
+import "gltf_release.ludic"
 import "skin.ludic"
 import "scatter.ludic"
 import "actor.ludic"

packages/ludic.render3d/scatter.ludic

@@ -24,6 +24,7 @@ property Layer {
   inst:     floats,          # INST_FLOATS per instance
   count:    int = 0,
   cap:      int = 0,
+  have:     int = 0,            # instances inst and scratch have room for now: grown toward cap as filled
   near:     float = 0.0,        # float bits; instances beyond it draw as impostors (or not at all)
   cull:     float = 0.0,        # float bits; instances beyond it are skipped (0 = never)
   buf:      int = 0,
@@ -386,8 +387,11 @@ function layer_new(render3d_st: mut Render3dState, model: Model, cap: int, folia
   l.near = near
   l.cull = cull
   l.tint = v3_new(1.0, 1.0, 1.0)
-  l.inst = floats(cap * INST_FLOATS)
-  l.scratch = floats(cap * INST_FLOATS)
+  # room for what the layer holds, not for what it might: 45 layers at full capacity up front were
+  # 0.4 GB, most of it never filled (plan 23 of maroon-lake)
+  l.have = min(cap, 256)
+  l.inst = floats(l.have * INST_FLOATS)
+  l.scratch = floats(l.have * INST_FLOATS)
   l.last_cam = v3_new(100000.0, 0.0, 0.0)
   l.buf = gpu_buffer_new(render3d_st)
   l.imp_buf = gpu_buffer_new(render3d_st)
@@ -398,8 +402,23 @@ function layer_new(render3d_st: mut Render3dState, model: Model, cap: int, folia
   return l
 }
 
+# room for n instances (at most the layer's cap), doubling what there is so a fill costs a few copies
+function layer_room(l: Layer, n: int) -> void {
+  if n <= l.have { return }
+  var want = max(l.have * 2, n)
+  if want > l.cap { want = l.cap }
+  let inst = floats(want * INST_FLOATS)
+  if l.count > 0 { mem_copy(data_of(inst), data_of(l.inst), l.count * INST_FLOATS * 4) }
+  free(l.inst)
+  free(l.scratch)
+  l.inst = inst
+  l.scratch = floats(want * INST_FLOATS)
+  l.have = want
+}
+
 function layer_add(l: Layer, x: float, y: float, z: float, scale: float, yaw: float, seed: float, wind: float) -> void {
   if l.count >= l.cap { return }
+  layer_room(l, l.count + 1)
   let o = l.count * INST_FLOATS
   l.inst[o] = x;  l.inst[o + 1] = y;  l.inst[o + 2] = z;  l.inst[o + 3] = scale
   l.inst[o + 4] = Math.sin(yaw);  l.inst[o + 5] = Math.cos(yaw);  l.inst[o + 6] = seed;  l.inst[o + 7] = wind

packages/ludic.render3d/stream.ludic

@@ -162,16 +162,24 @@ function stream_evict(render3d_st: mut Render3dState, s: Stream) -> void {
   }
   t = lo
   # everything wanted by the walk in progress stays whatever the threshold says
-  let kept = new []Chunk
+  # compacted in place, and an evicted chunk goes whole: a new list per eviction and the chunks'
+  # own records were never given back
+  var w = 0
   var i = 0
   while i < s.n {
     let c = s.chunks[i]
-    if c.used >= t or c.used == render3d_st.stream_walk_no { push(kept, c) }
-    else { if c.data != null { free(c.data) } }
+    if c.used >= t or c.used == render3d_st.stream_walk_no {
+      s.chunks[w] = c
+      w += 1
+    } else {
+      if c.data != null { free(c.data) }
+      free(c)
+    }
     i += 1
   }
-  s.chunks = kept
-  s.n = len(kept)
+  let ch = s.chunks
+  while len(ch) > w { List.pop(ch) }
+  s.n = w
   for h in 0 .. STREAM_HASH { s.htab[h] = 0 }
   i = 0
   while i < s.n { s.keys[i] = s.chunks[i].key;  stream_remember(s, s.chunks[i].key, i);  i += 1 }
@@ -246,6 +254,7 @@ function stream_update(render3d_st: mut Render3dState, s: Stream, cam_x: float,
             }
             if c != null and c.count > 0 and l.count + c.count <= l.cap and stream_chunk_visible(render3d_st, s, cx, cz, c) {
               let tg = gl_now_us()
+              layer_room(l, l.count + c.count)
               mem_copy(mem_off(l.inst, l.count * INST_FLOATS * 4), data_of(c.data), c.count * INST_FLOATS * 4)
               l.count += c.count
               render3d_st.stream_us_gather = render3d_st.stream_us_gather + (gl_now_us() - tg)

packages/ludic.render3d/terrain.ludic

@@ -145,6 +145,9 @@ function terrain_ortho_forest(render3d_st: mut Render3dState, x: float, z: float
 function terrain_use_ortho(render3d_st: mut Render3dState, path: string) -> void {
   let px = png_decode(render3d_st, path)
   if px == null { return }
+  # a map set up over another lets go of the last one's photograph first
+  if render3d_st.ter_ortho_px != null { free(render3d_st.ter_ortho_px) }
+  if render3d_st.ter_ortho_tex != 0 { gpu_tex_free(render3d_st, render3d_st.ter_ortho_tex) }
   render3d_st.ter_ortho_px = px
   render3d_st.ter_ortho_w = render3d_st.tex_w
   render3d_st.ter_ortho_c = render3d_st.tex_channels
@@ -154,6 +157,7 @@ function terrain_use_ortho(render3d_st: mut Render3dState, path: string) -> void
   gpu_tex_param(render3d_st, GPU_TEX2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
 }
 function terrain_use_dem(render3d_st: mut Render3dState, path: string, emin: float, emax: float, base: float, ox: float, oz: float) -> void {
+  if render3d_st.ter_dem_tex != 0 { gpu_tex_free(render3d_st, render3d_st.ter_dem_tex) }
   render3d_st.ter_dem_tex = tex_load(render3d_st, path, false)
   render3d_st.ter_dem_min = emin;  render3d_st.ter_dem_max = emax;  render3d_st.ter_dem_base = base
   render3d_st.ter_ox = ox;  render3d_st.ter_oz = oz
@@ -167,6 +171,8 @@ function terrain_generate(render3d_st: mut Render3dState) -> void {
   if render3d_st.ter_dem_tex != 0 { defs = "#define DEM\n" }
   if render3d_st.ter_smooth { defs = "#define SMOOTH\n" }
   let p = r3d_program(render3d_st, "fullscreen.vert", "heightgen.frag", defs)
+  # a map generated over another: its 256 MB height texture goes first
+  if render3d_st.ter_height_tex != 0 { gpu_tex_free(render3d_st, render3d_st.ter_height_tex);  render3d_st.ter_height_tex = 0 }
   render3d_st.ter_height_tex = tex_target(render3d_st, TERRAIN_RES, TERRAIN_RES, GL_R32F, GL_RED, GL_FLOAT, GL_LINEAR)
   let fbo = gpu_fb_new(render3d_st)
   gpu_fb_bind(render3d_st, fbo)
@@ -203,8 +209,9 @@ function terrain_generate(render3d_st: mut Render3dState) -> void {
   mesh_draw(render3d_st, render3d_st.sky_fullscreen)
   gpu_program_free(render3d_st, pn)
   gpu_tex_free(render3d_st, raw)
-  # read the heights back for placement
-  render3d_st.ter_heights = floats(TERRAIN_RES * TERRAIN_RES)
+  # read the heights back for placement, into the array a previous map had: always the same size,
+  # and made anew on every build it was 64 MB lost per world swap
+  if render3d_st.ter_heights == null { render3d_st.ter_heights = floats(TERRAIN_RES * TERRAIN_RES) }
   gpu_tex_bind(render3d_st, GPU_TEX2D, render3d_st.ter_height_tex)
   gpu_pixel_store(render3d_st, GL_PACK_ALIGNMENT, 4)
   gpu_tex_read(render3d_st, GPU_TEX2D, GL_RED, GL_FLOAT, data_of(render3d_st.ter_heights))

tools/ludic-cli/test.ludic

@@ -608,6 +608,13 @@ function headless_case(path: pointer, exp: pointer, label: pointer) -> void {
   let got = capture_line(`{out} < /dev/null`)
   if got == exp { ok(label) } else { bad2(label, `got [{got}] want [{exp}]`) }
 }
+# a program built headless whose verdict is one of its lines (the renderer logs before it)
+function headless_line_case(path: pointer, exp: pointer, label: pointer) -> void {
+  let out = `{tmp_dir()}/h_{flat(path)}`
+  if not shq(`bin/ludicc --headless examples/{path}.ludic -o {out} > {out}.log 2>&1`) { bad2(label, capture_line(`grep -i error {out}.log | head -1`));  return }
+  let got = capture_line(`{out} < /dev/null 2>&1 | grep -c '^{exp}$'`)
+  if got == "1" { ok(label) } else { bad2(label, capture_line(`{out} < /dev/null 2>&1 | grep -i fail | head -1`)) }
+}
 # ludic migrate state and components: the header names what every member needs, a field read in a
 # member is not edited, a module named like a package keeps a state of its own, and the program
 # runs as it did
@@ -1045,6 +1052,7 @@ function cmd_dev_test() -> int {
   reject_case("rejected/state_component_ro", "look is read-only here (look: Look)", "a component's state is read-only unless its header says mut")
   controller_case("state/component", "", " #10 big 10 1", "component.ludic (0.S: a component's header names its states; its getters, functions and events take them and the template never sees them)")
   headless_case("rendering/ui_render3d", "ok", "ui_render3d.ludic (ludic.ui's render3d backend builds against the renderer)")
+  headless_line_case("rendering/release", "RELEASE OK", "release.ludic (model_release: meshes go, a shared texture stays until its last user goes)")
   migrate_component_case()
   migrate_foreign_case()
   reject_case("rejected/runtime_type_clash", "PadButton is the runtime's enum", "a program's type named like one of the runtime's is refused")