Merge branch 'r3d/zero-leak' into lang/uifree

examples/rendering/steady.ludic

@@ -33,6 +33,26 @@ program Steady {
     return Vk.heap_bytes() - before
   }
 
+  # bytes gained over n loads and releases of one model: its parse, its meshes and its textures
+  function model_rounds(render3d_st: mut Render3dState, n: int) -> long {
+    let before = Vk.heap_bytes()
+    for i in 0 .. n {
+      let m = gltf_load(render3d_st, "packages/ludic.lab/plate", "plate.gltf", "plate")
+      model_release(render3d_st, m)
+    }
+    return Vk.heap_bytes() - before
+  }
+
+  # bytes gained over n parses of a glTF document, each freed whole (Json.free_all): strings too
+  function parse_rounds(text: string, n: int) -> long {
+    let before = Vk.heap_bytes()
+    for i in 0 .. n {
+      let doc = Json.parse(text)
+      Json.free_all(doc)
+    }
+    return Vk.heap_bytes() - before
+  }
+
   handler Boot(render3d_st: mut Render3dState) phase Start {
     spawn Anchor {}
     r3d_on_draw(render3d_st, fn scene_draw)
@@ -49,7 +69,12 @@ program Steady {
     let grew_b = buffer_rounds(render3d_st, b, 5000)
     frame_rounds(render3d_st, 120)
     let grew_f = frame_rounds(render3d_st, 600)
-    print(`steady: the buffer path gained {grew_b} bytes over 5000 rounds, the frame {grew_f} over 600`)
+    model_rounds(render3d_st, 20)
+    let grew_m = model_rounds(render3d_st, 200)
+    let text = Fs.read_text("packages/ludic.lab/plate/plate.gltf")
+    parse_rounds(text, 20)
+    let grew_p = parse_rounds(text, 200)
+    print(`steady: the buffer path gained {grew_b} bytes over 5000 rounds, the frame {grew_f} over 600, a glTF parsed and freed {grew_p} over 200, a model loaded and let go {grew_m} over 200`)
     # a few KB of slack for what the system's own libraries keep (Metal's caches, the allocator)
     var ok = grew_b < 16384
     if not ok { print("steady: FAILED - releasing and making a buffer again leaves memory behind") }
@@ -57,6 +82,16 @@ program Steady {
       ok = false
       print("steady: FAILED - a frame leaves memory behind")
     }
+    if grew_p >= 16384 {
+      ok = false
+      print("steady: FAILED - a parsed document freed whole leaves memory behind")
+    }
+    # a load keeps a few records of its own for now (handles are not reused); a document's strings
+    # left behind were ~640 KB a model, so a bound of 8 KB a round is what this holds
+    if grew_m >= 200 * 8192 {
+      ok = false
+      print("steady: FAILED - a model loaded and let go leaves memory behind")
+    }
     if ok { print("STEADY OK") } else { print("STEADY FAILED") }
     quit()
   }

packages/ludic.anim/read.ludic

@@ -62,7 +62,7 @@ function read_one(anim_st: AnimState, render3d_st: mut Render3dState, a: Val) ->
   }
   if n_keep == 0 { return null }
   let c = new Clip
-  c.name = nm
+  c.name = nm[0 .. len(nm)]              # a copy: the document goes, strings and all, at the next load
   c.n = n_keep
   c.cnode = words(n_keep)
   c.cpath = words(n_keep)

packages/ludic.render3d/env.ludic

@@ -206,6 +206,11 @@ export state Render3dState {
   gvk_nopool: bool = false
   gvk_bc_said: int = 0              # attachments refused for being compressed, said so far
   gvk_map_slot: []pointer = null    # gvk_mem_new's mapped-pointer slot, made once
+  gvk_lay_sig: words = null           # gvk_layout_id's scratch: one layout as numbers
+  gvk_lay_seen: words = null
+  gvk_lay_flat: []int = null          # every layout known, end to end; gvk_lay_off / _len find each
+  gvk_lay_off: []int = null
+  gvk_lay_len: []int = null
   gvk_has_bc: bool = false          # textureCompressionBC: .dds textures are uploaded compressed
   # R3D_VK_PROF: Vulkan objects made and destroyed, reported every 120 frames
   gvk_mk_img: int = 0
@@ -335,7 +340,6 @@ export state Render3dState {
   gvk_n_flush: int = 0
   gvk_prof_frames: int = 0
   gvk_n_pipe_new: int = 0           # pipelines made this profile window
-  gvk_layout_keys: []string = null
   gvk_pc_prog: []int = null
   gvk_pc_layout: []int = null
   gvk_pc_state: []int = null

packages/ludic.render3d/gltf.ludic

@@ -144,7 +144,8 @@ function gltf_prim(render3d_st: mut Render3dState, p: Val) -> Prim {
     if un != null { pr.nrm = gltf_texture(render3d_st, un, false) }
     if ua != null { pr.arm = gltf_texture(render3d_st, ua, false) }
     var mname: string = null
-    if value_has(mat, "name") != 0 { mname = value_as_str(value_get(mat, "name")) }
+    # a copy: the document goes, strings and all, at the next load
+    if value_has(mat, "name") != 0 { let s = value_as_str(value_get(mat, "name"));  mname = s[0 .. len(s)] }
     pr.name = mname
     if mname != null {
       if ud != null { gltf_mat_remember(render3d_st, mname, pr.diff, pr.nrm, pr.arm) }
@@ -168,7 +169,7 @@ function gltf_load(render3d_st: mut Render3dState, dir: string, file: string, no
   if text == null { print(`gltf: cannot read {dir}/{file}`);  return null }
   # the last file's tree goes first (it is read only until the next load: ludic.anim's read.ludic),
   # and the text once parsed - nothing gives memory back on its own (plan 23 of maroon-lake)
-  if render3d_st.gltf_doc != null { Json.free(render3d_st.gltf_doc) }
+  if render3d_st.gltf_doc != null { Json.free_all(render3d_st.gltf_doc) }
   render3d_st.gltf_doc = Json.parse(text)
   free(text)
   let buffers = value_get(render3d_st.gltf_doc, "buffers")

packages/ludic.render3d/gpu_vk.ludic

@@ -408,8 +408,17 @@ function gvk_mem_new(render3d_st: mut Render3dState, req: bytes, want: int, imag
       if host { bsize = GVK_BLOCK_HOST }
       let mem = gvk_mem_raw(render3d_st, t, bsize, host, mp)
       if mem != 0 {
-        push(render3d_st.gvk_blk_mem, mem);  push(render3d_st.gvk_blk_kind, kind);  push(render3d_st.gvk_blk_size, bsize);  push(render3d_st.gvk_blk_map, mp[0])
+        # into the record of a block given back, when there is one: a buffer made again every few
-        blk = len(render3d_st.gvk_blk_mem) - 1;  off = 0
+        # frames empties a block and makes the next, and each made a new record for good
+        blk = -1
+        for e in 0 .. len(render3d_st.gvk_blk_kind) { if blk < 0 and render3d_st.gvk_blk_kind[e] == -1 and render3d_st.gvk_blk_mem[e] == 0 { blk = e } }
+        if blk < 0 {
+          push(render3d_st.gvk_blk_mem, mem);  push(render3d_st.gvk_blk_kind, kind);  push(render3d_st.gvk_blk_size, bsize);  push(render3d_st.gvk_blk_map, mp[0])
+          blk = len(render3d_st.gvk_blk_mem) - 1
+        } else {
+          render3d_st.gvk_blk_mem[blk] = mem;  render3d_st.gvk_blk_kind[blk] = kind;  render3d_st.gvk_blk_size[blk] = bsize;  render3d_st.gvk_blk_map[blk] = mp[0]
+        }
+        off = 0
         gvk_fr_put(render3d_st, blk, size, bsize - size)
       }
     }

packages/ludic.render3d/gpu_vk_draw.ludic

@@ -1875,7 +1875,7 @@ function gvk_prof_frame(render3d_st: mut Render3dState) -> void {
 function gvk_prof_made(render3d_st: mut Render3dState) -> void {
   let r = render3d_st
   print(`r3d: vulkan made/destroyed: images {r.gvk_mk_img}/{r.gvk_mk_x_img}, views {r.gvk_mk_view}/{r.gvk_mk_x_view}, buffers {r.gvk_mk_buf}/{r.gvk_mk_x_buf}, memory {r.gvk_mk_mem}/{r.gvk_mk_x_mem}, samplers {r.gvk_mk_smp}, sets {r.gvk_mk_set}, pools {r.gvk_mk_dpool}, cmds {r.gvk_mk_cmd}`)
-  print(`r3d: vulkan caches: layer views {gvk_len_i(r.gvk_layer_views)}, pipelines {gvk_len_s(r.gvk_pipe_keys)}, layouts {gvk_len_s(r.gvk_layout_keys)}, pipe cache {gvk_len_i(r.gvk_pc_prog)}, retired {gvk_len_l(r.gvk_retired_buf)}, buffers {gvk_len_l(r.gvk_buf)}, allocs {r.gvk_n_allocs}`)
+  print(`r3d: vulkan caches: layer views {gvk_len_i(r.gvk_layer_views)}, pipelines {gvk_len_s(r.gvk_pipe_keys)}, layouts {gvk_len_i(r.gvk_lay_off)}, pipe cache {gvk_len_i(r.gvk_pc_prog)}, retired {gvk_len_l(r.gvk_retired_buf)}, buffers {gvk_len_l(r.gvk_buf)}, allocs {r.gvk_n_allocs}`)
   render3d_st.gvk_mk_img = 0;  render3d_st.gvk_mk_x_img = 0;  render3d_st.gvk_mk_view = 0;  render3d_st.gvk_mk_x_view = 0
   render3d_st.gvk_mk_buf = 0;  render3d_st.gvk_mk_x_buf = 0;  render3d_st.gvk_mk_mem = 0;  render3d_st.gvk_mk_x_mem = 0
   render3d_st.gvk_mk_smp = 0;  render3d_st.gvk_mk_set = 0;  render3d_st.gvk_mk_dpool = 0;  render3d_st.gvk_mk_cmd = 0
@@ -1893,13 +1893,44 @@ function gvk_len_l(xs: []long) -> int { if xs == null { return 0 };  return len(
 function gvk_layout_id(render3d_st: mut Render3dState, m: Mesh) -> int {
   if m == null or m.attrs == null { return 1 }
   if m.vk_layout > 0 { return m.vk_layout }
-  let key = gvk_layout_key(m)
+  # the layout as numbers in a buffer made once - per attribute its index, its buffer's order of
-  if render3d_st.gvk_layout_keys == null { render3d_st.gvk_layout_keys = new []string }
+  # first use and its six fields - matched against the layouts known: a new mesh costs nothing, and
-  var id = 0
+  # only a layout never seen before is kept (a key string per mesh was never given back)
-  for i in 0 .. len(render3d_st.gvk_layout_keys) { if id == 0 and render3d_st.gvk_layout_keys[i] == key { id = i + 2 } }
+  if render3d_st.gvk_lay_sig == null {
-  if id == 0 { push(render3d_st.gvk_layout_keys, key);  id = len(render3d_st.gvk_layout_keys) + 1 }
+    render3d_st.gvk_lay_sig = words(GPU_MAX_ATTRS * 8);  render3d_st.gvk_lay_seen = words(GPU_MAX_ATTRS)
-  m.vk_layout = id
+    render3d_st.gvk_lay_flat = new []int;  render3d_st.gvk_lay_off = new []int;  render3d_st.gvk_lay_len = new []int
-  return id
+  }
+  let sig = render3d_st.gvk_lay_sig
+  let seen = render3d_st.gvk_lay_seen
+  var n = 0
+  var ns = 0
+  for i in 0 .. m.n_attrs {
+    let o = i * GPU_ATTR_W
+    if m.attrs[o + 1] == 0 { continue }
+    var bi = -1
+    for q in 0 .. ns { if seen[q] == m.attrs[o] and bi < 0 { bi = q } }
+    if bi < 0 { bi = ns;  seen[ns] = m.attrs[o];  ns += 1 }
+    sig[n] = i;  sig[n + 1] = bi
+    for k in 1 .. 7 { sig[n + 1 + k] = m.attrs[o + k] }
+    n += 8
+  }
+  let flat = render3d_st.gvk_lay_flat
+  for e in 0 .. len(render3d_st.gvk_lay_off) {
+    if render3d_st.gvk_lay_len[e] == n {
+      let at = render3d_st.gvk_lay_off[e]
+      var same = true
+      var k = 0
+      while same and k < n { if flat[at + k] != sig[k] { same = false };  k += 1 }
+      if same {
+        m.vk_layout = e + 2
+        return m.vk_layout
+      }
+    }
+  }
+  push(render3d_st.gvk_lay_off, len(flat));  push(render3d_st.gvk_lay_len, n)
+  for k in 0 .. n { push(flat, sig[k]) }
+  m.vk_layout = len(render3d_st.gvk_lay_off) + 1
+  return m.vk_layout
 }
 function gvk_blend_index(f: int) -> int {
   if f == GL_ZERO { return 0 }

packages/ludic.render3d/skin.ludic

@@ -87,7 +87,8 @@ function skin_load(render3d_st: mut Render3dState, idx: int) -> Skin {
   for i in 0 .. n {
     let nd = value_at(nodes, i)
     var nm: string = ""
-    if value_has(nd, "name") != 0 { nm = value_as_str(value_get(nd, "name")) }
+    # a copy: the document goes, strings and all, at the next load
+    if value_has(nd, "name") != 0 { let s = value_as_str(value_get(nd, "name"));  nm = s[0 .. len(s)] }
     push(sk.names, nm)
     skin_jv3(sk.rest_t, i * 3, nd, "translation", 0.0, 0.0, 0.0)
     skin_jv3(sk.rest_s, i * 3, nd, "scale", 1.0, 1.0, 1.0)

runtime/native/namespaces.ludic

@@ -490,6 +490,7 @@ namespace Json {
   alias encode(value) = json_encode
   alias parse(text) = json_parse
   alias free(value) = json_free
+  alias free_all(value) = json_free_all
 }
 namespace Xml {
   alias parse(text) = xml_parse

runtime/native/value.ludic

@@ -403,6 +403,21 @@ function value_free(v: Val) -> void {
   free(v)
 }
 function json_free(v: Val) -> void { value_free(v) }
+# ... and its strings too - every string value and object key the parser made - for a tree whose
+# reader copied what it keeps (render3d's glTF documents): a parse left otherwise gives back the
+# nodes and keeps every name, ~640 KB a model
+function value_free_all(v: Val) -> void {
+  if v == null { return }
+  if v.tag == 4 and v.txt != null { free(v.txt) }
+  if v.kids != null {
+    for i in 0 .. len(v.kids) { value_free_all(v.kids[i]) }
+    if v.keys != null { for i in 0 .. len(v.keys) { if v.keys[i] != null { free(v.keys[i]) } } }
+    free(v.kids)
+    free(v.keys)
+  }
+  free(v)
+}
+function json_free_all(v: Val) -> void { value_free_all(v) }
 
 function jp_value(p: JP) -> Val {
   jp_skip_ws(p)
@@ -486,6 +501,7 @@ function jp_number(p: JP) -> Val {
       frac = (digits[di] * 65536 + frac) / 10
       di -= 1
     }
+    free(digits)                                 # one list per decimal in the document, never given back
     var raw = ip * 65536 + frac
     raw = jp_exponent(p, raw)
     if neg != 0 { raw = -raw }