render3d: ground densities as LGD2 - empty tiles free, one-value tiles a word, deflated tiles, quantized

The first cut (LGD1) stored every tile of every layer raw, scale channel and all: 288 MB a map, which
made Maroon's pack 803 MB. LGD2 keeps a tile index per layer (8 bytes a tile, read whole at the map's
load): 0 all zero, 1 one value in the word, else an offset and a length, the top bit set when the bake
deflated it (inflated into the tile cache with the runtime's z_inflate, a reused buffer; nothing per
frame beyond the cache). A layer keeps its scale only where it varies, the density 6 bits and the scale
4 (the same integer arithmetic as the bake). The bake is maroon-lake's tools/bake/ground_density.py;
the dev copy here writes the same format stored. Maroon 23.2 MB, Lamar 17.2 MB.

The deflated flag is tested as a negative length: `b & 0x80000000` compiled to broken IR (an i32
`and` with a bare `-`), a compiler fault to fix separately.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/ludic.render3d/env.ludic

@@ -854,6 +854,8 @@ export state Render3dState {
   gd_key: words = null
   gd_ref: words = null
   gd_slot_of: words = null
+  gd_idx: words = null               # every layer's tile index: (offset, or 0 empty / 1 one value; length or value)
+  gd_zbuf: []byte = null             # one tile's deflated bytes as read
   gd_hand: int = 0
   gd_trample_cb: fn(float, float) -> float = null
   gb_layer: int = -1                 # the densities' layer the blades grow by; -1: the rules (grass_density.ludic)

packages/ludic.render3d/ground_density.ludic

@@ -34,7 +34,7 @@ function gd_open_at(render3d_st: mut Render3dState, path: string, base: int) ->
   let head = words(GD_HEADW)
   file_seek(f, base, 0)
   let got = file_read(f, data_of(head), GD_HEADW * 4)
-  if got != GD_HEADW * 4 or head[0] != GD_MAGIC or head[1] != GD_VERSION or head[3] != GD_SIDE {
+  if got != GD_HEADW * 4 or head[0] != GD_MAGIC or head[1] != GD_VERSION or head[3] != GD_SIDE or head[4] > GD_LAYERS {
     free(head)
     file_close(f)
     return false
@@ -46,15 +46,31 @@ function gd_open_at(render3d_st: mut Render3dState, path: string, base: int) ->
   for l in 0 .. GD_LAYERS { render3d_st.gd_ch[l] = 0;  render3d_st.gd_off[l] = 0 }
   for l in 0 .. layers { render3d_st.gd_ch[l] = head[8 + l * 2];  render3d_st.gd_off[l] = head[9 + l * 2] }
   free(head)
+  gd_index_read(render3d_st, layers * n * n)
   gd_cache_reset(render3d_st, layers * n * n)
   return true
 }
 
+# every layer's tile index read whole (8 bytes a tile: 1.8 MB for fourteen layers on an 8 km map)
+@alloc_ok("a map being loaded: the densities' index, made once and reused")
+function gd_index_read(render3d_st: mut Render3dState, tiles: int) -> void {
+  if render3d_st.gd_idx == null or len(render3d_st.gd_idx) < tiles * 2 {
+    if render3d_st.gd_idx != null { free(render3d_st.gd_idx) }
+    render3d_st.gd_idx = words(max(tiles * 2, 2))
+  }
+  let per = render3d_st.gd_n * render3d_st.gd_n
+  for l in 0 .. render3d_st.gd_layers {
+    file_seek(render3d_st.gd_file, render3d_st.gd_base + render3d_st.gd_off[l], 0)
+    file_read(render3d_st.gd_file, mem_off(data_of(render3d_st.gd_idx), l * per * 8), per * 8)
+  }
+}
+
 # the cache made (once) and emptied, and the index of where each layer's tile is: -1 where it is not in
 @alloc_ok("a map being loaded: the densities' cache and index, once")
 function gd_cache_reset(render3d_st: mut Render3dState, tiles: int) -> void {
   if render3d_st.gd_pool == null {
     render3d_st.gd_pool = buffer(GD_SLOTS * GD_SIDE * GD_SIDE * 2)
+    render3d_st.gd_zbuf = buffer(GD_SIDE * GD_SIDE * 2)
     render3d_st.gd_key = words(GD_SLOTS);  render3d_st.gd_ref = words(GD_SLOTS)
   }
   if render3d_st.gd_slot_of == null or len(render3d_st.gd_slot_of) < tiles {
@@ -86,15 +102,42 @@ function gd_slot(render3d_st: mut Render3dState, l: int, t: int) -> int {
   }
   render3d_st.gd_hand = (s + 1) % GD_SLOTS
   if render3d_st.gd_key[s] >= 0 { render3d_st.gd_slot_of[render3d_st.gd_key[s]] = -1 }
-  let tb = GD_SIDE * GD_SIDE * render3d_st.gd_ch[l]
-  file_seek(render3d_st.gd_file, render3d_st.gd_base + render3d_st.gd_off[l] + t * tb, 0)
-  file_read(render3d_st.gd_file, mem_off(data_of(render3d_st.gd_pool), s * GD_SIDE * GD_SIDE * 2), tb)
+  gd_fill(render3d_st, l, key, s)
   render3d_st.gd_key[s] = key
   render3d_st.gd_slot_of[key] = s
   render3d_st.gd_ref[s] = 1
   return s
 }
 
+# layer l's tile `key` (its place in the index) into slot s: nothing (zeros), one value, or its bytes read
+# and - when the bake deflated them - inflated
+function gd_fill(render3d_st: mut Render3dState, l: int, key: int, s: int) -> void {
+  let ch = render3d_st.gd_ch[l]
+  let tb = GD_SIDE * GD_SIDE * ch
+  let o = s * GD_SIDE * GD_SIDE * 2
+  let pool = render3d_st.gd_pool
+  let a = render3d_st.gd_idx[key * 2]
+  let b = render3d_st.gd_idx[key * 2 + 1]
+  if a == 0 or a == 1 {
+    var v = 0
+    var g = 0
+    if a == 1 { v = b & 255;  g = (b >> 8) & 255 }
+    for p in 0 .. GD_SIDE * GD_SIDE {
+      pool[o + p * ch] = v
+      if ch == 2 { pool[o + p * 2 + 1] = g }
+    }
+    return
+  }
+  let n = b & 0x7FFFFFFF
+  file_seek(render3d_st.gd_file, render3d_st.gd_base + a, 0)
+  if b < 0 {
+    file_read(render3d_st.gd_file, data_of(render3d_st.gd_zbuf), min(n, len(render3d_st.gd_zbuf)))
+    z_inflate(data_of(render3d_st.gd_zbuf), n, mem_off(data_of(pool), o), tb)
+  } else {
+    file_read(render3d_st.gd_file, mem_off(data_of(pool), o), min(n, tb))
+  }
+}
+
 # layer l's texel (tx, tz) (clamped to the map), channel c (0 density, 1 scale), 0..255
 function gd_texel(render3d_st: mut Render3dState, l: int, tx: int, tz: int, c: int) -> int {
   let w = render3d_st.gd_n * GD_SIDE

packages/ludic.render3d/ground_density_write.ludic

@@ -1,61 +1,96 @@
-# ground_density_write.ludic — a map's painted ground densities (one PNG per ground layer: R the chance a cell
-# keeps one, G an optional scale) cut into the terrain's 64 m tiles and written as one file (LGD1), the
-# build-time bake's payload or a dev build's own copy. One layer is decoded, cut and let go at a time.
+# ground_density_write.ludic — a map's painted ground densities (one PNG per layer: R the chance a cell keeps
+# one, G an optional scale) cut into the terrain's 64 m tiles as LGD2, the format the build-time bake writes
+# (maroon-lake's tools/bake/ground_density.py): an all-zero tile is nothing and a one-value tile a word, the
+# scale is kept only where it varies, and the density is held to 6 bits and the scale to 4. This is a dev
+# build's own copy, made when the bake is missing: the same bytes the bake decodes to, never deflated.
 
-const GD_MAGIC: int = 826557260        # "LGD1" read as a little-endian u32: 0x3144474C
-const GD_VERSION: int = 1
-const GD_HEADW: int = 136              # words: magic, version, n, side, layers, 3 spare, then (channels, offset) x 64
+const GD_MAGIC: int = 843335500        # "LGD2" read as a little-endian u32: 0x3244474C
+const GD_VERSION: int = 2
+const GD_HEADW: int = 136              # words: magic, version, n, side, layers, 3 spare, (channels, index) x 64
 const GD_LAYERS: int = 64
+const GD_R_BITS: int = 6
+const GD_G_BITS: int = 4
 
-# the layers' PNGs (paths, in the rows' order) written to f from `base`; the payload's length, or -1.
-# n tiles a side of `side` texels each (the PNGs must be n * side square)
-@alloc_ok("a bake, or a dev build's map load: the densities cut once")
+# v (0..255) held to `bits`, a byte again - the integer arithmetic the bake uses, so both agree to the bit
+function gd_q(v: int, bits: int) -> int {
+  let top = (1 << bits) - 1
+  let q = (v * top + 127) / 255
+  return (q * 255 + top / 2) / top
+}
+
+# the layers' PNGs written to f from `base` (n tiles a side of `side` texels); the payload's length, or -1
+@alloc_ok("a dev build's map load: the densities cut once")
 function gd_write_payload(render3d_st: mut Render3dState, f: pointer, base: int, pngs: []string, n: int, side: int) -> int {
   let layers = min(len(pngs), GD_LAYERS)
   let head = words(GD_HEADW)
   for k in 0 .. GD_HEADW { head[k] = 0 }
-  let hbytes = GD_HEADW * 4
-  file_write(f, data_of(head), hbytes)
-  var at = hbytes
+  let idx = words(max(layers * n * n * 2, 1))
+  for k in 0 .. len(idx) { idx[k] = 0 }
+  let ib = GD_HEADW * 4
+  file_write(f, data_of(head), ib)
+  file_write(f, data_of(idx), layers * n * n * 8)
+  var at = ib + layers * n * n * 8
   let tile = buffer(side * side * 2)
   for l in 0 .. layers {
     let px = png_decode_bytes(render3d_st, pngs[l], null)
     if px == null or render3d_st.tex_w != n * side or render3d_st.tex_h != n * side { gd_say_bad(pngs[l], n * side);  return -1 }
     let c = render3d_st.tex_channels
     var ch = 1
-    if c >= 2 { ch = 2 }
-    head[8 + l * 2] = ch;  head[9 + l * 2] = at
-    at += gd_write_layer(f, px, c, ch, n, side, tile)
+    if c >= 2 and gd_g_varies(px, c, n * side) { ch = 2 }
+    head[8 + l * 2] = ch;  head[9 + l * 2] = ib + l * n * n * 8
+    at = gd_write_layer(f, px, c, ch, n, side, tile, idx, l * n * n * 2, at)
     free(px)
   }
   free(tile)
   head[0] = GD_MAGIC;  head[1] = GD_VERSION;  head[2] = n;  head[3] = side;  head[4] = layers
   file_seek(f, base, 0)
-  file_write(f, data_of(head), hbytes)
+  file_write(f, data_of(head), ib)
+  file_write(f, data_of(idx), layers * n * n * 8)
   file_seek(f, base + at, 0)
-  free(head)
+  free(head);  free(idx)
   return at
 }
 
-# one layer's tiles, tile after tile; its bytes
-function gd_write_layer(f: pointer, px: []byte, c: int, ch: int, n: int, side: int, tile: []byte) -> int {
+# does the PNG's G vary anywhere (else the layer keeps R alone)
+function gd_g_varies(px: []byte, c: int, w: int) -> bool {
+  let g0 = px[1]
+  for i in 0 .. w * w { if px[i * c + 1] != g0 { return true } }
+  return false
+}
+
+# one layer's tiles: each quantized, then nothing (all zero), a word (one value) or its bytes; the next offset
+function gd_write_layer(f: pointer, px: []byte, c: int, ch: int, n: int, side: int, tile: []byte, idx: words, i0: int, at0: int) -> int {
   let w = n * side
+  var at = at0
   for j in 0 .. n {
     for i in 0 .. n {
+      var same = true
       for r in 0 .. side {
         for k in 0 .. side {
           let o = ((j * side + r) * w + i * side + k) * c
-          tile[(r * side + k) * ch] = px[o]
-          if ch == 2 { tile[(r * side + k) * 2 + 1] = px[o + 1] }
+          let p = (r * side + k) * ch
+          tile[p] = gd_q(px[o], GD_R_BITS)
+          if ch == 2 { tile[p + 1] = gd_q(px[o + 1], GD_G_BITS) }
+          if tile[p] != tile[0] or (ch == 2 and tile[p + 1] != tile[1]) { same = false }
         }
       }
+      let e = i0 + (j * n + i) * 2
+      if same {
+        var g = 0
+        if ch == 2 { g = tile[1] }
+        if tile[0] != 0 or g != 0 { idx[e] = 1;  idx[e + 1] = tile[0] | (g << 8) }
+      } else {
         file_write(f, data_of(tile), side * side * ch)
+        idx[e] = at;  idx[e + 1] = side * side * ch
+        at += side * side * ch
       }
     }
-  return n * n * side * side * ch
+  }
+  return at
 }
 
-# The bake: the densities as a bake's file (ludic.base's LBAK header, as terrain_tiles_bake writes it)
+# The bake's own file from a build without Python (ludic.base's LBAK header, as terrain_tiles_bake writes it):
+# stored tiles, which the reader takes as it takes the deflated ones
 @alloc_ok("a bake: once, at build time")
 export function ground_density_bake(render3d_st: mut Render3dState, path: string, key: string, version: int, inputs_hash: long, pngs: []string) -> bool {
   Fs.mkdir(Path.dir(path))