feat(ecs): chunks as a grouping - IntIndex takes sparse values; ludic.things' chunk column and things_drop_chunk

A streamed world loads and lets go of a chunk's entities together, so a chunk is a first-class
group: chunk_of(x, z, size) packs a chunk's cell into an int, tb_remove_all(tb, ix, v) removes every
row an index files under one value (the observers told of each). IntIndex kept one list per value
up to the largest, which a packed chunk number (hundreds of millions) turned into hundreds of
millions of empty lists and a 12 GB test run; a value past 1024 now gets its slot through an IntMap,
so a sparse value costs one list.

ludic.things files every Thing under its 256 m chunk (TH_CHUNK), kept by every move;
things_chunk_count and things_drop_chunk (the port and the facts told of each removal, as
thing_remove tells them) are what the memory budget's streamed chunks ask. The moves are their own
file (moves.ludic). Tests: ludic.base 35, ludic.things 8, all under a 2 GB cap (24 MB peak).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/ludic.base/ecs_group.ludic

@@ -0,0 +1,20 @@
+# ludic.base/ecs_group.ludic - a group is every row an index files under one value: a kind, or a
+# chunk of the world, loaded and let go together. A chunk's number packs its two cell coordinates.
+export function chunk_of(x: float, z: float, size: float) -> int {
+  let cx = int(Math.floor(x / size))
+  let cz = int(Math.floor(z / size))
+  return ((cx & 32767) << 15) | (cz & 32767)
+}
+
+# every row filed under v in ix removed (the observers told of each); how many went. The index's
+# own list shrinks as each goes, so it is read from its end until it is empty.
+export function tb_remove_all<T>(tb: Table<T>, ix: IntIndex, v: int) -> int {
+  var n = 0
+  let rows = ix_rows(ix, v)
+  while len(rows) > 0 {
+    let h = tb.ent[rows[len(rows) - 1]]
+    if not tb_remove(tb, h) { return n }
+    n += 1
+  }
+  return n
+}

packages/ludic.base/ecs_index.ludic

@@ -5,10 +5,10 @@ export property IntIndex {
   gate: int = -1              # an int column: a row is filed only while it is non-zero (-1: always)
   lists: []words = null       # value -> its rows
   pos: words = null           # row -> where it stands in its value's list, -1 not filed
-  val: words = null           # row -> the value it is filed under
+  val: words = null           # row -> the slot of the value it is filed under
   empty: words = null         # what a value with no rows answers
+  sparse: IntMap = null       # a value past IX_DENSE (a chunk's packed cell) to its list's slot
 }
-
 export function tb_index<T>(tb: Table<T>, col: int, gate: int) -> IntIndex {
   let ix = new IntIndex
   ix.col = col
@@ -17,6 +17,7 @@ export function tb_index<T>(tb: Table<T>, col: int, gate: int) -> IntIndex {
   ix.pos = words(0)
   ix.val = words(0)
   ix.empty = words(0)
+  ix.sparse = imap_new()
   for r in 0 .. tb.n { ix_grow(ix) }
   push(tb.idx, ix)
   for r in 0 .. tb.n { tb_ix_refile(tb, ix, r) }
@@ -25,17 +26,16 @@ export function tb_index<T>(tb: Table<T>, col: int, gate: int) -> IntIndex {
 
 # the rows filed under v - the index's own list: read it, never push to it
 export function ix_rows(ix: IntIndex, v: int) -> words {
-  if v < 0 or v >= len(ix.lists) { return ix.empty }
-  return ix.lists[v]
-}
-export function ix_count(ix: IntIndex, v: int) -> int {
-  if v < 0 or v >= len(ix.lists) { return 0 }
-  return len(ix.lists[v])
+  let s = ix_slot(ix, v, false)
+  if s < 0 { return ix.empty }
+  return ix.lists[s]
 }
+export function ix_count(ix: IntIndex, v: int) -> int { return len(ix_rows(ix, v)) }
 # the first row filed under v, or -1
 export function ix_first(ix: IntIndex, v: int) -> int {
-  if ix_count(ix, v) == 0 { return -1 }
-  return ix.lists[v][0]
+  let rows = ix_rows(ix, v)
+  if len(rows) == 0 { return -1 }
+  return rows[0]
 }
 
 function ix_grow(ix: IntIndex) -> void {
@@ -49,13 +49,13 @@ function ix_shrink(ix: IntIndex) -> void {
 }
 
 function ix_file(ix: IntIndex, r: int, v: int) -> void {
-  if ix.pos[r] >= 0 and ix.val[r] == v { return }
+  let s = ix_slot(ix, v, true)
+  if ix.pos[r] >= 0 and ix.val[r] == s { return }
   ix_unfile(ix, r)
-  if v < 0 { return }
-  while len(ix.lists) <= v { push(ix.lists, words(0)) }
-  ix.pos[r] = len(ix.lists[v])
-  ix.val[r] = v
-  push(ix.lists[v], r)
+  if s < 0 { return }
+  ix.pos[r] = len(ix.lists[s])
+  ix.val[r] = s
+  push(ix.lists[s], r)
 }
 
 # out of its list: the list's last row takes its place

packages/ludic.base/ecs_index_slot.ludic

@@ -0,0 +1,24 @@
+# ludic.base/ecs_index_slot.ludic - where a value's list lives: a small value (a kind) is its own
+# slot; a large one (a chunk's packed cell) is given the next slot through a map, so a sparse
+# value costs one list and not every list up to it
+const IX_DENSE: int = 1024     # a value below this is its own slot; above, one is given it
+
+# the slot value v's list is in, -1 when it has none (and `make` did not ask for one)
+function ix_slot(ix: IntIndex, v: int, make: bool) -> int {
+  if v < 0 { return -1 }
+  if v < IX_DENSE {
+    if v >= len(ix.lists) {
+      if not make { return -1 }
+      while len(ix.lists) <= v { push(ix.lists, words(0)) }
+    }
+    return v
+  }
+  var s = imap_get(ix.sparse, v, -1)
+  if s < 0 and make {
+    while len(ix.lists) < IX_DENSE { push(ix.lists, words(0)) }
+    s = len(ix.lists)
+    push(ix.lists, words(0))
+    imap_put(ix.sparse, v, s)
+  }
+  return s
+}

packages/ludic.base/index.ludic

@@ -18,5 +18,7 @@ import "ecs_grid_query.ludic"
 import "ecs_grid_within.ludic"
 import "ecs_grid_where.ludic"
 import "ecs_index.ludic"
+import "ecs_index_slot.ludic"
 import "ecs_map.ludic"
 import "ecs_plan.ludic"
+import "ecs_group.ludic"

packages/ludic.base/tests/ecs_test.ludic

@@ -152,6 +152,30 @@ program EcsTest {
     expect_eq(tb_changed_since(tb, seen, out), 2)     # a removal is told by tb_on_remove
   }
 
+  # values far apart (a chunk's packed cell) each take one list, not every list up to them
+  function sparse_case() -> void {
+    let tb: Table<Toy> = table_new(2, 2)
+    let ix = tb_index(tb, KIND, -1)
+    let a = chunk_of(9000.0, -9000.0, 256.0)
+    let b = chunk_of(-512.0, 700.0, 256.0)
+    expect(a > 100000 and b > 100000)
+    put(tb, "a", 0.0, 0.0, a)
+    put(tb, "b", 0.0, 0.0, b)
+    let c = put(tb, "c", 0.0, 0.0, a)
+    put(tb, "d", 0.0, 0.0, 3)
+    expect_eq(ix_count(ix, a), 2)
+    expect_eq(ix_count(ix, b), 1)
+    expect_eq(ix_count(ix, 3), 1)
+    expect(len(ix.lists) < 1100)
+    tb_set_i(tb, KIND, tb_row(tb, c), b)
+    expect_eq(ix_count(ix, a), 1)
+    expect_eq(ix_count(ix, b), 2)
+    expect_eq(tb_remove_all(tb, ix, b), 2)
+    expect_eq(ix_count(ix, b), 0)
+    expect_eq(tb_len(tb), 2)
+  }
+
+  test "a sparse value takes one list" () { sparse_case() }
   test "observers hear each add, and a remove once" (hook_st: HookState) { hooks_case(hook_st) }
   test "change detection skips the blocks nobody wrote" () { blocks_case() }
   test "a removed handle misses, rows stay dense, a reused slot is a new handle" () { handles_case() }

packages/ludic.things/chunks.ludic

@@ -0,0 +1,17 @@
+# chunks.ludic - the Things of one chunk of the world, counted and let go together: what a streamed
+# world asks when a chunk is loaded or dropped. A chunk is chunk_of(x, z, TH_CHUNK_M).
+export function things_chunk(x: float, z: float) -> int { return chunk_of(x, z, TH_CHUNK_M) }
+
+export function things_chunk_count(things_st: ThingsState, chunk: int) -> int { return ix_count(things_st.th.chunks, chunk) }
+
+# every Thing standing in the chunk removed for good, the port and the facts told of each as
+# thing_remove tells them; how many went
+export function things_drop_chunk(things_st: mut ThingsState, chunk: int) -> int {
+  var n = 0
+  let rows = ix_rows(things_st.th.chunks, chunk)
+  while len(rows) > 0 {
+    thing_remove(things_st, things_st.th.tab.rec[rows[len(rows) - 1]])
+    n += 1
+  }
+  return n
+}

packages/ludic.things/index.ludic

@@ -7,7 +7,9 @@ import "kinds.ludic"
 import "port.ludic"
 import "store.ludic"
 import "verbs.ludic"
+import "moves.ludic"
 import "queries.ludic"
 import "near.ludic"
+import "chunks.ludic"
 import "dispatch.ludic"
 import "system.ludic"

packages/ludic.things/moves.ludic

@@ -0,0 +1,55 @@
+# moves.ludic - a Thing moved, shown or hidden: the record and its row together, so the grid, the
+# kind indexes and the chunk index never answer for where it was or what it was
+export function thing_move(t: Thing, x: float, z: float, yaw: float) -> void {
+  thing_move_to(t, x, ThingsWorld.ground(x, z), z, yaw)
+}
+
+export function thing_move_to(t: Thing, x: float, y: float, z: float, yaw: float) -> void {
+  thing_place_at(t, x, y, z, yaw)
+  ThingsWorld.moved(t)
+}
+
+# moved without telling the port: a stand-in the game draws itself, or nothing draws
+export function thing_place_at(t: Thing, x: float, y: float, z: float, yaw: float) -> void {
+  t.x = x
+  t.y = y
+  t.z = z
+  t.yaw = yaw
+  let r = th_row(t)
+  if r >= 0 { th_moved(t.tab, r, x, z) }
+}
+
+export function thing_set_active(t: Thing, on: bool) -> void {
+  if t == null { return }
+  thing_set_on(t, on)
+  ThingsWorld.shown(t)
+}
+export function thing_hide(t: Thing) -> void { thing_set_active(t, false) }
+export function thing_show(t: Thing) -> void { thing_set_active(t, true) }
+
+# active or not without telling the port
+export function thing_set_on(t: Thing, on: bool) -> void {
+  if t == null { return }
+  t.active = on
+  let r = th_row(t)
+  if r < 0 { return }
+  var v = 0
+  if on { v = 1 }
+  tb_set_i(t.tab, TH_ON, r, v)
+}
+
+# where it stands on the ground plane, without telling the port (its height is the caller's)
+export function thing_set_xz(t: Thing, x: float, z: float) -> void {
+  if t == null { return }
+  t.x = x
+  t.z = z
+  let r = th_row(t)
+  if r >= 0 { th_moved(t.tab, r, x, z) }
+}
+
+# where a row now stands: its columns, its cell in the grid and its chunk
+function th_moved(tb: Table<Thing>, r: int, x: float, z: float) -> void {
+  tb_set_xz(tb, TH_X, TH_Z, r, x, z)
+  let c = chunk_of(x, z, TH_CHUNK_M)
+  if tb.i[TH_CHUNK][r] != c { tb_set_i(tb, TH_CHUNK, r, c) }
+}

packages/ludic.things/store.ludic

@@ -5,6 +5,8 @@ export const TH_X: int = 0          # float columns
 export const TH_Z: int = 1
 export const TH_KIND: int = 0       # int columns
 export const TH_ON: int = 1         # 1 while active
+export const TH_CHUNK: int = 2      # the chunk of the world it stands in (chunk_of, TH_CHUNK_M)
+export const TH_CHUNK_M: float = 256.0
 const TH_CELL: float = 16.0
 
 # the table and the indexes kept over it, made together
@@ -13,6 +15,7 @@ export property ThingsTable {
   grid: Grid = null
   kinds: IntIndex = null            # active Things by kind
   every: IntIndex = null            # every Thing by kind, active or not
+  chunks: IntIndex = null           # every Thing by the chunk it stands in: loaded and let go together
 }
 
 export state ThingsState {
@@ -27,11 +30,12 @@ function th_facts__new() -> Queue<ThingFact> { return queue_new("things.facts")
 
 function th__new() -> ThingsTable {
   let th = new ThingsTable
-  let tb: Table<Thing> = table_new(2, 2)
+  let tb: Table<Thing> = table_new(2, 3)
   th.tab = tb
   th.grid = tb_grid(tb, TH_X, TH_Z, TH_ON, TH_CELL)
   th.kinds = tb_index(tb, TH_KIND, TH_ON)
   th.every = tb_index(tb, TH_KIND, -1)
+  th.chunks = tb_index(tb, TH_CHUNK, -1)
   return th
 }
 
@@ -82,7 +86,7 @@ export function things_verify(things_st: ThingsState) -> int {
     let t = tb.rec[r]
     var on = 0
     if t.active { on = 1 }
-    if tb.f[TH_X][r] != t.x or tb.f[TH_Z][r] != t.z or tb.i[TH_KIND][r] != t.kind or tb.i[TH_ON][r] != on or t.ent != tb.ent[r] { bad += 1 }
+    if tb.f[TH_X][r] != t.x or tb.f[TH_Z][r] != t.z or tb.i[TH_KIND][r] != t.kind or tb.i[TH_ON][r] != on or tb.i[TH_CHUNK][r] != chunk_of(t.x, t.z, TH_CHUNK_M) or t.ent != tb.ent[r] { bad += 1 }
   }
   return bad
 }

packages/ludic.things/tests/things_test.ludic

@@ -178,4 +178,23 @@ program ThingsTest {
     keep.x = 1.0
     expect_eq(things_verify(things_st), 1)
   }
+
+  test "a chunk's Things are counted and let go together; a move carries a Thing to its new chunk" (things_st: mut ThingsState, things_test_st: ThingsTestState) {
+    things_clear(things_st)
+    put(things_st, TH_BRANCH, 10.0, 10.0)
+    put(things_st, TH_BRANCH, 20.0, 30.0)
+    let far = put(things_st, TH_BRANCH, 600.0, 10.0)
+    let here = things_chunk(10.0, 10.0)
+    expect_eq(things_chunk_count(things_st, here), 2)
+    expect_eq(things_chunk_count(things_st, things_chunk(600.0, 10.0)), 1)
+    thing_move(far, 15.0, 15.0, 0.0)
+    expect_eq(things_chunk_count(things_st, here), 3)
+    facts(things_st)
+    let gone = things_test_st.gone
+    expect_eq(things_drop_chunk(things_st, here), 3)
+    expect_eq(things_count(things_st), 0)
+    expect_eq(things_test_st.gone, gone + 3)
+    expect_eq(len(facts(things_st)), 3)
+    expect_eq(things_verify(things_st), 0)
+  }
 }

packages/ludic.things/verbs.ludic

@@ -35,6 +35,7 @@ function th_columns(t: Thing) -> void {
   var on = 0
   if t.active { on = 1 }
   tb.i[TH_ON][r] = on
+  tb.i[TH_CHUNK][r] = chunk_of(t.x, t.z, TH_CHUNK_M)
   tb_refile(tb, r)
 }
 
@@ -66,50 +67,3 @@ export function things_drop(things_st: mut ThingsState, gone: fn(Thing) -> bool)
     r -= 1
   }
 }
-
-export function thing_move(t: Thing, x: float, z: float, yaw: float) -> void {
-  thing_move_to(t, x, ThingsWorld.ground(x, z), z, yaw)
-}
-
-export function thing_move_to(t: Thing, x: float, y: float, z: float, yaw: float) -> void {
-  thing_place_at(t, x, y, z, yaw)
-  ThingsWorld.moved(t)
-}
-
-# moved without telling the port: a stand-in the game draws itself, or nothing draws
-export function thing_place_at(t: Thing, x: float, y: float, z: float, yaw: float) -> void {
-  t.x = x
-  t.y = y
-  t.z = z
-  t.yaw = yaw
-  let r = th_row(t)
-  if r >= 0 { tb_set_xz(t.tab, TH_X, TH_Z, r, x, z) }
-}
-
-export function thing_set_active(t: Thing, on: bool) -> void {
-  if t == null { return }
-  thing_set_on(t, on)
-  ThingsWorld.shown(t)
-}
-export function thing_hide(t: Thing) -> void { thing_set_active(t, false) }
-export function thing_show(t: Thing) -> void { thing_set_active(t, true) }
-
-# active or not without telling the port
-export function thing_set_on(t: Thing, on: bool) -> void {
-  if t == null { return }
-  t.active = on
-  let r = th_row(t)
-  if r < 0 { return }
-  var v = 0
-  if on { v = 1 }
-  tb_set_i(t.tab, TH_ON, r, v)
-}
-
-# where it stands on the ground plane, without telling the port (its height is the caller's)
-export function thing_set_xz(t: Thing, x: float, z: float) -> void {
-  if t == null { return }
-  t.x = x
-  t.z = z
-  let r = th_row(t)
-  if r >= 0 { tb_set_xz(t.tab, TH_X, TH_Z, r, x, z) }
-}