feat(ecs): chunks as a grouping - IntIndex takes sparse values; ludic.things' chunk column and things_drop_chunk

A streamed world loads and lets go of a chunk's entities together, so a chunk is a first-class
group: chunk_of(x, z, size) packs a chunk's cell into an int, tb_remove_all(tb, ix, v) removes every
row an index files under one value (the observers told of each). IntIndex kept one list per value
up to the largest, which a packed chunk number (hundreds of millions) turned into hundreds of
millions of empty lists and a 12 GB test run; a value past 1024 now gets its slot through an IntMap,
so a sparse value costs one list.

ludic.things files every Thing under its 256 m chunk (TH_CHUNK), kept by every move;
things_chunk_count and things_drop_chunk (the port and the facts told of each removal, as
thing_remove tells them) are what the memory budget's streamed chunks ask. The moves are their own
file (moves.ludic). Tests: ludic.base 35, ludic.things 8, all under a 2 GB cap (24 MB peak).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/ludic.base/ecs_group.ludic

@@ -0,0 +1,20 @@
+# ludic.base/ecs_group.ludic - a group is every row an index files under one value: a kind, or a
+# chunk of the world, loaded and let go together. A chunk's number packs its two cell coordinates.
+export function chunk_of(x: float, z: float, size: float) -> int {
+  let cx = int(Math.floor(x / size))
+  let cz = int(Math.floor(z / size))
+  return ((cx & 32767) << 15) | (cz & 32767)
+}
+
+# every row filed under v in ix removed (the observers told of each); how many went. The index's
+# own list shrinks as each goes, so it is read from its end until it is empty.
+export function tb_remove_all<T>(tb: Table<T>, ix: IntIndex, v: int) -> int {
+  var n = 0
+  let rows = ix_rows(ix, v)
+  while len(rows) > 0 {
+    let h = tb.ent[rows[len(rows) - 1]]
+    if not tb_remove(tb, h) { return n }
+    n += 1
+  }
+  return n
+}

packages/ludic.base/ecs_index.ludic

@@ -5,10 +5,10 @@ export property IntIndex {
   gate: int = -1              # an int column: a row is filed only while it is non-zero (-1: always)
   lists: []words = null       # value -> its rows
   pos: words = null           # row -> where it stands in its value's list, -1 not filed
-  val: words = null           # row -> the value it is filed under
+  val: words = null           # row -> the slot of the value it is filed under
   empty: words = null         # what a value with no rows answers
+  sparse: IntMap = null       # a value past IX_DENSE (a chunk's packed cell) to its list's slot
 }
-
 export function tb_index<T>(tb: Table<T>, col: int, gate: int) -> IntIndex {
   let ix = new IntIndex
   ix.col = col
@@ -17,6 +17,7 @@ export function tb_index<T>(tb: Table<T>, col: int, gate: int) -> IntIndex {
   ix.pos = words(0)
   ix.val = words(0)
   ix.empty = words(0)
+  ix.sparse = imap_new()
   for r in 0 .. tb.n { ix_grow(ix) }
   push(tb.idx, ix)
   for r in 0 .. tb.n { tb_ix_refile(tb, ix, r) }
@@ -25,17 +26,16 @@ export function tb_index<T>(tb: Table<T>, col: int, gate: int) -> IntIndex {
 
 # the rows filed under v - the index's own list: read it, never push to it
 export function ix_rows(ix: IntIndex, v: int) -> words {
-  if v < 0 or v >= len(ix.lists) { return ix.empty }
-  return ix.lists[v]
-}
-export function ix_count(ix: IntIndex, v: int) -> int {
-  if v < 0 or v >= len(ix.lists) { return 0 }
-  return len(ix.lists[v])
+  let s = ix_slot(ix, v, false)
+  if s < 0 { return ix.empty }
+  return ix.lists[s]
 }
+export function ix_count(ix: IntIndex, v: int) -> int { return len(ix_rows(ix, v)) }
 # the first row filed under v, or -1
 export function ix_first(ix: IntIndex, v: int) -> int {
-  if ix_count(ix, v) == 0 { return -1 }
-  return ix.lists[v][0]
+  let rows = ix_rows(ix, v)
+  if len(rows) == 0 { return -1 }
+  return rows[0]
 }
 
 function ix_grow(ix: IntIndex) -> void {
@@ -49,13 +49,13 @@ function ix_shrink(ix: IntIndex) -> void {
 }
 
 function ix_file(ix: IntIndex, r: int, v: int) -> void {
-  if ix.pos[r] >= 0 and ix.val[r] == v { return }
+  let s = ix_slot(ix, v, true)
+  if ix.pos[r] >= 0 and ix.val[r] == s { return }
   ix_unfile(ix, r)
-  if v < 0 { return }
-  while len(ix.lists) <= v { push(ix.lists, words(0)) }
-  ix.pos[r] = len(ix.lists[v])
-  ix.val[r] = v
-  push(ix.lists[v], r)
+  if s < 0 { return }
+  ix.pos[r] = len(ix.lists[s])
+  ix.val[r] = s
+  push(ix.lists[s], r)
 }
 
 # out of its list: the list's last row takes its place

packages/ludic.base/ecs_index_slot.ludic

@@ -0,0 +1,24 @@
+# ludic.base/ecs_index_slot.ludic - where a value's list lives: a small value (a kind) is its own
+# slot; a large one (a chunk's packed cell) is given the next slot through a map, so a sparse
+# value costs one list and not every list up to it
+const IX_DENSE: int = 1024     # a value below this is its own slot; above, one is given it
+
+# the slot value v's list is in, -1 when it has none (and `make` did not ask for one)
+function ix_slot(ix: IntIndex, v: int, make: bool) -> int {
+  if v < 0 { return -1 }
+  if v < IX_DENSE {
+    if v >= len(ix.lists) {
+      if not make { return -1 }
+      while len(ix.lists) <= v { push(ix.lists, words(0)) }
+    }
+    return v
+  }
+  var s = imap_get(ix.sparse, v, -1)
+  if s < 0 and make {
+    while len(ix.lists) < IX_DENSE { push(ix.lists, words(0)) }
+    s = len(ix.lists)
+    push(ix.lists, words(0))
+    imap_put(ix.sparse, v, s)
+  }
+  return s
+}

packages/ludic.base/index.ludic

@@ -18,5 +18,7 @@ import "ecs_grid_query.ludic"
 import "ecs_grid_within.ludic"
 import "ecs_grid_where.ludic"
 import "ecs_index.ludic"
+import "ecs_index_slot.ludic"
 import "ecs_map.ludic"
 import "ecs_plan.ludic"
+import "ecs_group.ludic"

packages/ludic.base/tests/ecs_test.ludic

@@ -152,6 +152,30 @@ program EcsTest {
     expect_eq(tb_changed_since(tb, seen, out), 2)     # a removal is told by tb_on_remove
   }
 
+  # values far apart (a chunk's packed cell) each take one list, not every list up to them
+  function sparse_case() -> void {
+    let tb: Table<Toy> = table_new(2, 2)
+    let ix = tb_index(tb, KIND, -1)
+    let a = chunk_of(9000.0, -9000.0, 256.0)
+    let b = chunk_of(-512.0, 700.0, 256.0)
+    expect(a > 100000 and b > 100000)
+    put(tb, "a", 0.0, 0.0, a)
+    put(tb, "b", 0.0, 0.0, b)
+    let c = put(tb, "c", 0.0, 0.0, a)
+    put(tb, "d", 0.0, 0.0, 3)
+    expect_eq(ix_count(ix, a), 2)
+    expect_eq(ix_count(ix, b), 1)
+    expect_eq(ix_count(ix, 3), 1)
+    expect(len(ix.lists) < 1100)
+    tb_set_i(tb, KIND, tb_row(tb, c), b)
+    expect_eq(ix_count(ix, a), 1)
+    expect_eq(ix_count(ix, b), 2)
+    expect_eq(tb_remove_all(tb, ix, b), 2)
+    expect_eq(ix_count(ix, b), 0)
+    expect_eq(tb_len(tb), 2)
+  }
+
+  test "a sparse value takes one list" () { sparse_case() }
   test "observers hear each add, and a remove once" (hook_st: HookState) { hooks_case(hook_st) }
   test "change detection skips the blocks nobody wrote" () { blocks_case() }
   test "a removed handle misses, rows stay dense, a reused slot is a new handle" () { handles_case() }