chunked tables: a key is unique across its map and the slot's own, not interned - a slot keeps row i's key in its own buffer i, rewritten when the slot is refilled (interning ~92k map-unique keys as a player walks would fill the bounded intern table and keep them all); ludicc --check refuses a key written in two of a map's chunk files, naming both; reseeded

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

LANGUAGE.md

@@ -694,10 +694,12 @@ refills them in place - the rows list and each row's lists are emptied and refil
 back to its record's defaults (a template made once) - and a record is made only when a load needs
 more of its type than any load before it. Nothing is allocated past that high water, so a frame may
 call `instances_in`. Never keep a row, a row's list or a chunk's rows across a load or an `_out`: keep
-the key or the index, or copy the numbers. A string field (and a key) is interned, and safe to keep;
-the program's interned texts are bounded (49152 distinct, 4 MB, past which each is a heap copy the
-fence reports), so a chunked table's keys should repeat from chunk to chunk (`i0`, `i1`, ...) - its
-row id is `(cx, cz, key)` - rather than count across the whole map. A record may hold a record of its
+the key or the index, or copy the numbers. A string field, and a whole-map table's key, is interned
+and safe to keep. A CHUNKED table's keys are unique across its map (`t0` .. `t91842`: a row's id is
+`(map, key)`, and a tree moved into another chunk keeps it; `ludicc --check` refuses a key written in
+two chunk files, naming both), so they are NOT interned - interning every key a player walks past would
+fill the bounded intern table and keep them all: a slot's keys live in the slot's own buffers,
+rewritten when the slot is refilled. Keep such a key past `_out` with `intern(row.key)`. A record may hold a record of its
 own type only through a list. The reader
 (the runtime's `lres.ludic`) keeps its own buffers the same way: the file's bytes in one that grows
 only for a bigger file, its tree as parallel lists reused from file to file.

changes/chunked-keys.md

@@ -0,0 +1,7 @@
+bump: patch
+type: fix
+**A chunked table's keys are unique across its map, and not interned.** A row's id is `(map, key)`, so a
+tree moved into another chunk keeps it, and `ludicc --check` refuses a key written in two of a map's
+chunk files, naming both. The keys are no longer interned: interning every key a player walked past would
+have filled the bounded intern table and kept them all for good. A chunk slot keeps its keys in its own
+buffers, rewritten in place when the slot is refilled; `intern(row.key)` keeps one past `_out`.

examples/rejected/permap_maps/maps/m1/trees/-1_0.lres

@@ -0,0 +1,2 @@
+t3 { x: -1.0 }
+t1 { x: -2.0 }

examples/rejected/permap_maps/maps/m1/trees/0_0.lres

@@ -0,0 +1,2 @@
+t1 { x: 1.0 }
+t2 { x: 2.0 }

examples/rejected/permap_maps/permap_maps.ludic

@@ -1,6 +1,7 @@
 # permap_maps.ludic - `ludicc --check` reads every map directory and checks its tables against their
-# records. maps/m1 holds four mistakes, each refused at its file, line and column: a field the
-# record lacks, a value of the wrong type, a constant nobody declared, and a key naming no row.
+# records. maps/m1 holds five mistakes, each refused at its file, line and column: a field the
+# record lacks, a value of the wrong type, a constant nobody declared, a key naming no row, and a
+# chunked table's key written in two of the map's chunk files (t1, in trees/-1_0 and trees/0_0).
 program PerMapMaps {
   numbers float
   const MDL_TENT: int = 7
@@ -13,6 +14,8 @@ program PerMapMaps {
     @Ref(Spots) home: string = ""
   }
   @PerMap @ByKey registry Props of PropRow from "props.lres"
+  property Tree { key: string = "", x: float = 0.0 }
+  @PerMap @Chunked(64) registry Trees of Tree from "trees/{cx}_{cz}.lres"
   entry (props: mut Props) {
     print(`{props_load(props, "m1")}`)
   }

runtime/native/lres.ludic

@@ -43,6 +43,9 @@ property LresTree {
   fval: []float = new []float  # a float (an int's value too)
   sval: []string = new []string   # a string, a name, a fn's name (interned)
   name: []string = new []string   # an entry's key, a field's name ("" in a list)
+  local: bool = false          # a chunk's file: an entry's key is left in the file (koff, klen), never interned
+  koff: []int = new []int      # per entry, when local: where its key starts in buf ...
+  klen: []int = new []int      # ... and how long it is
   kid: []int = new []int       # the first child of a record or a list, -1
   nxt: []int = new []int       # the next sibling, -1
   line: []int = new []int
@@ -64,6 +67,42 @@ property LresTree {
   cfv: []float = new []float
 }
 
+# a chunk slot's own keys: a chunked table's keys are unique across a map (t0 .. t91842), so interning
+# them would fill the intern table as a player walks and keep every one for good. Row i's key lives
+# in buffer i, rewritten in place when the slot is refilled and grown only past the longest key it held.
+property LresKeys {
+  bufs: []pointer = new []pointer
+  caps: []int = new []int
+}
+# entry e's key as row i of a slot: in the slot's own buffer, or interned for a whole-map table (ks null)
+@alloc_ok("a slot's key buffer past the most rows or the longest key it has held (high water)")
+function lres_key_of(t: LresTree, e: int, ks: LresKeys, i: int) -> string {
+  if ks == null or not t.local { return t.name[e] }
+  let n = t.klen[e]
+  if i >= len(ks.bufs) {
+    var c = 16
+    while c < n + 1 { c = c * 2 }
+    push(ks.bufs, bytes(c))
+    push(ks.caps, c)
+  } else if ks.caps[i] < n + 1 {
+    var c = ks.caps[i] * 2
+    while c < n + 1 { c = c * 2 }
+    ks.bufs[i] = resize(ks.bufs[i], c)
+    ks.caps[i] = c
+  }
+  let kb = ks.bufs[i]
+  let buf = t.buf
+  let a = t.koff[e]
+  var j = 0
+  while j < n {
+    kb[j] = buf[a + j]
+    j += 1
+  }
+  kb[n] = 0
+  let k: string = kb
+  return k
+}
+
 # a key -> row index table over interned keys: open addressing, rebuilt in place on every load
 property LresHash {
   slot: []int = new []int      # a row's index + 1; 0 is empty
@@ -203,6 +242,8 @@ function lres_node(t: LresTree, kind: int, line: int, col: int) -> int {
     t.nxt[i] = -1
     t.line[i] = line
     t.col[i] = col
+    t.koff[i] = 0
+    t.klen[i] = 0
   } else {
     push(t.kind, kind)
     push(t.ival, 0)
@@ -213,6 +254,8 @@ function lres_node(t: LresTree, kind: int, line: int, col: int) -> int {
     push(t.nxt, -1)
     push(t.line, line)
     push(t.col, col)
+    push(t.koff, 0)
+    push(t.klen, 0)
   }
   t.nn = i + 1
   return i
@@ -267,15 +310,21 @@ function lres_intern(t: LresTree, a: int, b: int) -> string {
   return intern(sb)
 }
 function lres_ident(t: LresTree) -> string {
+  let a = lres_ident_at(t)
+  if a < 0 { return "" }
+  return lres_intern(t, a, t.p)
+}
+# a name's first byte, the cursor left past it; -1 (and the tree bad) when there is none
+function lres_ident_at(t: LresTree) -> int {
   let c = lres_ch(t)
   if not lres_is_alpha(c) {
     lres_fail(t, "a name was expected here")
-    return ""
+    return -1
   }
   let buf = t.buf
   let a = t.p
   while t.p < t.n and (lres_is_alpha(buf[t.p]) or lres_is_digit(buf[t.p])) { t.p += 1 }
-  return lres_intern(t, a, t.p)
+  return a
 }
 function lres_link(t: LresTree, parent: int, last: int, k: int) -> void {
   if last < 0 { t.kid[parent] = k } else { t.nxt[last] = k }
@@ -291,16 +340,22 @@ function lres_parse(t: LresTree) -> void {
     if t.p >= t.n { return }
     let line = t.ln
     let col = t.p - t.ls + 1
-    let key = lres_ident(t)
+    let ka = lres_ident_at(t)
     if t.bad { return }
+    let kb = t.p
+    var key = ""
+    if not t.local { key = lres_intern(t, ka, kb) }
     lres_skip(t)
     if lres_ch(t) != '{' {
+      if t.local { key = lres_intern(t, ka, kb) }   # a wrong file only: its message names the key
       lres_fail(t, `the entry {key} is its key and a record: {key} {{ field: value }}`)
       return
     }
     let e = lres_record(t)
     if t.bad { return }
     t.name[e] = key
+    t.koff[e] = ka
+    t.klen[e] = kb - ka
     t.line[e] = line
     t.col[e] = col
     if last < 0 { t.top = e } else { t.nxt[last] = e }
@@ -614,7 +669,9 @@ function lres_no_fn(t: LresTree, n: int, ty: string) -> void {
   lres_fail_node(t, n, `there is no function {t.sval[n]} of type {ty}`)
 }
 function lres_dup(t: LresTree, n: int) -> void {
-  lres_fail_node(t, n, `the key {t.name[n]} is written twice`)
+  var key = t.name[n]
+  if t.local { key = lres_intern(t, t.koff[n], t.koff[n] + t.klen[n]) }   # a wrong file only
+  lres_fail_node(t, n, `the key {key} is written twice`)
 }
 
 # ---- the program's constants, by name ------------------------------------------------------------

selfhost/frontend/permap_check.ludic

@@ -208,6 +208,7 @@ function pmk_map(md: pointer) -> void {
     r += 1
   }
   pmk_dups()
+  pmk_chunk_dups()
   var e = 0
   while e < len(g_pmk_rec) {
     let rr = g_pmk_reg[e]
@@ -237,6 +238,30 @@ function pmk_dups() -> void {
     i += 1
   }
 }
+# a chunked table's key is unique across its map (a row's id is (map, key), and a tree moved into
+# another chunk keeps it): a key in two of a map's chunk files is refused, naming both
+function pmk_chunk_dups() -> void {
+  let xs = new []int
+  let keys = new []pointer
+  var i = 0
+  while i < len(g_pmk_key) {
+    if g_rg_chunk[g_pmk_reg[i]] > 0 {
+      push(xs, i)
+      push(keys, `{g_rg_name[g_pmk_reg[i]]}\n{g_pmk_key[i]}`)
+    } else { push(keys, "") }
+    i += 1
+  }
+  sc_sort(xs, keys)
+  i = 1
+  while i < len(xs) {
+    let a = xs[i - 1]
+    let k = xs[i]
+    if (keys[k] == keys[a]) and not (g_pmk_file[k] == g_pmk_file[a]) {
+      pmk_err(g_pmk_file[k], g_pmk_kline[k], g_pmk_kcol[k], `the key {g_pmk_key[k]} is also written at {g_pmk_file[a]}:{itoa(g_pmk_kline[a])} - a chunked table's key is unique across its map`)
+    }
+    i += 1
+  }
+}
 # every cross-row @Ref met names a row of its table in this map
 function pmk_refs() -> void {
   let xs = new []int

selfhost/frontend/permap_gen.ludic

@@ -126,6 +126,7 @@ function pm_gen_chunk_types(b: Buf, R: pointer, T: pointer) -> void {
   pm_l(b, `  rows: []{T} = new []{T}`)
   pm_l(b, `  pl__: {R}__Pool = new {R}__Pool`)
   pm_l(b, `  hs__: LresHash = new LresHash`)
+  pm_l(b, `  ks__: LresKeys = new LresKeys`)
   pm_l(b, "}")
   pm_l(b, `state {R} {{`)
   pm_l(b, `  chunks: []{R}Chunk = new []{R}Chunk`)
@@ -331,7 +332,7 @@ function pm_gen_rows(b: Buf, sn: pointer, R: pointer, T: pointer) -> void {
   pm_l(b, "  return r")
   pm_l(b, "}")
   pm_l(b, `@alloc_ok("the rows list grows only past the most rows this table has held (high water)")`)
-  pm_l(b, `function {sn}__fill_rows(t: LresTree, pl: {R}__Pool, rows: []{T}, hs: LresHash) -> bool {{`)
+  pm_l(b, `function {sn}__fill_rows(t: LresTree, pl: {R}__Pool, rows: []{T}, hs: LresHash, ks: LresKeys) -> bool {{`)
   pm_l(b, "  List.clear(rows)")
   pm_l(b, `  {sn}__rewind(pl)`)
   pm_l(b, "  lres_hash_reset(hs, lres_entries(t))")
@@ -341,7 +342,7 @@ function pm_gen_rows(b: Buf, sn: pointer, R: pointer, T: pointer) -> void {
   pm_l(b, `    let r = {sn}__row(pl, i)`)
   pm_l(b, `    {sn}__reset_{T}(pl, r)`)
   pm_l(b, `    {sn}__fill_{T}(t, pl, r, e)`)
-  pm_l(b, "    r.key = lres_name(t, e)")
+  pm_l(b, "    r.key = lres_key_of(t, e, ks, i)")
   pm_l(b, "    if lres_hash_put(hs, r.key, i) >= 0 { lres_dup(t, e) }")
   pm_l(b, "    push(rows, r)")
   pm_l(b, "    i += 1")
@@ -381,10 +382,11 @@ function pm_gen_whole_verbs(b: Buf, r: int, sn: pointer, R: pointer) -> void {
   pm_l(b, `@alloc_ok("a load grows the table's rows and lists only past its high water; an error's message only when a file is wrong")`)
   pm_l(b, `function {sn}_load(st: mut {R}, map: string) -> bool {{`)
   pm_l(b, "  let t = st.lt__")
+  pm_l(b, "  t.local = false")
   pm_gen_path(b, r)
   pm_l(b, "  let got = lres_read(t)")
   pm_l(b, "  if got == 0 { lres_missing(t) }")
-  pm_l(b, `  if got <= 0 or not {sn}__fill_rows(t, st.pl__, st.rows, st.hs__) {{`)
+  pm_l(b, `  if got <= 0 or not {sn}__fill_rows(t, st.pl__, st.rows, st.hs__, null) {{`)
   pm_l(b, `    {sn}__drop(st.pl__, st.rows, st.hs__)`)
   pm_l(b, "    st.err = lres_error(t)")
   pm_l(b, "    st.map = \"\"")
@@ -444,6 +446,7 @@ function pm_gen_chunk_verbs(b: Buf, r: int, sn: pointer, R: pointer) -> void {
   pm_l(b, "  ch.cx = cx")
   pm_l(b, "  ch.cz = cz")
   pm_l(b, "  let t = st.lt__")
+  pm_l(b, "  t.local = true                           # the keys are the slot's own, never interned")
   pm_gen_path(b, r)
   pm_l(b, "  let got = lres_read(t)")
   pm_l(b, "  if got == 0 {")
@@ -451,7 +454,7 @@ function pm_gen_chunk_verbs(b: Buf, r: int, sn: pointer, R: pointer) -> void {
   pm_l(b, "    st.err = \"\"")
   pm_l(b, "    return s")
   pm_l(b, "  }")
-  pm_l(b, `  if got < 0 or not {sn}__fill_rows(t, ch.pl__, ch.rows, ch.hs__) {{`)
+  pm_l(b, `  if got < 0 or not {sn}__fill_rows(t, ch.pl__, ch.rows, ch.hs__, ch.ks__) {{`)
   pm_l(b, `    {sn}__off(ch)`)
   pm_l(b, "    st.err = lres_error(t)")
   pm_l(b, "    return -1")

tools/ludic-cli/test.ludic

@@ -1277,10 +1277,11 @@ function cmd_dev_test() -> int {
   schema_case("lang/permap/permap", "\"units\": [\"m\", \"m/s\", \"m/s2\"", "--emit-schema: the canonical @Unit spellings")
   reject_case("rejected/permap_ref_int", "names a @PerMap registry, whose rows have no index a program can hold - use a string key", "@Ref into a @PerMap registry is on a string field")
   reject_case("rejected/permap_prefix", "a @PerMap registry has no 'as PREFIX'", "a @PerMap registry has no prefix and no constants")
-  diag_json_case("rejected/permap_maps/permap_maps", 4, "\"line\": 2, \"col\": 18, \"severity\": \"error\", \"message\": \"PropRow has no field colour\"", "--check reads every map: a field its record lacks, at the map file's line and column")
-  diag_json_case("rejected/permap_maps/permap_maps", 4, "model wants an int, not a string", "--check reads every map: a value of the wrong type")
-  diag_json_case("rejected/permap_maps/permap_maps", 4, "unknown constant MDL_BENCH", "--check reads every map: a constant nobody declared")
-  diag_json_case("rejected/permap_maps/permap_maps", 4, "there is no Spots row nowhere in this map", "--check reads every map: a cross-row @Ref names a row of the same map")
+  diag_json_case("rejected/permap_maps/permap_maps", 5, "\"line\": 2, \"col\": 18, \"severity\": \"error\", \"message\": \"PropRow has no field colour\"", "--check reads every map: a field its record lacks, at the map file's line and column")
+  diag_json_case("rejected/permap_maps/permap_maps", 5, "model wants an int, not a string", "--check reads every map: a value of the wrong type")
+  diag_json_case("rejected/permap_maps/permap_maps", 5, "unknown constant MDL_BENCH", "--check reads every map: a constant nobody declared")
+  diag_json_case("rejected/permap_maps/permap_maps", 5, "there is no Spots row nowhere in this map", "--check reads every map: a cross-row @Ref names a row of the same map")
+  diag_json_case("rejected/permap_maps/permap_maps", 5, "the key t1 is also written at", "--check reads every map: a chunked table's key is unique across the map's chunk files, both places named")
   feat_case("lang/defaults", "", "a 300.0 8.0 / b 120.0 8.0 / c! 300.0 8.0 / d 300.0 2.0 / x / y", "defaults.ludic (L11: default parameters, positional then named arguments, in a generic too)")
   reject_case("rejected/default_missing", "this call to f leaves out a, which has no default", "a call leaves out only what has a default")
   reject_case("rejected/default_unknown", "f has no parameter c", "a named argument names a parameter the function has")