ludic.nav (23.5b): a baked mesh resident by chunk

nav_load_index indexes a saved mesh's tiles (cell, offset, size, ref) from their headers and keeps the file open, with no tile in. nav_tiles_keep reads in the tiles within rin of any player and removes those past rout of all; the file is read through Ludic's pack-aware file_open. The file format is unchanged. tiles_test: tiles come and go with the point, a path works across the seams once they are in, and a reset closes all. Measured on the baked maps (the 900 / 1100 m ring round the start): Maroon 617 of 1807 tiles in, 8.9 MB instead of about 27 MB for a person and 5.4 MB of about 14 for a large walker; Lamar 391 of 1147. Both libraries are rebuilt; nav 17/17 on the Mac and the PC, DLL KERNEL32 only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/ludic.nav/README.md

@@ -82,3 +82,12 @@ and `nav_crowd_us()` is what the stepping has cost. `nav_crowd_add_fixed` adds a
 but never steers (a player), which `nav_crowd_place` puts where it is, and how fast it goes, each frame. The mechanic decides and the crowd moves, and
 dropping a mesh drops its crowd first. On the test meadow, twenty walkers crossing head-on never come
 closer than their two radii, and a step costs about 12 us.
+
+## Tiles resident by chunk (phase 23.5b)
+
+`nav_load_index(kind, path)` loads a baked mesh as an index of its tiles and keeps its file open,
+with no tile in yet. `nav_tiles_keep(kind, xs, zs, n, rin, rout)` reads in the tiles within `rin`
+of any of the n points and lets go of those past `rout` of all of them. It is asked now and then,
+not every frame. The file is read through `file_open`, so a shipped game reads straight from its
+pack. `nav_tiles_in` / `_bytes` / `_all` say what is resident. Dropping the mesh closes the file.
+On Maroon the 900 / 1100 m ring round the start holds 617 of 1807 tiles, 8.9 MB of a person's mesh.

packages/ludic.nav/index.ludic

@@ -9,3 +9,4 @@ import "state.ludic"
 import "build.ludic"
 import "query.ludic"
 import "crowd.ludic"
+import "resident.ludic"

packages/ludic.nav/native.ludic

@@ -26,4 +26,14 @@ extern function nvc_crowd_speed(c: pointer, i: int, speed: float) -> void = "nav
 extern function nvc_crowd_update(c: pointer, dt: float) -> void = "nav_crowd_update"
 extern function nvc_crowd_read(c: pointer, i: int, out: pointer) -> int = "nav_crowd_read"
 extern function nvc_crowd_us(c: pointer) -> int = "nav_crowd_us"
+extern function nvc_unstage(buf: pointer) -> void = "nav_unstage"
+extern function nvc_index_open(head: pointer) -> pointer = "nav_index_open"
+extern function nvc_index_count_in(head: pointer) -> int = "nav_index_count_in"
+extern function nvc_index_put(h: pointer, rec: pointer, offset: long) -> int = "nav_index_put"
+extern function nvc_index_count(h: pointer) -> int = "nav_index_count"
+extern function nvc_index_tile(h: pointer, i: int, out: pointer) -> int = "nav_index_tile"
+extern function nvc_index_offset(h: pointer, i: int) -> long = "nav_index_offset"
+extern function nvc_index_size(h: pointer, i: int) -> int = "nav_index_size"
+extern function nvc_index_in(h: pointer, i: int, data: pointer, size: int) -> int = "nav_index_in"
+extern function nvc_index_out(h: pointer, i: int) -> int = "nav_index_out"
 extern function nvc_load_staged(buf: pointer, size: int) -> pointer = "nav_load_staged"

packages/ludic.nav/native/shim/nav_resident.inl

@@ -0,0 +1,72 @@
+// nav_resident.inl - a baked mesh whose tiles are resident by chunk (phase 23.5b): the file is
+// indexed once (each tile's grid cell, where its bytes are, its ref), and a tile's bytes are read
+// in when a player comes near and let go when all are far. The game reads the file (its reads are
+// the pack's); the shim only parses what it is handed and keeps the index.
+
+// a mesh with no tiles yet, from nav_save's 40-byte head; null for anything else
+NAV_SHIM void *nav_index_open(const unsigned char *head) {
+  int magic = 0, version = 0;
+  memcpy(&magic, head, 4); memcpy(&version, head + 4, 4);
+  if (magic != NAV_MAGIC || version != NAV_VERSION) return nullptr;
+  dtNavMeshParams p;
+  memset(&p, 0, sizeof(p));
+  memcpy(p.orig, head + 12, 12); memcpy(&p.tileWidth, head + 24, 4); memcpy(&p.tileHeight, head + 28, 4);
+  memcpy(&p.maxTiles, head + 32, 4); memcpy(&p.maxPolys, head + 36, 4);
+  dtNavMesh *mesh = dtAllocNavMesh();
+  if (!mesh || dtStatusFailed(mesh->init(&p))) { dtFreeNavMesh(mesh); return nullptr; }
+  return nav_open(mesh);
+}
+// how many tiles the file said it holds (from the same head)
+NAV_SHIM int nav_index_count_in(const unsigned char *head) {
+  int count = 0;
+  memcpy(&count, head + 8, 4);
+  return count;
+}
+
+// one tile's record, read at `offset`: its ref and size, and the start of its data (its header's
+// magic, version, x, y). Returns the data's size, so the caller can seek past it; -1 when not a tile
+NAV_SHIM int nav_index_put(void *h, const unsigned char *rec, long long offset) {
+  Nav *n = static_cast<Nav *>(h);
+  NavTileRec t;
+  memcpy(&t.ref, rec, 8); memcpy(&t.size, rec + 8, 4);
+  int magic = 0;
+  memcpy(&magic, rec + 12, 4); memcpy(&t.x, rec + 20, 4); memcpy(&t.y, rec + 24, 4);
+  if (magic != DT_NAVMESH_MAGIC || t.size <= 0) return -1;
+  t.offset = offset + 12;
+  n->tiles.push_back(t);
+  return t.size;
+}
+NAV_SHIM int nav_index_count(void *h) { return static_cast<int>(static_cast<Nav *>(h)->tiles.size()); }
+
+// tile i's centre and whether it is in: out[0] x, out[1] z, out[2] 1 in / 0 out, out[3] its bytes
+NAV_SHIM int nav_index_tile(void *h, int i, float *out) {
+  Nav *n = static_cast<Nav *>(h);
+  if (i < 0 || i >= (int)n->tiles.size()) return 0;
+  const NavTileRec &t = n->tiles[i];
+  const dtNavMeshParams *p = n->mesh->getParams();
+  out[0] = p->orig[0] + (t.x + 0.5f) * p->tileWidth;
+  out[1] = p->orig[2] + (t.y + 0.5f) * p->tileHeight;
+  out[2] = t.in ? 1.0f : 0.0f;
+  out[3] = (float)t.size;
+  return 1;
+}
+// where tile i's bytes are in the file (the caller seeks there and reads nav_index_size of them)
+NAV_SHIM long long nav_index_offset(void *h, int i) { return static_cast<Nav *>(h)->tiles[i].offset; }
+NAV_SHIM int nav_index_size(void *h, int i) { return static_cast<Nav *>(h)->tiles[i].size; }
+
+// tile i's bytes (staged by nav_stage, owned from here on) added to the mesh; 0 when refused
+NAV_SHIM int nav_index_in(void *h, int i, unsigned char *data, int size) {
+  Nav *n = static_cast<Nav *>(h);
+  if (i < 0 || i >= (int)n->tiles.size() || n->tiles[i].in || !data) { free(data); return 0; }
+  if (dtStatusFailed(n->mesh->addTile(data, size, DT_TILE_FREE_DATA, (dtTileRef)n->tiles[i].ref, nullptr))) { free(data); return 0; }
+  n->tiles[i].in = true;
+  return 1;
+}
+// tile i let go, its bytes with it
+NAV_SHIM int nav_index_out(void *h, int i) {
+  Nav *n = static_cast<Nav *>(h);
+  if (i < 0 || i >= (int)n->tiles.size() || !n->tiles[i].in) return 0;
+  n->mesh->removeTile((dtTileRef)n->tiles[i].ref, nullptr, nullptr);
+  n->tiles[i].in = false;
+  return 1;
+}

packages/ludic.nav/native/shim/nav_shim.cpp

@@ -43,6 +43,14 @@ const int NAV_MAX_POLYS = 512;     // polygons a path may cross; a longer one is
 const int NAV_MAX_NODES = 4096;    // the search's open list
 const int NAV_FILTERS = 8;         // costs per kind of ground: one set per kind of walker's taste
 
+// one tile of a file-backed mesh (nav_resident.inl): its cell, where its bytes are, whether it is in
+struct NavTileRec {
+  int x = 0, y = 0, size = 0;
+  long long offset = 0;
+  unsigned long long ref = 0;
+  bool in = false;
+};
+
 // a navmesh and the one query that walks it, with the scratch a path needs
 struct Nav {
   dtNavMesh *mesh = nullptr;
@@ -52,6 +60,7 @@ struct Nav {
   dtPolyRef polys[NAV_MAX_POLYS];
   int partial = 0;                     // the last path stopped short of its end
   long long ns = 0;                    // time spent finding paths, for the measure (17.10)
+  std::vector<NavTileRec> tiles;       // a file-backed mesh's index; empty for a built or loaded one
 };
 
 // a navmesh with its query, from params (a set of tiles) or from one tile's data
@@ -79,3 +88,4 @@ const dtQueryFilter *nav_filter(Nav *n, int f) { return &n->filters[f >= 0 && f
 #include "nav_tiles.inl"
 #include "nav_query.inl"
 #include "nav_crowd.inl"
+#include "nav_resident.inl"

packages/ludic.nav/native/shim/nav_tiles.inl

@@ -103,6 +103,7 @@ NAV_SHIM void *nav_load(const unsigned char *buf, int size) {
 
 // a file read straight into the shim's own memory rather than a Ludic slice (which is never given
 // back): nav_stage hands out the room, nav_load_staged parses it and frees it, whatever the outcome
+NAV_SHIM void nav_unstage(unsigned char *buf) { free(buf); }
 NAV_SHIM unsigned char *nav_stage(int size) { return size > 0 ? static_cast<unsigned char *>(malloc(size)) : nullptr; }
 NAV_SHIM void *nav_load_staged(unsigned char *buf, int size) {
   void *h = buf ? nav_load(buf, size) : nullptr;

packages/ludic.nav/resident.ludic

@@ -0,0 +1,85 @@
+# resident.ludic - a baked mesh whose tiles are resident by chunk (phase 23.5b): its file is indexed
+# once and kept open, and a tile is read in when a player comes within `rin` of it and let go when
+# every player is past `rout`. Only the index and the tiles near someone are in memory.
+
+# kind's mesh as an index of the file at `path`, with no tile in yet; false when it is not a mesh
+export function nav_load_index(nav_st: mut NavState, kind: int, path: string) -> bool {
+  let f = file_open(path, "rb")
+  if f == null { return false }
+  let rec = nvc_stage(40)
+  var ok = file_read(f, rec, 40) == 40
+  var h: pointer = null
+  if ok { h = nvc_index_open(rec) }
+  if h != null {
+    let count = nvc_index_count_in(rec)
+    var at = 40
+    for i in 0 .. count {
+      file_seek(f, at, 0)
+      if file_read(f, rec, 28) != 28 { break }
+      let n = nvc_index_put(h, rec, long(at))
+      if n < 0 { break }
+      at = at + 12 + n
+    }
+  }
+  nvc_unstage(rec)
+  if not nv_set(nav_st, kind, h) {
+    file_close(f)
+    return false
+  }
+  nav_st.nv_files[kind] = f
+  return true
+}
+
+# tiles within rin of any of the n points (xs, zs) read in, those past rout of all of them let go;
+# how many changed. Asked now and then (a second), not every frame
+export function nav_tiles_keep(nav_st: NavState, kind: int, xs: []float, zs: []float, n: int, rin: float, rout: float) -> int {
+  let h = nv_mesh(nav_st, kind)
+  let f = nav_st.nv_files[kind]
+  if h == null or f == null { return 0 }
+  var changed = 0
+  for i in 0 .. nvc_index_count(h) {
+    nvc_index_tile(h, i, nav_st.nv_tile)
+    var best = 1000000000000.0
+    for p in 0 .. n {
+      let dx = xs[p] - nav_st.nv_tile[0]
+      let dz = zs[p] - nav_st.nv_tile[1]
+      best = Math.min(best, dx * dx + dz * dz)
+    }
+    let is_in = nav_st.nv_tile[2] > 0.5
+    if is_in and best > rout * rout { changed += nvc_index_out(h, i) }
+    if not is_in and best < rin * rin { changed += nv_tile_read(h, f, i) }
+  }
+  return changed
+}
+function nv_tile_read(h: pointer, f: pointer, i: int) -> int {
+  let size = nvc_index_size(h, i)
+  let buf = nvc_stage(size)
+  if buf == null { return 0 }
+  file_seek(f, int(nvc_index_offset(h, i)), 0)
+  if file_read(f, buf, size) != size {
+    nvc_unstage(buf)
+    return 0
+  }
+  return nvc_index_in(h, i, buf, size)
+}
+
+# how many of kind's tiles are in, and (nav_tiles_bytes) how many bytes those hold
+export function nav_tiles_in(nav_st: NavState, kind: int) -> int { return nv_tiles_sum(nav_st, kind, false) }
+export function nav_tiles_bytes(nav_st: NavState, kind: int) -> int { return nv_tiles_sum(nav_st, kind, true) }
+function nv_tiles_sum(nav_st: NavState, kind: int, bytes: bool) -> int {
+  let h = nv_mesh(nav_st, kind)
+  if h == null { return 0 }
+  var n = 0
+  for i in 0 .. nvc_index_count(h) {
+    nvc_index_tile(h, i, nav_st.nv_tile)
+    if nav_st.nv_tile[2] > 0.5 and bytes { n += int(nav_st.nv_tile[3]) }
+    if nav_st.nv_tile[2] > 0.5 and not bytes { n += 1 }
+  }
+  return n
+}
+# how many tiles kind's file holds in all
+export function nav_tiles_all(nav_st: NavState, kind: int) -> int {
+  let h = nv_mesh(nav_st, kind)
+  if h == null { return 0 }
+  return nvc_index_count(h)
+}

packages/ludic.nav/state.ludic

@@ -47,6 +47,8 @@ export state NavState {
   nv_short: int = 0
   nv_crowds: []pointer = nv_none() # a crowd per kind, on that kind's mesh (crowd.ludic)
   nv_agent: []float = floats(6)    # the last walker read: position, velocity
+  nv_files: []pointer = nv_none()  # a file-backed mesh's file, kept open for its tiles (resident.ludic)
+  nv_tile: []float = floats(4)     # the last tile asked about: centre x, z, in, bytes
 }
 function nv_none() -> []pointer {
   let m = new []pointer
@@ -60,6 +62,8 @@ function nv_mesh(nav_st: NavState, kind: int) -> pointer {
 function nv_set(nav_st: mut NavState, kind: int, h: pointer) -> bool {
   if kind < 0 or kind >= NAV_KINDS { return false }
   nv_crowd_drop(nav_st, kind)
+  if nav_st.nv_files[kind] != null { file_close(nav_st.nv_files[kind]) }
+  nav_st.nv_files[kind] = null
   if nav_st.nv_meshes[kind] != null { nvc_free(nav_st.nv_meshes[kind]) }
   nav_st.nv_meshes[kind] = h
   return h != null

packages/ludic.nav/tests/tiles_test.ludic

@@ -63,4 +63,35 @@ program TilesTest {
     expect_eq(a, b)
     expect_near(last_x(nav_st), ax, 0.0001)
   }
+
+  test "a saved mesh as an index: only the tiles near a player are in, and they come and go" (nav_st: mut NavState) {
+    expect_eq(map(nav_st, false), 0)
+    let path = Os.temp_dir() + "/ludic_nav_resident.navmesh"
+    expect(nav_save_file(nav_st, NAV_PERSON, path))
+    nav_reset(nav_st)
+    expect(nav_load_index(nav_st, NAV_PERSON, path))
+    expect_eq(nav_tiles_in(nav_st, NAV_PERSON), 0)
+    expect(not nav_nearest(nav_st, NAV_PERSON, 20.0, 0.0, 20.0))
+    let xs = floats(1)
+    let zs = floats(1)
+    xs[0] = 20.0
+    zs[0] = 20.0
+    expect_eq(nav_tiles_keep(nav_st, NAV_PERSON, xs, zs, 1, 60.0, 90.0), 1)
+    expect_eq(nav_tiles_in(nav_st, NAV_PERSON), 1)
+    expect(nav_nearest(nav_st, NAV_PERSON, 20.0, 0.0, 20.0))
+    expect(not nav_nearest(nav_st, NAV_PERSON, 100.0, 0.0, 100.0))
+    xs[0] = 64.0
+    zs[0] = 64.0
+    expect_eq(nav_tiles_keep(nav_st, NAV_PERSON, xs, zs, 1, 60.0, 90.0), 3)
+    expect_eq(nav_tiles_in(nav_st, NAV_PERSON), 4)
+    expect(nav_path(nav_st, NAV_PERSON, 0, 10.0, 0.0, 20.0, 120.0, 0.0, 20.0) >= 3)
+    expect(not nav_partial(nav_st, NAV_PERSON))
+    xs[0] = 110.0
+    zs[0] = 110.0
+    expect_eq(nav_tiles_keep(nav_st, NAV_PERSON, xs, zs, 1, 60.0, 90.0), 1)
+    expect_eq(nav_tiles_in(nav_st, NAV_PERSON), 3)
+    expect(not nav_nearest(nav_st, NAV_PERSON, 20.0, 0.0, 20.0))
+    nav_reset(nav_st)
+    expect_eq(nav_tiles_in(nav_st, NAV_PERSON), 0)
+  }
 }