nav, physics: Recast's and Detour's memory counted (nav_heap_bytes / _peak / _allocs, a counted allocator registered at load, tile bytes staged from it); package tests that crossing the map and back holds the native bytes

The user's walk showed the footprint rising outside the Ludic heap. Jolt's bytes were already counted;
the navmesh's were not, so nothing could see them. Now:
- ludic.nav nav_heap_test: fifty crossings of a four-tile map by the resident index hold the tiles in and
  the native bytes to the first crossing's; a thousand crowd walkers in and out grow nothing; a reset
  gives back every byte the mesh took.
- ludic.physics cross_heap_test: a 1 km map of 64 m chunks kept to a ring round a player crossing
  corner to corner and back, each chunk a heightfield, twelve owned posts and six owned scaled hulls
  as the game makes them: six more crossings hold the bodies, the shapes and Jolt's bytes and peak
  exactly. (A post made as an offset of a cylinder, with only the offset owned, leaked the cylinder
  every time: the game's solid_pillar owns its cylinder directly.)

lib/macos-arm64 rebuilt; lib/windows-x64 needs native/build.sh run on the PC for the new exports.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/ludic.nav/README.md

@@ -72,6 +72,12 @@ across the seams, a tile under water with no polygons, and the set saved and loa
 and the shim (`native/shim/nav_shim.cpp`) into `lib/<target>/` - the same script on the Mac and on
 the PC (Git Bash, the LLVM installer's clang). `native/LICENSE-recastnavigation` ships with it.
 
+Every byte Recast and Detour allocate - the tiles in, the queries, the crowds - goes through a counted
+allocator registered when the library loads (`native/shim/nav_alloc.inl`, as ludic.physics counts
+Jolt's): `nav_heap_bytes()`, `nav_heap_peak()` and `nav_heap_allocs()`. A tile's bytes are staged from
+it too, since Detour frees them with the tile. `tests/nav_heap_test` holds fifty crossings of the map
+and a thousand crowd walkers to the bytes of the first, and a reset to the bytes before the mesh.
+
 ## Crowds (phase 18)
 
 `nav_crowd_start(kind, max, max_radius)` puts a DetourCrowd on a kind's mesh, with the mesh's tastes as

packages/ludic.nav/native.ludic

@@ -37,3 +37,11 @@ extern function nvc_index_size(h: pointer, i: int) -> int = "nav_index_size"
 extern function nvc_index_in(h: pointer, i: int, data: pointer, size: int) -> int = "nav_index_in"
 extern function nvc_index_out(h: pointer, i: int) -> int = "nav_index_out"
 extern function nvc_load_staged(buf: pointer, size: int) -> pointer = "nav_load_staged"
+extern function nvc_heap_bytes() -> long = "nav_heap_bytes"
+extern function nvc_heap_peak() -> long = "nav_heap_peak"
+extern function nvc_heap_allocs() -> long = "nav_heap_allocs"
+# the bytes Recast and Detour hold now (every tile in, the queries, the crowds), the most they ever
+# held, and how many blocks they asked for: counted in the shim, apart from the game's heap
+export function nav_heap_bytes() -> long { return nvc_heap_bytes() }
+export function nav_heap_peak() -> long { return nvc_heap_peak() }
+export function nav_heap_allocs() -> long { return nvc_heap_allocs() }

packages/ludic.nav/native/shim/nav_alloc.inl

@@ -0,0 +1,42 @@
+// nav_alloc.inl - Recast's and Detour's allocations counted apart from the game's (plan 25), as
+// ludic.physics counts Jolt's: libc's malloc underneath, each block carrying its size in a 16-byte
+// header, so the tiles, queries and crowds are measured exactly. Registered when the library loads.
+#include <atomic>
+
+namespace {
+constexpr size_t kNavHead = 16;
+std::atomic<long long> g_nav_live{0}, g_nav_peak{0}, g_nav_allocs{0};
+
+void *nav_heap_alloc(size_t n) {
+  char *b = static_cast<char *>(std::malloc(n + kNavHead));
+  if (!b) return nullptr;
+  std::memcpy(b, &n, sizeof n);
+  g_nav_allocs.fetch_add(1, std::memory_order_relaxed);
+  long long now = g_nav_live.fetch_add((long long)n, std::memory_order_relaxed) + (long long)n;
+  long long p = g_nav_peak.load(std::memory_order_relaxed);
+  while (now > p && !g_nav_peak.compare_exchange_weak(p, now, std::memory_order_relaxed)) {}
+  return b + kNavHead;
+}
+void nav_heap_free(void *p) {
+  if (!p) return;
+  char *b = static_cast<char *>(p) - kNavHead;
+  size_t n;
+  std::memcpy(&n, b, sizeof n);
+  g_nav_live.fetch_sub((long long)n, std::memory_order_relaxed);
+  std::free(b);
+}
+void *nav_dt_alloc(size_t n, dtAllocHint) { return nav_heap_alloc(n); }
+void *nav_rc_alloc(size_t n, rcAllocHint) { return nav_heap_alloc(n); }
+
+struct NavHeapRegister {
+  NavHeapRegister() {
+    dtAllocSetCustom(nav_dt_alloc, nav_heap_free);
+    rcAllocSetCustom(nav_rc_alloc, nav_heap_free);
+  }
+} g_nav_heap_register;
+}  // namespace
+
+// bytes Recast and Detour hold now, the most they ever held, and how many blocks they asked for
+NAV_SHIM long long nav_heap_bytes(void) { return g_nav_live.load(std::memory_order_relaxed); }
+NAV_SHIM long long nav_heap_peak(void) { return g_nav_peak.load(std::memory_order_relaxed); }
+NAV_SHIM long long nav_heap_allocs(void) { return g_nav_allocs.load(std::memory_order_relaxed); }

packages/ludic.nav/native/shim/nav_resident.inl

@@ -57,8 +57,8 @@ NAV_SHIM int nav_index_size(void *h, int i) { return static_cast<Nav *>(h)->tile
 // tile i's bytes (staged by nav_stage, owned from here on) added to the mesh; 0 when refused
 NAV_SHIM int nav_index_in(void *h, int i, unsigned char *data, int size) {
   Nav *n = static_cast<Nav *>(h);
-  if (i < 0 || i >= (int)n->tiles.size() || n->tiles[i].in || !data) { free(data); return 0; }
-  if (dtStatusFailed(n->mesh->addTile(data, size, DT_TILE_FREE_DATA, (dtTileRef)n->tiles[i].ref, nullptr))) { free(data); return 0; }
+  if (i < 0 || i >= (int)n->tiles.size() || n->tiles[i].in || !data) { dtFree(data); return 0; }
+  if (dtStatusFailed(n->mesh->addTile(data, size, DT_TILE_FREE_DATA, (dtTileRef)n->tiles[i].ref, nullptr))) { dtFree(data); return 0; }
   n->tiles[i].in = true;
   return 1;
 }

packages/ludic.nav/native/shim/nav_shim.cpp

@@ -7,6 +7,8 @@
 #include <DetourNavMeshQuery.h>
 #include <DetourCommon.h>
 #include <DetourCrowd.h>
+#include <RecastAlloc.h>
+#include <DetourAlloc.h>
 #include <cstring>
 #include <cstdlib>
 #include <cmath>
@@ -18,6 +20,8 @@
 #define NAV_SHIM extern "C" __attribute__((visibility("default")))
 #endif
 
+#include "nav_alloc.inl"
+
 #ifdef _WIN32
 extern "C" __declspec(dllimport) int __stdcall QueryPerformanceCounter(long long *);
 extern "C" __declspec(dllimport) int __stdcall QueryPerformanceFrequency(long long *);

packages/ludic.nav/native/shim/nav_tiles.inl

@@ -103,10 +103,11 @@ NAV_SHIM void *nav_load(const unsigned char *buf, int size) {
 
 // a file read straight into the shim's own memory rather than a Ludic slice (which is never given
 // back): nav_stage hands out the room, nav_load_staged parses it and frees it, whatever the outcome
-NAV_SHIM void nav_unstage(unsigned char *buf) { free(buf); }
-NAV_SHIM unsigned char *nav_stage(int size) { return size > 0 ? static_cast<unsigned char *>(malloc(size)) : nullptr; }
+// staged from Detour's own (counted) allocator, since a tile's bytes are handed to it to free
+NAV_SHIM void nav_unstage(unsigned char *buf) { dtFree(buf); }
+NAV_SHIM unsigned char *nav_stage(int size) { return size > 0 ? static_cast<unsigned char *>(dtAlloc(size, DT_ALLOC_PERM)) : nullptr; }
 NAV_SHIM void *nav_load_staged(unsigned char *buf, int size) {
   void *h = buf ? nav_load(buf, size) : nullptr;
-  free(buf);
+  dtFree(buf);
   return h;
 }

packages/ludic.nav/tests/nav_heap_test.ludic

@@ -0,0 +1,73 @@
+# nav_heap_test.ludic - Recast's and Detour's own memory, counted in the shim (plan 25): a player
+# crossing the map and coming back, again and again, leaves the resident tiles and the native bytes
+# where the first crossing left them; walkers in and out of a crowd grow nothing; a reset gives it back
+import "ludic.nav"
+import "ludic.base"
+import "fakes/meadow.ludic"
+program NavHeapTest {
+  numbers float
+
+  # a 128 m meadow as four 64 m tiles, saved as a file the index reads
+  function saved(nav_st: mut NavState, path: string) -> void {
+    let g = ground(128, 0)
+    expect(nav_tiled(nav_st, NAV_PERSON, 0.0, 0.0, 64.0, 16, 4096))
+    for tz in 0 .. 2 { for tx in 0 .. 2 { expect(nav_tile_build(nav_st, NAV_PERSON, tx, tz, g, new NavConfig) > 0) } }
+    expect(nav_save_file(nav_st, NAV_PERSON, path))
+    nav_reset(nav_st)
+  }
+  # one crossing: from the near corner to the far one and back, the ring kept at each end
+  function cross(nav_st: NavState, xs: []float, zs: []float) -> int {
+    var changed = 0
+    xs[0] = 110.0
+    zs[0] = 110.0
+    changed += nav_tiles_keep(nav_st, NAV_PERSON, xs, zs, 1, 60.0, 90.0)
+    xs[0] = 20.0
+    zs[0] = 20.0
+    changed += nav_tiles_keep(nav_st, NAV_PERSON, xs, zs, 1, 60.0, 90.0)
+    return changed
+  }
+
+  test "crossing the map and coming back fifty times holds the tiles and the native bytes of the first crossing" (nav_st: mut NavState) {
+    let path = Os.temp_dir() + "/ludic_nav_heap.navmesh"
+    saved(nav_st, path)
+    let before = nav_heap_bytes()
+    expect(nav_load_index(nav_st, NAV_PERSON, path))
+    let xs = floats(1)
+    let zs = floats(1)
+    xs[0] = 20.0
+    zs[0] = 20.0
+    nav_tiles_keep(nav_st, NAV_PERSON, xs, zs, 1, 60.0, 90.0)
+    expect(cross(nav_st, xs, zs) > 0)
+    let tiles = nav_tiles_in(nav_st, NAV_PERSON)
+    let held = nav_heap_bytes()
+    let peak = nav_heap_peak()
+    var changed = 0
+    for k in 0 .. 50 { changed += cross(nav_st, xs, zs) }
+    print(`nav heap: {before} bytes before the index, {held} after one crossing, {nav_heap_bytes()} after fifty more ({changed} tiles in or out), peak {peak} -> {nav_heap_peak()}`)
+    expect(changed >= 100)
+    expect_eq(nav_tiles_in(nav_st, NAV_PERSON), tiles)
+    expect(nav_heap_bytes() == held)
+    expect(nav_heap_peak() == peak)
+    nav_reset(nav_st)
+    expect(nav_heap_bytes() == before)
+  }
+
+  test "walkers in and out of a crowd a thousand times grow nothing, and the crowd goes with its mesh" (nav_st: mut NavState) {
+    let before = nav_heap_bytes()
+    expect(nav_build(nav_st, NAV_PERSON, ground(40, 0), new NavConfig))
+    expect(nav_crowd_start(nav_st, NAV_PERSON, 16, 0.6))
+    for k in 0 .. 10 { nav_crowd_remove(nav_st, NAV_PERSON, nav_crowd_add(nav_st, NAV_PERSON, 5.0, 0.0, 5.0, 0.35, 1.8, 1.4, 0, 2.0)) }
+    let held = nav_heap_bytes()
+    for k in 0 .. 1000 {
+      let a = nav_crowd_add(nav_st, NAV_PERSON, 5.0 + float(k % 30), 0.0, 5.0, 0.35, 1.8, 1.4, 0, 2.0)
+      expect(a >= 0)
+      nav_crowd_target(nav_st, NAV_PERSON, a, 30.0, 0.0, 30.0)
+      nav_crowd_step(nav_st, 1.0 / 60.0)
+      nav_crowd_remove(nav_st, NAV_PERSON, a)
+    }
+    print(`nav heap: a crowd's {held} bytes, {nav_heap_bytes()} after a thousand walkers came and went`)
+    expect(nav_heap_bytes() == held)
+    nav_reset(nav_st)
+    expect(nav_heap_bytes() == before)
+  }
+}

packages/ludic.physics/tests/cross_heap_test.ludic

@@ -0,0 +1,85 @@
+# cross_heap_test.ludic - a player crossing a 1 km map corner to corner and back, the ground and its
+# still things kept to a ring of 64 m chunks round them as the game keeps them: each chunk a heightfield
+# ground, twelve trunks (an owned post each, as solid_pillar) and six boulders (an owned scaled hull each). Crossing
+# again and again leaves the bodies, the shapes and Jolt's own bytes where the first crossing left them.
+import "ludic.physics"
+import "ludic.base"
+program CrossHeapTest {
+  numbers float
+  const N: int = 16                 # chunks a side
+  const RING: int = 2               # chunks either side of the player's that are in
+
+  state CrossState {
+    ground: []int = words(N * N)
+    shape: []int = words(N * N)
+    things: []int = words(N * N * 18)
+    on: []bool = new []bool
+    h: []float = floats(33 * 33)
+    hull: int = -1
+  }
+
+  function chunk_in(physics_st: mut PhysicsState, cs: mut CrossState, c: int) -> void {
+    let ox = float(c % N) * 64.0
+    let oz = float(c / N) * 64.0
+    cs.shape[c] = phys_heightfield(physics_st, cs.h, 33, ox, oz, 2.0)
+    cs.ground[c] = phys_ground_add(physics_st, cs.shape[c])
+    for k in 0 .. 12 {
+      let s = phys_cylinder(physics_st, 1.5, 0.12)
+      let b = phys_static_add(physics_st, s, ox + 4.0 + float(k) * 5.0, 1.5, oz + 10.0, 0.0)
+      phys_own(physics_st, b, s)
+      cs.things[c * 18 + k] = b
+    }
+    for k in 0 .. 6 {
+      let s = phys_scaled(physics_st, cs.hull, 0.5 + float(k) * 0.3)
+      let b = phys_static_add(physics_st, s, ox + 8.0 + float(k) * 9.0, 0.0, oz + 40.0, float(k))
+      phys_own(physics_st, b, s)
+      cs.things[c * 18 + 12 + k] = b
+    }
+    cs.on[c] = true
+  }
+  function chunk_out(physics_st: mut PhysicsState, cs: mut CrossState, c: int) -> void {
+    for k in 0 .. 18 { phys_remove(physics_st, cs.things[c * 18 + k]) }
+    phys_remove(physics_st, cs.ground[c])
+    phys_shape_free(physics_st, cs.shape[c])
+    cs.on[c] = false
+  }
+  # the ring kept round chunk (px, pz): those past it out, those in it in, and the broad phase rebuilt
+  function keep(physics_st: mut PhysicsState, cs: mut CrossState, px: int, pz: int) -> void {
+    for c in 0 .. N * N {
+      let near = Math.abs(c % N - px) <= RING and Math.abs(c / N - pz) <= RING
+      if cs.on[c] and not near { chunk_out(physics_st, cs, c) }
+      if not cs.on[c] and near { chunk_in(physics_st, cs, c) }
+    }
+    phys_settle(physics_st)
+    phys_step(physics_st)
+  }
+  function crossing(physics_st: mut PhysicsState, cs: mut CrossState) -> void {
+    for k in 0 .. N { keep(physics_st, cs, k, k) }
+    for k in 0 .. N { keep(physics_st, cs, N - 1 - k, N - 1 - k) }
+  }
+
+  test "crossing the map and back six times holds the bodies, the shapes and Jolt's bytes of the first crossing" (physics_st: mut PhysicsState, cross_st: mut CrossState) {
+    expect(phys_open(physics_st, 4096, 1))
+    for i in 0 .. N * N { push(cross_st.on, false) }
+    for j in 0 .. 33 { for i in 0 .. 33 { cross_st.h[j * 33 + i] = 0.02 * float(i + j) } }
+    let pts = floats(24)
+    for i in 0 .. 8 {
+      pts[i * 3] = float(i % 2) * 2.0 - 1.0
+      pts[i * 3 + 1] = float((i / 2) % 2)
+      pts[i * 3 + 2] = float(i / 4) * 2.0 - 1.0
+    }
+    cross_st.hull = phys_convex(physics_st, pts, 8)
+    crossing(physics_st, cross_st)
+    let bodies = phys_count(physics_st)
+    let shapes = phys_shapes(physics_st)
+    let held = phys_heap_bytes()
+    let peak = phys_heap_peak()
+    for k in 0 .. 6 { crossing(physics_st, cross_st) }
+    print(`cross heap: {bodies} bodies, {shapes} shapes and {held} Jolt bytes after one crossing; {phys_count(physics_st)}, {phys_shapes(physics_st)} and {phys_heap_bytes()} after six more; peak {peak} -> {phys_heap_peak()}`)
+    expect_eq(phys_count(physics_st), bodies)
+    expect_eq(phys_shapes(physics_st), shapes)
+    expect(phys_heap_bytes() == held)
+    expect(phys_heap_peak() == peak)
+    phys_close(physics_st)
+  }
+}