feat(pack): asset packs, so a built game is a thing you can hand someone

A Ludic game opened its assets by a path relative to the working directory, so
`build/mygame` ran only from the project root and there was nothing to give
anyone but "the binary, and also this whole tree". A game is not one file, but
shipping it has to be.

`ludic pack` writes a .lpak: a header, a name-sorted entry table, a name heap
and the blobs, 16-byte aligned. Entries are stored rather than compressed - PNG,
JPEG and glTF binary arrive compressed already, and a decompressor on the load
path would spend CPU to make the file no smaller.

The reader is spliced into the compiler at the one place every asset in a Ludic
program comes through: file_open. gltf_load, tex_load, Audio.load, Fs.read_text
and the renderer's own shader loads all bottom out there, so routing it through
@lp_pak_open reaches every one of them without any of them knowing. A read of a
packed path becomes an fmemopen over the mapped bytes; everything else is the
fopen it always was. Fs.exists and Fs.size consult the packs too, so a game that
guards a load with Fs.exists keeps finding its assets once they are packed.

The pack is mmap'd rather than read: 165 MB of textures costs one syscall at
boot and pages in only what is touched. @lp_pak_boot runs from
@llvm.global_ctors, before main, so a pack is mounted before the game's first
line - and it reads packs.index, a plain list, so the mount order is explicit
and a later pack shadows an earlier one.

Without a packs.index nothing mounts and every open goes to the filesystem
exactly as before, which is every `ludic run` during development. Packing is a
shipping step and is invisible until you ship.

The compiler reproduces itself byte-exactly and the C-free bootstrap from the
seed still holds.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tools/ludic-cli/pack.ludic

@@ -0,0 +1,357 @@
+# pack.ludic — `ludic pack`, the writer for the .lpak asset pack.
+#
+# The reader is in the compiler (selfhost/backend/stdlib/emit_pak.ludic), which
+# is also where the format is specified. This is the only thing that produces
+# one, and the two have exactly one contract between them: the entry table is
+# sorted by name, because the runtime binary-searches it.
+#
+#   ludic pack                      pack the manifest's `pack` roots to build/<name>.lpak
+#   ludic pack --out x.lpak a b     pack directories a and b to x.lpak
+#   ludic pack --list x.lpak        print what is in a pack
+#   ludic pack --verify x.lpak      re-hash every entry against the table
+#
+# Paths are stored exactly as the game asks for them — a file at ./assets/kit/x.png
+# is stored as "assets/kit/x.png" — so packing changes nothing about how the game
+# is written. That is the whole point: `gltf_load("assets/kit/hiker", ...)` is the
+# same line of code before and after.
+
+# ---- little-endian integers -------------------------------------------------
+#
+# The runtime reads these back with a plain i32 load, so the byte order here is
+# the byte order of the machine that runs the game. Both are little-endian on
+# every target Ludic supports today; a big-endian port would byte-swap in the
+# reader, not here.
+
+function put_u32(f: pointer, v: int) -> void {
+  let b = bytes(4)
+  b[0] = v & 255
+  b[1] = (v >> 8) & 255
+  b[2] = (v >> 16) & 255
+  b[3] = (v >> 24) & 255
+  file_write(f, b, 4)
+}
+
+# FNV-1a over `n` bytes of `buf` starting at `off`. Not a security hash - it is
+# there so `ludic pack --verify` can tell a truncated or bit-rotted pack from a
+# good one. The arithmetic wraps in 32 bits, which is exactly what FNV specifies
+# mod 2^32; the writer and the verifier compare the same bit pattern, so the
+# sign an int puts on it never matters.
+function fnv1a(buf: pointer, off: int, n: int) -> int {
+  var h = -2128831035                # 0x811C9DC5 as a signed 32-bit int
+  var i = 0
+  while i < n {
+    h = h ^ (buf[off + i] & 255)
+    h = h * 16777619
+    i += 1
+  }
+  return h
+}
+
+# ---- reading one file whole -------------------------------------------------
+#
+# read_file in the prelude NUL-terminates and does not report a length, which is
+# fine for source text and useless for a PNG. This keeps the length.
+
+var pk_len: int = 0
+
+function read_blob(path: pointer) -> pointer {
+  pk_len = 0
+  let f = file_open(path, "rb")
+  if f == null { return null }
+  file_seek(f, 0, 2)
+  let n = file_tell(f)
+  file_seek(f, 0, 0)
+  let buf = bytes(n + 1)
+  let got = file_read(f, buf, n)
+  file_close(f)
+  buf[got] = 0
+  pk_len = got
+  return buf
+}
+
+# ---- gathering the files ----------------------------------------------------
+
+# Every file under `root`, in byte order. `find | sort` rather than a walk of
+# Fs.list: the runtime binary-searches on strcmp order, and a recursive walk
+# emits "assets/a.png" after "assets/ab/x.png" because it descends per directory,
+# which is not that order. LC_ALL=C is what makes sort agree with strcmp.
+function pack_gather(root: pointer) -> []pointer {
+  var out = new []pointer
+  if not file_exists(root) {
+    err(`ludic pack: no such directory: {root}\n`)
+    return out
+  }
+  let listing = capture(`find {root} -type f ! -name '.DS_Store' | LC_ALL=C sort`)
+  let n = slen(listing)
+  var i = 0
+  while i < n {
+    let line = line_at(listing, i)
+    i = i + slen(line) + 1
+    let t = s_trim(line)
+    if slen(t) > 0 { push(out, t) }
+  }
+  return out
+}
+
+# every root's files, concatenated then re-sorted as one list, since the runtime
+# searches one table across all of them
+function pack_gather_all(roots: []pointer) -> []pointer {
+  var all = new []pointer
+  var i = 0
+  while i < len(roots) {
+    let one = pack_gather(roots[i])
+    var k = 0
+    while k < len(one) { push(all, one[k]);  k += 1 }
+    i += 1
+  }
+  return pack_sort(all)
+}
+
+# insertion sort by byte order. The lists are hundreds of entries, not millions,
+# and this keeps the one ordering guarantee the format makes in one readable place.
+function pack_sort(xs: []pointer) -> []pointer {
+  var i = 1
+  while i < len(xs) {
+    let v = xs[i]
+    var j = i - 1
+    while j >= 0 and pack_cmp(xs[j], v) > 0 {
+      xs[j + 1] = xs[j]
+      j -= 1
+    }
+    xs[j + 1] = v
+    i += 1
+  }
+  return xs
+}
+
+# strcmp order: negative, zero or positive, comparing unsigned bytes. This is the
+# ordering the format promises and the runtime's binary search assumes.
+function pack_cmp(a: pointer, b: pointer) -> int {
+  var i = 0
+  var r = 0
+  var done = false
+  while not done {
+    let ca = a[i] & 255
+    let cb = b[i] & 255
+    if ca != cb { r = ca - cb;  done = true }
+    else if ca == 0 { r = 0;  done = true }
+    else { i += 1 }
+  }
+  return r
+}
+
+# round `v` up to the next multiple of 16, so every blob starts aligned and a
+# reader can hand a mapped pointer straight to something that wants alignment
+function align16(v: int) -> int {
+  let r = v & 15
+  if r == 0 { return v }
+  return v + (16 - r)
+}
+
+# ---- writing ----------------------------------------------------------------
+
+function pack_write(out_path: pointer, files: []pointer) -> bool {
+  let n = len(files)
+  if n == 0 { err("ludic pack: nothing to pack\n");  return false }
+
+  # Layout is decided before a byte is written: the entry table has to carry
+  # absolute offsets, and those are only knowable once every size is known.
+  let names_at = 32 + 16 * n
+  var name_off = new []int
+  var names_len = 0
+  var i = 0
+  while i < n {
+    push(name_off, names_at + names_len)
+    names_len = names_len + slen(files[i]) + 1
+    i += 1
+  }
+  let data_at = align16(names_at + names_len)
+
+  var data_off = new []int
+  var data_len = new []int
+  var hash = new []int
+  var total = data_at
+  i = 0
+  while i < n {
+    let blob = read_blob(files[i])
+    if blob == null { err(`ludic pack: cannot read {files[i]}\n`);  return false }
+    push(data_off, total)
+    push(data_len, pk_len)
+    push(hash, fnv1a(blob, 0, pk_len))
+    total = align16(total + pk_len)
+    i += 1
+  }
+
+  run(`mkdir -p "$(dirname {out_path})"`)
+  let f = file_open(out_path, "wb")
+  if f == null { err(`ludic pack: cannot write {out_path}\n`);  return false }
+
+  # header
+  let magic = bytes(5);  magic[0] = 'L';  magic[1] = 'P';  magic[2] = 'A';  magic[3] = 'K';  magic[4] = 0
+  file_write(f, magic, 4)
+  put_u32(f, 1)                  # version
+  put_u32(f, n)                  # count
+  put_u32(f, names_at)
+  put_u32(f, data_at)
+  put_u32(f, 0);  put_u32(f, 0);  put_u32(f, 0)
+
+  # entry table, in the sorted order the runtime's binary search depends on
+  i = 0
+  while i < n {
+    put_u32(f, name_off[i])
+    put_u32(f, data_off[i])
+    put_u32(f, data_len[i])
+    put_u32(f, hash[i])
+    i += 1
+  }
+
+  # name heap
+  i = 0
+  while i < n {
+    file_write(f, files[i], slen(files[i]) + 1)
+    i += 1
+  }
+
+  # pad up to the first blob, then each blob followed by its alignment padding
+  pack_pad(f, data_at - (names_at + names_len))
+  var at = data_at
+  i = 0
+  while i < n {
+    let blob = read_blob(files[i])
+    if blob == null { err(`ludic pack: {files[i]} vanished mid-pack\n`);  file_close(f);  return false }
+    file_write(f, blob, pk_len)
+    at = at + pk_len
+    let want = align16(at)
+    pack_pad(f, want - at)
+    at = want
+    i += 1
+  }
+  file_close(f)
+
+  print(`packed {string(n)} files, {string(total / 1024)} KiB -> {out_path}`)
+  return true
+}
+
+function pack_pad(f: pointer, k: int) -> void {
+  if k <= 0 { return }
+  let z = bytes(k + 1)
+  var i = 0
+  while i < k { z[i] = 0;  i += 1 }
+  file_write(f, z, k)
+}
+
+# ---- reading a pack back (list / verify) ------------------------------------
+
+# the NUL-terminated name at `off`, copied out. Ludic's `+` on a pointer is
+# string concatenation, not address arithmetic, so an offset into a buffer has to
+# be sliced rather than added.
+function pack_name(buf: pointer, off: int) -> pointer {
+  var n = 0
+  while buf[off + n] != 0 { n += 1 }
+  return str_sub(buf, off, off + n)
+}
+
+function pack_u32(buf: pointer, at: int) -> int {
+  return (buf[at] & 255) | ((buf[at + 1] & 255) << 8) | ((buf[at + 2] & 255) << 16) | ((buf[at + 3] & 255) << 24)
+}
+
+function pack_open_read(path: pointer) -> pointer {
+  let buf = read_blob(path)
+  if buf == null { err(`ludic pack: cannot read {path}\n`);  return null }
+  if pk_len < 32 or buf[0] != 'L' or buf[1] != 'P' or buf[2] != 'A' or buf[3] != 'K' {
+    err(`ludic pack: {path} is not a pack\n`)
+    return null
+  }
+  return buf
+}
+
+function cmd_pack_list(path: pointer) -> int {
+  let buf = pack_open_read(path)
+  if buf == null { return 1 }
+  let n = pack_u32(buf, 8)
+  print(`{path}: {string(n)} entries`)
+  var i = 0
+  while i < n {
+    let e = 32 + 16 * i
+    let no = pack_u32(buf, e)
+    let dl = pack_u32(buf, e + 8)
+    print(`  {string(dl)}\t{pack_name(buf, no)}`)
+    i += 1
+  }
+  return 0
+}
+
+function cmd_pack_verify(path: pointer) -> int {
+  let buf = pack_open_read(path)
+  if buf == null { return 1 }
+  let n = pack_u32(buf, 8)
+  var bad = 0
+  var i = 0
+  while i < n {
+    let e = 32 + 16 * i
+    let no = pack_u32(buf, e)
+    let dof = pack_u32(buf, e + 4)
+    let dl = pack_u32(buf, e + 8)
+    let want = pack_u32(buf, e + 12)
+    if dof + dl > pk_len {
+      err(`  truncated: {pack_name(buf, no)}\n`);  bad += 1
+    } else if fnv1a(buf, dof, dl) != want {
+      err(`  corrupt: {pack_name(buf, no)}\n`);  bad += 1
+    }
+    i += 1
+  }
+  if bad > 0 { err(`ludic pack: {string(bad)} of {string(n)} entries failed\n`);  return 1 }
+  print(`OK {string(n)} entries verified`)
+  return 0
+}
+
+# ---- the command ------------------------------------------------------------
+
+# The roots to pack: an explicit list of directories on the command line, else
+# the manifest's `pack` lines, else assets/ when it exists. A game that keeps its
+# assets where the convention puts them needs no configuration at all.
+function pack_roots(m: Manifest, from: int) -> []pointer {
+  var roots = new []pointer
+  var i = from
+  while i < arg_count() {
+    let a = arg(i)
+    if a == "--out" { i += 1 }                 # its value is the pack, not a root
+    else if not s_starts(a, "--") { push(roots, a) }
+    i += 1
+  }
+  if len(roots) > 0 { return roots }
+  i = 0
+  while i < len(m.packs) { push(roots, m.packs[i]);  i += 1 }
+  if len(roots) > 0 { return roots }
+  if file_exists("assets") { push(roots, "assets") }
+  return roots
+}
+
+# the pack a project builds by default: build/<name>.lpak
+function pack_default_out(m: Manifest) -> pointer {
+  return `build/{project_name("")}.lpak`
+}
+
+function cmd_pack() -> int {
+  var out_path = ""
+  var i = 2
+  while i < arg_count() {
+    let a = arg(i)
+    if a == "--list" { return cmd_pack_list(argn(i + 1, "")) }
+    if a == "--verify" { return cmd_pack_verify(argn(i + 1, "")) }
+    if a == "--out" { out_path = argn(i + 1, "");  i += 1 }
+    i += 1
+  }
+  let m = read_root_manifest()
+  if out_path == "" { out_path = pack_default_out(m) }
+  let roots = pack_roots(m, 2)
+  if len(roots) == 0 {
+    err("ludic pack: nothing to pack.\n")
+    err("  put the assets under assets/, or name the roots in package.ludic:\n")
+    err("    pack \"assets/kit\"\n")
+    return 1
+  }
+  let files = pack_gather_all(roots)
+  if not pack_write(out_path, files) { return 1 }
+  return 0
+}