ludic/tools/ludic-cli/pack.ludic

# pack.ludic — `ludic pack`, the writer for the .lpak asset pack.
#
# The reader is in the compiler (selfhost/backend/stdlib/emit_pak.ludic), which
# is also where the format is specified. This is the only thing that produces
# one, and the two have exactly one contract between them: the entry table is
# sorted by name, because the runtime binary-searches it.
#
#   ludic pack                      pack the manifest's `pack` roots to build/<name>.lpak
#   ludic pack --out x.lpak a b     pack directories a and b to x.lpak
#   ludic pack --list x.lpak        print what is in a pack
#   ludic pack --verify x.lpak      re-hash every entry against the table
#
# Paths are stored exactly as the game asks for them — a file at ./assets/kit/x.png
# is stored as "assets/kit/x.png" — so packing changes nothing about how the game
# is written. That is the whole point: `gltf_load("assets/kit/hiker", ...)` is the
# same line of code before and after.

# ---- little-endian integers -------------------------------------------------
#
# The runtime reads these back with a plain i32 load, so the byte order here is
# the byte order of the machine that runs the game. Both are little-endian on
# every target Ludic supports today; a big-endian port would byte-swap in the
# reader, not here.

function put_u32(f: pointer, v: int) -> void {
  let b = bytes(4)
  b[0] = v & 255
  b[1] = (v >> 8) & 255
  b[2] = (v >> 16) & 255
  b[3] = (v >> 24) & 255
  file_write(f, b, 4)
}

# FNV-1a over `n` bytes of `buf` starting at `off`. Not a security hash - it is
# there so `ludic pack --verify` can tell a truncated or bit-rotted pack from a
# good one. The arithmetic wraps in 32 bits, which is exactly what FNV specifies
# mod 2^32; the writer and the verifier compare the same bit pattern, so the
# sign an int puts on it never matters.
function fnv1a(buf: pointer, off: int, n: int) -> int {
  var h = -2128831035                # 0x811C9DC5 as a signed 32-bit int
  var i = 0
  while i < n {
    h = h ^ (buf[off + i] & 255)
    h = h * 16777619
    i += 1
  }
  return h
}

# ---- reading one file whole -------------------------------------------------
#
# read_file in the prelude NUL-terminates and does not report a length, which is
# fine for source text and useless for a PNG. This keeps the length.

var pk_len: int = 0

function read_blob(path: pointer) -> pointer {
  pk_len = 0
  let f = file_open(path, "rb")
  if f == null { return null }
  file_seek(f, 0, 2)
  let n = file_tell(f)
  file_seek(f, 0, 0)
  let buf = bytes(n + 1)
  let got = file_read(f, buf, n)
  file_close(f)
  buf[got] = 0
  pk_len = got
  return buf
}

# ---- gathering the files ----------------------------------------------------
#
# A pack entry has a NAME (what the game asks for) and a PATH (where the bytes
# are right now). They are usually the same string, but not always: a package's
# shaders live in the toolchain install and have to be stored under the
# `packages/<module>/` name the renderer looks them up by, or a bundled game
# cannot find them and comes up with no shaders at all.

property PackList { names: []pointer, paths: []pointer }

# how many files `.packignore` kept out of the last gather, and whether to read one
# at all (`ludic pack --no-ignore` turns it off to see what a rule is costing)
var pk_skipped: int = 0
var pk_use_ignore: bool = true

function pack_list_new() -> PackList {
  let l = new PackList
  l.names = new []pointer
  l.paths = new []pointer
  return l
}

function pack_add(l: PackList, name: pointer, path: pointer) -> void {
  push(l.names, name)
  push(l.paths, path)
}

# Every file under `root`, in byte order. `find | sort` rather than a walk of
# Fs.list: the runtime binary-searches on strcmp order, and a recursive walk
# emits "assets/a.png" after "assets/ab/x.png" because it descends per directory,
# which is not that order. LC_ALL=C is what makes sort agree with strcmp.
#
# `base` is the directory find runs in and `prefix` what the results are stored
# under: with both empty the name is the path, which is the ordinary case.
function pack_gather_into(l: PackList, base: pointer, root: pointer, prefix: pointer) -> void {
  pack_gather_filtered(l, base, root, prefix, null)
}
# the same, with a `.packignore` set applied to the paths as they are gathered
function pack_gather_filtered(l: PackList, base: pointer, root: pointer, prefix: pointer, ig: PackIgnore) -> void {
  var cd = ""
  if base != "" { cd = `cd {base} && ` }
  # -L follows symlinks. An asset root is very often a link to a shared or
  # fetched tree - `ludic assets` writes one, and a game with two checkouts
  # sharing a texture set will have several - and a plain `find` walks straight
  # past them, packing nothing while reporting success.
  # `.packignore` is a build-time control file - nothing reads one at run time - so it
  # is dropped here rather than by the rules it carries, and `--no-ignore` does not
  # bring it back.
  let listing = capture(`{cd}find -L {root} -type f ! -name '.DS_Store' ! -name '.packignore' 2>/dev/null | LC_ALL=C sort`)
  let n = slen(listing)
  var i = 0
  while i < n {
    let line = line_at(listing, i)
    i = i + slen(line) + 1
    let rel = s_trim(line)
    if slen(rel) == 0 { continue }
    if ig != null and pi_ignored(ig, rel) { pk_skipped += 1;  continue }
    var name = rel
    if prefix != "" { name = `{prefix}/{rel}` }
    var path = rel
    if base != "" { path = `{base}/{rel}` }
    pack_add(l, name, path)
  }
}

# the project's own asset roots, stored under the names the game already uses
function pack_gather_all(roots: []pointer) -> PackList {
  let l = pack_list_new()
  pk_skipped = 0
  # `.packignore` (packignore.ludic): the project's own roots only. A package's
  # resources are added after this and are not the game's to exclude.
  var ig: PackIgnore = null
  if pk_use_ignore {
    ig = pi_new()
    pi_load_dir(ig, "")
    var r = 0
    while r < len(roots) { pi_load_root(ig, roots[r]);  r += 1 }
    if pi_count(ig) == 0 { ig = null }
  }
  var i = 0
  while i < len(roots) {
    if not file_exists(roots[i]) {
      err(`ludic pack: no such directory: {roots[i]}\n`)
    } else {
      let before = len(l.names)
      pack_gather_filtered(l, "", roots[i], "", ig)
      # A declared root that contributes nothing is almost always a mistake -
      # a typo, or a link into a tree that was never fetched - and the failure
      # it causes is a game that starts and then behaves as though half its
      # assets do not exist. Say so here rather than at the player.
      if len(l.names) == before and pk_skipped == 0 { err(`ludic pack: warning: {roots[i]} is empty, nothing packed from it\n`) }
    }
    i += 1
  }
  return pack_sort(l)
}

# insertion sort by byte order, carrying each name's path along with it. The
# lists are hundreds of entries, not millions, and this keeps the one ordering
# guarantee the format makes in one readable place.
function pack_sort(l: PackList) -> PackList {
  var i = 1
  while i < len(l.names) {
    let vn = l.names[i]
    let vp = l.paths[i]
    var j = i - 1
    while j >= 0 and pack_cmp(l.names[j], vn) > 0 {
      l.names[j + 1] = l.names[j]
      l.paths[j + 1] = l.paths[j]
      j -= 1
    }
    l.names[j + 1] = vn
    l.paths[j + 1] = vp
    i += 1
  }
  return l
}

# strcmp order: negative, zero or positive, comparing unsigned bytes. This is the
# ordering the format promises and the runtime's binary search assumes.
function pack_cmp(a: pointer, b: pointer) -> int {
  var i = 0
  var r = 0
  var done = false
  while not done {
    let ca = a[i] & 255
    let cb = b[i] & 255
    if ca != cb { r = ca - cb;  done = true }
    else if ca == 0 { r = 0;  done = true }
    else { i += 1 }
  }
  return r
}

# round `v` up to the next multiple of 16, so every blob starts aligned and a
# reader can hand a mapped pointer straight to something that wants alignment
function align16(v: int) -> int {
  let r = v & 15
  if r == 0 { return v }
  return v + (16 - r)
}

# ---- writing ----------------------------------------------------------------

function pack_write(out_path: pointer, l: PackList) -> bool {
  let names = l.names
  let paths = l.paths
  let n = len(names)
  if n == 0 { err("ludic pack: nothing to pack\n");  return false }

  # Layout is decided before a byte is written: the entry table has to carry
  # absolute offsets, and those are only knowable once every size is known.
  let names_at = 32 + 16 * n
  var name_off = new []int
  var names_len = 0
  var i = 0
  while i < n {
    push(name_off, names_at + names_len)
    names_len = names_len + slen(names[i]) + 1
    i += 1
  }
  let data_at = align16(names_at + names_len)

  var data_off = new []int
  var data_len = new []int
  var hash = new []int
  var total = data_at
  i = 0
  while i < n {
    let blob = read_blob(paths[i])
    if blob == null { err(`ludic pack: cannot read {paths[i]}\n`);  return false }
    push(data_off, total)
    push(data_len, pk_len)
    push(hash, fnv1a(blob, 0, pk_len))
    total = align16(total + pk_len)
    i += 1
  }

  shell(`mkdir -p "$(dirname {out_path})"`)
  let f = file_open(out_path, "wb")
  if f == null { err(`ludic pack: cannot write {out_path}\n`);  return false }

  # header
  let magic = bytes(5);  magic[0] = 'L';  magic[1] = 'P';  magic[2] = 'A';  magic[3] = 'K';  magic[4] = 0
  file_write(f, magic, 4)
  put_u32(f, 1)                  # version
  put_u32(f, n)                  # count
  put_u32(f, names_at)
  put_u32(f, data_at)
  put_u32(f, 0);  put_u32(f, 0);  put_u32(f, 0)

  # entry table, in the sorted order the runtime's binary search depends on
  i = 0
  while i < n {
    put_u32(f, name_off[i])
    put_u32(f, data_off[i])
    put_u32(f, data_len[i])
    put_u32(f, hash[i])
    i += 1
  }

  # name heap
  i = 0
  while i < n {
    file_write(f, names[i], slen(names[i]) + 1)
    i += 1
  }

  # pad up to the first blob, then each blob followed by its alignment padding
  pack_pad(f, data_at - (names_at + names_len))
  var at = data_at
  i = 0
  while i < n {
    let blob = read_blob(paths[i])
    if blob == null { err(`ludic pack: {paths[i]} vanished mid-pack\n`);  file_close(f);  return false }
    file_write(f, blob, pk_len)
    at = at + pk_len
    let want = align16(at)
    pack_pad(f, want - at)
    at = want
    i += 1
  }
  file_close(f)

  var note = ""
  if pk_skipped > 0 { note = `, {string(pk_skipped)} skipped by .packignore` }
  print(`packed {string(n)} files, {string(total / 1024)} KiB{note} -> {out_path}`)
  return true
}

function pack_pad(f: pointer, k: int) -> void {
  if k <= 0 { return }
  let z = bytes(k + 1)
  var i = 0
  while i < k { z[i] = 0;  i += 1 }
  file_write(f, z, k)
}

# ---- reading a pack back (list / verify) ------------------------------------

# the NUL-terminated name at `off`, copied out. Ludic's `+` on a pointer is
# string concatenation, not address arithmetic, so an offset into a buffer has to
# be sliced rather than added.
function pack_name(buf: pointer, off: int) -> pointer {
  var n = 0
  while buf[off + n] != 0 { n += 1 }
  return str_sub(buf, off, off + n)
}

function pack_u32(buf: pointer, at: int) -> int {
  return (buf[at] & 255) | ((buf[at + 1] & 255) << 8) | ((buf[at + 2] & 255) << 16) | ((buf[at + 3] & 255) << 24)
}

function pack_open_read(path: pointer) -> pointer {
  let buf = read_blob(path)
  if buf == null { err(`ludic pack: cannot read {path}\n`);  return null }
  if pk_len < 32 or buf[0] != 'L' or buf[1] != 'P' or buf[2] != 'A' or buf[3] != 'K' {
    err(`ludic pack: {path} is not a pack\n`)
    return null
  }
  return buf
}

function cmd_pack_list(path: pointer) -> int {
  let buf = pack_open_read(path)
  if buf == null { return 1 }
  let n = pack_u32(buf, 8)
  print(`{path}: {string(n)} entries`)
  var i = 0
  while i < n {
    let e = 32 + 16 * i
    let no = pack_u32(buf, e)
    let dl = pack_u32(buf, e + 8)
    print(`  {string(dl)}\t{pack_name(buf, no)}`)
    i += 1
  }
  return 0
}

function cmd_pack_verify(path: pointer) -> int {
  let buf = pack_open_read(path)
  if buf == null { return 1 }
  let n = pack_u32(buf, 8)
  var bad = 0
  var i = 0
  while i < n {
    let e = 32 + 16 * i
    let no = pack_u32(buf, e)
    let dof = pack_u32(buf, e + 4)
    let dl = pack_u32(buf, e + 8)
    let want = pack_u32(buf, e + 12)
    if dof + dl > pk_len {
      err(`  truncated: {pack_name(buf, no)}\n`);  bad += 1
    } else if fnv1a(buf, dof, dl) != want {
      err(`  corrupt: {pack_name(buf, no)}\n`);  bad += 1
    }
    i += 1
  }
  if bad > 0 { err(`ludic pack: {string(bad)} of {string(n)} entries failed\n`);  return 1 }
  print(`OK {string(n)} entries verified`)
  return 0
}

# ---- the packages a shipped game still needs --------------------------------
#
# A package can carry runtime resources, not just code: ludic.render3d loads its
# shaders from `packages/ludic.render3d/shaders/` at startup, resolving that
# against the project or the toolchain install. A bundled .app has neither, so
# without this a shipped 3D game starts, finds no shaders, and draws nothing -
# which is a spectacular way to fail and an easy one to ship by accident.
#
# So the resources go in the pack under exactly the name the package looks them
# up by. Source (.ludic) and documentation are left out: the game is already
# compiled, and nobody reads a package's FEATURES.md out of a .app.
function pack_add_packages(l: PackList) -> void {
  var base = "packages"
  if not file_exists(base) { base = `{ludic_home()}packages` }
  if not file_exists(base) { return }
  let mods = capture(`ls {base} 2>/dev/null`)
  let n = slen(mods)
  var i = 0
  while i < n {
    let line = line_at(mods, i)
    i = i + slen(line) + 1
    let mod = s_trim(line)
    if slen(mod) == 0 { continue }
    if not shq(`test -d {base}/{mod}`) { continue }      # packages/README.md is an index, not a package
    pack_gather_pkg(l, base, mod)
  }
}

# one package's resource files, named `packages/<module>/<path>`
function pack_gather_pkg(l: PackList, base: pointer, mod: pointer) -> void {
  let listing = capture(`cd {base}/{mod} 2>/dev/null && find -L . -type f ! -name '*.ludic' ! -name '*.md' ! -path './build/*' ! -name '.DS_Store' 2>/dev/null | LC_ALL=C sort`)
  let n = slen(listing)
  var i = 0
  while i < n {
    let line = line_at(listing, i)
    i = i + slen(line) + 1
    var rel = s_trim(line)
    if slen(rel) == 0 { continue }
    if s_starts(rel, "./") { rel = sslice(rel, 2, slen(rel)) }
    pack_add(l, `packages/{mod}/{rel}`, `{base}/{mod}/{rel}`)
  }
}

# ---- the command ------------------------------------------------------------

# The roots to pack: an explicit list of directories on the command line, else
# the manifest's `pack` lines, else assets/ when it exists. A game that keeps its
# assets where the convention puts them needs no configuration at all.
function pack_roots(m: Manifest, from: int) -> []pointer {
  var roots = new []pointer
  var i = from
  while i < arg_count() {
    let a = arg(i)
    if a == "--out" { i += 1 }                 # its value is the pack, not a root
    else if not s_starts(a, "--") { push(roots, a) }
    i += 1
  }
  if len(roots) > 0 { return roots }
  i = 0
  while i < len(m.packs) { push(roots, m.packs[i]);  i += 1 }
  if len(roots) > 0 { return roots }
  if file_exists("assets") { push(roots, "assets") }
  return roots
}

# the pack a project builds by default: build/<name>.lpak
function pack_default_out(m: Manifest) -> string {
  return `build/{project_name("")}.lpak`
}

function cmd_pack() -> int {
  var out_path = ""
  var i = 2
  while i < arg_count() {
    let a = arg(i)
    if a == "--list" { return cmd_pack_list(argn(i + 1, "")) }
    if a == "--verify" { return cmd_pack_verify(argn(i + 1, "")) }
    if a == "--out" { out_path = argn(i + 1, "");  i += 1 }
    if a == "--no-ignore" { pk_use_ignore = false }
    i += 1
  }
  let m = read_root_manifest()
  if out_path == "" { out_path = pack_default_out(m) }
  let roots = pack_roots(m, 2)
  if len(roots) == 0 {
    err("ludic pack: nothing to pack.\n")
    err("  put the assets under assets/, or name the roots in package.ludic:\n")
    err("    pack \"assets/kit\"\n")
    return 1
  }
  let files = pack_gather_all(roots)
  if not pack_write(out_path, files) { return 1 }
  return 0
}