Merge commit 'c592974' into lang/foundations

packages/ludic.base/baked.ludic

@@ -0,0 +1,65 @@
+# baked.ludic - what is deterministic made at build time (`ludic bake`) by the game's own code, and
+# read at run time. Every baked file: a 32-byte header - "LBAK", the header's format, the bake's
+# generator version, the payload's offset, a hash of its inputs, the payload's length - its key, and
+# the payload, whose layout is the bake's own. The runtime checks the header, never the inputs:
+# `ludic bake --check` holds a baked file to its inputs at build time.
+export const BAKE_FORMAT: int = 1
+const BAKE_HEAD: int = 32
+
+export state BakeState {
+  said: []string = new []string       # the keys a dev build has said it made at run time
+}
+
+# assets/baked/<map>/<file>, or assets/baked/<file> for no map
+export function baked_path(map: string, file: string) -> string {
+  if len(map) == 0 { return "assets/baked/" + file }
+  return "assets/baked/" + map + "/" + file
+}
+
+# the payload of a baked file that is this bake at this version, or null (missing, not a baked file,
+# another format, another key or version: the caller makes it at run time)
+export function baked_open(path: string, key: string, version: int) -> []byte {
+  let b = Fs.read_bytes(path)
+  if b == null or len(b) < BAKE_HEAD { return null }
+  let off = bk_head_ok(b, len(b), key, version)
+  if off < 0 or long(off) + bk_u64(b, 24) > long(len(b)) { return null }
+  return view(b, off, int(bk_u64(b, 24)))        # the payload, in place: no copy of a large bake
+}
+
+# the payload's offset when `b` (its first `have` bytes) is this bake's header and key, else -1
+function bk_head_ok(b: []byte, have: int, key: string, version: int) -> int {
+  if have < BAKE_HEAD or b[0] != 76 or b[1] != 66 or b[2] != 65 or b[3] != 75 { return -1 }   # "LBAK"
+  if bk_u32(b, 4) != BAKE_FORMAT or bk_u32(b, 8) != version { return -1 }
+  let off = bk_u32(b, 12)
+  if off < BAKE_HEAD + len(key) + 1 or have < BAKE_HEAD + len(key) + 1 { return -1 }
+  for i in 0 .. len(key) { if b[BAKE_HEAD + i] != key[i] { return -1 } }
+  if b[BAKE_HEAD + len(key)] != 0 { return -1 }
+  return off
+}
+
+# a dev build's line, once a key: the bake is missing or stale, so it is made at run time
+export function bake_missing(bake_st: mut BakeState, key: string) -> void {
+  for i in 0 .. len(bake_st.said) { if bake_st.said[i] == key { return } }
+  push(bake_st.said, key)
+  print(`bake: {key} missing or stale - made at run time (ludic bake)`)
+}
+
+# a baked file written: its header, key and `n` bytes of payload; the directories made
+export function bake_write(path: string, key: string, version: int, inputs_hash: long, payload: []byte, n: int) -> bool {
+  let off = BAKE_HEAD + ((len(key) + 1 + 7) / 8) * 8
+  let b = buffer(off + n)
+  b[0] = 76
+  b[1] = 66
+  b[2] = 65
+  b[3] = 75
+  bk_put32(b, 4, BAKE_FORMAT)
+  bk_put32(b, 8, version)
+  bk_put32(b, 12, off)
+  bk_put64(b, 16, inputs_hash)
+  bk_put64(b, 24, long(n))
+  for i in 0 .. len(key) { b[BAKE_HEAD + i] = key[i] }
+  for i in 0 .. n { b[off + i] = payload[i] }
+  let slash = bk_last_slash(path)
+  if slash > 0 { Fs.mkdir(path[0 .. slash]) }
+  return Fs.write_bytes(path, b, off + n)
+}

packages/ludic.base/baked_hash.ludic

@@ -0,0 +1,40 @@
+# baked_hash.ludic - a bake's inputs as one number, and the header's little-endian fields. The hash
+# is FNV-1a 64 over each input in order: its path's bytes, a 0, then its bytes (tools/bake/baked.py
+# and the check compute the same)
+const BK_FNV_OFFSET: long = -3750763034362895579    # 0xcbf29ce484222325
+const BK_FNV_PRIME: long = 1099511628211
+
+# `inputs` space-separated, as a Bakes row lists them; a missing input hashes as its path alone
+export function bake_inputs_hash(inputs: string) -> long {
+  var h = BK_FNV_OFFSET
+  var a = 0
+  while a < len(inputs) {
+    while a < len(inputs) and inputs[a] == 32 { a += 1 }
+    var e = a
+    while e < len(inputs) and inputs[e] != 32 { e += 1 }
+    if e > a {
+      let p = inputs[a .. e]
+      for i in 0 .. len(p) { h = bk_fnv(h, p[i]) }
+      h = bk_fnv(h, 0)
+      let b = Fs.read_bytes(p)
+      if b != null { for i in 0 .. len(b) { h = bk_fnv(h, b[i]) } }
+    }
+    a = e
+  }
+  return h
+}
+function bk_fnv(h: long, c: int) -> long { return (h ^ long(c & 255)) * BK_FNV_PRIME }
+
+function bk_u32(b: []byte, at: int) -> int { return (b[at] & 255) | ((b[at + 1] & 255) << 8) | ((b[at + 2] & 255) << 16) | ((b[at + 3] & 255) << 24) }
+function bk_u64(b: []byte, at: int) -> long { return long(bk_u32(b, at)) & 4294967295 | (long(bk_u32(b, at + 4)) << 32) }
+function bk_put32(b: []byte, at: int, v: int) -> void {
+  for i in 0 .. 4 { b[at + i] = (v >> (i * 8)) & 255 }
+}
+function bk_put64(b: []byte, at: int, v: long) -> void {
+  for i in 0 .. 8 { b[at + i] = int((v >> long(i * 8)) & 255) }
+}
+function bk_last_slash(p: string) -> int {
+  var k = -1
+  for i in 0 .. len(p) { if p[i] == 47 { k = i } }
+  return k
+}

packages/ludic.base/baked_stream.ludic

@@ -0,0 +1,49 @@
+# baked_stream.ludic - a large bake read a piece at a time (placements by chunk, terrain by tile): the
+# header checked once, the file kept open, and each read into the caller's own buffer at a payload
+# offset, so a ring streaming chunks in makes nothing per chunk. Pack-aware (file_open)
+export property BakedFile {
+  f: pointer = null
+  off: int = 0                  # where the payload starts in the file
+  n: long = 0                   # the payload's length
+}
+
+# this bake's file kept open for reading, or null (missing, or another format, key or version)
+@alloc_ok("once per baked file opened: its header and record")
+export function baked_open_range(path: string, key: string, version: int) -> BakedFile {
+  let f = file_open(path, "rb")
+  if f == null { return null }
+  let want = 32 + len(key) + 1
+  let h = buffer(want)
+  if file_read(f, h, want) != want or bk_head_ok(h, want, key, version) < 0 {
+    file_close(f)
+    return null
+  }
+  let bf = new BakedFile
+  bf.f = f
+  bf.off = bk_u32(h, 12)
+  bf.n = bk_u64(h, 24)
+  return bf
+}
+
+# `n` bytes from payload offset `at` into `into` (from its start): the count read, 0 on a failure;
+# held to the buffer and to the payload
+export function baked_read(bf: BakedFile, at: int, n: int, into: []byte) -> int {
+  if bf == null or bf.f == null or at < 0 or n <= 0 { return 0 }
+  var k = n
+  if k > len(into) { k = len(into) }
+  if long(at) + long(k) > bf.n { k = int(bf.n - long(at)) }
+  if k <= 0 { return 0 }
+  if file_seek(bf.f, bf.off + at, 0) != 0 { return 0 }
+  return file_read(bf.f, into, k)
+}
+
+export function baked_len(bf: BakedFile) -> long {
+  if bf == null { return 0 }
+  return bf.n
+}
+
+export function baked_close(bf: BakedFile) -> void {
+  if bf == null or bf.f == null { return }
+  file_close(bf.f)
+  bf.f = null
+}

packages/ludic.base/index.ludic

@@ -10,6 +10,9 @@ import "queue_ring.ludic"
 import "text_buf.ludic"
 import "text_intern.ludic"
 import "text_ring.ludic"
+import "baked.ludic"
+import "baked_hash.ludic"
+import "baked_stream.ludic"
 import "rng.ludic"
 import "save.ludic"
 import "save_fields.ludic"

packages/ludic.base/tests/baked_test.ludic

@@ -0,0 +1,47 @@
+# baked_test.ludic - a baked file round-trips its payload; another version or key reads as null; the
+# inputs hash is FNV-1a 64 over path, 0 and bytes (tools/bake/baked.py computes the same)
+import "ludic.base"
+program BakedTest {
+  function dir() -> string { return Os.temp_dir() + "/ludic-baked-test" }
+  test "a baked file's payload comes back, for its key and version only" {
+    let p = buffer(5)
+    for i in 0 .. 5 { p[i] = 65 + i }
+    let path = dir() + "/a/b.bin"
+    expect(bake_write(path, "tiles", 3, 42, p, 5))
+    let b = baked_open(path, "tiles", 3)
+    expect(b != null)
+    expect_eq(len(b), 5)
+    expect_eq(b[4], 69)
+    expect(baked_open(path, "tiles", 4) == null)
+    expect(baked_open(path, "tile", 3) == null)
+    expect(baked_open(dir() + "/none.bin", "tiles", 3) == null)
+  }
+  test "a streamed bake reads a piece of its payload at an offset into the caller's buffer" {
+    let p = buffer(10)
+    for i in 0 .. 10 { p[i] = 48 + i }
+    let path = dir() + "/s.bin"
+    expect(bake_write(path, "chunks", 1, 0, p, 10))
+    let bf = baked_open_range(path, "chunks", 1)
+    expect(bf != null)
+    expect(baked_len(bf) == 10)
+    let into = buffer(4)
+    expect_eq(baked_read(bf, 6, 4, into), 4)
+    expect_eq(into[0], 54)
+    expect_eq(baked_read(bf, 8, 4, into), 2)
+    baked_close(bf)
+    expect(baked_open_range(path, "chunks", 2) == null)
+  }
+
+  test "the inputs hash follows the path and the bytes" {
+    Fs.write_text(dir() + "/in.txt", "abc")
+    let h1 = bake_inputs_hash(dir() + "/in.txt")
+    Fs.write_text(dir() + "/in.txt", "abd")
+    expect(bake_inputs_hash(dir() + "/in.txt") != h1)
+    expect(bake_inputs_hash("") == -3750763034362895579)
+  }
+  test "a dev build says a missing bake once" (bake_st: mut BakeState) {
+    bake_missing(bake_st, "k")
+    bake_missing(bake_st, "k")
+    expect_eq(len(bake_st.said), 1)
+  }
+}