Merge commit 'f0096ae' into lang/foundations

changes/i18n-keys.md

@@ -0,0 +1,10 @@
+bump: minor
+type: feature
+**`ludic.i18n`: keys (phase 26).** A key names what a text is for, and `en.po` says it in English like any
+other language. A key is a string with a marker byte (`I18N_KEY`, `I18N_PLURAL`), its arguments after
+byte 31, so the code that makes text never takes `I18nState`: `tr(k)`, `trf(k, a, b, c, d)` and
+`trn(k, n, a, b, c)` build it, and `L` makes it into text where it is drawn - the language in use, else
+en.po (read the first time a key is asked for), else the key itself, `[[key]]` in a developer's build
+(`i18n_loud`). Holes take their arguments in the language's order, a key argument made first; plural
+keys go by each language's rule. A string with no marker takes the old English path, so a game can
+move over a file at a time.

packages/ludic.i18n/README.md

@@ -37,6 +37,26 @@ the `.po`, for a script the default atlas does not carry; `Plural-Forms` picks t
 
 Whatever has no translation draws in English, so a partial file is always playable.
 
+## Keys (phase 26: the form that replaces English in the code)
+
+A key names what a text is FOR (`pause.resume`), and `assets/lang/en.po` says what it says in English,
+like any other language. A key travels as a string with a marker byte in front (the compiler's
+`k"pause.resume"` is exactly that; `kn"..."` a plural key), so code that makes text never takes the
+language's state: the text is made where it is drawn, by `L`.
+
+```ludic
+L(tr(k"pause.resume"))                             # "Resume"; "Devam" in Turkish
+L(trf(k"catch.saw", tr(k"species.elk.a_name"), "5:30", "", ""))   # "You saw an elk at 5:30."
+L(trn(kn"pack.items", n, "", "", ""))              # "{1} items" by the language's plural rule
+```
+
+- A key's text is the language in use's, else en.po's (read from the languages' directory the first
+  time a key is asked for), else the key itself - `[[key]]` in a developer's build (`i18n_loud`),
+  said once.
+- Holes `{1}` .. `{4}` take `trf`'s arguments in the language's own order; an argument that is a key
+  is made into its text first. A plural's count is `{1}`.
+- A string with no marker takes the English path below, so a game moves over a file at a time.
+
 ## Config and port
 
 ```ludic
@@ -54,7 +74,10 @@ port I18nWorld {
 | | |
 | --- | --- |
 | `L(s) -> string` | what the game draws, in the language in use (cached) |
-| `Ln(one, many, n) -> string` | a count's form, with `{1}` the count; English is one for 1, many otherwise |
+| `Ln(one, many, n) -> string` | a count's form, with `{1}` the count; English is one for 1, many otherwise (the old form) |
+| `tr(k)`, `trf(k, a, b, c, d)`, `trn(k, n, a, b, c) -> string` | a key, a key with its holes' arguments, a plural key by `n`: made into text by `L` |
+| `i18n_key(name)`, `i18n_key_plural(name)`, `key_text(k)`, `i18n_is_key(s)` | a key named at run time, its name without the marker, whether a string is one |
+| `i18n_base_text(text)`, `i18n_loud(on)` | the base (en.po) from text, for a test; a developer's build's loud missing key |
 | `i18n_init()`, `i18n_count()`, `i18n_index(code)`, `i18n_code_at(i)`, `i18n_name_at(i)`, `i18n_font_at(i)`, `i18n_mod_dir()` | the languages found, "en" first |
 | `i18n_use(code)`, `i18n_off()`, `i18n_on()`, `i18n_current()` | choosing one; whether anything is translating |
 | `i18n_font_base(dir)`, `i18n_font_now()` | the atlas the overlay was opened with, and the one it has now |

packages/ludic.i18n/index.ludic

@@ -1,4 +1,4 @@
-# ludic.i18n - English in the code, every other language a .po looked up at the moment of drawing
+# ludic.i18n - a key in the code (the old form: English), every language a .po looked up at the moment of drawing
 module ludic_i18n uses
 numbers float
 import "ports.ludic"
@@ -11,4 +11,7 @@ import "use.ludic"
 import "patterns.ludic"
 import "match.ludic"
 import "translate.ludic"
+import "keys.ludic"
+import "keyed.ludic"
+import "keyed_render.ludic"
 import "plural.ludic"

packages/ludic.i18n/keyed.ludic

@@ -0,0 +1,83 @@
+# keyed.ludic - en.po, the base under every language, and a key's text: the language in use, else en.po,
+# else the key itself (in a developer's build [[key]], said once)
+
+# en.po from the languages' directory, read the first time a key is asked for (none: every key is itself)
+@alloc_ok("once: en.po read")
+function kb_ensure(i18n_st: mut I18nState) -> void {
+  if i18n_st.base_tried { return }
+  i18n_st.base_tried = true
+  let path = i18n_st.lang_dir + "/en.po"
+  if not Fs.exists(path) { return }
+  let text = Fs.read_text(path)
+  if text != null { i18n_base_text(i18n_st, text) }
+}
+
+# a .po's text as the base (en.po; a test's own)
+@alloc_ok("once: the base language's messages read in")
+export function i18n_base_text(i18n_st: mut I18nState, text: string) -> void {
+  let es = i18n_parse(text)
+  i18n_st.base_tried = true
+  i18n_st.base_exact = Dict.new()
+  i18n_st.base_out = new []string
+  i18n_st.base_plural = Dict.new()
+  i18n_st.base_forms = new []string
+  i18n_st.base_rule = plural_rule_of(i18n_header(text, "Plural-Forms"))
+  for i in 0 .. len(es) {
+    let e = es[i]
+    if len(e.plural) > 0 and len(e.forms) > 0 {
+      Dict.set(i18n_st.base_plural, e.id, len(i18n_st.base_forms) / 3)
+      for k in 0 .. 3 {
+        if k < len(e.forms) { push(i18n_st.base_forms, e.forms[k]) } else { push(i18n_st.base_forms, e.forms[len(e.forms) - 1]) }
+      }
+    } else if not (blank(e.id) or blank(e.str)) {
+      Dict.set(i18n_st.base_exact, e.id, len(i18n_st.base_out))
+      push(i18n_st.base_out, e.str)
+    }
+  }
+  kc_clear(i18n_st)
+}
+
+# a developer's build draws a key en.po lacks as [[key]] and says it once; a player's draws the key
+export function i18n_loud(i18n_st: mut I18nState, on: bool) -> void { i18n_st.loud = on }
+
+# the text a key names, in the language in use, else English, else the key
+function kb_text(i18n_st: mut I18nState, key: string) -> string {
+  if i18n_st.on and i18n_st.exact != null {
+    let e = Dict.get_or(i18n_st.exact, key, -1)
+    if e >= 0 { return i18n_st.out[e] }
+  }
+  kb_ensure(i18n_st)
+  if i18n_st.base_exact != null {
+    let b = Dict.get_or(i18n_st.base_exact, key, -1)
+    if b >= 0 { return i18n_st.base_out[b] }
+  }
+  return kb_missing(i18n_st, key)
+}
+
+# a plural key's form for n, in the language in use (by its rule), else English's, else the key
+function kb_plural(i18n_st: mut I18nState, key: string, n: int) -> string {
+  if i18n_st.on and i18n_st.plural != null {
+    let k = Dict.get_or(i18n_st.plural, key, -1)
+    if k >= 0 {
+      let f = i18n_st.forms[k * 3 + i18n_plural_form(i18n_st.plural_rule, n)]
+      if not blank(f) { return f }
+    }
+  }
+  kb_ensure(i18n_st)
+  if i18n_st.base_plural != null {
+    let b = Dict.get_or(i18n_st.base_plural, key, -1)
+    if b >= 0 { return i18n_st.base_forms[b * 3 + i18n_plural_form(i18n_st.base_rule, n)] }
+  }
+  return kb_missing(i18n_st, key)
+}
+
+@alloc_ok("a key en.po lacks: the marker made and the line said, once per key")
+function kb_missing(i18n_st: mut I18nState, key: string) -> string {
+  if not i18n_st.loud { return key }
+  if i18n_st.said == null { i18n_st.said = Dict.new() }
+  if Dict.get_or(i18n_st.said, key, -1) < 0 {
+    Dict.set(i18n_st.said, key, 1)
+    print(`i18n: no English for the key {key} (assets/lang/en.po)`)   # not drawn
+  }
+  return "[[" + key + "]]"
+}

packages/ludic.i18n/keyed_render.ludic

@@ -0,0 +1,74 @@
+# keyed_render.ludic - a marked string made into text where it is drawn: its key looked up (a plural's by
+# its count), its holes filled with its arguments, each argument that is itself a key made first
+
+# what L does with a key: made once per distinct marked string in this language, then kept
+function keyed(i18n_st: mut I18nState, s: string) -> string {
+  if i18n_st.kc == null { kc_clear(i18n_st) }
+  let ci = Dict.get_or(i18n_st.kc, s, -1)
+  if ci >= 0 { return i18n_st.kc_out[ci] }
+  let r = kr_make(i18n_st, s)
+  if Dict.size(i18n_st.kc) >= CACHE_MAX { kc_clear(i18n_st) }
+  @alloc_ok("a key made into text: the result kept, up to CACHE_MAX")
+  Dict.set(i18n_st.kc, s, len(i18n_st.kc_out))
+  @alloc_ok("a key made into text: as above")
+  push(i18n_st.kc_out, r)
+  return r
+}
+
+# a language's change makes every kept text stale
+@alloc_ok("on a language change: the made keys let go")
+function kc_clear(i18n_st: mut I18nState) -> void {
+  i18n_st.kc = Dict.new()
+  i18n_st.kc_out = new []string
+}
+
+@alloc_ok("a key made into text (a cache miss): once per distinct marked string")
+function kr_make(i18n_st: mut I18nState, s: string) -> string {
+  let args = new []string
+  var start = 1
+  var i = 1
+  while i <= len(s) {
+    if i == len(s) or s[i] == I18N_SEP {
+      push(args, s[start..i])
+      start = i + 1
+    }
+    i += 1
+  }
+  let key = args[0]
+  var t = ""
+  if s[0] == I18N_PLURAL and len(args) > 1 { t = kb_plural(i18n_st, key, kr_int(args[1])) } else { t = kb_text(i18n_st, key) }
+  return kr_fill(i18n_st, t, args)
+}
+
+# {k} is argument k (args[0] is the key); a missing one is empty, a key is made into its own text
+@alloc_ok("a key made into text (a cache miss): its holes filled")
+function kr_fill(i18n_st: mut I18nState, t: string, args: []string) -> string {
+  var r = ""
+  var seg = 0
+  var i = 0
+  while i < len(t) {
+    let h = hole_at(t, i)
+    if h >= 1 {
+      var v = ""
+      if h < len(args) { v = args[h] }
+      if i18n_is_key(v) { v = keyed(i18n_st, v) }
+      r = r + t[seg..i] + v
+      i = hole_end(t, i)
+      seg = i
+      continue
+    }
+    i += 1
+  }
+  return r + t[seg..len(t)]
+}
+
+# a count written as decimal digits (a leading - allowed); anything else is 0
+function kr_int(s: string) -> int {
+  var v = 0
+  var neg = false
+  for i in 0 .. len(s) {
+    if i == 0 and s[i] == 45 { neg = true } else if s[i] >= 48 and s[i] <= 57 { v = v * 10 + (s[i] - 48) }
+  }
+  if neg { return -v }
+  return v
+}

packages/ludic.i18n/keys.ludic

@@ -0,0 +1,67 @@
+# keys.ludic - a key as the game carries it: its text behind a marker byte (the compiler's k"..." is
+# exactly that), holes as byte 31 then each argument, a plural's count first. Nothing here needs the
+# language: the text is made where it is drawn (L), so code that makes text never takes I18nState.
+export const I18N_KEY: int = 1         # k"pause.resume" is "\\x01pause.resume"
+export const I18N_PLURAL: int = 2      # kn"catch.count": a count chooses the form
+export const I18N_SEP: int = 31        # between the key and each argument
+
+# is this a key (or a key with its arguments), rather than English
+export function i18n_is_key(s: string) -> bool { return len(s) > 0 and (s[0] == I18N_KEY or s[0] == I18N_PLURAL) }
+
+# a key named at run time (a test, a table's own key): "pause.resume" marked
+@alloc_ok("a key made from its name: a test's, or a table's once")
+export function i18n_key(name: string) -> string { return kx_join(I18N_KEY, name, "", "", "", "", 0) }
+@alloc_ok("a plural key made from its name: a test's")
+export function i18n_key_plural(name: string) -> string { return kx_join(I18N_PLURAL, name, "", "", "", "", 0) }
+
+# the key's name without its marker ("pause.resume"), or the string as it is
+export function key_text(k: string) -> string {
+  if not i18n_is_key(k) { return k }
+  var e = 1
+  while e < len(k) and k[e] != I18N_SEP { e += 1 }
+  return k[1..e]
+}
+
+# the text of a key: drawn translated where it is drawn (a free cast once the key is typed)
+export function tr(k: string) -> string { return k }
+# a key's text with its holes, {1} .. {4}, filled by strings already made (a number, a name, another key)
+export function trf(k: string, a: string, b: string, c: string, d: string) -> string {
+  var mark = I18N_KEY
+  if i18n_is_key(k) { mark = k[0] }
+  return kx_join(mark, key_text(k), a, b, c, d, kx_count(a, b, c, d))
+}
+# a plural key's text by n: the count is {1}, a .. c are {2} .. {4}
+export function trn(k: string, n: int, a: string, b: string, c: string) -> string { return kx_join(I18N_PLURAL, key_text(k), `{n}`, a, b, c, 1 + kx_count(a, b, c, "")) }
+
+function kx_count(a: string, b: string, c: string, d: string) -> int {
+  if len(d) > 0 { return 4 }
+  if len(c) > 0 { return 3 }
+  if len(b) > 0 { return 2 }
+  if len(a) > 0 { return 1 }
+  return 0
+}
+
+# the marker, the name, then each of the first `n` arguments after a byte 31
+function kx_join(mark: int, name: string, a: string, b: string, c: string, d: string, n: int) -> string {
+  var size = 1 + len(name)
+  if n > 0 { size += 1 + len(a) }
+  if n > 1 { size += 1 + len(b) }
+  if n > 2 { size += 1 + len(c) }
+  if n > 3 { size += 1 + len(d) }
+  let buf = buffer(size)
+  buf[0] = mark
+  var at = kx_put(buf, 1, name)
+  if n > 0 { at = kx_put(buf, kx_sep(buf, at), a) }
+  if n > 1 { at = kx_put(buf, kx_sep(buf, at), b) }
+  if n > 2 { at = kx_put(buf, kx_sep(buf, at), c) }
+  if n > 3 { at = kx_put(buf, kx_sep(buf, at), d) }
+  return text_of(buf, size)
+}
+function kx_sep(buf: []byte, at: int) -> int {
+  buf[at] = I18N_SEP
+  return at + 1
+}
+function kx_put(buf: []byte, at: int, s: string) -> int {
+  for i in 0 .. len(s) { buf[at + i] = s[i] }
+  return at + len(s)
+}

packages/ludic.i18n/ports.ludic

@@ -45,6 +45,16 @@ export state I18nState {
   forms: []string = null              # three forms per entry, "" where the file has fewer
   plural_rule: int = 0                # PL_*
   tabs: []I18nTables = null           # by language: its lookups once read, kept for the next switch
+  base_exact: Dict = null             # en.po, under every language: a key -> index into base_out
+  base_out: []string = null
+  base_plural: Dict = null            # a plural key -> index into base_forms / 3
+  base_forms: []string = null
+  base_rule: int = 0
+  base_tried: bool = false
+  kc: Dict = null                     # a marked string (key and args) -> index into kc_out, in this language
+  kc_out: []string = null
+  loud: bool = false                  # a developer's build: a key en.po lacks draws as [[key]], said once
+  said: Dict = null
 }
 
 # where the shipped languages are (a directory with languages.txt and <code>.po), and the atlas a

packages/ludic.i18n/tests/keys_test.ludic

@@ -0,0 +1,77 @@
+# keys_test.ludic - keys where English was: en.po as the base under every language, a key's text from the
+# language in use else English else the key ([[key]] in a developer's build), holes filled by arguments
+# that may be keys themselves, plural forms by each language's rule, and the old English form beside them
+import "ludic.i18n"
+program I18nKeysTest {
+  numbers float
+  function q(s: string) -> string { return "\"" + s + "\"" }
+  function po_line(id: string, tr: string) -> string { return "msgid " + q(id) + "\nmsgstr " + q(tr) + "\n\n" }
+  function plural(id: string, one: string, many: string, three: string) -> string {
+    var s = "msgid " + q(id) + "\nmsgid_plural " + q(id) + "\nmsgstr[0] " + q(one) + "\nmsgstr[1] " + q(many) + "\n"
+    if len(three) > 0 { s = s + "msgstr[2] " + q(three) + "\n" }
+    return s + "\n"
+  }
+  function en() -> string {
+    var s = "msgid \"\"\nmsgstr \"\"\n\"Plural-Forms: nplurals=2; plural=(n != 1);\\n\"\n\n"
+    s = s + po_line("pause.resume", "Resume") + po_line("pause.quit", "Quit") + po_line("hud.day", "Day {1}")
+    s = s + po_line("species.elk.a_name", "an elk") + po_line("catch.saw", "You saw {1} at {2}.")
+    return s + plural("catch.count", "{1} trout", "{1} trout", "") + plural("pack.items", "{1} item", "{1} items", "")
+  }
+  # Turkish-shaped, with a Russian-shaped plural so the third form is chosen by the rule
+  function tt() -> string {
+    var s = "msgid \"\"\nmsgstr \"\"\n\"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : 2);\\n\"\n\n"
+    s = s + po_line("pause.resume", "Devam") + po_line("hud.day", "{1}. Gun") + po_line("species.elk.a_name", "bir geyik")
+    s = s + po_line("catch.saw", "{2} saatinde {1} gordun.") + po_line("Settings", "Ayarlar")
+    return s + plural("pack.items", "{1} esya", "{1} esyaa", "{1} esyalar")
+  }
+  function start (i18n_st: mut I18nState) -> void {
+    i18n_off(i18n_st)
+    i18n_loud(i18n_st, false)
+    i18n_base_text(i18n_st, en())
+  }
+
+  test "a key is its English under en.po, its own language's text when there is one, else English, else itself" (i18n_st: mut I18nState) {
+    start (i18n_st)
+    expect(L(i18n_st, i18n_key("pause.resume")) == "Resume")
+    expect(L(i18n_st, tr(i18n_key("pause.quit"))) == "Quit")
+    i18n_load_text(i18n_st, tt())
+    expect(L(i18n_st, i18n_key("pause.resume")) == "Devam")
+    expect(L(i18n_st, i18n_key("pause.quit")) == "Quit")                  # the language lacks it: English
+    expect(L(i18n_st, i18n_key("nowhere.at_all")) == "nowhere.at_all")    # en.po lacks it: the key
+    i18n_loud(i18n_st, true)
+    expect(L(i18n_st, i18n_key("nowhere.else")) == "[[nowhere.else]]")   # a developer's build says so
+    i18n_off(i18n_st)
+    expect(L(i18n_st, i18n_key("pause.resume")) == "Resume")              # back to English: nothing stale kept
+  }
+
+  test "holes take their arguments in the language's order, and an argument that is a key is made first" (i18n_st: mut I18nState) {
+    start (i18n_st)
+    expect(L(i18n_st, trf(i18n_key("hud.day"), "12", "", "", "")) == "Day 12")
+    let saw = trf(i18n_key("catch.saw"), i18n_key("species.elk.a_name"), "5:30", "", "")
+    expect(L(i18n_st, saw) == "You saw an elk at 5:30.")
+    i18n_load_text(i18n_st, tt())
+    expect(L(i18n_st, trf(i18n_key("hud.day"), "12", "", "", "")) == "12. Gun")
+    expect(L(i18n_st, saw) == "5:30 saatinde bir geyik gordun.")
+    expect(L(i18n_st, trf(i18n_key("hud.day"), "", "", "", "")) == ". Gun")    # a missing argument is empty
+    expect(key_text(saw) == "catch.saw" and key_text("Settings") == "Settings")
+  }
+
+  test "a plural key takes its form by the count, by each language's own rule, falling back to English's" (i18n_st: mut I18nState) {
+    start (i18n_st)
+    expect(L(i18n_st, trn(i18n_key("pack.items"), 1, "", "", "")) == "1 item")
+    expect(L(i18n_st, trn(i18n_key("pack.items"), 3, "", "", "")) == "3 items")
+    i18n_load_text(i18n_st, tt())
+    expect(L(i18n_st, trn(i18n_key("pack.items"), 21, "", "", "")) == "21 esya")     # n%10 == 1
+    expect(L(i18n_st, trn(i18n_key("pack.items"), 5, "", "", "")) == "5 esyalar")
+    expect(L(i18n_st, trn(i18n_key("catch.count"), 2, "", "", "")) == "2 trout")      # the language lacks it
+  }
+
+  test "the old English form still works beside keys, so a game can move over a file at a time" (i18n_st: mut I18nState) {
+    start (i18n_st)
+    expect(L(i18n_st, "Settings") == "Settings")
+    i18n_load_text(i18n_st, tt())
+    expect(L(i18n_st, "Settings") == "Ayarlar")
+    expect(L(i18n_st, i18n_key("pause.resume")) == "Devam")
+    expect(not i18n_is_key("Settings") and i18n_is_key(i18n_key("x")) and i18n_is_key(i18n_key_plural("x")))
+  }
+}

packages/ludic.i18n/translate.ludic

@@ -1,6 +1,7 @@
-# translate.ludic - what the game draws, in the language in use: exact, else a pattern, else the
-# padding a layout put round it taken off, else a paragraph a sentence at a time; English otherwise
+# translate.ludic - what the game draws, in the language in use: a key (keyed_render.ludic); else, the old
+# English form, exact, else a pattern, else padding taken off, else a paragraph a sentence at a time
 export function L(i18n_st: mut I18nState, s: string) -> string {
+  if i18n_is_key(s) { return keyed(i18n_st, s) }
   if not i18n_st.on { return s }
   let ci = Dict.get_or(i18n_st.cache, s, -1)
   if ci >= 0 { return i18n_st.cache_out[ci] }

packages/ludic.i18n/use.ludic

@@ -3,6 +3,7 @@ export function i18n_use(i18n_st: mut I18nState, code: string) -> void {
   if i18n_st.codes == null { i18n_init(i18n_st) }
   i18n_st.on = false
   i18n_st.cur = "en"
+  kc_clear(i18n_st)
   var k = i18n_index(i18n_st, code)
   let forced = I18nWorld.forced()
   if len(forced) > 0 { k = i18n_index(i18n_st, forced) }
@@ -36,6 +37,7 @@ export function i18n_use(i18n_st: mut I18nState, code: string) -> void {
 @alloc_ok("on a language change: the language's messages read in")
 export function i18n_load_text(i18n_st: mut I18nState, text: string) -> void {
   let es = i18n_parse(text)
+  kc_clear(i18n_st)
   i18n_st.exact = Dict.new()
   i18n_st.out = new []string
   i18n_st.cache = Dict.new()
@@ -82,6 +84,7 @@ export function i18n_load_text(i18n_st: mut I18nState, text: string) -> void {
 
 # English again: nothing loaded, nothing translated
 export function i18n_off(i18n_st: mut I18nState) -> void {
+  kc_clear(i18n_st)
   i18n_st.on = false
   i18n_st.cur = "en"
 }