diff --git a/changes/i18n-keys.md b/changes/i18n-keys.md new file mode 100644 index 00000000..7bc7a970 --- /dev/null +++ b/changes/i18n-keys.md @@ -0,0 +1,10 @@ +bump: minor +type: feature +**`ludic.i18n`: keys (phase 26).** A key names what a text is for, and `en.po` says it in English like any +other language. A key is a string with a marker byte (`I18N_KEY`, `I18N_PLURAL`), its arguments after +byte 31, so the code that makes text never takes `I18nState`: `tr(k)`, `trf(k, a, b, c, d)` and +`trn(k, n, a, b, c)` build it, and `L` makes it into text where it is drawn - the language in use, else +en.po (read the first time a key is asked for), else the key itself, `[[key]]` in a developer's build +(`i18n_loud`). Holes take their arguments in the language's order, a key argument made first; plural +keys go by each language's rule. A string with no marker takes the old English path, so a game can +move over a file at a time. diff --git a/packages/ludic.i18n/README.md b/packages/ludic.i18n/README.md index 29d87232..38a989b4 100644 --- a/packages/ludic.i18n/README.md +++ b/packages/ludic.i18n/README.md @@ -37,6 +37,26 @@ the `.po`, for a script the default atlas does not carry; `Plural-Forms` picks t Whatever has no translation draws in English, so a partial file is always playable. +## Keys (phase 26: the form that replaces English in the code) + +A key names what a text is FOR (`pause.resume`), and `assets/lang/en.po` says what it says in English, +like any other language. A key travels as a string with a marker byte in front (the compiler's +`k"pause.resume"` is exactly that; `kn"..."` a plural key), so code that makes text never takes the +language's state: the text is made where it is drawn, by `L`. + +```ludic +L(tr(k"pause.resume")) # "Resume"; "Devam" in Turkish +L(trf(k"catch.saw", tr(k"species.elk.a_name"), "5:30", "", "")) # "You saw an elk at 5:30." +L(trn(kn"pack.items", n, "", "", "")) # "{1} items" by the language's plural rule +``` + +- A key's text is the language in use's, else en.po's (read from the languages' directory the first + time a key is asked for), else the key itself - `[[key]]` in a developer's build (`i18n_loud`), + said once. +- Holes `{1}` .. `{4}` take `trf`'s arguments in the language's own order; an argument that is a key + is made into its text first. A plural's count is `{1}`. +- A string with no marker takes the English path below, so a game moves over a file at a time. + ## Config and port ```ludic @@ -54,7 +74,10 @@ port I18nWorld { | | | | --- | --- | | `L(s) -> string` | what the game draws, in the language in use (cached) | -| `Ln(one, many, n) -> string` | a count's form, with `{1}` the count; English is one for 1, many otherwise | +| `Ln(one, many, n) -> string` | a count's form, with `{1}` the count; English is one for 1, many otherwise (the old form) | +| `tr(k)`, `trf(k, a, b, c, d)`, `trn(k, n, a, b, c) -> string` | a key, a key with its holes' arguments, a plural key by `n`: made into text by `L` | +| `i18n_key(name)`, `i18n_key_plural(name)`, `key_text(k)`, `i18n_is_key(s)` | a key named at run time, its name without the marker, whether a string is one | +| `i18n_base_text(text)`, `i18n_loud(on)` | the base (en.po) from text, for a test; a developer's build's loud missing key | | `i18n_init()`, `i18n_count()`, `i18n_index(code)`, `i18n_code_at(i)`, `i18n_name_at(i)`, `i18n_font_at(i)`, `i18n_mod_dir()` | the languages found, "en" first | | `i18n_use(code)`, `i18n_off()`, `i18n_on()`, `i18n_current()` | choosing one; whether anything is translating | | `i18n_font_base(dir)`, `i18n_font_now()` | the atlas the overlay was opened with, and the one it has now | diff --git a/packages/ludic.i18n/index.ludic b/packages/ludic.i18n/index.ludic index 732d5f5b..cba83ae4 100644 --- a/packages/ludic.i18n/index.ludic +++ b/packages/ludic.i18n/index.ludic @@ -1,4 +1,4 @@ -# ludic.i18n - English in the code, every other language a .po looked up at the moment of drawing +# ludic.i18n - a key in the code (the old form: English), every language a .po looked up at the moment of drawing module ludic_i18n uses numbers float import "ports.ludic" @@ -11,4 +11,7 @@ import "use.ludic" import "patterns.ludic" import "match.ludic" import "translate.ludic" +import "keys.ludic" +import "keyed.ludic" +import "keyed_render.ludic" import "plural.ludic" diff --git a/packages/ludic.i18n/keyed.ludic b/packages/ludic.i18n/keyed.ludic new file mode 100644 index 00000000..ae52b3ad --- /dev/null +++ b/packages/ludic.i18n/keyed.ludic @@ -0,0 +1,83 @@ +# keyed.ludic - en.po, the base under every language, and a key's text: the language in use, else en.po, +# else the key itself (in a developer's build [[key]], said once) + +# en.po from the languages' directory, read the first time a key is asked for (none: every key is itself) +@alloc_ok("once: en.po read") +function kb_ensure(i18n_st: mut I18nState) -> void { + if i18n_st.base_tried { return } + i18n_st.base_tried = true + let path = i18n_st.lang_dir + "/en.po" + if not Fs.exists(path) { return } + let text = Fs.read_text(path) + if text != null { i18n_base_text(i18n_st, text) } +} + +# a .po's text as the base (en.po; a test's own) +@alloc_ok("once: the base language's messages read in") +export function i18n_base_text(i18n_st: mut I18nState, text: string) -> void { + let es = i18n_parse(text) + i18n_st.base_tried = true + i18n_st.base_exact = Dict.new() + i18n_st.base_out = new []string + i18n_st.base_plural = Dict.new() + i18n_st.base_forms = new []string + i18n_st.base_rule = plural_rule_of(i18n_header(text, "Plural-Forms")) + for i in 0 .. len(es) { + let e = es[i] + if len(e.plural) > 0 and len(e.forms) > 0 { + Dict.set(i18n_st.base_plural, e.id, len(i18n_st.base_forms) / 3) + for k in 0 .. 3 { + if k < len(e.forms) { push(i18n_st.base_forms, e.forms[k]) } else { push(i18n_st.base_forms, e.forms[len(e.forms) - 1]) } + } + } else if not (blank(e.id) or blank(e.str)) { + Dict.set(i18n_st.base_exact, e.id, len(i18n_st.base_out)) + push(i18n_st.base_out, e.str) + } + } + kc_clear(i18n_st) +} + +# a developer's build draws a key en.po lacks as [[key]] and says it once; a player's draws the key +export function i18n_loud(i18n_st: mut I18nState, on: bool) -> void { i18n_st.loud = on } + +# the text a key names, in the language in use, else English, else the key +function kb_text(i18n_st: mut I18nState, key: string) -> string { + if i18n_st.on and i18n_st.exact != null { + let e = Dict.get_or(i18n_st.exact, key, -1) + if e >= 0 { return i18n_st.out[e] } + } + kb_ensure(i18n_st) + if i18n_st.base_exact != null { + let b = Dict.get_or(i18n_st.base_exact, key, -1) + if b >= 0 { return i18n_st.base_out[b] } + } + return kb_missing(i18n_st, key) +} + +# a plural key's form for n, in the language in use (by its rule), else English's, else the key +function kb_plural(i18n_st: mut I18nState, key: string, n: int) -> string { + if i18n_st.on and i18n_st.plural != null { + let k = Dict.get_or(i18n_st.plural, key, -1) + if k >= 0 { + let f = i18n_st.forms[k * 3 + i18n_plural_form(i18n_st.plural_rule, n)] + if not blank(f) { return f } + } + } + kb_ensure(i18n_st) + if i18n_st.base_plural != null { + let b = Dict.get_or(i18n_st.base_plural, key, -1) + if b >= 0 { return i18n_st.base_forms[b * 3 + i18n_plural_form(i18n_st.base_rule, n)] } + } + return kb_missing(i18n_st, key) +} + +@alloc_ok("a key en.po lacks: the marker made and the line said, once per key") +function kb_missing(i18n_st: mut I18nState, key: string) -> string { + if not i18n_st.loud { return key } + if i18n_st.said == null { i18n_st.said = Dict.new() } + if Dict.get_or(i18n_st.said, key, -1) < 0 { + Dict.set(i18n_st.said, key, 1) + print(`i18n: no English for the key {key} (assets/lang/en.po)`) # not drawn + } + return "[[" + key + "]]" +} diff --git a/packages/ludic.i18n/keyed_render.ludic b/packages/ludic.i18n/keyed_render.ludic new file mode 100644 index 00000000..825a4ae9 --- /dev/null +++ b/packages/ludic.i18n/keyed_render.ludic @@ -0,0 +1,74 @@ +# keyed_render.ludic - a marked string made into text where it is drawn: its key looked up (a plural's by +# its count), its holes filled with its arguments, each argument that is itself a key made first + +# what L does with a key: made once per distinct marked string in this language, then kept +function keyed(i18n_st: mut I18nState, s: string) -> string { + if i18n_st.kc == null { kc_clear(i18n_st) } + let ci = Dict.get_or(i18n_st.kc, s, -1) + if ci >= 0 { return i18n_st.kc_out[ci] } + let r = kr_make(i18n_st, s) + if Dict.size(i18n_st.kc) >= CACHE_MAX { kc_clear(i18n_st) } + @alloc_ok("a key made into text: the result kept, up to CACHE_MAX") + Dict.set(i18n_st.kc, s, len(i18n_st.kc_out)) + @alloc_ok("a key made into text: as above") + push(i18n_st.kc_out, r) + return r +} + +# a language's change makes every kept text stale +@alloc_ok("on a language change: the made keys let go") +function kc_clear(i18n_st: mut I18nState) -> void { + i18n_st.kc = Dict.new() + i18n_st.kc_out = new []string +} + +@alloc_ok("a key made into text (a cache miss): once per distinct marked string") +function kr_make(i18n_st: mut I18nState, s: string) -> string { + let args = new []string + var start = 1 + var i = 1 + while i <= len(s) { + if i == len(s) or s[i] == I18N_SEP { + push(args, s[start..i]) + start = i + 1 + } + i += 1 + } + let key = args[0] + var t = "" + if s[0] == I18N_PLURAL and len(args) > 1 { t = kb_plural(i18n_st, key, kr_int(args[1])) } else { t = kb_text(i18n_st, key) } + return kr_fill(i18n_st, t, args) +} + +# {k} is argument k (args[0] is the key); a missing one is empty, a key is made into its own text +@alloc_ok("a key made into text (a cache miss): its holes filled") +function kr_fill(i18n_st: mut I18nState, t: string, args: []string) -> string { + var r = "" + var seg = 0 + var i = 0 + while i < len(t) { + let h = hole_at(t, i) + if h >= 1 { + var v = "" + if h < len(args) { v = args[h] } + if i18n_is_key(v) { v = keyed(i18n_st, v) } + r = r + t[seg..i] + v + i = hole_end(t, i) + seg = i + continue + } + i += 1 + } + return r + t[seg..len(t)] +} + +# a count written as decimal digits (a leading - allowed); anything else is 0 +function kr_int(s: string) -> int { + var v = 0 + var neg = false + for i in 0 .. len(s) { + if i == 0 and s[i] == 45 { neg = true } else if s[i] >= 48 and s[i] <= 57 { v = v * 10 + (s[i] - 48) } + } + if neg { return -v } + return v +} diff --git a/packages/ludic.i18n/keys.ludic b/packages/ludic.i18n/keys.ludic new file mode 100644 index 00000000..91022f8b --- /dev/null +++ b/packages/ludic.i18n/keys.ludic @@ -0,0 +1,67 @@ +# keys.ludic - a key as the game carries it: its text behind a marker byte (the compiler's k"..." is +# exactly that), holes as byte 31 then each argument, a plural's count first. Nothing here needs the +# language: the text is made where it is drawn (L), so code that makes text never takes I18nState. +export const I18N_KEY: int = 1 # k"pause.resume" is "\\x01pause.resume" +export const I18N_PLURAL: int = 2 # kn"catch.count": a count chooses the form +export const I18N_SEP: int = 31 # between the key and each argument + +# is this a key (or a key with its arguments), rather than English +export function i18n_is_key(s: string) -> bool { return len(s) > 0 and (s[0] == I18N_KEY or s[0] == I18N_PLURAL) } + +# a key named at run time (a test, a table's own key): "pause.resume" marked +@alloc_ok("a key made from its name: a test's, or a table's once") +export function i18n_key(name: string) -> string { return kx_join(I18N_KEY, name, "", "", "", "", 0) } +@alloc_ok("a plural key made from its name: a test's") +export function i18n_key_plural(name: string) -> string { return kx_join(I18N_PLURAL, name, "", "", "", "", 0) } + +# the key's name without its marker ("pause.resume"), or the string as it is +export function key_text(k: string) -> string { + if not i18n_is_key(k) { return k } + var e = 1 + while e < len(k) and k[e] != I18N_SEP { e += 1 } + return k[1..e] +} + +# the text of a key: drawn translated where it is drawn (a free cast once the key is typed) +export function tr(k: string) -> string { return k } +# a key's text with its holes, {1} .. {4}, filled by strings already made (a number, a name, another key) +export function trf(k: string, a: string, b: string, c: string, d: string) -> string { + var mark = I18N_KEY + if i18n_is_key(k) { mark = k[0] } + return kx_join(mark, key_text(k), a, b, c, d, kx_count(a, b, c, d)) +} +# a plural key's text by n: the count is {1}, a .. c are {2} .. {4} +export function trn(k: string, n: int, a: string, b: string, c: string) -> string { return kx_join(I18N_PLURAL, key_text(k), `{n}`, a, b, c, 1 + kx_count(a, b, c, "")) } + +function kx_count(a: string, b: string, c: string, d: string) -> int { + if len(d) > 0 { return 4 } + if len(c) > 0 { return 3 } + if len(b) > 0 { return 2 } + if len(a) > 0 { return 1 } + return 0 +} + +# the marker, the name, then each of the first `n` arguments after a byte 31 +function kx_join(mark: int, name: string, a: string, b: string, c: string, d: string, n: int) -> string { + var size = 1 + len(name) + if n > 0 { size += 1 + len(a) } + if n > 1 { size += 1 + len(b) } + if n > 2 { size += 1 + len(c) } + if n > 3 { size += 1 + len(d) } + let buf = buffer(size) + buf[0] = mark + var at = kx_put(buf, 1, name) + if n > 0 { at = kx_put(buf, kx_sep(buf, at), a) } + if n > 1 { at = kx_put(buf, kx_sep(buf, at), b) } + if n > 2 { at = kx_put(buf, kx_sep(buf, at), c) } + if n > 3 { at = kx_put(buf, kx_sep(buf, at), d) } + return text_of(buf, size) +} +function kx_sep(buf: []byte, at: int) -> int { + buf[at] = I18N_SEP + return at + 1 +} +function kx_put(buf: []byte, at: int, s: string) -> int { + for i in 0 .. len(s) { buf[at + i] = s[i] } + return at + len(s) +} diff --git a/packages/ludic.i18n/ports.ludic b/packages/ludic.i18n/ports.ludic index 93e099fb..f2f8cc50 100644 --- a/packages/ludic.i18n/ports.ludic +++ b/packages/ludic.i18n/ports.ludic @@ -45,6 +45,16 @@ export state I18nState { forms: []string = null # three forms per entry, "" where the file has fewer plural_rule: int = 0 # PL_* tabs: []I18nTables = null # by language: its lookups once read, kept for the next switch + base_exact: Dict = null # en.po, under every language: a key -> index into base_out + base_out: []string = null + base_plural: Dict = null # a plural key -> index into base_forms / 3 + base_forms: []string = null + base_rule: int = 0 + base_tried: bool = false + kc: Dict = null # a marked string (key and args) -> index into kc_out, in this language + kc_out: []string = null + loud: bool = false # a developer's build: a key en.po lacks draws as [[key]], said once + said: Dict = null } # where the shipped languages are (a directory with languages.txt and .po), and the atlas a diff --git a/packages/ludic.i18n/tests/keys_test.ludic b/packages/ludic.i18n/tests/keys_test.ludic new file mode 100644 index 00000000..5ed7de3e --- /dev/null +++ b/packages/ludic.i18n/tests/keys_test.ludic @@ -0,0 +1,77 @@ +# keys_test.ludic - keys where English was: en.po as the base under every language, a key's text from the +# language in use else English else the key ([[key]] in a developer's build), holes filled by arguments +# that may be keys themselves, plural forms by each language's rule, and the old English form beside them +import "ludic.i18n" +program I18nKeysTest { + numbers float + function q(s: string) -> string { return "\"" + s + "\"" } + function po_line(id: string, tr: string) -> string { return "msgid " + q(id) + "\nmsgstr " + q(tr) + "\n\n" } + function plural(id: string, one: string, many: string, three: string) -> string { + var s = "msgid " + q(id) + "\nmsgid_plural " + q(id) + "\nmsgstr[0] " + q(one) + "\nmsgstr[1] " + q(many) + "\n" + if len(three) > 0 { s = s + "msgstr[2] " + q(three) + "\n" } + return s + "\n" + } + function en() -> string { + var s = "msgid \"\"\nmsgstr \"\"\n\"Plural-Forms: nplurals=2; plural=(n != 1);\\n\"\n\n" + s = s + po_line("pause.resume", "Resume") + po_line("pause.quit", "Quit") + po_line("hud.day", "Day {1}") + s = s + po_line("species.elk.a_name", "an elk") + po_line("catch.saw", "You saw {1} at {2}.") + return s + plural("catch.count", "{1} trout", "{1} trout", "") + plural("pack.items", "{1} item", "{1} items", "") + } + # Turkish-shaped, with a Russian-shaped plural so the third form is chosen by the rule + function tt() -> string { + var s = "msgid \"\"\nmsgstr \"\"\n\"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : 2);\\n\"\n\n" + s = s + po_line("pause.resume", "Devam") + po_line("hud.day", "{1}. Gun") + po_line("species.elk.a_name", "bir geyik") + s = s + po_line("catch.saw", "{2} saatinde {1} gordun.") + po_line("Settings", "Ayarlar") + return s + plural("pack.items", "{1} esya", "{1} esyaa", "{1} esyalar") + } + function start (i18n_st: mut I18nState) -> void { + i18n_off(i18n_st) + i18n_loud(i18n_st, false) + i18n_base_text(i18n_st, en()) + } + + test "a key is its English under en.po, its own language's text when there is one, else English, else itself" (i18n_st: mut I18nState) { + start (i18n_st) + expect(L(i18n_st, i18n_key("pause.resume")) == "Resume") + expect(L(i18n_st, tr(i18n_key("pause.quit"))) == "Quit") + i18n_load_text(i18n_st, tt()) + expect(L(i18n_st, i18n_key("pause.resume")) == "Devam") + expect(L(i18n_st, i18n_key("pause.quit")) == "Quit") # the language lacks it: English + expect(L(i18n_st, i18n_key("nowhere.at_all")) == "nowhere.at_all") # en.po lacks it: the key + i18n_loud(i18n_st, true) + expect(L(i18n_st, i18n_key("nowhere.else")) == "[[nowhere.else]]") # a developer's build says so + i18n_off(i18n_st) + expect(L(i18n_st, i18n_key("pause.resume")) == "Resume") # back to English: nothing stale kept + } + + test "holes take their arguments in the language's order, and an argument that is a key is made first" (i18n_st: mut I18nState) { + start (i18n_st) + expect(L(i18n_st, trf(i18n_key("hud.day"), "12", "", "", "")) == "Day 12") + let saw = trf(i18n_key("catch.saw"), i18n_key("species.elk.a_name"), "5:30", "", "") + expect(L(i18n_st, saw) == "You saw an elk at 5:30.") + i18n_load_text(i18n_st, tt()) + expect(L(i18n_st, trf(i18n_key("hud.day"), "12", "", "", "")) == "12. Gun") + expect(L(i18n_st, saw) == "5:30 saatinde bir geyik gordun.") + expect(L(i18n_st, trf(i18n_key("hud.day"), "", "", "", "")) == ". Gun") # a missing argument is empty + expect(key_text(saw) == "catch.saw" and key_text("Settings") == "Settings") + } + + test "a plural key takes its form by the count, by each language's own rule, falling back to English's" (i18n_st: mut I18nState) { + start (i18n_st) + expect(L(i18n_st, trn(i18n_key("pack.items"), 1, "", "", "")) == "1 item") + expect(L(i18n_st, trn(i18n_key("pack.items"), 3, "", "", "")) == "3 items") + i18n_load_text(i18n_st, tt()) + expect(L(i18n_st, trn(i18n_key("pack.items"), 21, "", "", "")) == "21 esya") # n%10 == 1 + expect(L(i18n_st, trn(i18n_key("pack.items"), 5, "", "", "")) == "5 esyalar") + expect(L(i18n_st, trn(i18n_key("catch.count"), 2, "", "", "")) == "2 trout") # the language lacks it + } + + test "the old English form still works beside keys, so a game can move over a file at a time" (i18n_st: mut I18nState) { + start (i18n_st) + expect(L(i18n_st, "Settings") == "Settings") + i18n_load_text(i18n_st, tt()) + expect(L(i18n_st, "Settings") == "Ayarlar") + expect(L(i18n_st, i18n_key("pause.resume")) == "Devam") + expect(not i18n_is_key("Settings") and i18n_is_key(i18n_key("x")) and i18n_is_key(i18n_key_plural("x"))) + } +} diff --git a/packages/ludic.i18n/translate.ludic b/packages/ludic.i18n/translate.ludic index 3d5bf7b9..e5a2d70c 100644 --- a/packages/ludic.i18n/translate.ludic +++ b/packages/ludic.i18n/translate.ludic @@ -1,6 +1,7 @@ -# translate.ludic - what the game draws, in the language in use: exact, else a pattern, else the -# padding a layout put round it taken off, else a paragraph a sentence at a time; English otherwise +# translate.ludic - what the game draws, in the language in use: a key (keyed_render.ludic); else, the old +# English form, exact, else a pattern, else padding taken off, else a paragraph a sentence at a time export function L(i18n_st: mut I18nState, s: string) -> string { + if i18n_is_key(s) { return keyed(i18n_st, s) } if not i18n_st.on { return s } let ci = Dict.get_or(i18n_st.cache, s, -1) if ci >= 0 { return i18n_st.cache_out[ci] } diff --git a/packages/ludic.i18n/use.ludic b/packages/ludic.i18n/use.ludic index 08fdce80..b0533f3b 100644 --- a/packages/ludic.i18n/use.ludic +++ b/packages/ludic.i18n/use.ludic @@ -3,6 +3,7 @@ export function i18n_use(i18n_st: mut I18nState, code: string) -> void { if i18n_st.codes == null { i18n_init(i18n_st) } i18n_st.on = false i18n_st.cur = "en" + kc_clear(i18n_st) var k = i18n_index(i18n_st, code) let forced = I18nWorld.forced() if len(forced) > 0 { k = i18n_index(i18n_st, forced) } @@ -36,6 +37,7 @@ export function i18n_use(i18n_st: mut I18nState, code: string) -> void { @alloc_ok("on a language change: the language's messages read in") export function i18n_load_text(i18n_st: mut I18nState, text: string) -> void { let es = i18n_parse(text) + kc_clear(i18n_st) i18n_st.exact = Dict.new() i18n_st.out = new []string i18n_st.cache = Dict.new() @@ -82,6 +84,7 @@ export function i18n_load_text(i18n_st: mut I18nState, text: string) -> void { # English again: nothing loaded, nothing translated export function i18n_off(i18n_st: mut I18nState) -> void { + kc_clear(i18n_st) i18n_st.on = false i18n_st.cur = "en" }