From 9e713bc39c99ce80dc4ffe9bc90d6b6b56aecedd Mon Sep 17 00:00:00 2001 From: Orkuncakilkaya Date: Fri, 25 Sep 2026 08:20:46 +0300 Subject: [PATCH] feat(packages): ludic.i18n - gettext languages from a shipped list and a player's folder, exact lines, patterns whose holes are translated in turn, paragraphs a sentence at a time, plurals by Plural-Forms, X-Font atlases through a port Co-Authored-By: Claude Opus 5.5 --- packages/README.md | 1 + packages/ludic.i18n/README.md | 79 +++++++++++ packages/ludic.i18n/bytes.ludic | 76 +++++++++++ packages/ludic.i18n/index.ludic | 13 ++ packages/ludic.i18n/langs.ludic | 82 ++++++++++++ packages/ludic.i18n/match.ludic | 95 ++++++++++++++ packages/ludic.i18n/package.ludic | 6 + packages/ludic.i18n/patterns.ludic | 83 ++++++++++++ packages/ludic.i18n/plural.ludic | 74 +++++++++++ packages/ludic.i18n/po.ludic | 94 ++++++++++++++ packages/ludic.i18n/ports.ludic | 24 ++++ packages/ludic.i18n/state.ludic | 47 +++++++ packages/ludic.i18n/tests/i18n_test.ludic | 151 ++++++++++++++++++++++ packages/ludic.i18n/translate.ludic | 75 +++++++++++ packages/ludic.i18n/use.ludic | 80 ++++++++++++ 15 files changed, 980 insertions(+) create mode 100644 packages/ludic.i18n/README.md create mode 100644 packages/ludic.i18n/bytes.ludic create mode 100644 packages/ludic.i18n/index.ludic create mode 100644 packages/ludic.i18n/langs.ludic create mode 100644 packages/ludic.i18n/match.ludic create mode 100644 packages/ludic.i18n/package.ludic create mode 100644 packages/ludic.i18n/patterns.ludic create mode 100644 packages/ludic.i18n/plural.ludic create mode 100644 packages/ludic.i18n/po.ludic create mode 100644 packages/ludic.i18n/ports.ludic create mode 100644 packages/ludic.i18n/state.ludic create mode 100644 packages/ludic.i18n/tests/i18n_test.ludic create mode 100644 packages/ludic.i18n/translate.ludic create mode 100644 packages/ludic.i18n/use.ludic diff --git a/packages/README.md b/packages/README.md index 92e5a063..76b20b48 100644 --- a/packages/README.md +++ b/packages/README.md @@ -19,6 +19,7 @@ section. The rules are in [ludic.base](ludic.base/README.md). | [ludic.effects](ludic.effects/README.md) | timed modifiers that run down in game time (a meal's warmth, a drink's legs) | | [ludic.fire](ludic.fire/README.md) | a camp fire: fuel, the rain on it, warmth at a distance, whether it will cook | | [ludic.fishing](ludic.fishing/README.md) | a rod at the water: cast, bite, the reel's fight, the landing; species as an open registry | +| [ludic.i18n](ludic.i18n/README.md) | a game in any language: gettext `.po` files, patterns with holes, plurals, a mod folder, a font per language | | [ludic.inventory](ludic.inventory/README.md) | a pack: a count per kind, with room the game decides | | [ludic.needs](ludic.needs/README.md) | a body's warmth, food, water and energy, and the countdown to a collapse | | [ludic.settings](ludic.settings/README.md) | a game's settings as data: one store, a fact per change, ranges, a safe set | diff --git a/packages/ludic.i18n/README.md b/packages/ludic.i18n/README.md new file mode 100644 index 00000000..29d87232 --- /dev/null +++ b/packages/ludic.i18n/README.md @@ -0,0 +1,79 @@ +# ludic.i18n + +A game in any language, from gettext files anyone can add. English is the source and stays in the +code: nothing here, and nothing a translator does, rewrites a string in the game. A language is a +`.po` keyed by the English exactly as the game draws it, and text is translated at the moment it is +drawn, so a string that reaches the screen is covered wherever it was built. Uses nothing - not +even `ludic.base`; the font atlas and the mod folder are the game's, through a port. + +```ludic +import "ludic.i18n" +i18n_use("tr") # from settings; the atlas follows +ov_text(x, y, 16, L("Settings"), ...) +ov_text(x, y, 16, L(`Day {day}`), ...) # "Day {1}" in the file matches "Day 12" +ov_text(x, y, 16, Ln("{1} trout", "{1} trout", n), ...) +``` + +## Where languages come from + +Later ones replace earlier ones with the same code: + +- `/languages.txt` - the ones that ship, one code per line (`#` a comment), each `/.po`. +- every `*.po` in the mod folder the port names - a player drops a file in and picks it. No build. + +`X-Language-Name` in the header is what a menu shows; `X-Font` names a font atlas directory beside +the `.po`, for a script the default atlas does not carry; `Plural-Forms` picks the plural rule. + +## What a msgid can be + +- `"Hold to break off deadwood"` - an exact string. +- `"Day {1}"` -> `"{1}. Gün"` - a pattern: the English around the holes must match, the holes are + whatever the game put there, and a translation may reorder or drop them. A hole's own text is + translated too when the file knows it - an item name, or "a deer" with its English article taken + off. Of several patterns that match, the one pinning down the most English wins. +- A paragraph the game assembled from sentences is tried whole, then a sentence at a time; the + padding a layout put round a word (" owned") is taken off and put back. +- `msgid_plural` / `msgstr[k]` - `Ln(one, many, n)`, by the file's `Plural-Forms`. + +Whatever has no translation draws in English, so a partial file is always playable. + +## Config and port + +```ludic +i18n_config(dir, default_font) # "assets/lang", "assets/kit/font" unless said +port I18nWorld { + mod_dir: fn() -> string # a player's own .po files; "" none (the default) + forced: fn() -> string # a code that wins over the one asked for (a test); "" none + font_ready: fn() -> bool # is there an overlay to load an atlas into (default no) + set_font: fn(string) -> bool # load that atlas directory +} +``` + +## API + +| | | +| --- | --- | +| `L(s) -> string` | what the game draws, in the language in use (cached) | +| `Ln(one, many, n) -> string` | a count's form, with `{1}` the count; English is one for 1, many otherwise | +| `i18n_init()`, `i18n_count()`, `i18n_index(code)`, `i18n_code_at(i)`, `i18n_name_at(i)`, `i18n_font_at(i)`, `i18n_mod_dir()` | the languages found, "en" first | +| `i18n_use(code)`, `i18n_off()`, `i18n_on()`, `i18n_current()` | choosing one; whether anything is translating | +| `i18n_font_base(dir)`, `i18n_font_now()` | the atlas the overlay was opened with, and the one it has now | +| `i18n_load_text(text)`, `i18n_parse(text) -> []PoEntry`, `i18n_header(text, field)` | a `.po` from text, outright | +| `i18n_exact_count()`, `i18n_pattern_count()`, `i18n_plural_rule()`, `i18n_plural_form(rule, n)`, `PL_*` | what the file gave | +| `i18n_find`, `i18n_starts`, `i18n_ends` | the byte helpers, for a game that wants them | + +## Keeping an extractor working + +A game's extractor finds the English by reading its own source for literals (Maroon Lake's +`tools/i18n/extract.py`); nothing here changes what it reads. Build sentences a translator can +find: one template literal with holes, not fragments glued with `+`. + +## Tests + +```bash +ludic test packages/ludic.i18n +``` + +A shipped list and a mod folder in the temp directory: exact lines, patterns with translated holes +and articles, paragraphs, padding, plurals (a Russian-shaped rule), the font and its fallback, a mod +replacing a shipped code, a forced code, and English untouched. diff --git a/packages/ludic.i18n/bytes.ludic b/packages/ludic.i18n/bytes.ludic new file mode 100644 index 00000000..f02d180c --- /dev/null +++ b/packages/ludic.i18n/bytes.ludic @@ -0,0 +1,76 @@ +# bytes.ludic - the byte-level helpers the parser and the matcher share +export function i18n_find(s: string, n: int, from: int, lit: string, m: int) -> int { + if m == 0 { return from } + var i = from + while i + m <= n { + if s[i] == lit[0] { + var k = 1 + while k < m and s[i + k] == lit[k] { k += 1 } + if k == m { return i } + } + i += 1 + } + return -1 +} + +export function i18n_starts(s: string, p: string) -> bool { + let n = len(p) + if len(s) < n { return false } + for i in 0 .. n { if s[i] != p[i] { return false } } + return true +} + +export function i18n_ends(s: string, p: string) -> bool { + let n = len(p) + let m = len(s) + if m < n { return false } + for i in 0 .. n { if s[m - n + i] != p[i] { return false } } + return true +} + +function blank(s: string) -> bool { return len(s) == 0 } + +function trim(s: string) -> string { + let n = len(s) + var a = 0 + var b = n + while a < n and (s[a] == 32 or s[a] == 9 or s[a] == 13) { a += 1 } + while b > a and (s[b - 1] == 32 or s[b - 1] == 9 or s[b - 1] == 13) { b -= 1 } + if a == 0 and b == n { return s } + return s[a..b] +} + +function lines_of(text: string) -> []string { + let r = new []string + let n = len(text) + var start = 0 + for i in 0 .. n { + if text[i] == 10 { + push(r, text[start..i]) + start = i + 1 + } + } + if start < n { push(r, text[start..n]) } + return r +} + +# the hole numbers of "{1}" / "{12}" in a string, each seen at i: its number, or -1 and no hole +function hole_at(s: string, i: int) -> int { + let n = len(s) + if s[i] != 123 { return -1 } + var j = i + 1 + var num = 0 + while j < n and s[j] >= 48 and s[j] <= 57 { + num = num * 10 + (s[j] - 48) + j += 1 + } + if j > i + 1 and j < n and s[j] == 125 { return num } + return -1 +} + +# where the hole at i ends (one past its closing brace) +function hole_end(s: string, i: int) -> int { + var j = i + 1 + while s[j] != 125 { j += 1 } + return j + 1 +} diff --git a/packages/ludic.i18n/index.ludic b/packages/ludic.i18n/index.ludic new file mode 100644 index 00000000..65ffb6ca --- /dev/null +++ b/packages/ludic.i18n/index.ludic @@ -0,0 +1,13 @@ +# ludic.i18n - English in the code, every other language a .po looked up at the moment of drawing +module ludic_i18n uses +numbers float +import "ports.ludic" +import "state.ludic" +import "bytes.ludic" +import "po.ludic" +import "langs.ludic" +import "use.ludic" +import "patterns.ludic" +import "match.ludic" +import "translate.ludic" +import "plural.ludic" diff --git a/packages/ludic.i18n/langs.ludic b/packages/ludic.i18n/langs.ludic new file mode 100644 index 00000000..c216ffc9 --- /dev/null +++ b/packages/ludic.i18n/langs.ludic @@ -0,0 +1,82 @@ +# langs.ludic - where languages come from, later ones replacing earlier ones with the same code: +# /languages.txt (one code per line, # a comment) and every .po in the mod folder +export function i18n_init() -> void { + codes = new []string + names = new []string + paths = new []string + fonts = new []string + lang_add("en", "English", "", "") + let idx = Fs.read_text(lang_dir + "/languages.txt") + if idx != null { + let ls = lines_of(idx) + for i in 0 .. len(ls) { + let ln = trim(ls[i]) + if len(ln) == 0 or ln[0] == 35 { continue } + probe(ln, `{lang_dir}/{ln}.po`, lang_dir) + } + } + let md = i18n_mod_dir() + if len(md) == 0 { return } + Fs.mkdir(md) + let files = Fs.list(md) + if files == null { return } + for i in 0 .. len(files) { + let nm = files[i] + let n = len(nm) + if n > 3 and i18n_ends(nm, ".po") { probe(nm[0..n - 3], md + "/" + nm, md) } + } +} + +# where a player's own .po files go; "" when the game has none +export function i18n_mod_dir() -> string { return I18nWorld.mod_dir() } + +function lang_add(code: string, name: string, path: string, font: string) -> void { + for i in 0 .. len(codes) { + if codes[i] == code { + names[i] = name + paths[i] = path + fonts[i] = font + return + } + } + push(codes, code) + push(names, name) + push(paths, path) + push(fonts, font) +} + +function probe(code: string, path: string, dir: string) -> void { + let text = Fs.read_text(path) + if text == null { return } + var name = i18n_header(text, "X-Language-Name") + if len(name) == 0 { name = code } + var font = i18n_header(text, "X-Font") + if len(font) > 0 { font = dir + "/" + font } + lang_add(code, name, path, font) +} + +export function i18n_count() -> int { + if codes == null { return 1 } + return len(codes) +} + +export function i18n_index(code: string) -> int { + if codes != null { for i in 0 .. len(codes) { if codes[i] == code { return i } } } + return 0 +} + +export function i18n_code_at(i: int) -> string { + if codes == null or i < 0 or i >= len(codes) { return "en" } + return codes[i] +} + +export function i18n_name_at(i: int) -> string { + if names == null or i < 0 or i >= len(names) { return "English" } + return names[i] +} + +# the font atlas a language names with X-Font, beside its .po; "" is the default +export function i18n_font_at(i: int) -> string { + if fonts == null or i < 0 or i >= len(fonts) { return "" } + return fonts[i] +} diff --git a/packages/ludic.i18n/match.ludic b/packages/ludic.i18n/match.ludic new file mode 100644 index 00000000..08d4c7cb --- /dev/null +++ b/packages/ludic.i18n/match.ludic @@ -0,0 +1,95 @@ +# match.ludic - which pattern produces a string, the best one being the one that pins down the most +# English, and the text that filled each hole translated in turn + +# does pattern p produce s? The holes land in caps, in hole order +function try_pattern(p: int, s: string) -> bool { + let n = len(s) + let l0 = pat_l0[p] + let nl = pat_nl[p] + let first = lits[l0] + let f_n = len(first) + let last = lits[l0 + nl - 1] + let l_n = len(last) + if nl - 1 > CAPS { return false } + if n < f_n + l_n + nl - 1 { return false } # every hole takes at least a byte + for k in 0 .. f_n { if s[k] != first[k] { return false } } + for k in 0 .. l_n { if s[n - l_n + k] != last[k] { return false } } + let lim = n - l_n + var pos = f_n + for g in 0 .. nl - 1 { + var stop = lim + var adv = 0 + if g < nl - 2 { + let mid = lits[l0 + g + 1] + stop = i18n_find(s, lim, pos + 1, mid, len(mid)) + if stop < 0 { return false } + adv = len(mid) + } + if stop <= pos { return false } + caps[g] = s[pos..stop] + pos = stop + adv + } + return true +} + +# the best of a chain that produces s, or `best` as it was +function best_of(head: int, next: words, s: string, best: int) -> int { + var b = best + var c = head + while c > 0 { + let p = c - 1 + var sc = -1 + if b >= 0 { sc = pat_sc[b] } + if pat_sc[p] > sc and try_pattern(p, s) { b = p } + c = next[p] + } + return b +} + +function pattern(s: string, depth: int) -> string { + hit = false + if pat_n == 0 { return s } + let n = len(s) + var best = -1 + if n >= 4 { + best = best_of(Dict.get_or(pfx, s[0..4], 0), nx_p, s, best) + best = best_of(Dict.get_or(sfx, s[n - 4..n], 0), nx_s, s, best) + } + if n < 80 { best = best_of(gen_head, nx_g, s, best) } + if best < 0 { return s } + try_pattern(best, s) + let l0 = pat_l0[best] + let nl = pat_nl[best] + # copied out first: translating a hole may match another pattern and reuse caps + let raw = new []string + for g in 0 .. nl - 1 { push(raw, caps[g]) } + let cs = new []string + for g in 0 .. nl - 1 { push(cs, piece(raw[g], depth + 1)) } + let r = fill_holes(pat_out[best], l0, nl, cs) + # a pattern that gives the English straight back has translated nothing, and must not stop + # the sentence-by-sentence pass + if r == s { return s } + hit = true + return r +} + +# the text that filled a hole: known as it stands, known without the English article the game +# glued on ("a deer", "the Dark King"), or itself a pattern +function piece(c: string, depth: int) -> string { + let e = Dict.get_or(exact, c, -1) + if e >= 0 { return out[e] } + let n = len(c) + var bare = -1 + if n > 4 and (i18n_starts(c, "the ") or i18n_starts(c, "The ")) { bare = 4 } + else if n > 3 and (i18n_starts(c, "an ") or i18n_starts(c, "An ")) { bare = 3 } + else if n > 2 and (i18n_starts(c, "a ") or i18n_starts(c, "A ")) { bare = 2 } + if bare > 0 { + let e2 = Dict.get_or(exact, c[bare..n], -1) + if e2 >= 0 { return out[e2] } + } + if depth < 3 { + let r = pattern(c, depth) + if hit { return r } + } + return c +} diff --git a/packages/ludic.i18n/package.ludic b/packages/ludic.i18n/package.ludic new file mode 100644 index 00000000..6ed2c76d --- /dev/null +++ b/packages/ludic.i18n/package.ludic @@ -0,0 +1,6 @@ +# ludic.i18n - a game in any language from gettext files anyone can add: the English stays in the +# code, a .po is keyed by the English as drawn, exact lines, patterns with holes, paragraphs a +# sentence at a time, plurals, a mod folder and a font per language. Uses nothing. See README.md. +package "ludic.i18n" +version "0.1.0" +kind source diff --git a/packages/ludic.i18n/patterns.ludic b/packages/ludic.i18n/patterns.ludic new file mode 100644 index 00000000..2fb90f51 --- /dev/null +++ b/packages/ludic.i18n/patterns.ludic @@ -0,0 +1,83 @@ +# patterns.ludic - a msgid with holes ("Day {1}"): its literal pieces, how much English they pin +# down, and a chain by its first (else last) four bytes so a lookup tries only its own kind +function holes(s: string) -> int { + let n = len(s) + var h = 0 + var i = 0 + while i < n { + if hole_at(s, i) >= 0 { + h += 1 + i = hole_end(s, i) + continue + } + i += 1 + } + return h +} + +function add_pattern(id: string, tr: string) -> void { + let p = pat_n + pat_n += 1 + let l0 = len(lits) + let n = len(id) + var seg = 0 + var i = 0 + var nl = 0 + var score = 0 + while i < n { + let num = hole_at(id, i) + if num >= 0 { + push(lits, id[seg..i]) + score += i - seg + pat_ph[l0 + nl] = num + nl += 1 + i = hole_end(id, i) + seg = i + continue + } + i += 1 + } + push(lits, id[seg..n]) + score += n - seg + nl += 1 + pat_l0[p] = l0 + pat_nl[p] = nl + pat_sc[p] = score + push(pat_out, tr) + chain(p, lits[l0], lits[l0 + nl - 1]) +} + +function chain(p: int, first: string, last: string) -> void { + if len(first) >= 4 { + let key: string = first[0..4] + nx_p[p] = Dict.get_or(pfx, key, 0) + Dict.set(pfx, key, p + 1) + } else if len(last) >= 4 { + let key: string = last[len(last) - 4..len(last)] + nx_s[p] = Dict.get_or(sfx, key, 0) + Dict.set(sfx, key, p + 1) + } else { + nx_g[p] = gen_head + gen_head = p + 1 + } +} + +# a translation with its holes filled from caps, by number: a translation may reorder or drop them +function fill_holes(t: string, l0: int, nl: int, cs: []string) -> string { + let n = len(t) + var r = "" + var seg = 0 + var i = 0 + while i < n { + let num = hole_at(t, i) + if num >= 0 { + r = r + t[seg..i] + for g in 0 .. nl - 1 { if pat_ph[l0 + g] == num { r = r + cs[g] } } + i = hole_end(t, i) + seg = i + continue + } + i += 1 + } + return r + t[seg..n] +} diff --git a/packages/ludic.i18n/plural.ludic b/packages/ludic.i18n/plural.ludic new file mode 100644 index 00000000..504b0f46 --- /dev/null +++ b/packages/ludic.i18n/plural.ludic @@ -0,0 +1,74 @@ +# plural.ludic - msgid_plural / msgstr[k], chosen by the file's Plural-Forms. The rules a game meets +# are named rather than evaluated: one form, n != 1, n > 1 and the three East Slavic forms +export const PL_NOT_ONE: int = 0 # English, German, Turkish as most files write it +export const PL_ABOVE_ONE: int = 1 # French, Portuguese (0 and 1 are singular) +export const PL_ONE_FORM: int = 2 # Japanese, Chinese +export const PL_SLAVIC: int = 3 # Russian, Ukrainian: one, few, many + +function plural_rule_of(h: string) -> int { + let k = i18n_find(h, len(h), 0, "plural=", 7) + if k < 0 { return PL_NOT_ONE } + let e: string = h[k + 7..len(h)] + if i18n_find(e, len(e), 0, "n%10", 4) >= 0 or i18n_find(e, len(e), 0, "n % 10", 6) >= 0 { return PL_SLAVIC } + if i18n_find(e, len(e), 0, ">", 1) >= 0 { return PL_ABOVE_ONE } + if i18n_find(e, len(e), 0, "n", 1) < 0 { return PL_ONE_FORM } + return PL_NOT_ONE +} + +export function i18n_plural_rule() -> int { return plural_rule } + +# which msgstr[k] a count takes under a rule +export function i18n_plural_form(rule: int, n: int) -> int { + if rule == PL_ONE_FORM { return 0 } + if rule == PL_ABOVE_ONE { + if n > 1 { return 1 } + return 0 + } + if rule == PL_SLAVIC { + if n % 10 == 1 and n % 100 != 11 { return 0 } + if n % 10 >= 2 and n % 10 <= 4 and (n % 100 < 10 or n % 100 >= 20) { return 1 } + return 2 + } + if n != 1 { return 1 } + return 0 +} + +function plural_add(e: PoEntry) -> void { + if len(e.forms) == 0 or blank(e.forms[0]) { return } + Dict.set(plural, e.id, len(forms) / 3) + for k in 0 .. 3 { + if k < len(e.forms) { push(forms, e.forms[k]) } else { push(forms, e.forms[len(e.forms) - 1]) } + } +} + +# `one` or `many` by the count, in the language in use, with every {1} the count: Ln("{1} trout", +# "{1} trout", 3). English (or a line the file lacks) is one for 1 and many otherwise +export function Ln(one: string, many: string, n: int) -> string { + var t = many + if n == 1 { t = one } + if on { + let k = Dict.get_or(plural, one, -1) + if k >= 0 { + let f = forms[k * 3 + i18n_plural_form(plural_rule, n)] + if not blank(f) { t = f } + } + } + return count_in(t, n) +} + +function count_in(t: string, n: int) -> string { + let m = len(t) + var r = "" + var seg = 0 + var i = 0 + while i < m { + if hole_at(t, i) == 1 { + r = r + t[seg..i] + `{n}` + i = hole_end(t, i) + seg = i + continue + } + i += 1 + } + return r + t[seg..m] +} diff --git a/packages/ludic.i18n/po.ludic b/packages/ludic.i18n/po.ludic new file mode 100644 index 00000000..dc77ff54 --- /dev/null +++ b/packages/ludic.i18n/po.ludic @@ -0,0 +1,94 @@ +# po.ludic - a gettext file: msgid / msgid_plural / msgstr / msgstr[k] and their continuation lines; +# comments, flags and msgctxt are read past +property PoEntry { + id: string = "", + plural: string = "", + str: string = "", + forms: []string +} + +function entry_new() -> PoEntry { + let e = new PoEntry + e.forms = new []string + return e +} + +function entry_put(e: PoEntry, mode: int, s: string) -> void { + if mode == 1 { e.id = e.id + s } + if mode == 2 { e.str = e.str + s } + if mode == 3 { e.plural = e.plural + s } + if mode == 4 { e.forms[len(e.forms) - 1] = e.forms[len(e.forms) - 1] + s } +} + +export function i18n_parse(text: string) -> []PoEntry { + let r = new []PoEntry + let lines = lines_of(text) + var mode = 0 # 1 msgid, 2 msgstr, 3 msgid_plural, 4 msgstr[k] + var e: PoEntry = null + for li in 0 .. len(lines) { + let line = trim(lines[li]) + if i18n_starts(line, "msgid ") { + if e != null { push(r, e) } + e = entry_new() + e.id = unquote(line, 6) + mode = 1 + } else if e != null and i18n_starts(line, "msgid_plural ") { + e.plural = unquote(line, 13) + mode = 3 + } else if e != null and i18n_starts(line, "msgstr[") { + push(e.forms, unquote(line, 7)) + mode = 4 + } else if e != null and i18n_starts(line, "msgstr ") { + e.str = unquote(line, 7) + mode = 2 + } else if len(line) > 0 and line[0] == 34 { + if e != null { entry_put(e, mode, unquote(line, 0)) } + } else { + if e != null and (mode == 2 or mode == 4) { + push(r, e) + e = null + } + mode = 0 + } + } + if e != null { push(r, e) } + return r +} + +# the string whose opening quote is at or after `from`, with \" \\ \n \t undone +function unquote(line: string, from: int) -> string { + let n = len(line) + var i = from + while i < n and line[i] != 34 { i += 1 } + i += 1 + var seg = i + var s = "" + while i < n { + let c = line[i] + if c == 92 and i + 1 < n { + if i > seg { s = s + line[seg..i] } + let e = line[i + 1] + if e == 110 { s = s + "\n" } else if e == 116 { s = s + "\t" } else { s = s + line[i + 1..i + 2] } + i += 2 + seg = i + continue + } + if c == 34 { break } + i += 1 + } + if i > seg { s = s + line[seg..i] } + return s +} + +# a header field out of the msgstr "" block: "X-Language-Name: Türkçe\n" +export function i18n_header(text: string, field: string) -> string { + var n = len(text) + if n > 4000 { n = 4000 } + let key = field + ": " + let at = i18n_find(text, n, 0, key, len(key)) + if at < 0 { return "" } + let s0 = at + len(key) + var e = s0 + while e < n and text[e] != 92 and text[e] != 34 and text[e] != 10 { e += 1 } + return trim(text[s0..e]) +} diff --git a/packages/ludic.i18n/ports.ludic b/packages/ludic.i18n/ports.ludic new file mode 100644 index 00000000..9dece674 --- /dev/null +++ b/packages/ludic.i18n/ports.ludic @@ -0,0 +1,24 @@ +# ports.ludic - what the languages ask the game: where a player's own .po files are, a code that +# overrides the choice (a test in a language), and the overlay's font atlas, which is the game's +export port I18nWorld { + mod_dir: fn() -> string = fn i18n__no_dir # "" is no mod folder + forced: fn() -> string = fn i18n__no_code # "" is the code asked for + font_ready: fn() -> bool = fn i18n__no # is there an overlay to load a font into + set_font: fn(string) -> bool = fn i18n__no_font # load this atlas directory; false if it could not +} + +function i18n__no_dir() -> string { return "" } +function i18n__no_code() -> string { return "" } +function i18n__no() -> bool { return false } +function i18n__no_font(dir: string) -> bool { return false } + +var lang_dir: string = "assets/lang" +var font_default: string = "assets/kit/font" + +# where the shipped languages are (a directory with languages.txt and .po), and the atlas a +# language with no X-Font draws in +export function i18n_config(dir: string, default_font: string) -> void { + lang_dir = dir + font_default = default_font + codes = null +} diff --git a/packages/ludic.i18n/state.ludic b/packages/ludic.i18n/state.ludic new file mode 100644 index 00000000..8fce2317 --- /dev/null +++ b/packages/ludic.i18n/state.ludic @@ -0,0 +1,47 @@ +# state.ludic - the languages found, the one in use, and the loaded file's lookups +var on: bool = false # a language other than English is drawing +var cur: string = "en" +var codes: []string = null # every language found, "en" first +var names: []string = null +var paths: []string = null +var fonts: []string = null # a font atlas directory, "" for the game's own +var font_now: string = "" + +var exact: Dict = null # msgid -> index into out +var out: []string = null +var lits: []string = null # every pattern's literal pieces, one run per pattern +var pat_out: []string = null +var pat_l0: words = null # where a pattern's pieces start in lits +var pat_nl: words = null # how many pieces (holes + 1) +var pat_sc: words = null # bytes of English it pins down: more is a better match +var pat_ph: words = null # by piece: the hole number that follows it +var pat_n: int = 0 +var pfx: Dict = null # a pattern's first four bytes -> its chain head + 1 +var sfx: Dict = null # else its last four +var gen_head: int = 0 # else the short list, tried on short strings only +var nx_p: words = null +var nx_s: words = null +var nx_g: words = null +var cache: Dict = null # English as drawn -> index into cache_out +var cache_out: []string = null +var caps: []string = null +var hit: bool = false + +var plural: Dict = null # a plural msgid (the singular) -> index into forms / 3 +var forms: []string = null # three forms per entry, "" where the file has fewer +var plural_rule: int = 0 # PL_* + +const CACHE_MAX: int = 4000 +const CAPS: int = 16 + +export function i18n_on() -> bool { return on } +export function i18n_current() -> string { return cur } +export function i18n_font_now() -> string { return font_now } +export function i18n_exact_count() -> int { + if out == null { return 0 } + return len(out) +} +export function i18n_pattern_count() -> int { return pat_n } + +# the overlay's own font, as the overlay was opened with it +export function i18n_font_base(path: string) -> void { font_now = path } diff --git a/packages/ludic.i18n/tests/i18n_test.ludic b/packages/ludic.i18n/tests/i18n_test.ludic new file mode 100644 index 00000000..dd525380 --- /dev/null +++ b/packages/ludic.i18n/tests/i18n_test.ludic @@ -0,0 +1,151 @@ +# i18n_test.ludic - languages from a shipped list and a mod folder, exact lines, patterns with their +# holes translated, a paragraph a sentence at a time, padding, plurals, the font a language names, +# and English untouched throughout; every file written to the temp directory +import "ludic.i18n" +program I18nTest { + numbers float + var forced_code: string = "" + var fonts_set: []string = null + var font_ok: bool = true + + function dir() -> string { return Os.temp_dir() + "/ludic-i18n-test" } + function mods() -> string { return dir() + "/mods" } + function t_mods() -> string { return mods() } + function t_forced() -> string { return forced_code } + function t_ready() -> bool { return true } + function t_font(d: string) -> bool { + push(fonts_set, d) + return font_ok + } + bind I18nWorld { mod_dir: fn t_mods, forced: fn t_forced, font_ready: fn t_ready, set_font: fn t_font } + + function q(s: string) -> string { return "\"" + s + "\"" } + function po_line(id: string, tr: string) -> string { return "msgid " + q(id) + "\nmsgstr " + q(tr) + "\n\n" } + function header(name: string, extra: string) -> string { + return "msgid \"\"\nmsgstr \"\"\n\"X-Language-Name: " + name + "\\n\"\n" + extra + "\n" + } + + # a shipped "tt" with a font of its own, and a Russian-shaped plural + function tt() -> string { + var s = header("Test Tongue", "\"X-Font: font_tt\\n\"\n\"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : 2);\\n\"") + s = s + po_line("Settings", "Zettings") + po_line("Plant fiber", "Lif") + po_line("a deer", "") + po_line("deer", "geyik") + s = s + po_line("Day {1}", "{1}. Gun") + po_line("+{1} {2}", "{2} +{1}") + po_line("You saw {1}.", "{1} gordun.") + s = s + po_line("You need to eat.", "Yemek lazim.") + po_line("owned", "senin") + s = s + "msgid \"{1} trout\"\nmsgid_plural \"{1} trout\"\nmsgstr[0] \"{1} alabalik\"\nmsgstr[1] \"{1} alabaliga\"\nmsgstr[2] \"{1} alabaliklar\"\n\n" + s = s + "msgid \"Multi\"\n\"line\"\nmsgstr \"Cok\"\n\"satir\\n\"\n" + return s + } + + function fresh() -> void { + forced_code = "" + fonts_set = new []string + font_ok = true + Fs.mkdir(dir()) + Fs.mkdir(mods()) + Fs.write_text(dir() + "/languages.txt", "# shipped\ntt\nnope\n") + Fs.write_text(dir() + "/tt.po", tt()) + Fs.remove(mods() + "/zz.po") + i18n_config(dir(), "font_default") + i18n_init() + } + + test "the shipped list and English are found, a missing file is not" { + fresh() + expect_eq(i18n_count(), 2) + expect(i18n_code_at(0) == "en") + expect_eq(i18n_index("tt"), 1) + expect_eq(i18n_index("nope"), 0) + expect(i18n_name_at(1) == "Test Tongue") + expect(i18n_font_at(1) == dir() + "/font_tt") + expect(L("Settings") == "Settings") + } + + test "exact lines, patterns with translated holes, and English where unknown" { + fresh() + i18n_use("tt") + expect(i18n_on()) + expect(i18n_current() == "tt") + expect(L("Settings") == "Zettings") + expect(L("Day 12") == "12. Gun") + expect(L("+2 Plant fiber") == "Lif +2") + expect(L("You saw a deer.") == "geyik gordun.") + expect(L("zzqx words nobody wrote") == "zzqx words nobody wrote") + expect(L("Multiline") == "Coksatir\n") + expect_eq(i18n_exact_count(), 6) + expect_eq(i18n_pattern_count(), 3) + } + + test "a paragraph a sentence at a time, and a layout's padding kept" { + fresh() + i18n_use("tt") + expect(L("You need to eat. zzqx words nobody wrote.") == "Yemek lazim. zzqx words nobody wrote.") + expect(L("zzqx. zzqy.") == "zzqx. zzqy.") + expect(L(" owned") == " senin") + expect(L(" zzqx") == " zzqx") + } + + test "plurals by the file's rule, and English by one and many" { + fresh() + expect(Ln("{1} trout", "{1} trouts", 1) == "1 trout") + expect(Ln("{1} trout", "{1} trouts", 3) == "3 trouts") + i18n_use("tt") + expect_eq(i18n_plural_rule(), PL_SLAVIC) + expect(Ln("{1} trout", "{1} trouts", 21) == "21 alabalik") + expect(Ln("{1} trout", "{1} trouts", 5) == "5 alabaliklar") + expect(Ln("{1} fish", "{1} fishes", 2) == "2 fishes") + expect_eq(i18n_plural_form(PL_NOT_ONE, 0), 1) + expect_eq(i18n_plural_form(PL_ABOVE_ONE, 0), 0) + expect_eq(i18n_plural_form(PL_SLAVIC, 3), 1) + expect_eq(i18n_plural_form(PL_ONE_FORM, 7), 0) + } + + test "the font follows the language, and falls back when it will not load" { + fresh() + i18n_font_base("font_default") + i18n_use("tt") + expect_eq(len(fonts_set), 1) + expect(i18n_font_now() == dir() + "/font_tt") + i18n_use("en") + expect(i18n_font_now() == "font_default") + font_ok = false + i18n_use("tt") + expect(i18n_font_now() == "font_default") + } + + test "a mod in the player's folder is found, replaces a shipped code, and goes when removed" { + fresh() + Fs.write_text(mods() + "/zz.po", header("Mod Tongue", "") + po_line("Settings", "Mettings")) + Fs.write_text(mods() + "/tt.po", header("Tongue Two", "") + po_line("Settings", "Z2")) + i18n_init() + expect_eq(i18n_count(), 3) + expect(i18n_name_at(i18n_index("zz")) == "Mod Tongue") + i18n_use("zz") + expect(L("Settings") == "Mettings") + expect(L("Day 3") == "Day 3") + i18n_use("tt") + expect(L("Settings") == "Z2") + Fs.remove(mods() + "/zz.po") + Fs.remove(mods() + "/tt.po") + i18n_init() + expect_eq(i18n_index("zz"), 0) + i18n_use("en") + expect(not i18n_on()) + expect(L("Settings") == "Settings") + } + + test "a forced code wins over the one asked for" { + fresh() + forced_code = "tt" + i18n_use("en") + expect(i18n_current() == "tt") + } + + test "a header field, and a .po loaded from text" { + expect(i18n_header(tt(), "X-Language-Name") == "Test Tongue") + expect(i18n_header(tt(), "X-Nothing") == "") + i18n_load_text(po_line("Quit", "Cik")) + expect(L("Quit") == "Cik") + i18n_off() + expect(L("Quit") == "Quit") + } +} diff --git a/packages/ludic.i18n/translate.ludic b/packages/ludic.i18n/translate.ludic new file mode 100644 index 00000000..b8032a77 --- /dev/null +++ b/packages/ludic.i18n/translate.ludic @@ -0,0 +1,75 @@ +# translate.ludic - what the game draws, in the language in use: exact, else a pattern, else the +# padding a layout put round it taken off, else a paragraph a sentence at a time; English otherwise +export function L(s: string) -> string { + if not on { return s } + let ci = Dict.get_or(cache, s, -1) + if ci >= 0 { return cache_out[ci] } + let r = whole(s) + if Dict.size(cache) >= CACHE_MAX { + Dict.clear(cache) + cache_out = new []string + } + Dict.set(cache, s, len(cache_out)) + push(cache_out, r) + return r +} + +function whole(s: string) -> string { + let n = len(s) + if n == 0 { return s } + let e = Dict.get_or(exact, s, -1) + if e >= 0 { return out[e] } + let r = pattern(s, 0) + if hit { return r } + # the padding a layout put round a word: " owned" + var a = 0 + var b = n + while a < n and s[a] == 32 { a += 1 } + while b > a and s[b - 1] == 32 { b -= 1 } + if a > 0 or b < n { + if b <= a { return s } + let core: string = s[a..b] + let t = whole(core) + if t == core { return s } + return s[0..a] + t + s[b..n] + } + return sentences(s) +} + +# a paragraph built from sentences, one at a time; English stays where a sentence is unknown +function sentences(s: string) -> string { + let n = len(s) + var r = "" + var start = 0 + var any = false + var pieces = 0 + var i = 0 + while i < n - 1 { + let c = s[i] + if (c == 46 or c == 33 or c == 63) and s[i + 1] == 32 { + let t = sentence(s[start..i + 1]) + if hit { any = true } + r = r + t + " " + start = i + 2 + pieces += 1 + i += 2 + continue + } + i += 1 + } + if pieces == 0 { return s } + let t = sentence(s[start..n]) + if hit { any = true } + if not any { return s } + return r + t +} + +function sentence(p: string) -> string { + hit = false + let e = Dict.get_or(exact, p, -1) + if e >= 0 { + hit = true + return out[e] + } + return pattern(p, 0) +} diff --git a/packages/ludic.i18n/use.ludic b/packages/ludic.i18n/use.ludic new file mode 100644 index 00000000..09fa44b7 --- /dev/null +++ b/packages/ludic.i18n/use.ludic @@ -0,0 +1,80 @@ +# use.ludic - choosing a language: its file loaded into the lookups, and the font atlas it names +export function i18n_use(code: string) -> void { + if codes == null { i18n_init() } + on = false + cur = "en" + var k = i18n_index(code) + let forced = I18nWorld.forced() + if len(forced) > 0 { k = i18n_index(forced) } + var font = "" + if k > 0 { + let text = Fs.read_text(paths[k]) + if text != null { + i18n_load_text(text) + on = true + cur = codes[k] + font = fonts[k] + } + } + # the atlas follows the language, and is only reloaded when it actually changes + if len(font) == 0 { font = font_default } + if I18nWorld.font_ready() and not (font == font_now) { + if I18nWorld.set_font(font) { font_now = font } else if not (font_now == font_default) { + I18nWorld.set_font(font_default) + font_now = font_default + } + } +} + +# a .po's text as the lookups, whatever the language is called (a test, a file from elsewhere) +export function i18n_load_text(text: string) -> void { + let es = i18n_parse(text) + exact = Dict.new() + out = new []string + cache = Dict.new() + cache_out = new []string + caps = new []string + for c in 0 .. CAPS { push(caps, "") } + plural = Dict.new() + forms = new []string + plural_rule = plural_rule_of(i18n_header(text, "Plural-Forms")) + var np = 0 + var nlit = 0 + for i in 0 .. len(es) { + if blank(es[i].id) or blank(es[i].str) { continue } + let h = holes(es[i].id) + if h > 0 { + np += 1 + nlit += h + 1 + } + } + pat_out = new []string + lits = new []string + pat_l0 = words(np + 1) + pat_nl = words(np + 1) + pat_sc = words(np + 1) + pat_ph = words(nlit + 1) + nx_p = words(np + 1) + nx_s = words(np + 1) + nx_g = words(np + 1) + pfx = Dict.new() + sfx = Dict.new() + gen_head = 0 + pat_n = 0 + for i in 0 .. len(es) { + let e = es[i] + if len(e.plural) > 0 { plural_add(e) } else if not (blank(e.id) or blank(e.str)) { + if holes(e.id) > 0 { add_pattern(e.id, e.str) } else { + Dict.set(exact, e.id, len(out)) + push(out, e.str) + } + } + } + on = true +} + +# English again: nothing loaded, nothing translated +export function i18n_off() -> void { + on = false + cur = "en" +}