workshopsoft/ludic
2
0
Fork 0
Code Issues Pull requests Projects Releases 26 Packages Wiki Activity Actions
ludic/selfhost/backend/stdlib/emit_unicode.ludic
Orkuncakilkaya 23726afa90
All checks were successful
bootstrap / cfree-fixpoint (push) Successful in 12s
Details
ci / build-and-test (push) Successful in 50s
Details
commit-lint / conventional-commits (push) Successful in 3s
Details
docs / build-and-deploy (push) Successful in 2s
Details
refactor(selfhost): reorganise into concern-based subdirectories 
Split the flat 38-file selfhost/ into concern-based subdirectories:

  frontend/        lex, parse, parse_game, ast
  support/         str, buf, io
  backend/         core IR + expression/statement lowering
  backend/game/    ECS/scene/event/world lowering
  backend/stdlib/  the namespaced Math.*/Text.*/Crypto.*/… intrinsics

and split the three oversized emitters at responsibility boundaries so
no file mixes concerns:

  emit_game.ludic  -> + emit_world.ludic         (reflection world table,
                                                  tick helpers, @main synthesis)
  emit_expr.ludic  -> + emit_call.ludic          (namespaced builtins, call
                                                  lowering, expr dispatch)
  emit_text.ludic  -> + emit_text_prelude.ludic  (emitted string-builder runtime)

FRAGS in tools/x/selfhost.ludic is updated to the new paths with the link
order preserved, and the Python doc/vocabulary tooling is updated to walk
the new layout. Because the build is a plain in-order concatenation and
every split lands on a blank-line boundary, the regenerated seed is
byte-identical: `x reseed` leaves selfhost/ludicc.seed.ll unchanged,
`x bootstrap-cfree` still reaches its fixed point, and both `x test` (56)
and `x selfhost-test` (29, incl. golden renders) stay green.

Closes #29

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-31 00:26:02 +03:00

318 lines
25 KiB
Text
Raw Blame History

# emit_unicode.ludic — the Unicode.* namespace: correct-by-default text over
# UTF-8, so player names, translated UI, and chat behave for every language
# instead of counting bytes and splitting characters in half. The byte-oriented
# Text.* stays for speed; Unicode.* is the layer that understands code points and
# (approximately) grapheme clusters.
#
# Unicode.len(s) -> int number of code points (not bytes)
# Unicode.byte_len(s) -> int number of bytes (the contrast to len)
# Unicode.is_valid_utf8(s) -> bool strict UTF-8 validation of untrusted input
# Unicode.char_at(s, i) -> int the i-th code point (-1 if out of range)
# Unicode.chars(s) -> []int every code point, in order
# Unicode.upper(s) -> string uppercased (ASCII + Latin-1)
# Unicode.lower(s) -> string lowercased (ASCII + Latin-1)
# Unicode.truncate(s, n) -> string first n code points, never a half-char
# Unicode.grapheme_len(s) -> int user-perceived characters (approx UAX#29)
#
# Unicode version / coverage: this v1 decodes and validates the full UTF-8 range.
# Case mapping covers ASCII and the Latin-1 Supplement letters (correct for
# Western-European text); Latin-Extended/Greek/Cyrillic case, locale rules (e.g.
# Turkish i, German ß->SS), and normalization (NFC) are documented follow-ups.
# grapheme_len is an approximation of UAX#29 that handles combining marks,
# variation selectors, ZWJ sequences (e.g. family emoji), and regional-indicator
# flag pairs — enough for measuring and truncating real player text; full
# segmentation with its data tables is a follow-up.
#
# Every function is pure over a NUL-terminated UTF-8 buffer: no global state, no
# platform dependence, so results are identical on native, headless, and wasm.
function is_unicode_ns(meth: pointer) -> bool {
if (meth == "len") or (meth == "byte_len") or (meth == "is_valid_utf8") { return true }
if (meth == "char_at") or (meth == "chars") { return true }
if (meth == "upper") or (meth == "lower") or (meth == "truncate") { return true }
if (meth == "grapheme_len") { return true }
return false
}
function emit_unicode_ns(meth: pointer, e: Node) -> Val {
g_uses_unicodert = true
if (meth == "byte_len") { # bytes, the contrast to len()
let s = emit_expr(e.kids[0])
let r = emit_bind(`call i64 @strlen(ptr {s.code})`)
return val(emit_bind(`trunc i64 {r} to i32`), "int")
}
if (meth == "len") {
let s = emit_expr(e.kids[0])
return val(emit_bind(`call i32 @fn_uni_len(ptr {s.code})`), "int")
}
if (meth == "is_valid_utf8") {
let s = emit_expr(e.kids[0])
return val(emit_bind(`call i32 @fn_uni_valid(ptr {s.code})`), "bool")
}
if (meth == "char_at") {
let s = emit_expr(e.kids[0]); let i = emit_expr(e.kids[1])
return val(emit_bind(`call i32 @fn_uni_char_at(ptr {s.code}, i32 {i.code})`), "int")
}
if (meth == "chars") {
let s = emit_expr(e.kids[0])
return val(emit_bind(`call ptr @fn_uni_chars(ptr {s.code})`), "[]int")
}
if (meth == "upper") {
let s = emit_expr(e.kids[0])
return val(emit_bind(`call ptr @fn_uni_case(ptr {s.code}, i32 1)`), "string")
}
if (meth == "lower") {
let s = emit_expr(e.kids[0])
return val(emit_bind(`call ptr @fn_uni_case(ptr {s.code}, i32 0)`), "string")
}
if (meth == "truncate") {
let s = emit_expr(e.kids[0]); let n = emit_expr(e.kids[1])
return val(emit_bind(`call ptr @fn_uni_truncate(ptr {s.code}, i32 {n.code})`), "string")
}
# grapheme_len
let s = emit_expr(e.kids[0])
return val(emit_bind(`call i32 @fn_uni_grapheme_len(ptr {s.code})`), "int")
}
# emit_unicode_prelude — the UTF-8 runtime, emitted once per program that uses
# Unicode.* (g_uses_unicodert). Pure integer/byte IR over NUL-terminated buffers.
function emit_unicode_prelude() -> void {
emit_uni_core()
emit_uni_case_fns()
emit_uni_str_fns()
}
# ---- decode + length + validation ------------------------------------------
function emit_uni_core() -> void {
# code-point count: every byte that is NOT a UTF-8 continuation byte
# (0b10xxxxxx) begins a new code point. Walks byte-by-byte, so it is safe on
# truncated/invalid input and stops exactly at the NUL.
emith("define i32 @fn_uni_len(ptr %s) {\n")
emith("entry:\n %ip = alloca i32\n %np = alloca i32\n store i32 0, ptr %ip\n store i32 0, ptr %np\n br label %lp\n")
emith("lp:\n %i = load i32, ptr %ip\n %p = getelementptr i8, ptr %s, i32 %i\n %b = load i8, ptr %p\n %c = zext i8 %b to i32\n")
emith(" %z = icmp eq i32 %c, 0\n br i1 %z, label %done, label %go\n")
emith("go:\n %m = and i32 %c, 192\n %cont = icmp eq i32 %m, 128\n br i1 %cont, label %skip, label %count\n")
emith("count:\n %n = load i32, ptr %np\n %n1 = add i32 %n, 1\n store i32 %n1, ptr %np\n br label %skip\n")
emith("skip:\n %i1 = add i32 %i, 1\n store i32 %i1, ptr %ip\n br label %lp\n")
emith("done:\n %r = load i32, ptr %np\n ret i32 %r\n}\n")
# decode one code point at byte offset %i; store it to %cpout; return the byte
# offset just past it. Lenient and overrun-safe: a truncated multibyte sequence
# (a continuation byte that is NUL) or an invalid lead byte decodes as a single
# byte, so the walk always makes progress and never reads past the terminator.
emith("define i32 @fn_uni_decode(ptr %s, i32 %i, ptr %cpout) {\n")
emith("entry:\n %p0 = getelementptr i8, ptr %s, i32 %i\n %b0 = load i8, ptr %p0\n %c0 = zext i8 %b0 to i32\n")
emith(" %a1 = icmp ult i32 %c0, 128\n br i1 %a1, label %one, label %multi\n")
emith("one:\n store i32 %c0, ptr %cpout\n %oi = add i32 %i, 1\n ret i32 %oi\n")
emith("multi:\n %h2 = and i32 %c0, 224\n %is2 = icmp eq i32 %h2, 192\n br i1 %is2, label %two, label %c3\n")
emith("c3:\n %h3 = and i32 %c0, 240\n %is3 = icmp eq i32 %h3, 224\n br i1 %is3, label %three, label %c4\n")
emith("c4:\n %h4 = and i32 %c0, 248\n %is4 = icmp eq i32 %h4, 240\n br i1 %is4, label %four, label %bad\n")
emith("bad:\n store i32 %c0, ptr %cpout\n %bi = add i32 %i, 1\n ret i32 %bi\n")
# two-byte
emith("two:\n %t1i = add i32 %i, 1\n %t1p = getelementptr i8, ptr %s, i32 %t1i\n %t1b = load i8, ptr %t1p\n %t1 = zext i8 %t1b to i32\n")
emith(" %t1z = icmp eq i32 %t1, 0\n br i1 %t1z, label %tbad, label %tok\n")
emith("tbad:\n store i32 %c0, ptr %cpout\n %tbi = add i32 %i, 1\n ret i32 %tbi\n")
emith("tok:\n %tm = and i32 %c0, 31\n %tsh = shl i32 %tm, 6\n %tl = and i32 %t1, 63\n %tcp = or i32 %tsh, %tl\n store i32 %tcp, ptr %cpout\n %ti = add i32 %i, 2\n ret i32 %ti\n")
# three-byte
emith("three:\n %r1i = add i32 %i, 1\n %r1p = getelementptr i8, ptr %s, i32 %r1i\n %r1b = load i8, ptr %r1p\n %r1 = zext i8 %r1b to i32\n")
emith(" %r1z = icmp eq i32 %r1, 0\n br i1 %r1z, label %rbad, label %rc2\n")
emith("rbad:\n store i32 %c0, ptr %cpout\n %rbi = add i32 %i, 1\n ret i32 %rbi\n")
emith("rc2:\n %r2i = add i32 %i, 2\n %r2p = getelementptr i8, ptr %s, i32 %r2i\n %r2b = load i8, ptr %r2p\n %r2 = zext i8 %r2b to i32\n")
emith(" %r2z = icmp eq i32 %r2, 0\n br i1 %r2z, label %rbad, label %rok\n")
emith("rok:\n %rm = and i32 %c0, 15\n %rsh = shl i32 %rm, 12\n %ram = and i32 %r1, 63\n %rash = shl i32 %ram, 6\n %rbm = and i32 %r2, 63\n")
emith(" %rp1 = or i32 %rsh, %rash\n %rcp = or i32 %rp1, %rbm\n store i32 %rcp, ptr %cpout\n %ri = add i32 %i, 3\n ret i32 %ri\n")
# four-byte
emith("four:\n %f1i = add i32 %i, 1\n %f1p = getelementptr i8, ptr %s, i32 %f1i\n %f1b = load i8, ptr %f1p\n %f1 = zext i8 %f1b to i32\n")
emith(" %f1z = icmp eq i32 %f1, 0\n br i1 %f1z, label %fbad, label %fc2\n")
emith("fbad:\n store i32 %c0, ptr %cpout\n %fbi = add i32 %i, 1\n ret i32 %fbi\n")
emith("fc2:\n %f2i = add i32 %i, 2\n %f2p = getelementptr i8, ptr %s, i32 %f2i\n %f2b = load i8, ptr %f2p\n %f2 = zext i8 %f2b to i32\n")
emith(" %f2z = icmp eq i32 %f2, 0\n br i1 %f2z, label %fbad, label %fc3\n")
emith("fc3:\n %f3i = add i32 %i, 3\n %f3p = getelementptr i8, ptr %s, i32 %f3i\n %f3b = load i8, ptr %f3p\n %f3 = zext i8 %f3b to i32\n")
emith(" %f3z = icmp eq i32 %f3, 0\n br i1 %f3z, label %fbad, label %fok\n")
emith("fok:\n %fm = and i32 %c0, 7\n %fsh = shl i32 %fm, 18\n %fam = and i32 %f1, 63\n %fash = shl i32 %fam, 12\n")
emith(" %fbm = and i32 %f2, 63\n %fbsh = shl i32 %fbm, 6\n %fcm = and i32 %f3, 63\n")
emith(" %fp1 = or i32 %fsh, %fash\n %fp2 = or i32 %fp1, %fbsh\n %fcp = or i32 %fp2, %fcm\n store i32 %fcp, ptr %cpout\n %fi = add i32 %i, 4\n ret i32 %fi\n}\n")
# strict UTF-8 validation: correct continuation bytes, no overlong encodings,
# no surrogates (U+D800..U+DFFF), and nothing above U+10FFFF. Returns 1/0.
emith("define i32 @fn_uni_valid(ptr %s) {\n")
emith("entry:\n %ip = alloca i32\n store i32 0, ptr %ip\n br label %lp\n")
emith("lp:\n %i = load i32, ptr %ip\n %p = getelementptr i8, ptr %s, i32 %i\n %b = load i8, ptr %p\n %c = zext i8 %b to i32\n")
emith(" %z = icmp eq i32 %c, 0\n br i1 %z, label %good, label %g\n")
emith("g:\n %a1 = icmp ult i32 %c, 128\n br i1 %a1, label %adv1, label %m2\n")
emith("adv1:\n %n1 = add i32 %i, 1\n store i32 %n1, ptr %ip\n br label %lp\n")
emith("m2:\n %h2 = and i32 %c, 224\n %is2 = icmp eq i32 %h2, 192\n br i1 %is2, label %do2, label %m3\n")
emith("m3:\n %h3 = and i32 %c, 240\n %is3 = icmp eq i32 %h3, 224\n br i1 %is3, label %do3, label %m4\n")
emith("m4:\n %h4 = and i32 %c, 248\n %is4 = icmp eq i32 %h4, 240\n br i1 %is4, label %do4, label %bad\n")
emith("bad:\n ret i32 0\n")
# 2-byte: cp in [0x80,0x7FF]
emith("do2:\n %v1i = add i32 %i, 1\n %v1p = getelementptr i8, ptr %s, i32 %v1i\n %v1b = load i8, ptr %v1p\n %v1 = zext i8 %v1b to i32\n")
emith(" %v1m = and i32 %v1, 192\n %v1ok = icmp eq i32 %v1m, 128\n br i1 %v1ok, label %do2b, label %bad\n")
emith("do2b:\n %q2m = and i32 %c, 31\n %q2s = shl i32 %q2m, 6\n %q2l = and i32 %v1, 63\n %cp2 = or i32 %q2s, %q2l\n")
emith(" %ov2 = icmp ult i32 %cp2, 128\n br i1 %ov2, label %bad, label %adv2\n")
emith("adv2:\n %n2 = add i32 %i, 2\n store i32 %n2, ptr %ip\n br label %lp\n")
# 3-byte: cp in [0x800,0xFFFF], excluding surrogates
emith("do3:\n %w1i = add i32 %i, 1\n %w1p = getelementptr i8, ptr %s, i32 %w1i\n %w1b = load i8, ptr %w1p\n %w1 = zext i8 %w1b to i32\n")
emith(" %w1m = and i32 %w1, 192\n %w1ok = icmp eq i32 %w1m, 128\n br i1 %w1ok, label %do3b, label %bad\n")
emith("do3b:\n %w2i = add i32 %i, 2\n %w2p = getelementptr i8, ptr %s, i32 %w2i\n %w2b = load i8, ptr %w2p\n %w2 = zext i8 %w2b to i32\n")
emith(" %w2m = and i32 %w2, 192\n %w2ok = icmp eq i32 %w2m, 128\n br i1 %w2ok, label %do3c, label %bad\n")
emith("do3c:\n %e3m = and i32 %c, 15\n %e3s = shl i32 %e3m, 12\n %e3am = and i32 %w1, 63\n %e3as = shl i32 %e3am, 6\n %e3bm = and i32 %w2, 63\n")
emith(" %cp3p = or i32 %e3s, %e3as\n %cp3 = or i32 %cp3p, %e3bm\n")
emith(" %ov3 = icmp ult i32 %cp3, 2048\n br i1 %ov3, label %bad, label %do3d\n")
emith("do3d:\n %sg1 = icmp uge i32 %cp3, 55296\n %sg2 = icmp ule i32 %cp3, 57343\n %sg = and i1 %sg1, %sg2\n br i1 %sg, label %bad, label %adv3\n")
emith("adv3:\n %n3 = add i32 %i, 3\n store i32 %n3, ptr %ip\n br label %lp\n")
# 4-byte: cp in [0x10000,0x10FFFF]
emith("do4:\n %x1i = add i32 %i, 1\n %x1p = getelementptr i8, ptr %s, i32 %x1i\n %x1b = load i8, ptr %x1p\n %x1 = zext i8 %x1b to i32\n")
emith(" %x1m = and i32 %x1, 192\n %x1ok = icmp eq i32 %x1m, 128\n br i1 %x1ok, label %do4b, label %bad\n")
emith("do4b:\n %x2i = add i32 %i, 2\n %x2p = getelementptr i8, ptr %s, i32 %x2i\n %x2b = load i8, ptr %x2p\n %x2 = zext i8 %x2b to i32\n")
emith(" %x2m = and i32 %x2, 192\n %x2ok = icmp eq i32 %x2m, 128\n br i1 %x2ok, label %do4c, label %bad\n")
emith("do4c:\n %x3i = add i32 %i, 3\n %x3p = getelementptr i8, ptr %s, i32 %x3i\n %x3b = load i8, ptr %x3p\n %x3 = zext i8 %x3b to i32\n")
emith(" %x3m = and i32 %x3, 192\n %x3ok = icmp eq i32 %x3m, 128\n br i1 %x3ok, label %do4d, label %bad\n")
emith("do4d:\n %y0 = and i32 %c, 7\n %y0s = shl i32 %y0, 18\n %y1m = and i32 %x1, 63\n %y1s = shl i32 %y1m, 12\n %y2m = and i32 %x2, 63\n %y2s = shl i32 %y2m, 6\n %y3m = and i32 %x3, 63\n")
emith(" %cp4p = or i32 %y0s, %y1s\n %cp4q = or i32 %cp4p, %y2s\n %cp4 = or i32 %cp4q, %y3m\n")
emith(" %ov4 = icmp ult i32 %cp4, 65536\n br i1 %ov4, label %bad, label %do4e\n")
emith("do4e:\n %hi4 = icmp ugt i32 %cp4, 1114111\n br i1 %hi4, label %bad, label %adv4\n")
emith("adv4:\n %n4 = add i32 %i, 4\n store i32 %n4, ptr %ip\n br label %lp\n")
emith("good:\n ret i32 1\n}\n")
}
# ---- case mapping + encode helpers -----------------------------------------
function emit_uni_case_fns() -> void {
# uppercase one code point: ASCII a-z and Latin-1 a-with-diacritic .. thorn
# (0xE0..0xFE except 0xF7). Other code points pass through unchanged (v1).
emith("define i32 @fn_uni_upcp(i32 %c) {\n")
emith(" %la = icmp uge i32 %c, 97\n %lb = icmp ule i32 %c, 122\n %asc = and i1 %la, %lb\n")
emith(" %da = icmp uge i32 %c, 224\n %db = icmp ule i32 %c, 254\n %dd = icmp ne i32 %c, 247\n %d1 = and i1 %da, %db\n %d2 = and i1 %d1, %dd\n")
emith(" %map = or i1 %asc, %d2\n %up = sub i32 %c, 32\n %r = select i1 %map, i32 %up, i32 %c\n ret i32 %r\n}\n")
# lowercase one code point: ASCII A-Z and Latin-1 A-with-diacritic .. Thorn
# (0xC0..0xDE except 0xD7). Other code points pass through unchanged (v1).
emith("define i32 @fn_uni_locp(i32 %c) {\n")
emith(" %ua = icmp uge i32 %c, 65\n %ub = icmp ule i32 %c, 90\n %asc = and i1 %ua, %ub\n")
emith(" %da = icmp uge i32 %c, 192\n %db = icmp ule i32 %c, 222\n %dd = icmp ne i32 %c, 215\n %d1 = and i1 %da, %db\n %d2 = and i1 %d1, %dd\n")
emith(" %map = or i1 %asc, %d2\n %lo = add i32 %c, 32\n %r = select i1 %map, i32 %lo, i32 %c\n ret i32 %r\n}\n")
# UTF-8 byte width needed to encode a code point
emith("define i32 @fn_uni_cpwidth(i32 %cp) {\n")
emith(" %a = icmp ult i32 %cp, 128\n %b = icmp ult i32 %cp, 2048\n %c = icmp ult i32 %cp, 65536\n")
emith(" %w34 = select i1 %c, i32 3, i32 4\n %w234 = select i1 %b, i32 2, i32 %w34\n %w = select i1 %a, i32 1, i32 %w234\n ret i32 %w\n}\n")
# encode a code point into %dst at byte offset %off; return the new offset
emith("define i32 @fn_uni_encode(ptr %dst, i32 %off, i32 %cp) {\n")
emith("entry:\n %w = call i32 @fn_uni_cpwidth(i32 %cp)\n %is1 = icmp eq i32 %w, 1\n br i1 %is1, label %e1, label %k2\n")
emith("e1:\n %d0 = getelementptr i8, ptr %dst, i32 %off\n %b0 = trunc i32 %cp to i8\n store i8 %b0, ptr %d0\n %o1 = add i32 %off, 1\n ret i32 %o1\n")
emith("k2:\n %is2 = icmp eq i32 %w, 2\n br i1 %is2, label %e2, label %k3\n")
emith("e2:\n %hi2 = lshr i32 %cp, 6\n %by0 = or i32 %hi2, 192\n %lo2 = and i32 %cp, 63\n %by1 = or i32 %lo2, 128\n")
emith(" %p20 = getelementptr i8, ptr %dst, i32 %off\n %t20 = trunc i32 %by0 to i8\n store i8 %t20, ptr %p20\n")
emith(" %o21 = add i32 %off, 1\n %p21 = getelementptr i8, ptr %dst, i32 %o21\n %t21 = trunc i32 %by1 to i8\n store i8 %t21, ptr %p21\n %o2 = add i32 %off, 2\n ret i32 %o2\n")
emith("k3:\n %is3 = icmp eq i32 %w, 3\n br i1 %is3, label %e3, label %e4\n")
emith("e3:\n %hi3 = lshr i32 %cp, 12\n %by30 = or i32 %hi3, 224\n %m31 = lshr i32 %cp, 6\n %m31b = and i32 %m31, 63\n %by31 = or i32 %m31b, 128\n %m32 = and i32 %cp, 63\n %by32 = or i32 %m32, 128\n")
emith(" %p30 = getelementptr i8, ptr %dst, i32 %off\n %t30 = trunc i32 %by30 to i8\n store i8 %t30, ptr %p30\n")
emith(" %o31 = add i32 %off, 1\n %p31 = getelementptr i8, ptr %dst, i32 %o31\n %t31 = trunc i32 %by31 to i8\n store i8 %t31, ptr %p31\n")
emith(" %o32 = add i32 %off, 2\n %p32 = getelementptr i8, ptr %dst, i32 %o32\n %t32 = trunc i32 %by32 to i8\n store i8 %t32, ptr %p32\n %o3 = add i32 %off, 3\n ret i32 %o3\n")
emith("e4:\n %hi4 = lshr i32 %cp, 18\n %by40 = or i32 %hi4, 240\n %m41 = lshr i32 %cp, 12\n %m41b = and i32 %m41, 63\n %by41 = or i32 %m41b, 128\n")
emith(" %m42 = lshr i32 %cp, 6\n %m42b = and i32 %m42, 63\n %by42 = or i32 %m42b, 128\n %m43 = and i32 %cp, 63\n %by43 = or i32 %m43, 128\n")
emith(" %p40 = getelementptr i8, ptr %dst, i32 %off\n %t40 = trunc i32 %by40 to i8\n store i8 %t40, ptr %p40\n")
emith(" %o41 = add i32 %off, 1\n %p41 = getelementptr i8, ptr %dst, i32 %o41\n %t41 = trunc i32 %by41 to i8\n store i8 %t41, ptr %p41\n")
emith(" %o42 = add i32 %off, 2\n %p42 = getelementptr i8, ptr %dst, i32 %o42\n %t42 = trunc i32 %by42 to i8\n store i8 %t42, ptr %p42\n")
emith(" %o43 = add i32 %off, 3\n %p43 = getelementptr i8, ptr %dst, i32 %o43\n %t43 = trunc i32 %by43 to i8\n store i8 %t43, ptr %p43\n %o4 = add i32 %off, 4\n ret i32 %o4\n}\n")
# is this code point a grapheme "extend" (a combining mark or variation
# selector that joins the preceding base)? An approximation of the common
# ranges; ZWJ and regional indicators are handled by grapheme_len itself.
emith("define i32 @fn_uni_is_extend(i32 %c) {\n")
emith(" %r1 = call i32 @fn_uni_inrange(i32 %c, i32 768, i32 879)\n") # 0300-036F combining diacritics
emith(" %r2 = call i32 @fn_uni_inrange(i32 %c, i32 6832, i32 6911)\n") # 1AB0-1AFF
emith(" %r3 = call i32 @fn_uni_inrange(i32 %c, i32 7616, i32 7679)\n") # 1DC0-1DFF
emith(" %r4 = call i32 @fn_uni_inrange(i32 %c, i32 8400, i32 8447)\n") # 20D0-20FF
emith(" %r5 = call i32 @fn_uni_inrange(i32 %c, i32 65056, i32 65071)\n") # FE20-FE2F
emith(" %r6 = call i32 @fn_uni_inrange(i32 %c, i32 65024, i32 65039)\n") # FE00-FE0F variation selectors
emith(" %r7 = call i32 @fn_uni_inrange(i32 %c, i32 917760, i32 917999)\n") # E0100-E01EF
emith(" %o1 = or i32 %r1, %r2\n %o2 = or i32 %o1, %r3\n %o3 = or i32 %o2, %r4\n %o4 = or i32 %o3, %r5\n %o5 = or i32 %o4, %r6\n %o6 = or i32 %o5, %r7\n ret i32 %o6\n}\n")
emith("define i32 @fn_uni_inrange(i32 %c, i32 %lo, i32 %hi) {\n")
emith(" %a = icmp uge i32 %c, %lo\n %b = icmp ule i32 %c, %hi\n %x = and i1 %a, %b\n %r = zext i1 %x to i32\n ret i32 %r\n}\n")
}
# ---- code-point access + string builders -----------------------------------
function emit_uni_str_fns() -> void {
# the idx-th code point, or -1 when idx is past the end
emith("define i32 @fn_uni_char_at(ptr %s, i32 %idx) {\n")
emith("entry:\n %ip = alloca i32\n %kp = alloca i32\n %cp = alloca i32\n store i32 0, ptr %ip\n store i32 0, ptr %kp\n br label %lp\n")
emith("lp:\n %i = load i32, ptr %ip\n %p = getelementptr i8, ptr %s, i32 %i\n %b = load i8, ptr %p\n %c = zext i8 %b to i32\n")
emith(" %z = icmp eq i32 %c, 0\n br i1 %z, label %none, label %go\n")
emith("go:\n %ni = call i32 @fn_uni_decode(ptr %s, i32 %i, ptr %cp)\n %k = load i32, ptr %kp\n %hit = icmp eq i32 %k, %idx\n br i1 %hit, label %found, label %next\n")
emith("found:\n %v = load i32, ptr %cp\n ret i32 %v\n")
emith("next:\n %k1 = add i32 %k, 1\n store i32 %k1, ptr %kp\n store i32 %ni, ptr %ip\n br label %lp\n")
emith("none:\n ret i32 -1\n}\n")
# chars(s) -> []int : a fresh %LSlice of every code point, in order
emith("define ptr @fn_uni_chars(ptr %s) {\n")
emith("entry:\n %n = call i32 @fn_uni_len(ptr %s)\n %h = call ptr @malloc(i64 16)\n")
emith(" %nz = zext i32 %n to i64\n %bytes = mul i64 %nz, 4\n %data = call ptr @malloc(i64 %bytes)\n")
emith(" %d0 = getelementptr inbounds %LSlice, ptr %h, i32 0, i32 0\n store ptr %data, ptr %d0\n")
emith(" %d1 = getelementptr inbounds %LSlice, ptr %h, i32 0, i32 1\n store i32 %n, ptr %d1\n")
emith(" %d2 = getelementptr inbounds %LSlice, ptr %h, i32 0, i32 2\n store i32 %n, ptr %d2\n")
emith(" %ip = alloca i32\n %kp = alloca i32\n %cp = alloca i32\n store i32 0, ptr %ip\n store i32 0, ptr %kp\n br label %lp\n")
emith("lp:\n %i = load i32, ptr %ip\n %p = getelementptr i8, ptr %s, i32 %i\n %b = load i8, ptr %p\n %c = zext i8 %b to i32\n")
emith(" %z = icmp eq i32 %c, 0\n br i1 %z, label %done, label %go\n")
emith("go:\n %ni = call i32 @fn_uni_decode(ptr %s, i32 %i, ptr %cp)\n %v = load i32, ptr %cp\n %k = load i32, ptr %kp\n")
emith(" %slot = getelementptr i32, ptr %data, i32 %k\n store i32 %v, ptr %slot\n %k1 = add i32 %k, 1\n store i32 %k1, ptr %kp\n store i32 %ni, ptr %ip\n br label %lp\n")
emith("done:\n ret ptr %h\n}\n")
# case map the whole string. %up != 0 -> uppercase, else lowercase. Decodes,
# maps each code point, and re-encodes into a fresh buffer (worst case 4 bytes
# per code point, though ASCII/Latin-1 mapping preserves byte length).
emith("define ptr @fn_uni_case(ptr %s, i32 %up) {\n")
emith("entry:\n %bl = call i64 @strlen(ptr %s)\n %cap0 = mul i64 %bl, 4\n %cap = add i64 %cap0, 4\n %out = call ptr @malloc(i64 %cap)\n")
emith(" %ip = alloca i32\n %op = alloca i32\n %cp = alloca i32\n store i32 0, ptr %ip\n store i32 0, ptr %op\n br label %lp\n")
emith("lp:\n %i = load i32, ptr %ip\n %p = getelementptr i8, ptr %s, i32 %i\n %b = load i8, ptr %p\n %c = zext i8 %b to i32\n")
emith(" %z = icmp eq i32 %c, 0\n br i1 %z, label %done, label %go\n")
emith("go:\n %ni = call i32 @fn_uni_decode(ptr %s, i32 %i, ptr %cp)\n %v = load i32, ptr %cp\n")
emith(" %mu = call i32 @fn_uni_upcp(i32 %v)\n %ml = call i32 @fn_uni_locp(i32 %v)\n %isup = icmp ne i32 %up, 0\n %m = select i1 %isup, i32 %mu, i32 %ml\n")
emith(" %o = load i32, ptr %op\n %no = call i32 @fn_uni_encode(ptr %out, i32 %o, i32 %m)\n store i32 %no, ptr %op\n store i32 %ni, ptr %ip\n br label %lp\n")
emith("done:\n %fo = load i32, ptr %op\n %endp = getelementptr i8, ptr %out, i32 %fo\n store i8 0, ptr %endp\n ret ptr %out\n}\n")
# truncate(s, n) -> the first n code points as a fresh string (never splits a
# multibyte character). n <= 0 yields the empty string.
emith("define ptr @fn_uni_truncate(ptr %s, i32 %n) {\n")
emith("entry:\n %ip = alloca i32\n %kp = alloca i32\n %cp = alloca i32\n store i32 0, ptr %ip\n store i32 0, ptr %kp\n br label %lp\n")
emith("lp:\n %k = load i32, ptr %kp\n %enough = icmp sge i32 %k, %n\n br i1 %enough, label %cut, label %chk\n")
emith("chk:\n %i = load i32, ptr %ip\n %p = getelementptr i8, ptr %s, i32 %i\n %b = load i8, ptr %p\n %c = zext i8 %b to i32\n")
emith(" %z = icmp eq i32 %c, 0\n br i1 %z, label %cut, label %go\n")
emith("go:\n %ni = call i32 @fn_uni_decode(ptr %s, i32 %i, ptr %cp)\n %k1 = add i32 %k, 1\n store i32 %k1, ptr %kp\n store i32 %ni, ptr %ip\n br label %lp\n")
emith("cut:\n %len = load i32, ptr %ip\n %lz = zext i32 %len to i64\n %cap = add i64 %lz, 1\n %out = call ptr @malloc(i64 %cap)\n %lz2 = zext i32 %len to i64\n call ptr @memcpy(ptr %out, ptr %s, i64 %lz2)\n")
emith(" %endp = getelementptr i8, ptr %out, i32 %len\n store i8 0, ptr %endp\n ret ptr %out\n}\n")
emit_uni_grapheme()
}
# grapheme_len: an approximate UAX#29 count. A new cluster starts on each code
# point except: a combining/variation "extend"; the code point after a ZWJ (so
# ZWJ emoji sequences count as one); and the second regional indicator of a flag
# pair. Simple state carried in allocas.
function emit_uni_grapheme() -> void {
emith("define i32 @fn_uni_grapheme_len(ptr %s) {\n")
emith("entry:\n %ip = alloca i32\n %np = alloca i32\n %zp = alloca i32\n %rp = alloca i32\n %cp = alloca i32\n")
emith(" store i32 0, ptr %ip\n store i32 0, ptr %np\n store i32 0, ptr %zp\n store i32 0, ptr %rp\n br label %lp\n")
emith("lp:\n %i = load i32, ptr %ip\n %p = getelementptr i8, ptr %s, i32 %i\n %b = load i8, ptr %p\n %c = zext i8 %b to i32\n")
emith(" %z = icmp eq i32 %c, 0\n br i1 %z, label %done, label %go\n")
emith("go:\n %ni = call i32 @fn_uni_decode(ptr %s, i32 %i, ptr %cp)\n %v = load i32, ptr %cp\n store i32 %ni, ptr %ip\n")
# ZWJ (U+200D): extends the cluster and arms the join for the next code point
emith(" %iszwj = icmp eq i32 %v, 8205\n br i1 %iszwj, label %zwj, label %notzwj\n")
emith("zwj:\n %n0 = load i32, ptr %np\n %n0z = icmp eq i32 %n0, 0\n %n0b = zext i1 %n0z to i32\n %n0n = add i32 %n0, %n0b\n store i32 %n0n, ptr %np\n") # leading ZWJ still opens one cluster
emith(" store i32 1, ptr %zp\n store i32 0, ptr %rp\n br label %lp\n")
emith("notzwj:\n %n = load i32, ptr %np\n %first = icmp eq i32 %n, 0\n br i1 %first, label %open, label %cont\n")
# first cluster
emith("open:\n store i32 1, ptr %np\n store i32 0, ptr %zp\n %ri0 = call i32 @fn_uni_inrange(i32 %v, i32 127462, i32 127487)\n store i32 %ri0, ptr %rp\n br label %lp\n")
emith("cont:\n %zj = load i32, ptr %zp\n %afterz = icmp ne i32 %zj, 0\n br i1 %afterz, label %joinz, label %chkext\n")
# code point right after a ZWJ joins the current cluster
emith("joinz:\n store i32 0, ptr %zp\n store i32 0, ptr %rp\n br label %lp\n")
emith("chkext:\n %ext = call i32 @fn_uni_is_extend(i32 %v)\n %isext = icmp ne i32 %ext, 0\n br i1 %isext, label %joinext, label %chkri\n")
emith("joinext:\n store i32 0, ptr %rp\n br label %lp\n")
# regional indicator: joins only as the second of a pair
emith("chkri:\n %ri = call i32 @fn_uni_inrange(i32 %v, i32 127462, i32 127487)\n %isri = icmp ne i32 %ri, 0\n %ropen = load i32, ptr %rp\n %ropenb = icmp ne i32 %ropen, 0\n %pair = and i1 %isri, %ropenb\n br i1 %pair, label %joinri, label %newcl\n")
emith("joinri:\n store i32 0, ptr %rp\n br label %lp\n")
emith("newcl:\n %nn = load i32, ptr %np\n %nn1 = add i32 %nn, 1\n store i32 %nn1, ptr %np\n store i32 0, ptr %zp\n %riset = select i1 %isri, i32 1, i32 0\n store i32 %riset, ptr %rp\n br label %lp\n")
emith("done:\n %r = load i32, ptr %np\n ret i32 %r\n}\n")
}
Reference in a new issue View git blame Copy permalink