Json.parse decodes \uXXXX to UTF-8 - a surrogate pair joined, a lone surrogate or bad hex as U+FFFD - and \t \r \b \f, where it dropped the backslash and kept the hex as text; examples/lang/json_unicode checks it byte by byte

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Orkun ÇAKILKAYA 2026-09-29 21:07:10 +03:00
parent 313f95cd1f
commit 4376ddf0ec
4 changed files with 97 additions and 0 deletions

View file

@ -616,6 +616,16 @@ function jp_string_esc(p: JP) -> string {
if p.i < p.n {
let e = p.s[p.i]
if e == 'n' { out += "\n" }
else if e == 't' { out += "\t" }
else if e == 'r' { out += "\r" }
else if e == 'b' or e == 'f' or e == 'u' {
var code = 8
if e == 'f' { code = 12 }
if e == 'u' { code = jp_code(p) } # leaves p.i on the escape's last hex digit
let u = jp_utf8(code)
out += u
free(u)
}
else { out += p.s[p.i..p.i + 1] } # \" \\ \/ -> the literal char
p.i += 1
}
@ -627,6 +637,53 @@ function jp_string_esc(p: JP) -> string {
return out
}
# \uXXXX, p.i on the 'u': the code point, a surrogate pair joined (\ud83d\ude00 is one emoji); a
# lone or broken surrogate, or bad hex, is U+FFFD. Python's json.dump writes non-ASCII this way.
function jp_code(p: JP) -> int {
let hi = jp_hex4(p, p.i + 1)
if hi < 0 { return 65533 } # p.i stays on the 'u': what follows is read as text
p.i += 4
if hi >= 55296 and hi <= 56319 { # a high surrogate: a low one must follow
if p.i + 6 < p.n and p.s[p.i + 1] == '\\' and p.s[p.i + 2] == 'u' {
let lo = jp_hex4(p, p.i + 3)
if lo >= 56320 and lo <= 57343 {
p.i += 6
return 65536 + ((hi - 55296) << 10) + (lo - 56320)
}
}
return 65533
}
if hi >= 56320 and hi <= 57343 { return 65533 } # a low surrogate on its own
return hi
}
# four hex digits at `at`, or -1
function jp_hex4(p: JP, at: int) -> int {
if at + 4 > p.n { return -1 }
var v = 0
for k in 0 .. 4 {
let c = p.s[at + k]
var d = -1
if c >= '0' and c <= '9' { d = c - 48 }
else if c >= 'a' and c <= 'f' { d = c - 87 }
else if c >= 'A' and c <= 'F' { d = c - 55 }
if d < 0 { return -1 }
v = v * 16 + d
}
return v
}
# one code point as a UTF-8 string
function jp_utf8(c: int) -> string {
let out = bytes(5)
var n = 0
if c < 128 { out[0] = c; n = 1 }
else if c < 2048 { out[0] = 192 | (c >> 6); out[1] = 128 | (c & 63); n = 2 }
else if c < 65536 { out[0] = 224 | (c >> 12); out[1] = 128 | ((c >> 6) & 63); out[2] = 128 | (c & 63); n = 3 }
else { out[0] = 240 | (c >> 18); out[1] = 128 | ((c >> 12) & 63); out[2] = 128 | ((c >> 6) & 63); out[3] = 128 | (c & 63); n = 4 }
out[n] = 0
let t: string = out
return t
}
# read a number; a '.' makes it a fixed node, otherwise an int node.
function jp_number(p: JP) -> Val {
var neg = 0