Json.parse decodes \uXXXX to UTF-8 - a surrogate pair joined, a lone surrogate or bad hex as U+FFFD - and \t \r \b \f, where it dropped the backslash and kept the hex as text; examples/lang/json_unicode checks it byte by byte
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
313f95cd1f
commit
4376ddf0ec
4 changed files with 97 additions and 0 deletions
|
|
@ -616,6 +616,16 @@ function jp_string_esc(p: JP) -> string {
|
|||
if p.i < p.n {
|
||||
let e = p.s[p.i]
|
||||
if e == 'n' { out += "\n" }
|
||||
else if e == 't' { out += "\t" }
|
||||
else if e == 'r' { out += "\r" }
|
||||
else if e == 'b' or e == 'f' or e == 'u' {
|
||||
var code = 8
|
||||
if e == 'f' { code = 12 }
|
||||
if e == 'u' { code = jp_code(p) } # leaves p.i on the escape's last hex digit
|
||||
let u = jp_utf8(code)
|
||||
out += u
|
||||
free(u)
|
||||
}
|
||||
else { out += p.s[p.i..p.i + 1] } # \" \\ \/ -> the literal char
|
||||
p.i += 1
|
||||
}
|
||||
|
|
@ -627,6 +637,53 @@ function jp_string_esc(p: JP) -> string {
|
|||
return out
|
||||
}
|
||||
|
||||
# \uXXXX, p.i on the 'u': the code point, a surrogate pair joined (\ud83d\ude00 is one emoji); a
|
||||
# lone or broken surrogate, or bad hex, is U+FFFD. Python's json.dump writes non-ASCII this way.
|
||||
function jp_code(p: JP) -> int {
|
||||
let hi = jp_hex4(p, p.i + 1)
|
||||
if hi < 0 { return 65533 } # p.i stays on the 'u': what follows is read as text
|
||||
p.i += 4
|
||||
if hi >= 55296 and hi <= 56319 { # a high surrogate: a low one must follow
|
||||
if p.i + 6 < p.n and p.s[p.i + 1] == '\\' and p.s[p.i + 2] == 'u' {
|
||||
let lo = jp_hex4(p, p.i + 3)
|
||||
if lo >= 56320 and lo <= 57343 {
|
||||
p.i += 6
|
||||
return 65536 + ((hi - 55296) << 10) + (lo - 56320)
|
||||
}
|
||||
}
|
||||
return 65533
|
||||
}
|
||||
if hi >= 56320 and hi <= 57343 { return 65533 } # a low surrogate on its own
|
||||
return hi
|
||||
}
|
||||
# four hex digits at `at`, or -1
|
||||
function jp_hex4(p: JP, at: int) -> int {
|
||||
if at + 4 > p.n { return -1 }
|
||||
var v = 0
|
||||
for k in 0 .. 4 {
|
||||
let c = p.s[at + k]
|
||||
var d = -1
|
||||
if c >= '0' and c <= '9' { d = c - 48 }
|
||||
else if c >= 'a' and c <= 'f' { d = c - 87 }
|
||||
else if c >= 'A' and c <= 'F' { d = c - 55 }
|
||||
if d < 0 { return -1 }
|
||||
v = v * 16 + d
|
||||
}
|
||||
return v
|
||||
}
|
||||
# one code point as a UTF-8 string
|
||||
function jp_utf8(c: int) -> string {
|
||||
let out = bytes(5)
|
||||
var n = 0
|
||||
if c < 128 { out[0] = c; n = 1 }
|
||||
else if c < 2048 { out[0] = 192 | (c >> 6); out[1] = 128 | (c & 63); n = 2 }
|
||||
else if c < 65536 { out[0] = 224 | (c >> 12); out[1] = 128 | ((c >> 6) & 63); out[2] = 128 | (c & 63); n = 3 }
|
||||
else { out[0] = 240 | (c >> 18); out[1] = 128 | ((c >> 12) & 63); out[2] = 128 | ((c >> 6) & 63); out[3] = 128 | (c & 63); n = 4 }
|
||||
out[n] = 0
|
||||
let t: string = out
|
||||
return t
|
||||
}
|
||||
|
||||
# read a number; a '.' makes it a fixed node, otherwise an int node.
|
||||
function jp_number(p: JP) -> Val {
|
||||
var neg = 0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue