# ============================================================================ # regex.ludic — a regular-expression engine, written in Ludic. # # PCRE/PECL-compatible syntax over a LINEAR-TIME Thompson NFA (a Pike VM with # capture slots), so a bad pattern from a modder can never cause catastrophic # backtracking — `(a+)+$` on non-matching input is O(n·m), not exponential. The # pattern compiles to a small bytecode program (an unanchored lazy `.*?` prefix # makes a plain search match anywhere); the VM runs every alive thread in # lockstep per input byte, deduped by program counter so work stays bounded. # # Supported: literals, `.`, character classes `[...]` (ranges, negation, and the # \d \w \s \D \W \S shorthands), anchors `^` `$`, alternation `|`, capturing # `(...)` and non-capturing `(?:...)` groups, and the quantifiers `* + ?` and # `{n} {n,} {n,m}` in greedy or lazy (`?`-suffixed) form, plus the common # escapes. Numbered capture groups. Out of scope for a linear engine (and so # unsupported): backreferences and look-around. On the degenerate case of a # nullable subpattern under an unbounded quantifier (e.g. `(a*)*`), match/ # capture positions may differ from a backtracking engine like Python's — the # price of the linear-time guarantee. # # ludicc splices this file into any program that mentions `Regex.*` (like the # game runtime in core.ludic); it is a fragment, not a `program`/`module` block. # The Regex.* namespace (emit_call.ludic) aliases each method to the matching # `regex_*` function below. # ============================================================================ # ---- small helpers ---------------------------------------------------------- function rx_slen(s: pointer) -> int { var n = 0; while s[n] != 0 { n += 1 }; return n } # ---- growable int vector ---------------------------------------------------- property IVec { d: words, n: int, cap: int } function iv_new() -> IVec { let v = new IVec v.cap = 16; v.d = words(16); v.n = 0 return v } function iv_grow(v: IVec, need: int) -> void { if v.n + need <= v.cap { return } var nc = v.cap * 2 while nc < v.n + need { nc *= 2 } let nd = words(nc) var i = 0 while i < v.n { nd[i] = v.d[i]; i += 1 } v.d = nd; v.cap = nc } function iv_push(v: IVec, x: int) -> int { iv_grow(v, 1) let idx = v.n v.d[v.n] = x v.n += 1 return idx } # ---- instruction opcodes ---------------------------------------------------- const OP_CHAR: int = 1 const OP_ANY: int = 2 # any byte except '\n' const OP_CLASS: int = 3 const OP_MATCH: int = 4 const OP_JMP: int = 5 const OP_SPLIT: int = 6 const OP_SAVE: int = 7 const OP_BOL: int = 8 const OP_EOL: int = 9 const OP_ANYNL: int = 10 # any byte incl '\n' (the .*? search prefix) # ---- AST node types --------------------------------------------------------- const N_LIT: int = 1 const N_ANY: int = 2 const N_CLASS: int = 3 const N_CONCAT: int = 4 const N_ALT: int = 5 const N_STAR: int = 6 const N_PLUS: int = 7 const N_QUEST: int = 8 const N_REP: int = 9 const N_GROUP: int = 10 const N_BOL: int = 11 const N_EOL: int = 12 const N_EMPTY: int = 13 property RNode { op: int = 0, ch: int = 0, cls: int = 0, lo: int = 0, hi: int = 0, greedy: int = 1, gidx: int = -1, kids: []RNode } function rx_node(op: int) -> RNode { let n = new RNode n.op = op n.greedy = 1 n.gidx = -1 n.kids = new []RNode return n } property Prog { code: IVec, cls: IVec, ngroups: int, ok: int } # ---- parser state ----------------------------------------------------------- export state RtRegexState { rx_pat: pointer = "" rx_pos: int = 0 rx_len: int = 0 rx_err: int = 0 rx_ngroup: int = 0 rx_prog: Prog = null rx_named_gidx: int = 0 } function rx_peek(rt_regex_st: RtRegexState) -> int { if rt_regex_st.rx_pos < rt_regex_st.rx_len { return rt_regex_st.rx_pat[rt_regex_st.rx_pos] & 255 }; return -1 } function rx_peek2(rt_regex_st: RtRegexState) -> int { if rt_regex_st.rx_pos + 1 < rt_regex_st.rx_len { return rt_regex_st.rx_pat[rt_regex_st.rx_pos + 1] & 255 }; return -1 } function rx_adv(rt_regex_st: mut RtRegexState) -> int { let c = rx_peek(rt_regex_st); rt_regex_st.rx_pos += 1; return c } # ---- character classes ------------------------------------------------------ # a class is 8 i32 words (256 bits) in prog.cls; class k occupies cls[8k .. 8k+8) function rx_class_new(rt_regex_st: RtRegexState) -> int { let idx = rt_regex_st.rx_prog.cls.n / 8 var i = 0 while i < 8 { iv_push(rt_regex_st.rx_prog.cls, 0); i += 1 } return idx } function rx_class_set(rt_regex_st: mut RtRegexState, idx: int, c: int) -> void { let w = idx * 8 + (c >> 5) rt_regex_st.rx_prog.cls.d[w] = rt_regex_st.rx_prog.cls.d[w] | (1 << (c & 31)) } function rx_class_set_range(rt_regex_st: mut RtRegexState, idx: int, a: int, b: int) -> void { var c = a while c <= b { rx_class_set(rt_regex_st, idx, c); c += 1 } } function rx_class_negate(rt_regex_st: mut RtRegexState, idx: int) -> void { var i = 0 while i < 8 { let w = idx * 8 + i; rt_regex_st.rx_prog.cls.d[w] = ~rt_regex_st.rx_prog.cls.d[w]; i += 1 } } function rx_class_unset(rt_regex_st: mut RtRegexState, idx: int, c: int) -> void { let w = idx * 8 + (c >> 5) rt_regex_st.rx_prog.cls.d[w] = rt_regex_st.rx_prog.cls.d[w] & ~(1 << (c & 31)) } function rx_class_unset_range(rt_regex_st: mut RtRegexState, idx: int, a: int, b: int) -> void { var c = a while c <= b { rx_class_unset(rt_regex_st, idx, c); c += 1 } } function rx_class_set_all(rt_regex_st: mut RtRegexState, idx: int) -> void { var i = 0 while i < 8 { rt_regex_st.rx_prog.cls.d[idx * 8 + i] = -1; i += 1 } } function rx_class_set_word(rt_regex_st: mut RtRegexState, idx: int) -> void { rx_class_set_range(rt_regex_st, idx, 48, 57) rx_class_set_range(rt_regex_st, idx, 65, 90) rx_class_set_range(rt_regex_st, idx, 97, 122) rx_class_set(rt_regex_st, idx, 95) } function rx_class_set_ws(rt_regex_st: mut RtRegexState, idx: int) -> void { rx_class_set(rt_regex_st, idx, 32); rx_class_set(rt_regex_st, idx, 9); rx_class_set(rt_regex_st, idx, 10) rx_class_set(rt_regex_st, idx, 13); rx_class_set(rt_regex_st, idx, 12); rx_class_set(rt_regex_st, idx, 11) } function rx_class_has(prog: Prog, idx: int, c: int) -> bool { let w = prog.cls.d[idx * 8 + (c >> 5)] return (w & (1 << (c & 31))) != 0 } function rx_is_digit(c: int) -> bool { return c >= '0' and c <= '9' } function rx_is_word(c: int) -> bool { return (c >= '0' and c <= '9') or (c >= 'A' and c <= 'Z') or (c >= 'a' and c <= 'z') or c == '_' } function rx_is_ws(c: int) -> bool { return c == ' ' or c == '\t' or c == '\n' or c == '\r' or c == 12 or c == 11 } # OR the set named by \d \D \w \W \s \S into class idx. The negated forms OR in # the complement set bit-by-bit (never via set-all+unset, which would clobber a # previously-set member — e.g. the literal `1` in [1\Da] must survive \D). function rx_class_add_pre(rt_regex_st: mut RtRegexState, idx: int, kind: int) -> void { if kind == 100 { rx_class_set_range(rt_regex_st, idx, 48, 57); return } # \d if kind == 119 { rx_class_set_word(rt_regex_st, idx); return } # \w if kind == 115 { rx_class_set_ws(rt_regex_st, idx); return } # \s var c = 0 while c < 256 { if kind == 68 { if not rx_is_digit(c) { rx_class_set(rt_regex_st, idx, c) } } # \D else if kind == 87 { if not rx_is_word(c) { rx_class_set(rt_regex_st, idx, c) } } # \W else if kind == 83 { if not rx_is_ws(c) { rx_class_set(rt_regex_st, idx, c) } } # \S c += 1 } } # parse a [...] class starting at '['; returns an N_CLASS node function rx_parse_class(rt_regex_st: mut RtRegexState) -> RNode { rx_adv(rt_regex_st) # consume '[' let idx = rx_class_new(rt_regex_st) var neg = false if rx_peek(rt_regex_st) == 94 { neg = true; rx_adv(rt_regex_st) } # [^ ...] # a ']' as the first char is a literal if rx_peek(rt_regex_st) == 93 { rx_class_set(rt_regex_st, idx, 93); rx_adv(rt_regex_st) } while rx_peek(rt_regex_st) != 93 and rx_peek(rt_regex_st) >= 0 { var lo = rx_adv(rt_regex_st) if lo == 92 { # escape inside class let e = rx_adv(rt_regex_st) if e == 'd' or e == 'D' or e == 'w' or e == 'W' or e == 's' or e == 'S' { rx_class_add_pre(rt_regex_st, idx, e) continue } lo = rx_class_escape_char(e) } # a range a-b (but a trailing '-' before ']' is literal) if rx_peek(rt_regex_st) == 45 and rx_peek2(rt_regex_st) != 93 and rx_peek2(rt_regex_st) >= 0 { rx_adv(rt_regex_st) # consume '-' var hi = rx_adv(rt_regex_st) if hi == 92 { hi = rx_class_escape_char(rx_adv(rt_regex_st)) } rx_class_set_range(rt_regex_st, idx, lo, hi) } else { rx_class_set(rt_regex_st, idx, lo) } } if rx_peek(rt_regex_st) != 93 { rt_regex_st.rx_err = 1 } else { rx_adv(rt_regex_st) } # consume ']' if neg { rx_class_negate(rt_regex_st, idx) } let node = rx_node(N_CLASS) node.cls = idx return node } # map an escaped char inside a class to its byte function rx_class_escape_char(e: int) -> int { if e == 'n' { return 10 } if e == 't' { return 9 } if e == 'r' { return 13 } if e == 'f' { return 12 } if e == 'v' { return 11 } if e == '0' { return 0 } return e } # ---- escapes outside a class ------------------------------------------------ function rx_parse_escape(rt_regex_st: mut RtRegexState) -> RNode { rx_adv(rt_regex_st) # consume '\' let e = rx_adv(rt_regex_st) if e == 'd' or e == 'D' or e == 'w' or e == 'W' or e == 's' or e == 'S' { let idx = rx_class_new(rt_regex_st) rx_class_add_pre(rt_regex_st, idx, e) let node = rx_node(N_CLASS) node.cls = idx return node } if e >= '1' and e <= '9' { rt_regex_st.rx_err = 1; return rx_node(N_EMPTY) } # backrefs unsupported var ch = e if e == 'n' { ch = 10 } else if e == 't' { ch = 9 } else if e == 'r' { ch = 13 } else if e == 'f' { ch = 12 } else if e == 'v' { ch = 11 } else if e == '0' { ch = 0 } else if e < 0 { rt_regex_st.rx_err = 1; return rx_node(N_EMPTY) } let node = rx_node(N_LIT) node.ch = ch return node } # ---- parser ----------------------------------------------------------------- function rx_parse_alt(rt_regex_st: mut RtRegexState) -> RNode { let first = rx_parse_concat(rt_regex_st) if rx_peek(rt_regex_st) != 124 { return first } let alt = rx_node(N_ALT) push(alt.kids, first) while rx_peek(rt_regex_st) == 124 { rx_adv(rt_regex_st) push(alt.kids, rx_parse_concat(rt_regex_st)) if rt_regex_st.rx_err == 1 { break } } return alt } function rx_parse_concat(rt_regex_st: mut RtRegexState) -> RNode { let cat = rx_node(N_CONCAT) while true { let c = rx_peek(rt_regex_st) if c < 0 or c == '|' or c == ')' { break } push(cat.kids, rx_parse_repeat(rt_regex_st)) if rt_regex_st.rx_err == 1 { break } } if len(cat.kids) == 1 { return cat.kids[0] } if len(cat.kids) == 0 { return rx_node(N_EMPTY) } return cat } # read the optional lazy '?' after a quantifier; returns greedy flag (0 = lazy) function rx_lazy(rt_regex_st: mut RtRegexState) -> int { if rx_peek(rt_regex_st) == 63 { rx_adv(rt_regex_st); return 0 } return 1 } function rx_parse_repeat(rt_regex_st: mut RtRegexState) -> RNode { let atom = rx_parse_atom(rt_regex_st) if rt_regex_st.rx_err == 1 { return atom } let c = rx_peek(rt_regex_st) if c == '*' or c == '+' or c == '?' { rx_adv(rt_regex_st) var op = N_STAR if c == '+' { op = N_PLUS } if c == '?' { op = N_QUEST } let r = rx_node(op) r.greedy = rx_lazy(rt_regex_st) push(r.kids, atom) return r } if c == '{' { return rx_parse_brace(rt_regex_st, atom) } return atom } # {n} {n,} {n,m} function rx_parse_brace(rt_regex_st: mut RtRegexState, atom: RNode) -> RNode { let save = rt_regex_st.rx_pos rx_adv(rt_regex_st) # consume '{' var lo = 0 var haslo = false while rx_peek(rt_regex_st) >= 48 and rx_peek(rt_regex_st) <= 57 { lo = lo * 10 + (rx_adv(rt_regex_st) - 48); haslo = true } var hi = lo var hasComma = false if rx_peek(rt_regex_st) == 44 { hasComma = true; rx_adv(rt_regex_st); hi = -1 var hashi = false while rx_peek(rt_regex_st) >= 48 and rx_peek(rt_regex_st) <= 57 { if not hashi { hi = 0 }; hi = hi * 10 + (rx_adv(rt_regex_st) - 48); hashi = true } } if rx_peek(rt_regex_st) != 125 or not haslo { # not a valid brace -> literal '{' rt_regex_st.rx_pos = save rx_adv(rt_regex_st) let n = rx_node(N_LIT); n.ch = 123; return n } rx_adv(rt_regex_st) # consume '}' let r = rx_node(N_REP) r.lo = lo r.hi = hi r.greedy = rx_lazy(rt_regex_st) push(r.kids, atom) return r } function rx_parse_atom(rt_regex_st: mut RtRegexState) -> RNode { let c = rx_peek(rt_regex_st) if c == '(' { # '(' rx_adv(rt_regex_st) var gidx = -1 if rx_peek(rt_regex_st) == 63 { # (? ... rx_adv(rt_regex_st) let d = rx_peek(rt_regex_st) if d == ':' { rx_adv(rt_regex_st) } # (?: non-capturing else if d == 'P' { rx_adv(rt_regex_st); rx_skip_name(rt_regex_st) } # (?P (captured, name ignored for now) else if d == '<' { rx_skip_name(rt_regex_st) } # (? else { rt_regex_st.rx_err = 1; return rx_node(N_EMPTY) } } else { rt_regex_st.rx_ngroup += 1 gidx = rt_regex_st.rx_ngroup } let inner = rx_parse_alt(rt_regex_st) if rx_peek(rt_regex_st) != 41 { rt_regex_st.rx_err = 1; return rx_node(N_EMPTY) } rx_adv(rt_regex_st) # consume ')' let g = rx_node(N_GROUP) g.gidx = gidx push(g.kids, inner) return g } if c == '[' { return rx_parse_class(rt_regex_st) } if c == '.' { rx_adv(rt_regex_st); return rx_node(N_ANY) } if c == '^' { rx_adv(rt_regex_st); return rx_node(N_BOL) } if c == '$' { rx_adv(rt_regex_st); return rx_node(N_EOL) } if c == '\\' { return rx_parse_escape(rt_regex_st) } if c == '*' or c == '+' or c == '?' or c == ')' { rt_regex_st.rx_err = 1; return rx_node(N_EMPTY) } if c < 0 { rt_regex_st.rx_err = 1; return rx_node(N_EMPTY) } rx_adv(rt_regex_st) let n = rx_node(N_LIT) n.ch = c return n } # skip a (?P / (? group name up to '>'; leaves a capturing group function rx_skip_name(rt_regex_st: mut RtRegexState) -> void { if rx_peek(rt_regex_st) == 80 { rx_adv(rt_regex_st) } # already consumed by caller in (?P case? guard if rx_peek(rt_regex_st) == 60 { rx_adv(rt_regex_st) } # consume '<' while rx_peek(rt_regex_st) != 62 and rx_peek(rt_regex_st) >= 0 { rx_adv(rt_regex_st) } if rx_peek(rt_regex_st) == 62 { rx_adv(rt_regex_st) } # consume '>' rt_regex_st.rx_ngroup += 1 # note: the caller set gidx = -1; fix it up to a real capture index rt_regex_st.rx_named_gidx = rt_regex_st.rx_ngroup } # ---- compile AST -> program ------------------------------------------------- function pg_emit(rt_regex_st: RtRegexState, op: int, a: int, b: int) -> int { let pc = rt_regex_st.rx_prog.code.n / 3 iv_push(rt_regex_st.rx_prog.code, op) iv_push(rt_regex_st.rx_prog.code, a) iv_push(rt_regex_st.rx_prog.code, b) return pc } function pg_set_a(rt_regex_st: mut RtRegexState, pc: int, a: int) -> void { rt_regex_st.rx_prog.code.d[3 * pc + 1] = a } function pg_set_b(rt_regex_st: mut RtRegexState, pc: int, b: int) -> void { rt_regex_st.rx_prog.code.d[3 * pc + 2] = b } function pg_pc(rt_regex_st: RtRegexState) -> int { return rt_regex_st.rx_prog.code.n / 3 } function rx_compile(rt_regex_st: mut RtRegexState, node: RNode) -> void { let op = node.op if op == N_EMPTY { return } if op == N_LIT { pg_emit(rt_regex_st, OP_CHAR, node.ch, 0); return } if op == N_ANY { pg_emit(rt_regex_st, OP_ANY, 0, 0); return } if op == N_CLASS { pg_emit(rt_regex_st, OP_CLASS, node.cls, 0); return } if op == N_BOL { pg_emit(rt_regex_st, OP_BOL, 0, 0); return } if op == N_EOL { pg_emit(rt_regex_st, OP_EOL, 0, 0); return } if op == N_CONCAT { var i = 0 while i < len(node.kids) { rx_compile(rt_regex_st, node.kids[i]); i += 1 } return } if op == N_GROUP { if node.gidx >= 0 { pg_emit(rt_regex_st, OP_SAVE, 2 * node.gidx, 0) rx_compile(rt_regex_st, node.kids[0]) pg_emit(rt_regex_st, OP_SAVE, 2 * node.gidx + 1, 0) } else { rx_compile(rt_regex_st, node.kids[0]) } return } if op == N_ALT { let jmps = iv_new() var i = 0 while i < len(node.kids) { if i < len(node.kids) - 1 { let sp = pg_emit(rt_regex_st, OP_SPLIT, 0, 0) pg_set_a(rt_regex_st, sp, pg_pc(rt_regex_st)) rx_compile(rt_regex_st, node.kids[i]) let j = pg_emit(rt_regex_st, OP_JMP, 0, 0) iv_push(jmps, j) pg_set_b(rt_regex_st, sp, pg_pc(rt_regex_st)) } else { rx_compile(rt_regex_st, node.kids[i]) } i += 1 } let end = pg_pc(rt_regex_st) i = 0 while i < jmps.n { pg_set_a(rt_regex_st, jmps.d[i], end); i += 1 } return } if op == N_STAR { let l1 = pg_pc(rt_regex_st) let sp = pg_emit(rt_regex_st, OP_SPLIT, 0, 0) let l2 = pg_pc(rt_regex_st) rx_compile(rt_regex_st, node.kids[0]) pg_emit(rt_regex_st, OP_JMP, l1, 0) let l3 = pg_pc(rt_regex_st) if node.greedy == 1 { pg_set_a(rt_regex_st, sp, l2); pg_set_b(rt_regex_st, sp, l3) } else { pg_set_a(rt_regex_st, sp, l3); pg_set_b(rt_regex_st, sp, l2) } return } if op == N_PLUS { let l1 = pg_pc(rt_regex_st) rx_compile(rt_regex_st, node.kids[0]) let sp = pg_emit(rt_regex_st, OP_SPLIT, 0, 0) let l3 = pg_pc(rt_regex_st) if node.greedy == 1 { pg_set_a(rt_regex_st, sp, l1); pg_set_b(rt_regex_st, sp, l3) } else { pg_set_a(rt_regex_st, sp, l3); pg_set_b(rt_regex_st, sp, l1) } return } if op == N_QUEST { let sp = pg_emit(rt_regex_st, OP_SPLIT, 0, 0) let l2 = pg_pc(rt_regex_st) rx_compile(rt_regex_st, node.kids[0]) let l3 = pg_pc(rt_regex_st) if node.greedy == 1 { pg_set_a(rt_regex_st, sp, l2); pg_set_b(rt_regex_st, sp, l3) } else { pg_set_a(rt_regex_st, sp, l3); pg_set_b(rt_regex_st, sp, l2) } return } if op == N_REP { let kid = node.kids[0] var i = 0 while i < node.lo { rx_compile(rt_regex_st, kid); i += 1 } if node.hi < 0 { let st = rx_node(N_STAR) st.greedy = node.greedy push(st.kids, kid) rx_compile(rt_regex_st, st) } else { i = 0 while i < node.hi - node.lo { let q = rx_node(N_QUEST) q.greedy = node.greedy push(q.kids, kid) rx_compile(rt_regex_st, q) i += 1 } } return } } # regex_compile(pattern) -> Prog (null on a syntax error) function regex_compile(rt_regex_st: mut RtRegexState, pattern: pointer) -> Prog { let p = new Prog p.code = iv_new() p.cls = iv_new() p.ngroups = 0 p.ok = 1 rt_regex_st.rx_prog = p rt_regex_st.rx_pat = pattern rt_regex_st.rx_pos = 0 rt_regex_st.rx_len = rx_slen(pattern) rt_regex_st.rx_err = 0 rt_regex_st.rx_ngroup = 0 let root = rx_parse_alt(rt_regex_st) if rt_regex_st.rx_err == 1 or rt_regex_st.rx_pos != rt_regex_st.rx_len { return null } p.ngroups = rt_regex_st.rx_ngroup # unanchored lazy .*? prefix so a match may start at any position let sp0 = pg_emit(rt_regex_st, OP_SPLIT, 0, 0) let consume = pg_pc(rt_regex_st) pg_emit(rt_regex_st, OP_ANYNL, 0, 0) pg_emit(rt_regex_st, OP_JMP, sp0, 0) let body = pg_pc(rt_regex_st) pg_set_a(rt_regex_st, sp0, body) pg_set_b(rt_regex_st, sp0, consume) pg_emit(rt_regex_st, OP_SAVE, 0, 0) rx_compile(rt_regex_st, root) pg_emit(rt_regex_st, OP_SAVE, 1, 0) pg_emit(rt_regex_st, OP_MATCH, 0, 0) return p }