feat(lang): strict numbers in float files; render3d on float
A numbers float file adapts decimal literals to a fixed operand or slot, and refuses to promote a computed int to a float implicitly: there it is almost always float bits. Explicit float(x) is always allowed. render3d's numbers are float, converted by tools/migrate/floatbits.py - a whole-program inference of which ints carried IEEE bits (union-find over flows, calls, returns, buffers, nested buffers and lexical scopes) and a rewriter to operators, Math.* and float literals, with float_bits / float_from_bits left only where bits really cross (runtime scratch buffers, mixed buffers). Seed regenerated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
3ac0d8d5cb
commit
cc89fc37a4
35 changed files with 57282 additions and 54382 deletions
332
tools/migrate/floatbits.py
Normal file
332
tools/migrate/floatbits.py
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
"""Rewrite IEEE-float-bits-in-an-int code to Ludic's `float`.
|
||||
python3 floatbits.py <game dir> <ludic home> [--dry]
|
||||
Converts <game>/src, <game>/lab and <home>/packages/ludic.render3d; <home>/runtime stays as it is
|
||||
and is where raw bits remain (float_bits / float_from_bits at the calls into it)."""
|
||||
import struct
|
||||
import sys
|
||||
from floatinfer import Program, FLOAT_OPS
|
||||
from ludic_ast import PREC, walk
|
||||
|
||||
BIN = {'f_add': ('+', 9), 'f_sub': ('-', 9), 'f_mul': ('*', 10), 'f_div': ('/', 10), 'f_mod': ('%', 10)}
|
||||
CMP = {'f_ls': ('<', 3), 'f_gt': ('>', 3)}
|
||||
MATH = {'f_abs': 'abs', 'f_min': 'min', 'f_max': 'max', 'f_clamp': 'clamp', 'f_sqrt': 'sqrt', 'f_sin': 'sin',
|
||||
'f_cos': 'cos', 'f_tan': 'tan', 'f_atan2': 'atan2', 'f_floor': 'floor', 'f_pow': 'pow', 'f_exp': 'exp',
|
||||
'f_log': 'log', 'f_lerp': 'lerp', 'f_rad': 'deg_to_rad'}
|
||||
CONST = {'F_ZERO': '0.0', 'F_ONE': '1.0', 'F_TWO': '2.0', 'F_HALF': '0.5', 'F_PI': 'PI'}
|
||||
CONV = {'f_to_int': 'int', 'f_fx': 'fixed', 'f32_to_fx': 'fixed', 'fx_to_f32': 'float', 'f_from_int': 'float'}
|
||||
|
||||
|
||||
def plain(v):
|
||||
"""a float as decimal text Ludic reads: no exponent, the shortest that round-trips as f32"""
|
||||
import math
|
||||
if not math.isfinite(v): return '1000000000.0' # a NaN or infinite sentinel: never equal to a real value
|
||||
for digits in range(1, 12):
|
||||
t = f'{v:.{digits}g}'
|
||||
if struct.unpack('>f', struct.pack('>f', float(t)))[0] == struct.unpack('>f', struct.pack('>f', v))[0]: break
|
||||
from decimal import Decimal
|
||||
t = format(Decimal(t), 'f')
|
||||
return t if '.' in t else t + '.0'
|
||||
|
||||
|
||||
def flit(text, bits=False):
|
||||
"""a literal's text as a float literal; with bits, a big int or hex is a float's bit pattern"""
|
||||
t = text.replace('_', '')
|
||||
if t.startswith('0x') and bits:
|
||||
return plain(struct.unpack('>f', struct.pack('>I', int(t, 16) & 0xffffffff))[0])
|
||||
if t.isdigit() and int(t) > (1 << 20) and bits:
|
||||
return plain(struct.unpack('>f', struct.pack('>I', int(t) & 0xffffffff))[0])
|
||||
if t.startswith('0x'): return plain(float(int(t, 16)))
|
||||
return t if '.' in t else t + '.0'
|
||||
|
||||
|
||||
INT_OPS = ('+', '-', '*', '/', '%', '|', '&', '^', '<<', '>>')
|
||||
|
||||
|
||||
class Rewriter:
|
||||
def __init__(self, prog):
|
||||
self.p = prog
|
||||
uf = prog.uf
|
||||
self.fl = lambda k: k is not None and uf.is_float(k) and not uf.is_int(k)
|
||||
self.to_float_lit = set() # num nodes that stand where a float is
|
||||
self.wrap_bits = set() # nodes whose float value leaves as bits
|
||||
self.wrap_from = set() # nodes whose bits arrive as a float
|
||||
self.plan()
|
||||
|
||||
# ---- where literals and bits cross --------------------------------------
|
||||
def value_into(self, node, target_float):
|
||||
"""a value node flows into a slot: note a literal to float, or a crossing"""
|
||||
if node is None: return
|
||||
inner = node.kids[0] if node.kind == 'paren' else node
|
||||
lit = inner if inner.kind == 'num' else (inner.kids[0] if inner.kind == 'unary' and inner.val == '-' and inner.kids and inner.kids[0].kind == 'num' else None)
|
||||
k = self.p.node_key.get(id(node))
|
||||
if self.mixed_read(inner):
|
||||
if target_float: self.wrap_from.add(id(node))
|
||||
return
|
||||
if k is not None and k[0] == 'RC' and inner.kind == 'call' and inner.kids[0].kind == 'id':
|
||||
f = self.p.funcs.get(inner.kids[0].val)
|
||||
if f and f.ret and self.fl(f.ret.key) != self.fl(k): k = f.ret.key
|
||||
if target_float:
|
||||
if lit is not None: self.to_float_lit.add(id(lit))
|
||||
elif k is not None and not self.fl(k) and (self.p.uf.is_int(k) or k[0] in ('B', 'RC', 'RD')): self.wrap_from.add(id(node))
|
||||
else:
|
||||
if k is not None and self.fl(k): self.wrap_bits.add(id(node))
|
||||
|
||||
def plan(self):
|
||||
p = self.p
|
||||
for fp in p.files:
|
||||
if fp.runtime: continue
|
||||
for s in fp.stmts:
|
||||
fn = s.fn
|
||||
if s.kind == 'let' and s.b is not None:
|
||||
self.value_into(s.b, self.fl(s.decl.key))
|
||||
elif s.kind == 'assign':
|
||||
tk = p.node_key.get(id(s.a))
|
||||
if s.a.kind == 'index':
|
||||
ck = p.node_key.get(id(s.a.kids[0]))
|
||||
tf = ck is not None and ck not in p.mixed and self.fl(('E', ck))
|
||||
self.value_into(s.b, tf)
|
||||
else:
|
||||
self.value_into(s.b, self.fl(tk))
|
||||
elif s.kind == 'return':
|
||||
f = p.funcs.get(fn)
|
||||
if f and f.ret: self.value_into(s.a, self.fl(f.ret.key))
|
||||
for d in fp.globals.values():
|
||||
if d.init is not None: self.value_into(d.init, self.fl(d.key))
|
||||
for fields in fp.props.values():
|
||||
for d in fields.values():
|
||||
if d.init is not None: self.value_into(d.init, self.fl(d.key))
|
||||
for f, i, k, a in p.calls:
|
||||
self.value_into(a, self.fl(f.params[i].key))
|
||||
for call, a, k in p.bound_args:
|
||||
if self.fl(k) and a.kind != 'named': self.wrap_bits.add(id(a))
|
||||
for n_id, k in p.bound_rets.items():
|
||||
if self.fl(k): self.wrap_from.add(n_id)
|
||||
# comparisons between a float and a literal
|
||||
for fp in p.files:
|
||||
if fp.runtime: continue
|
||||
for s in fp.stmts:
|
||||
for e in (s.a, s.b):
|
||||
if e is None: continue
|
||||
for n in walk(e):
|
||||
if n.kind == 'bin' and n.val in ('==', '!=', '<', '>', '<=', '>='):
|
||||
a, b = n.kids
|
||||
ka, kb = p.node_key.get(id(a)), p.node_key.get(id(b))
|
||||
if self.fl(ka) and b.kind == 'num': self.to_float_lit.add(id(b))
|
||||
if self.fl(kb) and a.kind == 'num': self.to_float_lit.add(id(a))
|
||||
if n.kind == 'call' and n.kids[0].kind == 'id' and (n.kids[0].val in BIN or n.kids[0].val in CMP or n.kids[0].val in MATH or n.kids[0].val == 'f_neg'):
|
||||
for arg in n.kids[1:]:
|
||||
if self.mixed_read(arg): self.wrap_from.add(id(arg))
|
||||
if n.kind == 'bin' and n.val in ('==', '!=', '<', '>', '<=', '>='):
|
||||
a, b = n.kids
|
||||
ka, kb = self.kind(a), self.kind(b)
|
||||
if ka == 'f' and kb == 'i' and b.kind != 'num': self.wrap_bits.add(id(a))
|
||||
if kb == 'f' and ka == 'i' and a.kind != 'num': self.wrap_bits.add(id(b))
|
||||
if n.kind == 'bin' and n.val in INT_OPS:
|
||||
a, b = n.kids
|
||||
ka, kb = self.kind(a), self.kind(b)
|
||||
bitwise = n.val in ('|', '&', '^', '<<', '>>')
|
||||
if ka == 'f' and (bitwise or kb == 'i'): self.wrap_bits.add(id(a))
|
||||
if kb == 'f' and (bitwise or ka == 'i'): self.wrap_bits.add(id(b))
|
||||
if n.kind == 'call' and n.kids[0].kind == 'id' and n.kids[0].val in ('push',):
|
||||
args = n.kids[1:]
|
||||
if len(args) == 2:
|
||||
ck = p.node_key.get(id(args[0]))
|
||||
if ck is not None and ck not in p.mixed and self.fl(('E', ck)): self.value_into(args[1], True)
|
||||
|
||||
# ---- rendering ----------------------------------------------------------
|
||||
def render(self, n, src):
|
||||
"""(text, precedence) of a node after the rewrite"""
|
||||
t, pr = self.render_inner(n, src)
|
||||
if id(n) in self.wrap_bits: return f'float_bits({t})', 12
|
||||
if id(n) in self.wrap_from: return f'float_from_bits({t})', 12
|
||||
return t, pr
|
||||
|
||||
def need(self, kid, src, minp):
|
||||
t, pr = self.render(kid, src)
|
||||
return f'({t})' if pr < minp else t
|
||||
|
||||
def render_inner(self, n, src):
|
||||
k = n.kind
|
||||
if k == 'num':
|
||||
if id(n) in self.to_float_lit: return flit(n.val, bits=True), 12
|
||||
return n.val, 12
|
||||
if k == 'id':
|
||||
if n.val in CONST: return CONST[n.val], 12
|
||||
return n.val, 12
|
||||
if k == 'call' and n.kids[0].kind == 'id':
|
||||
name = n.kids[0].val
|
||||
args = n.kids[1:]
|
||||
if name in BIN and len(args) == 2:
|
||||
op, pr = BIN[name]
|
||||
return f'{self.need(args[0], src, pr)} {op} {self.need(args[1], src, pr + 1)}', pr
|
||||
if name in CMP and len(args) == 2:
|
||||
op, pr = CMP[name]
|
||||
return f'{self.need(args[0], src, pr + 1)} {op} {self.need(args[1], src, pr + 1)}', pr
|
||||
if name == 'f_neg' and len(args) == 1:
|
||||
return f'-{self.need(args[0], src, 11)}', 11
|
||||
if name in MATH:
|
||||
return f'Math.{MATH[name]}(' + ', '.join(self.render(a, src)[0] for a in args) + ')', 12
|
||||
if name == 'fx_to_f32' and len(args) == 1 and not self.is_fixed(args[0]):
|
||||
# Q16.16 bits in an int: float(x) of it would be the bits as a number
|
||||
return f'float({self.render(args[0], src)[0]}) / 65536.0', 10
|
||||
if name in CONV and len(args) == 1:
|
||||
return f'{CONV[name]}({self.render(args[0], src)[0]})', 12
|
||||
if name == 'fl' and len(args) == 1:
|
||||
a = args[0]
|
||||
if a.kind == 'num': return flit(a.val), 12
|
||||
if a.kind == 'unary' and a.val == '-' and a.kids[0].kind == 'num': return '-' + flit(a.kids[0].val), 11
|
||||
return f'float({self.render(a, src)[0]})', 12
|
||||
if name == 'fi' and len(args) == 1:
|
||||
a = args[0]
|
||||
if a.kind == 'num': return flit(a.val), 12
|
||||
if a.kind == 'unary' and a.val == '-' and a.kids[0].kind == 'num': return '-' + flit(a.kids[0].val), 11
|
||||
return f'float({self.render(a, src)[0]})', 12
|
||||
if name == 'fr' and len(args) == 2:
|
||||
parts = []
|
||||
for a in args:
|
||||
parts.append(flit(a.val) if a.kind == 'num' else f'float({self.render(a, src)[0]})')
|
||||
return f'{parts[0]} / {parts[1]}', 10
|
||||
if name == 'f_neg1' and not args: return '-1.0', 11
|
||||
if name in ('words', 'ints_n') and self.float_container(n):
|
||||
return f'{"floats" if name == "words" else "floats_n"}(' + ', '.join(self.render(a, src)[0] for a in args) + ')', 12
|
||||
if k == 'new' and n.val == '[]int' and self.float_container(n): return 'new []float', 12
|
||||
if k == 'new' and n.val in ('[]words', '[][]int') and self.nested_float(('A', id(n))):
|
||||
return ('new []floats' if n.val == '[]words' else 'new [][]float'), 12
|
||||
if not n.kids: return src[n.s:n.e], self.prec_of(n)
|
||||
# splice the kids into the node's own text
|
||||
out, pos = [], n.s
|
||||
for idx, kid in enumerate(n.kids):
|
||||
if kid.s < pos or kid.s < 0: continue
|
||||
out.append(src[pos:kid.s])
|
||||
if n.kind == 'bin':
|
||||
pr = PREC[n.val]
|
||||
t = self.need(kid, src, pr if idx == 0 else pr + 1)
|
||||
elif n.kind == 'unary':
|
||||
t = self.need(kid, src, 11)
|
||||
elif n.kind in ('member', 'index', 'slice') and idx == 0 or n.kind == 'call' and idx == 0:
|
||||
t = self.need(kid, src, 12)
|
||||
else:
|
||||
t = self.render(kid, src)[0]
|
||||
out.append(t)
|
||||
pos = kid.e
|
||||
out.append(src[pos:n.e])
|
||||
return ''.join(out), self.prec_of(n)
|
||||
|
||||
def nested_float(self, k):
|
||||
ik = ('E', k)
|
||||
return ik not in self.p.mixed and self.fl(('E', ik))
|
||||
|
||||
def mixed_read(self, n):
|
||||
"""an element of a buffer that holds both kinds: its bits, whatever the inference joined it to"""
|
||||
if n.kind != 'index': return False
|
||||
ck = self.p.node_key.get(id(n.kids[0]))
|
||||
return ck is not None and ck in self.p.mixed
|
||||
|
||||
def is_fixed(self, n):
|
||||
k = self.p.node_key.get(id(n))
|
||||
if k is not None and self.p.types.get(k, '') == 'fixed': return True
|
||||
return n.kind == 'call' and n.kids[0].kind == 'id' and n.kids[0].val in ('f_fx', 'f32_to_fx')
|
||||
|
||||
def kind(self, n):
|
||||
"""'f', 'i', 'lit' or None: what a node carries, as far as the inference knows"""
|
||||
if n.kind == 'num': return 'i' if '.' not in n.val else 'lit'
|
||||
if n.kind == 'paren': return self.kind(n.kids[0])
|
||||
if self.mixed_read(n): return 'i'
|
||||
if n.kind == 'bin' and n.val in INT_OPS:
|
||||
ks = {self.kind(k) for k in n.kids} - {'lit', None}
|
||||
return ks.pop() if len(ks) == 1 else None
|
||||
k = self.p.node_key.get(id(n))
|
||||
if k is None: return None
|
||||
if self.fl(k): return 'f'
|
||||
if self.p.uf.is_int(k): return 'i'
|
||||
return None
|
||||
|
||||
def float_container(self, n):
|
||||
k = self.p.node_key.get(id(n))
|
||||
return k is not None and k not in self.p.mixed and self.fl(('E', k))
|
||||
|
||||
def prec_of(self, n):
|
||||
if n.kind == 'bin': return PREC[n.val]
|
||||
if n.kind == 'unary': return 11
|
||||
return 12
|
||||
|
||||
# ---- files --------------------------------------------------------------
|
||||
def roots(self, fp):
|
||||
"""the outermost expression nodes of a file, each once"""
|
||||
seen = []
|
||||
for s in fp.stmts:
|
||||
for e in (s.a, s.b):
|
||||
if e is not None: seen.append(e)
|
||||
if s.decl is not None and s.decl.init is not None and s.decl.init is not s.b: seen.append(s.decl.init)
|
||||
for d in fp.globals.values():
|
||||
if d.init is not None: seen.append(d.init)
|
||||
for fields in fp.props.values():
|
||||
for d in fields.values():
|
||||
if d.init is not None: seen.append(d.init)
|
||||
seen.sort(key=lambda n: (n.s, -n.e))
|
||||
out, end = [], -1
|
||||
for n in seen:
|
||||
if n.s >= end:
|
||||
out.append(n)
|
||||
end = n.e
|
||||
return out
|
||||
|
||||
def decl_edits(self, fp):
|
||||
p = self.p
|
||||
edits = []
|
||||
decls = list(fp.globals.values())
|
||||
for fields in fp.props.values(): decls += list(fields.values())
|
||||
for f in fp.funcs.values():
|
||||
if f.file != fp.path: continue
|
||||
decls += list(f.params)
|
||||
if f.ret: decls.append(f.ret)
|
||||
decls += list(f.locals.values())
|
||||
for d in decls:
|
||||
if d.ts is None or d.ts < 0 or not d.ty: continue
|
||||
if d.ty == 'int' and self.fl(d.key): edits.append((d.ts, d.te, 'float'))
|
||||
elif d.ty in ('words', '[]int') and d.key not in p.mixed and self.fl(('E', d.key)):
|
||||
edits.append((d.ts, d.te, 'floats' if d.ty == 'words' else '[]float'))
|
||||
elif d.ty in ('[]words', '[][]int') and self.nested_float(d.key):
|
||||
edits.append((d.ts, d.te, '[]floats' if d.ty == '[]words' else '[][]float'))
|
||||
return edits
|
||||
|
||||
def rewrite(self, fp):
|
||||
src = fp.src
|
||||
edits = self.decl_edits(fp)
|
||||
for n in self.roots(fp):
|
||||
t, _ = self.render(n, src)
|
||||
if t != src[n.s:n.e]: edits.append((n.s, n.e, t))
|
||||
edits.sort()
|
||||
out, pos = [], 0
|
||||
for s, e, t in edits:
|
||||
if s < pos: continue
|
||||
out.append(src[pos:s])
|
||||
out.append(t)
|
||||
pos = e
|
||||
out.append(src[pos:])
|
||||
return ''.join(out)
|
||||
|
||||
|
||||
def main():
|
||||
game, home = sys.argv[1], sys.argv[2]
|
||||
dry = '--dry' in sys.argv
|
||||
prog = Program([game + '/src', game + '/lab', home + '/packages/ludic.render3d'], [home + '/runtime/native']).run()
|
||||
rw = Rewriter(prog)
|
||||
changed = 0
|
||||
for fp in prog.files:
|
||||
if fp.runtime: continue
|
||||
new = rw.rewrite(fp)
|
||||
if new != fp.src:
|
||||
changed += 1
|
||||
if not dry: open(fp.path, 'w').write(new)
|
||||
print(f'{changed} files rewritten')
|
||||
if dry:
|
||||
import difflib
|
||||
for fp in prog.files:
|
||||
if fp.runtime or not any(x in fp.path for x in sys.argv[3:] if not x.startswith('--')): continue
|
||||
new = rw.rewrite(fp)
|
||||
for l in difflib.unified_diff(fp.src.split('\n'), new.split('\n'), lineterm='', n=0): print(l)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
523
tools/migrate/floatinfer.py
Normal file
523
tools/migrate/floatinfer.py
Normal file
|
|
@ -0,0 +1,523 @@
|
|||
"""Which Ludic ints are IEEE float bit patterns: union-find over every name a value can live in,
|
||||
seeded by the fl/fi/f_* helpers (float) and by integer arithmetic (int)."""
|
||||
import glob
|
||||
import os
|
||||
from ludic_decls import FileParse
|
||||
from ludic_ast import walk
|
||||
|
||||
FLOAT_OPS = {'f_add': 2, 'f_sub': 2, 'f_mul': 2, 'f_div': 2, 'f_neg': 1, 'f_abs': 1, 'f_min': 2,
|
||||
'f_max': 2, 'f_clamp': 3, 'f_sqrt': 1, 'f_sin': 1, 'f_cos': 1, 'f_tan': 1, 'f_atan2': 2,
|
||||
'f_floor': 1, 'f_mod': 2, 'f_lerp': 3, 'f_rad': 1, 'f_pow': 2, 'f_exp': 1, 'f_log': 1}
|
||||
FLOAT_CMP = {'f_ls', 'f_gt'}
|
||||
FLOAT_CONST = {'F_ZERO', 'F_ONE', 'F_TWO', 'F_HALF', 'F_PI'}
|
||||
CONTAINER_MAKERS = {'words', 'ints_n', 'ints_n1', 'floats'}
|
||||
|
||||
|
||||
class UF:
|
||||
def __init__(self):
|
||||
self.p = {}
|
||||
self.sz = {}
|
||||
self.fl = {}
|
||||
self.it = {}
|
||||
|
||||
def find(self, k):
|
||||
root = k
|
||||
while True:
|
||||
p = self.p.setdefault(root, root)
|
||||
if p == root: break
|
||||
root = p
|
||||
while k != root:
|
||||
nxt = self.p[k]
|
||||
self.p[k] = root
|
||||
k = nxt
|
||||
return root
|
||||
|
||||
def union(self, a, b):
|
||||
if a is None or b is None: return
|
||||
ra, rb = self.find(a), self.find(b)
|
||||
if ra == rb: return
|
||||
if self.sz.get(ra, 1) > self.sz.get(rb, 1): ra, rb = rb, ra
|
||||
self.p[ra] = rb
|
||||
self.sz[rb] = self.sz.get(rb, 1) + self.sz.get(ra, 1)
|
||||
for d in (self.fl, self.it):
|
||||
if ra in d:
|
||||
n, ex = d.pop(ra)
|
||||
m, ex2 = d.get(rb, (0, []))
|
||||
d[rb] = (n + m, (ex2 + ex)[:4])
|
||||
|
||||
def mark(self, k, which, why):
|
||||
if k is None: return
|
||||
r = self.find(k)
|
||||
d = self.fl if which == 'f' else self.it
|
||||
n, ex = d.get(r, (0, []))
|
||||
d[r] = (n + 1, ex if len(ex) >= 4 else ex + [why])
|
||||
|
||||
def is_float(self, k):
|
||||
return k is not None and self.find(k) in self.fl
|
||||
|
||||
def is_int(self, k):
|
||||
return k is not None and self.find(k) in self.it
|
||||
|
||||
|
||||
class Program:
|
||||
def __init__(self, convert_roots, readonly_roots):
|
||||
self.files = []
|
||||
for root, runtime in [(r, False) for r in convert_roots] + [(r, True) for r in readonly_roots]:
|
||||
paths = [root] if root.endswith('.ludic') else sorted(glob.glob(root + '/**/*.ludic', recursive=True))
|
||||
for p in paths:
|
||||
self.files.append(FileParse(p, open(p).read(), runtime=runtime))
|
||||
self.funcs, self.globals, self.props = {}, {}, {}
|
||||
for fp in self.files:
|
||||
for n, f in fp.funcs.items():
|
||||
if n not in self.funcs or not f.runtime: self.funcs.setdefault(n, f)
|
||||
for n, d in fp.globals.items():
|
||||
d.key = ('G', n)
|
||||
self.globals.setdefault(n, d)
|
||||
for r, fields in fp.props.items():
|
||||
for fn_, d in fields.items():
|
||||
d.key = ('F', r, fn_)
|
||||
self.props.setdefault(r, fields)
|
||||
for f in self.funcs.values():
|
||||
for i, d in enumerate(f.params): d.key = ('P', f.name, i)
|
||||
if f.ret: f.ret.key = ('R', f.name)
|
||||
for n, d in f.locals.items(): d.key = ('L', f.name, n)
|
||||
self.field_owners = {}
|
||||
for r, fields in self.props.items():
|
||||
for fn_ in fields: self.field_owners.setdefault(fn_, []).append(r)
|
||||
self.uf = UF()
|
||||
self.types = {} # key -> declared type text
|
||||
for d in self.all_decls(): self.types[d.key] = d.ty
|
||||
self.calls = [] # (callee Func, arg index, arg key, node)
|
||||
self.bound_args = [] # (call node, arg node, arg key): a float going into a runtime call
|
||||
self.bound_rets = {} # call node id -> key of a runtime result
|
||||
self.node_key = {} # id(node) -> key
|
||||
self.stores = [] # (container key, value key, value node): a[i] = v, push(a, v)
|
||||
self.elinks = [] # (container, container): one buffer's contents flow into another's
|
||||
self.flows = [] # (a, b, where): a value moves between a and b (=, let, return, ==)
|
||||
self.splits = [] # the flows whose two ends turned out different kinds: bits cross there
|
||||
self.reads = [] # (container key, read-site key): v = a[i]
|
||||
self.rets = [] # (callee, call-site key)
|
||||
self.poly_ret = set()
|
||||
self.cur = ''
|
||||
self.scope = None
|
||||
self.mixed = set() # containers holding both floats and ints: they keep their bits
|
||||
|
||||
def all_decls(self):
|
||||
for d in self.globals.values(): yield d
|
||||
for fields in self.props.values():
|
||||
yield from fields.values()
|
||||
for f in self.funcs.values():
|
||||
yield from f.params
|
||||
if f.ret: yield f.ret
|
||||
yield from f.locals.values()
|
||||
|
||||
# ---- names --------------------------------------------------------------
|
||||
def lookup(self, name, fn):
|
||||
f = self.funcs.get(fn)
|
||||
if f:
|
||||
sc = self.scope or {}
|
||||
if name in sc: return f.locals[sc[name]].key
|
||||
if name in f.locals and not sc: return f.locals[name].key
|
||||
for d in f.params:
|
||||
if d.name == name: return d.key
|
||||
if name in self.globals: return self.globals[name].key
|
||||
return None
|
||||
|
||||
def type_of_key(self, k):
|
||||
return self.types.get(k, '')
|
||||
|
||||
def rec_of(self, node, fn):
|
||||
"""the record type an expression evaluates to, as far as declarations say"""
|
||||
if node.kind == 'id':
|
||||
k = self.lookup(node.val, fn)
|
||||
return self.type_of_key(k) if k else ''
|
||||
if node.kind == 'member':
|
||||
r = self.rec_of(node.kids[0], fn)
|
||||
d = self.props.get(r, {}).get(node.val)
|
||||
return d.ty if d else ''
|
||||
if node.kind == 'index':
|
||||
t = self.rec_of(node.kids[0], fn)
|
||||
return t[2:] if t.startswith('[]') else ''
|
||||
if node.kind == 'call' and node.kids[0].kind == 'id':
|
||||
f = self.funcs.get(node.kids[0].val)
|
||||
return f.ret.ty if f and f.ret else ''
|
||||
if node.kind == 'new': return node.val
|
||||
if node.kind == 'paren': return self.rec_of(node.kids[0], fn)
|
||||
return ''
|
||||
|
||||
def member_key(self, node, fn):
|
||||
r = self.rec_of(node.kids[0], fn)
|
||||
if r in self.props and node.val in self.props[r]: return self.props[r][node.val].key
|
||||
owners = self.field_owners.get(node.val, [])
|
||||
if len(owners) == 1: return self.props[owners[0]][node.val].key
|
||||
return ('FU', id(node))
|
||||
|
||||
CONTAINER_TYPES = ('words', 'floats', 'pointers')
|
||||
|
||||
def is_container(self, k):
|
||||
if k is None: return False
|
||||
if k[0] == 'A': return True
|
||||
if k[0] == 'RC': return self.types.get(k, '') in self.CONTAINER_TYPES or self.types.get(k, '').startswith('[]')
|
||||
t = self.types.get(k, '')
|
||||
return t in self.CONTAINER_TYPES or t.startswith('[]')
|
||||
|
||||
def flow(self, a, b):
|
||||
if a is not None and b is not None: self.flows.append((a, b, self.cur))
|
||||
|
||||
def settle_flows(self):
|
||||
keep = []
|
||||
for a, b, where in self.flows:
|
||||
d = self.decide([a, b])
|
||||
if d == 'split':
|
||||
self.splits.append((a, b, where))
|
||||
elif d == 'join':
|
||||
self.link(a, b)
|
||||
else:
|
||||
keep.append((a, b, where))
|
||||
self.flows = keep
|
||||
|
||||
def link(self, a, b):
|
||||
"""a value flows between a and b. Buffers are not merged: their elements are linked, later,
|
||||
and only if the two carry the same kind of number"""
|
||||
if a is None or b is None: return
|
||||
if self.is_container(a) or self.is_container(b):
|
||||
self.elinks.append((a, b))
|
||||
return
|
||||
self.uf.union(a, b)
|
||||
|
||||
def inner_of(self, k):
|
||||
"""the key every inner buffer of nested buffer k shares, or None when k is not nested"""
|
||||
t = self.types.get(k, '')
|
||||
if not t.startswith('[]'): return None
|
||||
it = t[2:]
|
||||
if not (it in self.CONTAINER_TYPES or it.startswith('[]')): return None
|
||||
ik = ('E', k)
|
||||
self.types[ik] = it
|
||||
return ik
|
||||
|
||||
def settle_elinks(self):
|
||||
for a, b in list(self.elinks):
|
||||
ia, ib = self.inner_of(a), self.inner_of(b)
|
||||
if ia is not None and ib is not None and (ia, ib) not in self.elinks: self.elinks.append((ia, ib))
|
||||
for a, b in self.elinks:
|
||||
ea, eb = ('E', a), ('E', b)
|
||||
if a in self.mixed or b in self.mixed: continue
|
||||
d = self.decide([ea, eb])
|
||||
if d == 'split':
|
||||
self.mixed.add(a)
|
||||
self.mixed.add(b)
|
||||
continue
|
||||
if d == 'join': self.uf.union(ea, eb)
|
||||
|
||||
# ---- expressions -------------------------------------------------------
|
||||
def ev(self, n, fn):
|
||||
k = self._ev(n, fn)
|
||||
if k is not None: self.node_key[id(n)] = k
|
||||
return k
|
||||
|
||||
def _ev(self, n, fn):
|
||||
uf = self.uf
|
||||
kind = n.kind
|
||||
if kind == 'num':
|
||||
if '.' in n.val: return None
|
||||
if n.val in ('0',) or n.val.startswith('0x'): return None
|
||||
if int(n.val.replace('_', '')) > (1 << 20): return None # a float's bits, written out
|
||||
k = ('N', id(n))
|
||||
uf.mark(k, 'i', 'literal')
|
||||
return k
|
||||
if kind == 'id':
|
||||
if n.val in FLOAT_CONST:
|
||||
k = ('C', id(n))
|
||||
uf.mark(k, 'f', n.val)
|
||||
return k
|
||||
return self.lookup(n.val, fn)
|
||||
if kind == 'paren': return self.ev(n.kids[0], fn)
|
||||
if kind == 'tmpl':
|
||||
for x in n.kids: self.ev(x, fn)
|
||||
return None
|
||||
if kind == 'member': return self.member_key(n, fn) if n.kids[0].kind != 'id' or n.kids[0].val[:1].islower() else None
|
||||
if kind == 'index':
|
||||
ck = self.ev(n.kids[0], fn)
|
||||
ik = self.ev(n.kids[1], fn)
|
||||
uf.mark(ik, 'i', 'index')
|
||||
if ck is None: return None
|
||||
inner = self.inner_of(ck)
|
||||
if inner is not None: return inner
|
||||
rk = ('RD', id(n))
|
||||
self.reads.append((ck, rk))
|
||||
return rk
|
||||
if kind == 'slice':
|
||||
for x in n.kids: self.ev(x, fn)
|
||||
return None
|
||||
if kind == 'unary':
|
||||
x = self.ev(n.kids[0], fn)
|
||||
if n.val == '-':
|
||||
uf.mark(x, 'i', 'negated with -')
|
||||
return x
|
||||
return None
|
||||
if kind == 'bin':
|
||||
a, b = self.ev(n.kids[0], fn), self.ev(n.kids[1], fn)
|
||||
if n.val in ('==', '!='):
|
||||
self.flow(a, b)
|
||||
return None
|
||||
if n.val in ('and', 'or', '&&', '||'): return None
|
||||
if n.val in ('<', '>', '<=', '>='):
|
||||
self.flow(a, b)
|
||||
return None
|
||||
uf.mark(a, 'i', f'operand of {n.val}')
|
||||
uf.mark(b, 'i', f'operand of {n.val}')
|
||||
if n.val == '..': return None
|
||||
k = ('X', id(n))
|
||||
uf.mark(k, 'i', f'result of {n.val}')
|
||||
return k
|
||||
if kind == 'list':
|
||||
k = ('A', id(n))
|
||||
for x in n.kids:
|
||||
v = self.ev(x, fn)
|
||||
if v is not None: self.stores.append((k, v, x))
|
||||
return k
|
||||
if kind == 'new':
|
||||
if not n.val.startswith('[]'): return None
|
||||
self.types[('A', id(n))] = n.val
|
||||
return ('A', id(n))
|
||||
if kind == 'call': return self.ev_call(n, fn)
|
||||
for x in n.kids: self.ev(x, fn)
|
||||
return None
|
||||
|
||||
def ev_call(self, n, fn):
|
||||
uf = self.uf
|
||||
callee = n.kids[0]
|
||||
args = n.kids[1:]
|
||||
name = callee.val if callee.kind == 'id' else None
|
||||
keys = [self.ev(a.kids[0] if a.kind == 'named' else a, fn) for a in args]
|
||||
if name in FLOAT_OPS:
|
||||
for k in keys: uf.mark(k, 'f', name)
|
||||
k = ('X', id(n))
|
||||
uf.mark(k, 'f', name)
|
||||
return k
|
||||
if name in FLOAT_CMP:
|
||||
for k in keys: uf.mark(k, 'f', name)
|
||||
return None
|
||||
if name in ('fl', 'fi', 'fr', 'f_from_int', 'fx_to_f32', 'f_neg1'):
|
||||
if name in ('fi', 'fr', 'f_from_int'):
|
||||
for k in keys: uf.mark(k, 'i', name)
|
||||
k = ('X', id(n))
|
||||
uf.mark(k, 'f', name)
|
||||
return k
|
||||
if name in ('f_to_int',):
|
||||
for k in keys: uf.mark(k, 'f', name)
|
||||
k = ('X', id(n))
|
||||
uf.mark(k, 'i', name)
|
||||
return k
|
||||
if name in ('f_fx', 'f32_to_fx'):
|
||||
for k in keys: uf.mark(k, 'f', name)
|
||||
return None
|
||||
if name == 'f_lt':
|
||||
for k in keys: uf.mark(k, 'f', name)
|
||||
return None
|
||||
if name in CONTAINER_MAKERS: return ('A', id(n))
|
||||
if name == 'push' and len(keys) == 2 and keys[0] is not None:
|
||||
inner = self.inner_of(keys[0])
|
||||
if inner is not None and keys[1] is not None:
|
||||
self.elinks.append((inner, keys[1]))
|
||||
return None
|
||||
if keys[1] is not None: self.stores.append((keys[0], keys[1], args[1]))
|
||||
return None
|
||||
if name == 'len':
|
||||
k = ('X', id(n))
|
||||
uf.mark(k, 'i', 'len')
|
||||
return k
|
||||
f = self.funcs.get(name) if name else None
|
||||
if f and not f.runtime and not f.extern:
|
||||
for i, k in enumerate(keys):
|
||||
if i < len(f.params): self.calls.append((f, i, k, args[i]))
|
||||
if not f.ret: return None
|
||||
rk = ('RC', id(n))
|
||||
self.types[rk] = f.ret.ty
|
||||
self.rets.append((f, rk))
|
||||
return rk
|
||||
# a runtime or engine call: a float going in leaves as bits, a result coming out is bits
|
||||
for i, a in enumerate(args):
|
||||
if keys[i] is not None: self.bound_args.append((n, a, keys[i]))
|
||||
k = ('B', id(n))
|
||||
self.bound_rets[id(n)] = k
|
||||
return k
|
||||
|
||||
# ---- statements -------------------------------------------------------
|
||||
def run(self):
|
||||
uf = self.uf
|
||||
for d in self.globals.values():
|
||||
if d.init is not None: self.flow(d.key, self.ev(d.init, None))
|
||||
for fields in self.props.values():
|
||||
for d in fields.values():
|
||||
if d.init is not None: self.flow(d.key, self.ev(d.init, None))
|
||||
for fp in self.files:
|
||||
for s in fp.stmts:
|
||||
fn = s.fn
|
||||
self.scope = s.scope
|
||||
self.cur = f'{fp.path.split("/")[-1]}:{fn}:{s.kind}:{fp.src.count(chr(10), 0, (s.a or s.b).s if (s.a or s.b) else 0) + 1}'
|
||||
if s.kind == 'let':
|
||||
if s.b is not None:
|
||||
if not s.decl.ty: self.types[s.decl.key] = self.rec_of(s.b, fn)
|
||||
self.flow(s.decl.key, self.ev(s.b, fn))
|
||||
elif s.kind == 'assign':
|
||||
if s.a.kind == 'index':
|
||||
ck = self.ev(s.a.kids[0], fn)
|
||||
uf.mark(self.ev(s.a.kids[1], fn), 'i', 'index')
|
||||
vk = self.ev(s.b, fn)
|
||||
inner = self.inner_of(ck) if ck is not None else None
|
||||
if inner is not None and vk is not None: self.elinks.append((inner, vk))
|
||||
elif ck is not None and vk is not None: self.stores.append((ck, vk, s.b))
|
||||
else:
|
||||
self.flow(self.ev(s.a, fn), self.ev(s.b, fn))
|
||||
elif s.kind == 'opassign':
|
||||
uf.mark(self.ev(s.a, fn), 'i', s.op)
|
||||
uf.mark(self.ev(s.b, fn), 'i', s.op)
|
||||
elif s.kind == 'return':
|
||||
f = self.funcs.get(fn)
|
||||
k = self.ev(s.a, fn)
|
||||
if f and f.ret: self.flow(f.ret.key, k)
|
||||
elif s.kind == 'forrange':
|
||||
uf.mark(self.ev(s.a, fn), 'i', 'for range')
|
||||
uf.mark(self.ev(s.b, fn), 'i', 'for range')
|
||||
uf.mark(self.lookup(s.var, fn), 'i', 'for range')
|
||||
elif s.kind == 'foreach':
|
||||
ck = self.ev(s.a, fn)
|
||||
if ck is not None:
|
||||
vk = self.lookup(s.var, fn)
|
||||
et = self.types.get(ck, '')
|
||||
if et.startswith('[]'): self.types[vk] = et[2:]
|
||||
self.reads.append((ck, vk))
|
||||
else:
|
||||
if s.a is not None: self.ev(s.a, fn)
|
||||
# parameters: joined to what is passed, unless a function is fed both kinds
|
||||
self.poly = set()
|
||||
self.cur = 'settle'
|
||||
self.final = False
|
||||
for _ in range(12):
|
||||
self.settle_flows()
|
||||
self.settle_params()
|
||||
self.settle_stores()
|
||||
self.settle_reads()
|
||||
self.settle_rets()
|
||||
self.settle_elinks()
|
||||
self.final = True # what never gained a kind joins whatever it flows into
|
||||
for _ in range(3):
|
||||
self.settle_flows()
|
||||
self.settle_params()
|
||||
self.settle_stores()
|
||||
self.settle_reads()
|
||||
self.settle_rets()
|
||||
self.settle_elinks()
|
||||
self.spread_mixed()
|
||||
return self
|
||||
|
||||
def spread_mixed(self):
|
||||
"""a buffer shared with one that holds both kinds holds both too"""
|
||||
grew = True
|
||||
while grew:
|
||||
grew = False
|
||||
for a, b in self.elinks:
|
||||
if (a in self.mixed) != (b in self.mixed):
|
||||
self.mixed.add(a)
|
||||
self.mixed.add(b)
|
||||
grew = True
|
||||
|
||||
def decide(self, keys):
|
||||
"""'join', 'wait' or 'split' for a set of keys a value flows between"""
|
||||
k = self.kinds_of(keys)
|
||||
if k == {'f', 'i'}: return 'split'
|
||||
if k or self.final: return 'join'
|
||||
return 'wait'
|
||||
|
||||
def settle_reads(self):
|
||||
for ck, rk in self.reads:
|
||||
root = ck
|
||||
if root in self.mixed: continue
|
||||
e = ('E', root)
|
||||
d = self.decide([e, rk])
|
||||
if d == 'split':
|
||||
self.mixed.add(root)
|
||||
continue
|
||||
if d == 'join': self.uf.union(e, rk)
|
||||
|
||||
def settle_rets(self):
|
||||
by = {}
|
||||
for f, rk in self.rets: by.setdefault(f.name, []).append(rk)
|
||||
for fn_, rks in by.items():
|
||||
if fn_ in self.poly_ret: continue
|
||||
rkey = self.funcs[fn_].ret.key
|
||||
probe = [('E', k) for k in rks + [rkey]] if self.is_container(rkey) else rks + [rkey]
|
||||
d = self.decide(probe)
|
||||
if d == 'split':
|
||||
self.poly_ret.add(fn_)
|
||||
continue
|
||||
if d == 'join':
|
||||
for rk in rks: self.link(rkey, rk)
|
||||
|
||||
def kinds_of(self, keys):
|
||||
uf, kinds = self.uf, set()
|
||||
for k in keys:
|
||||
if k is None: continue
|
||||
if uf.is_float(k) and not uf.is_int(k): kinds.add('f')
|
||||
elif uf.is_int(k) and not uf.is_float(k): kinds.add('i')
|
||||
return kinds
|
||||
|
||||
def settle_stores(self):
|
||||
by = {}
|
||||
for ck, vk, node in self.stores: by.setdefault(ck, []).append(vk)
|
||||
for root, vks in by.items():
|
||||
if root in self.mixed: continue
|
||||
d = self.decide(vks + [('E', root)])
|
||||
if d == 'split':
|
||||
self.mixed.add(root)
|
||||
continue
|
||||
if d == 'join':
|
||||
for vk in vks: self.uf.union(('E', root), vk)
|
||||
|
||||
def settle_params(self):
|
||||
uf = self.uf
|
||||
by = {}
|
||||
for f, i, k, a in self.calls: by.setdefault((f.name, i), []).append(k)
|
||||
for (fn_, i), ks in by.items():
|
||||
f = self.funcs[fn_]
|
||||
pk = f.params[i].key
|
||||
cont = self.is_container(pk)
|
||||
kinds = set()
|
||||
for k in ks + [pk]:
|
||||
if k is None: continue
|
||||
q = ('E', k) if cont or self.is_container(k) else k
|
||||
if uf.is_float(q) and not uf.is_int(q): kinds.add('f')
|
||||
elif uf.is_int(q) and not uf.is_float(q): kinds.add('i')
|
||||
if kinds == {'f', 'i'}:
|
||||
self.poly.add((fn_, i))
|
||||
if self.final and not cont:
|
||||
# an argument with no evidence of its own takes the parameter's kind
|
||||
for k in ks:
|
||||
if k is not None and not uf.is_float(k) and not uf.is_int(k) and not self.is_container(k): self.link(pk, k)
|
||||
continue
|
||||
if not kinds and not self.final: continue
|
||||
for k in ks: self.link(pk, k)
|
||||
|
||||
def conflicts(self):
|
||||
uf = self.uf
|
||||
roots = set(uf.fl) & set(uf.it)
|
||||
return [(r, uf.fl[r], uf.it[r]) for r in roots]
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
game = sys.argv[1]
|
||||
home = sys.argv[2]
|
||||
prog = Program([game + '/src', game + '/lab', home + '/packages/ludic.render3d'], [home + '/runtime/native']).run()
|
||||
fl = sum(1 for d in prog.all_decls() if prog.uf.is_float(d.key) and not prog.uf.is_int(d.key))
|
||||
print(f'{len(prog.files)} files, {len(prog.funcs)} functions; {fl} declarations are float')
|
||||
cs = prog.conflicts()
|
||||
print(f'{len(cs)} groups are both; {len(prog.splits)} flows cross between kinds; {len(prog.poly)} params and {len(prog.poly_ret)} returns take both; {len(prog.mixed)} buffers hold both')
|
||||
for a, b, w in prog.splits[:25]: print(' split', w, a, b)
|
||||
names = {}
|
||||
for d in prog.all_decls():
|
||||
r = prog.uf.find(d.key)
|
||||
names.setdefault(r, []).append(d.key)
|
||||
for r, f_, i_ in sorted(cs, key=lambda c: -len(names.get(c[0], [])))[:25]:
|
||||
print(len(names.get(r, [])), names.get(r, [])[:6], 'F:', f_[:2], 'I:', i_[:3])
|
||||
242
tools/migrate/ludic_ast.py
Normal file
242
tools/migrate/ludic_ast.py
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
"""A small Ludic parser for migration tools: tokens with source offsets, statements and
|
||||
expression trees, enough of the grammar to rewrite expressions in place and know which
|
||||
declaration each name refers to. It does not check anything; it tolerates what it does not
|
||||
understand by keeping it as raw text."""
|
||||
import re
|
||||
|
||||
KEYWORDS = {'let', 'var', 'const', 'if', 'else', 'while', 'for', 'in', 'return', 'break', 'continue',
|
||||
'function', 'handler', 'property', 'model', 'program', 'import', 'event', 'emit', 'not',
|
||||
'and', 'or', 'new', 'true', 'false', 'null', 'extern', 'spawn', 'enum', 'match', 'fn',
|
||||
'scene', 'layer', 'ui', 'test', 'namespace', 'entry', 'become', 'numbers', 'on'}
|
||||
|
||||
|
||||
class Tok:
|
||||
__slots__ = ('k', 't', 's', 'e', 'holes')
|
||||
|
||||
def __init__(self, k, t, s, e):
|
||||
self.k, self.t, self.s, self.e, self.holes = k, t, s, e, None
|
||||
|
||||
def __repr__(self):
|
||||
return f'{self.k}:{self.t!r}'
|
||||
|
||||
|
||||
OPS = ['..', '->', '==', '!=', '<=', '>=', '+=', '-=', '*=', '/=', '%=', '<<', '>>', '&&', '||',
|
||||
'+', '-', '*', '/', '%', '<', '>', '=', '(', ')', '[', ']', '{', '}', ',', '.', ':', ';',
|
||||
'@', '&', '|', '^', '!', '?', '~']
|
||||
|
||||
|
||||
NUM_RE = re.compile(r'0x[0-9a-fA-F_]+|\d[\d_]*(\.\d+(?!\.))?')
|
||||
ID_RE = re.compile(r'[A-Za-z_][A-Za-z0-9_]*')
|
||||
|
||||
|
||||
def tokenize(src, base=0):
|
||||
"""Tokens of src; offsets are relative to the whole file (base added)."""
|
||||
toks = []
|
||||
i, n = 0, len(src)
|
||||
while i < n:
|
||||
c = src[i]
|
||||
if c == '\n':
|
||||
toks.append(Tok('NL', '\n', base + i, base + i + 1))
|
||||
i += 1
|
||||
elif c in ' \t\r':
|
||||
i += 1
|
||||
elif c == '#':
|
||||
j = src.find('\n', i)
|
||||
j = n if j < 0 else j
|
||||
i = j
|
||||
elif c == '"':
|
||||
j = i + 1
|
||||
while j < n and src[j] != '"':
|
||||
j += 2 if src[j] == '\\' else 1
|
||||
toks.append(Tok('STR', src[i:j + 1], base + i, base + j + 1))
|
||||
i = j + 1
|
||||
elif c == "'":
|
||||
j = i + 1
|
||||
while j < n and src[j] != "'":
|
||||
j += 2 if src[j] == '\\' else 1
|
||||
toks.append(Tok('CHR', src[i:j + 1], base + i, base + j + 1))
|
||||
i = j + 1
|
||||
elif c == '`':
|
||||
j = i + 1
|
||||
holes = []
|
||||
while j < n and src[j] != '`':
|
||||
if src[j] == '{':
|
||||
d, k = 1, j + 1
|
||||
while k < n and d:
|
||||
if src[k] == '{': d += 1
|
||||
elif src[k] == '}': d -= 1
|
||||
k += 1
|
||||
holes.append((j + 1, k - 1))
|
||||
j = k
|
||||
else:
|
||||
j += 1
|
||||
t = Tok('TMPL', src[i:j + 1], base + i, base + j + 1)
|
||||
t.holes = [tokenize(src[a:b], base + a) for a, b in holes]
|
||||
toks.append(t)
|
||||
i = j + 1
|
||||
elif c.isdigit():
|
||||
m = NUM_RE.match(src, i)
|
||||
toks.append(Tok('NUM', m.group(0), base + i, base + i + len(m.group(0))))
|
||||
i += len(m.group(0))
|
||||
elif c.isalpha() or c == '_':
|
||||
m = ID_RE.match(src, i)
|
||||
toks.append(Tok('ID', m.group(0), base + i, base + i + len(m.group(0))))
|
||||
i += len(m.group(0))
|
||||
else:
|
||||
for op in OPS:
|
||||
if src.startswith(op, i):
|
||||
toks.append(Tok('OP', op, base + i, base + i + len(op)))
|
||||
i += len(op)
|
||||
break
|
||||
else:
|
||||
toks.append(Tok('OP', c, base + i, base + i + 1))
|
||||
i += 1
|
||||
return toks
|
||||
|
||||
|
||||
class Node:
|
||||
"""kind: num str chr tmpl id call member index slice unary bin paren list new fnref raw named"""
|
||||
__slots__ = ('kind', 'val', 'kids', 's', 'e', 'fn', 'sym', 'tmpl_holes')
|
||||
|
||||
def __init__(self, kind, val, kids, s, e):
|
||||
self.kind, self.val, self.kids, self.s, self.e = kind, val, kids, s, e
|
||||
self.fn = None
|
||||
self.sym = None
|
||||
self.tmpl_holes = None
|
||||
|
||||
|
||||
PREC = {'or': 1, '||': 1, 'and': 2, '&&': 2, '==': 3, '!=': 3, '<': 3, '<=': 3, '>': 3, '>=': 3,
|
||||
'..': 4, '|': 5, '^': 6, '&': 7, '<<': 8, '>>': 8, '+': 9, '-': 9, '*': 10, '/': 10, '%': 10}
|
||||
|
||||
|
||||
class ExprParser:
|
||||
def __init__(self, toks, i=0):
|
||||
self.t, self.i = toks, i
|
||||
|
||||
def peek(self, o=0):
|
||||
j = self.i + o
|
||||
return self.t[j] if j < len(self.t) else Tok('EOF', '', -1, -1)
|
||||
|
||||
def eat(self):
|
||||
tk = self.peek()
|
||||
self.i += 1
|
||||
return tk
|
||||
|
||||
def is_op(self, v):
|
||||
tk = self.peek()
|
||||
return tk.k == 'OP' and tk.t == v
|
||||
|
||||
def binop(self):
|
||||
tk = self.peek()
|
||||
if tk.k == 'OP' and tk.t in PREC: return tk.t
|
||||
if tk.k == 'ID' and tk.t in ('and', 'or'): return tk.t
|
||||
return None
|
||||
|
||||
def expr(self, minp=0):
|
||||
left = self.unary()
|
||||
while True:
|
||||
op = self.binop()
|
||||
if op is None or PREC[op] < minp:
|
||||
return left
|
||||
self.eat()
|
||||
while self.peek().k == 'NL': self.eat()
|
||||
right = self.expr(PREC[op] + 1)
|
||||
left = Node('bin', op, [left, right], left.s, right.e)
|
||||
|
||||
def unary(self):
|
||||
tk = self.peek()
|
||||
if tk.k == 'OP' and tk.t in ('-', '!', '~'):
|
||||
self.eat()
|
||||
x = self.unary()
|
||||
return Node('unary', tk.t, [x], tk.s, x.e)
|
||||
if tk.k == 'ID' and tk.t == 'not':
|
||||
self.eat()
|
||||
x = self.unary()
|
||||
return Node('unary', 'not', [x], tk.s, x.e)
|
||||
return self.postfix(self.primary())
|
||||
|
||||
def primary(self):
|
||||
tk = self.eat()
|
||||
if tk.k == 'NUM': return Node('num', tk.t, [], tk.s, tk.e)
|
||||
if tk.k == 'STR': return Node('str', tk.t, [], tk.s, tk.e)
|
||||
if tk.k == 'CHR': return Node('chr', tk.t, [], tk.s, tk.e)
|
||||
if tk.k == 'TMPL':
|
||||
n = Node('tmpl', tk.t, [], tk.s, tk.e)
|
||||
for ht in tk.holes:
|
||||
if ht:
|
||||
p = ExprParser(ht)
|
||||
try:
|
||||
n.kids.append(p.expr())
|
||||
except Exception:
|
||||
pass
|
||||
return n
|
||||
if tk.k == 'OP' and tk.t == '(':
|
||||
x = self.expr()
|
||||
end = self.eat()
|
||||
return Node('paren', None, [x], tk.s, end.e)
|
||||
if tk.k == 'OP' and tk.t == '[':
|
||||
kids = []
|
||||
while not self.is_op(']') and self.peek().k != 'EOF':
|
||||
if self.peek().k == 'NL':
|
||||
self.eat()
|
||||
continue
|
||||
kids.append(self.expr())
|
||||
if self.is_op(','): self.eat()
|
||||
end = self.eat()
|
||||
return Node('list', None, kids, tk.s, end.e)
|
||||
if tk.k == 'ID' and tk.t == 'new':
|
||||
s = tk.s
|
||||
ty = ''
|
||||
while self.is_op('[') or self.is_op(']'): ty += self.eat().t
|
||||
nm = self.eat()
|
||||
return Node('new', ty + nm.t, [], s, nm.e)
|
||||
if tk.k == 'ID' and tk.t == 'fn' and self.peek().k == 'ID':
|
||||
nm = self.eat()
|
||||
return Node('fnref', nm.t, [], tk.s, nm.e)
|
||||
if tk.k == 'ID': return Node('id', tk.t, [], tk.s, tk.e)
|
||||
if tk.k == 'EOF': raise SyntaxError('end of input')
|
||||
return Node('raw', tk.t, [], tk.s, tk.e)
|
||||
|
||||
def postfix(self, x):
|
||||
while True:
|
||||
if self.is_op('('):
|
||||
self.eat()
|
||||
args = []
|
||||
while not self.is_op(')') and self.peek().k != 'EOF':
|
||||
if self.peek().k == 'NL':
|
||||
self.eat()
|
||||
continue
|
||||
if self.peek().k == 'ID' and self.peek(1).k == 'OP' and self.peek(1).t == ':':
|
||||
nm = self.eat()
|
||||
self.eat()
|
||||
v = self.expr()
|
||||
args.append(Node('named', nm.t, [v], nm.s, v.e))
|
||||
else:
|
||||
args.append(self.expr())
|
||||
if self.is_op(','): self.eat()
|
||||
end = self.eat()
|
||||
n = Node('call', None, [x] + args, x.s, end.e)
|
||||
x = n
|
||||
elif self.is_op('.') and self.peek(1).k == 'ID':
|
||||
self.eat()
|
||||
nm = self.eat()
|
||||
x = Node('member', nm.t, [x], x.s, nm.e)
|
||||
elif self.is_op('['):
|
||||
self.eat()
|
||||
a = self.expr()
|
||||
if self.is_op('..'):
|
||||
self.eat()
|
||||
b = self.expr()
|
||||
end = self.eat()
|
||||
x = Node('slice', None, [x, a, b], x.s, end.e)
|
||||
else:
|
||||
end = self.eat()
|
||||
x = Node('index', None, [x, a], x.s, end.e)
|
||||
else:
|
||||
return x
|
||||
|
||||
|
||||
def walk(n):
|
||||
yield n
|
||||
for k in n.kids:
|
||||
yield from walk(k)
|
||||
347
tools/migrate/ludic_decls.py
Normal file
347
tools/migrate/ludic_decls.py
Normal file
|
|
@ -0,0 +1,347 @@
|
|||
"""Statements and declarations of a Ludic file, over ludic_ast's tokens and expressions."""
|
||||
from ludic_ast import tokenize, ExprParser, Node, Tok
|
||||
|
||||
STMT_END = ('NL', ';')
|
||||
|
||||
|
||||
class Decl:
|
||||
"""A typed name: its type text and where that text is, so a rewrite can change it."""
|
||||
__slots__ = ('name', 'ty', 'ts', 'te', 'file', 'init', 'key')
|
||||
|
||||
def __init__(self, name, ty, ts, te, file, init=None):
|
||||
self.name, self.ty, self.ts, self.te, self.file, self.init = name, ty, ts, te, file, init
|
||||
self.key = None
|
||||
|
||||
|
||||
class Func:
|
||||
__slots__ = ('name', 'params', 'ret', 'file', 'extern', 'runtime', 'locals')
|
||||
|
||||
def __init__(self, name, file):
|
||||
self.name, self.file = name, file
|
||||
self.params, self.ret, self.extern, self.runtime, self.locals = [], None, False, False, {}
|
||||
|
||||
|
||||
class Stmt:
|
||||
"""kind: let assign opassign return expr cond forrange foreach
|
||||
a, b: expression nodes (lhs / rhs); decl: the Decl a let/var introduced"""
|
||||
__slots__ = ('kind', 'fn', 'a', 'b', 'decl', 'op', 'file', 'var', 'scope')
|
||||
|
||||
def __init__(self, kind, fn, a=None, b=None, decl=None, op=None, file=None, var=None):
|
||||
self.kind, self.fn, self.a, self.b, self.decl, self.op, self.file, self.var = kind, fn, a, b, decl, op, file, var
|
||||
self.scope = None
|
||||
|
||||
|
||||
class FileParse:
|
||||
def __init__(self, path, src, runtime=False):
|
||||
self.path, self.src, self.runtime = path, src, runtime
|
||||
self.t = [x for x in tokenize(src)]
|
||||
self.i = 0
|
||||
self.funcs, self.globals, self.props, self.stmts = {}, {}, {}, []
|
||||
self.float_directive = False
|
||||
self.parse()
|
||||
|
||||
# ---- cursor -------------------------------------------------------------
|
||||
def pk(self, o=0):
|
||||
j = self.i + o
|
||||
return self.t[j] if j < len(self.t) else Tok('EOF', '', len(self.src), len(self.src))
|
||||
|
||||
def at(self, k, v=None, o=0):
|
||||
tk = self.pk(o)
|
||||
return tk.k == k and (v is None or tk.t == v)
|
||||
|
||||
def skip_nl(self):
|
||||
while self.at('NL') or self.at('OP', ';'): self.i += 1
|
||||
|
||||
def skip_to_eol(self):
|
||||
d = 0
|
||||
while not self.at('EOF'):
|
||||
tk = self.pk()
|
||||
if tk.k == 'OP' and tk.t in '([{': d += 1
|
||||
if tk.k == 'OP' and tk.t in ')]}':
|
||||
if d == 0: return
|
||||
d -= 1
|
||||
if d == 0 and (tk.k == 'NL' or (tk.k == 'OP' and tk.t == ';')): return
|
||||
self.i += 1
|
||||
|
||||
def skip_block(self):
|
||||
"""from a `{`, past its matching `}`"""
|
||||
while not self.at('OP', '{') and not self.at('EOF'): self.i += 1
|
||||
d = 0
|
||||
while not self.at('EOF'):
|
||||
tk = self.pk()
|
||||
if tk.k == 'OP' and tk.t == '{': d += 1
|
||||
if tk.k == 'OP' and tk.t == '}':
|
||||
d -= 1
|
||||
if d == 0:
|
||||
self.i += 1
|
||||
return
|
||||
self.i += 1
|
||||
|
||||
def type_here(self):
|
||||
"""the type text starting at the cursor, up to = , ) { NL; returns (text, start, end)"""
|
||||
s = self.pk().s
|
||||
e = s
|
||||
d = 0
|
||||
parts = []
|
||||
while not self.at('EOF'):
|
||||
tk = self.pk()
|
||||
if tk.k == 'NL': break
|
||||
if tk.k == 'OP' and tk.t in ('=', ',', '{') and d == 0: break
|
||||
if tk.k == 'OP' and tk.t == ')' and d == 0: break
|
||||
if tk.k == 'OP' and tk.t in '([': d += 1
|
||||
if tk.k == 'OP' and tk.t in ')]': d -= 1
|
||||
parts.append(tk.t)
|
||||
e = tk.e
|
||||
self.i += 1
|
||||
return ''.join(parts), s, e
|
||||
|
||||
def expr(self):
|
||||
p = ExprParser(self.t, self.i)
|
||||
n = p.expr()
|
||||
self.i = p.i
|
||||
return n
|
||||
|
||||
# ---- the file -----------------------------------------------------------
|
||||
def parse(self):
|
||||
while not self.at('EOF'):
|
||||
self.skip_nl()
|
||||
if self.at('EOF'): break
|
||||
self.top()
|
||||
|
||||
def top(self):
|
||||
tk = self.pk()
|
||||
if tk.k == 'OP' and tk.t == '@':
|
||||
self.i += 1
|
||||
self.i += 1
|
||||
if self.at('OP', '('):
|
||||
d = 0
|
||||
while True:
|
||||
if self.at('OP', '('): d += 1
|
||||
if self.at('OP', ')'): d -= 1
|
||||
self.i += 1
|
||||
if d == 0: break
|
||||
return
|
||||
if tk.k == 'OP' and tk.t == '}':
|
||||
self.i += 1
|
||||
return
|
||||
if tk.k != 'ID':
|
||||
self.skip_to_eol()
|
||||
if self.at('OP', '}') or self.at('OP', ')') or self.at('OP', ']'): self.i += 1
|
||||
return
|
||||
w = tk.t
|
||||
if w == 'program':
|
||||
self.i += 2
|
||||
self.skip_nl()
|
||||
if self.at('OP', '{'): self.i += 1
|
||||
return
|
||||
if w == 'numbers':
|
||||
self.float_directive = self.at('ID', 'float', 1)
|
||||
self.skip_to_eol()
|
||||
return
|
||||
if w == 'import' or w == 'model' and False:
|
||||
self.skip_to_eol()
|
||||
return
|
||||
if w == 'extern':
|
||||
self.i += 1
|
||||
self.function(extern=True)
|
||||
return
|
||||
if w == 'function':
|
||||
self.function()
|
||||
return
|
||||
if w == 'handler':
|
||||
nm = self.pk(1).t
|
||||
f = Func(nm, self.path)
|
||||
self.funcs.setdefault(nm, f)
|
||||
while not self.at('OP', '{') and not self.at('EOF'): self.i += 1
|
||||
self.i += 1
|
||||
self.body(f)
|
||||
return
|
||||
if w in ('var', 'const'):
|
||||
self.i += 1
|
||||
d = self.decl_here()
|
||||
if d: self.globals[d.name] = d
|
||||
return
|
||||
if w == 'property':
|
||||
self.prop()
|
||||
return
|
||||
if w in ('model', 'event', 'enum', 'ui', 'scene', 'layer', 'test', 'namespace', 'entry'):
|
||||
self.skip_block()
|
||||
return
|
||||
self.skip_to_eol()
|
||||
|
||||
def decl_here(self):
|
||||
"""after var/let/const: NAME [: TYPE] [= EXPR]"""
|
||||
nm = self.pk()
|
||||
if nm.k != 'ID':
|
||||
self.skip_to_eol()
|
||||
return None
|
||||
self.i += 1
|
||||
ty, ts, te = '', nm.e, nm.e
|
||||
if self.at('OP', ':'):
|
||||
self.i += 1
|
||||
ty, ts, te = self.type_here()
|
||||
init = None
|
||||
if self.at('OP', '='):
|
||||
self.i += 1
|
||||
init = self.expr()
|
||||
return Decl(nm.t, ty, ts, te, self.path, init)
|
||||
|
||||
def prop(self):
|
||||
self.i += 1
|
||||
nm = self.pk().t
|
||||
self.i += 1
|
||||
while not self.at('OP', '{') and not self.at('EOF'): self.i += 1
|
||||
self.i += 1
|
||||
fields = self.props.setdefault(nm, {})
|
||||
while not self.at('OP', '}') and not self.at('EOF'):
|
||||
self.skip_nl()
|
||||
if self.at('ID') and self.at('OP', ':', 1):
|
||||
d = self.decl_here()
|
||||
if d: fields[d.name] = d
|
||||
if self.at('OP', ','): self.i += 1
|
||||
elif not self.at('OP', '}'):
|
||||
self.i += 1
|
||||
self.i += 1
|
||||
|
||||
def function(self, extern=False):
|
||||
self.i += 1 # function
|
||||
nm = self.pk().t
|
||||
self.i += 1
|
||||
f = Func(nm, self.path)
|
||||
f.extern, f.runtime = extern, self.runtime
|
||||
self.funcs[nm] = f
|
||||
if self.at('OP', '('):
|
||||
self.i += 1
|
||||
while not self.at('OP', ')') and not self.at('EOF'):
|
||||
self.skip_nl()
|
||||
pn = self.pk()
|
||||
self.i += 1
|
||||
if self.at('OP', ':'):
|
||||
self.i += 1
|
||||
ty, ts, te = self.type_here()
|
||||
f.params.append(Decl(pn.t, ty, ts, te, self.path))
|
||||
if self.at('OP', ','): self.i += 1
|
||||
self.i += 1
|
||||
if self.at('OP', '->'):
|
||||
self.i += 1
|
||||
ty, ts, te = self.type_here()
|
||||
f.ret = Decl('', ty, ts, te, self.path)
|
||||
if extern:
|
||||
self.skip_to_eol()
|
||||
return
|
||||
while not self.at('OP', '{') and not self.at('EOF') and not self.at('NL'): self.i += 1
|
||||
if self.at('OP', '{'):
|
||||
self.i += 1
|
||||
self.body(f)
|
||||
|
||||
def body(self, f):
|
||||
"""statements until the `}` that closes the function"""
|
||||
depth = 1
|
||||
self.scopes = [{}]
|
||||
self.pending = {}
|
||||
while not self.at('EOF'):
|
||||
self.skip_nl()
|
||||
if self.at('OP', '}'):
|
||||
self.i += 1
|
||||
depth -= 1
|
||||
if len(self.scopes) > 1: self.scopes.pop()
|
||||
if depth == 0: return
|
||||
# `} else if cond {` / `} else {`
|
||||
if self.at('ID', 'else'):
|
||||
self.i += 1
|
||||
if self.at('ID', 'if'):
|
||||
self.i += 1
|
||||
c = self.expr()
|
||||
self.stmts.append(Stmt('cond', f.name, c, file=self.path))
|
||||
if self.at('OP', '{'):
|
||||
self.i += 1
|
||||
depth += 1
|
||||
self.scopes.append({})
|
||||
continue
|
||||
if self.at('OP', '{'):
|
||||
self.i += 1
|
||||
depth += 1
|
||||
self.scopes.append({})
|
||||
continue
|
||||
n0 = len(self.stmts)
|
||||
opened = self.statement(f)
|
||||
for st in self.stmts[n0:]:
|
||||
if st.scope is None: st.scope = self.visible()
|
||||
if opened:
|
||||
self.scopes.append(self.pending)
|
||||
self.pending = {}
|
||||
depth += opened
|
||||
|
||||
def visible(self):
|
||||
out = {}
|
||||
for sc in self.scopes: out.update(sc)
|
||||
return out
|
||||
|
||||
def bind(self, f, d):
|
||||
"""a local declared here: its own entry, so a name reused in another block is another name"""
|
||||
uniq = d.name if d.name not in f.locals else f'{d.name}@{len(f.locals)}'
|
||||
f.locals[uniq] = d
|
||||
return uniq
|
||||
|
||||
def statement(self, f):
|
||||
"""one statement; returns how many blocks it opened"""
|
||||
tk = self.pk()
|
||||
w = tk.t if tk.k == 'ID' else None
|
||||
opened = 0
|
||||
if w in ('let', 'var', 'const'):
|
||||
self.i += 1
|
||||
d = self.decl_here()
|
||||
if d:
|
||||
uniq = self.bind(f, d)
|
||||
self.stmts.append(Stmt('let', f.name, decl=d, b=d.init, file=self.path))
|
||||
self.stmts[-1].scope = self.visible()
|
||||
self.scopes[-1][d.name] = uniq
|
||||
elif w in ('if', 'while'):
|
||||
self.i += 1
|
||||
c = self.expr()
|
||||
self.stmts.append(Stmt('cond', f.name, c, file=self.path))
|
||||
elif w == 'else':
|
||||
self.i += 1
|
||||
if self.at('ID', 'if'):
|
||||
self.i += 1
|
||||
c = self.expr()
|
||||
self.stmts.append(Stmt('cond', f.name, c, file=self.path))
|
||||
elif w == 'for':
|
||||
self.i += 1
|
||||
var = self.pk().t
|
||||
self.i += 2 # name in
|
||||
a = self.expr()
|
||||
if self.at('OP', '..'):
|
||||
self.i += 1
|
||||
b = self.expr()
|
||||
self.stmts.append(Stmt('forrange', f.name, a, b, var=var, file=self.path))
|
||||
else:
|
||||
self.stmts.append(Stmt('foreach', f.name, a, var=var, file=self.path))
|
||||
self.pending[var] = self.bind(f, Decl(var, '', -1, -1, self.path))
|
||||
self.stmts[-1].scope = dict(self.visible(), **{var: self.pending[var]})
|
||||
elif w == 'return':
|
||||
self.i += 1
|
||||
if not (self.at('NL') or self.at('OP', '}') or self.at('OP', ';')):
|
||||
self.stmts.append(Stmt('return', f.name, self.expr(), file=self.path))
|
||||
elif w in ('break', 'continue', 'quit'):
|
||||
self.i += 1
|
||||
elif w == 'spawn':
|
||||
self.skip_to_eol()
|
||||
else:
|
||||
try:
|
||||
a = self.expr()
|
||||
except Exception:
|
||||
self.skip_to_eol()
|
||||
return 0
|
||||
if self.at('OP') and self.pk().t in ('=', '+=', '-=', '*=', '/=', '%='):
|
||||
op = self.pk().t
|
||||
self.i += 1
|
||||
b = self.expr()
|
||||
self.stmts.append(Stmt('assign' if op == '=' else 'opassign', f.name, a, b, op=op, file=self.path))
|
||||
else:
|
||||
self.stmts.append(Stmt('expr', f.name, a, file=self.path))
|
||||
# a block opened on this line
|
||||
if self.at('OP', '{'):
|
||||
self.i += 1
|
||||
opened = 1
|
||||
return opened
|
||||
Loading…
Add table
Add a link
Reference in a new issue