ludic/tools/ludic-cli/migrate.ludic

# ---- ludic migrate state -------------------------------------------------------
# 0.S2: module-level vars become states. The compiler works out the edits (selfhost/frontend/
# migrate.ludic, `ludicc --migrate-state <plan>`); this applies them to the source, file by file,
# and prints what it could not decide.
#
#   ludic migrate state [file] [--runtime] [--unsafe] [--dry-run]
#
# Run it once per program that uses what changes (a package's test programs, the game's entry,
# its lab): a later run finds the states an earlier one made and passes them on.
var mgx_file: []pointer = null        # an edit: its file, where, how much it replaces, with what
var mgx_pos: []int = null
var mgx_len: []int = null
var mgx_text: []pointer = null
var mgx_texts: []pointer = null       # the files' texts, read once
var mgx_names: []pointer = null
var mgx_roots: []pointer = null       # where it may write: the programs and directories it was given

function mgx_unesc(s: pointer) -> pointer {
  var out = ""
  var i = 0
  let n = slen(s)
  while i < n {
    if s[i] == 92 and i + 1 < n {
      let c = s[i + 1]
      if c == 's' { out = out + " " } else if c == 'n' { out = out + "\n" } else { out = out + sslice(s, i + 1, i + 2) }
      i += 2
    } else {
      out = out + sslice(s, i, i + 1)
      i += 1
    }
  }
  return out
}
function mgx_text_of(path: pointer) -> pointer {
  for i in 0 .. len(mgx_names) { if mgx_names[i] == path { return mgx_texts[i] } }
  let t = read_file(path)
  push(mgx_names, path)
  push(mgx_texts, t)
  return t
}
function mgx_edit(file: pointer, pos: int, n: int, text: pointer) -> void {
  push(mgx_file, file)
  push(mgx_pos, pos)
  push(mgx_len, n)
  push(mgx_text, text)
}
function mgx_line_start(t: pointer, pos: int) -> int {
  var i = pos
  while i > 0 and t[i - 1] != 10 { i -= 1 }
  return i
}
function mgx_line_end(t: pointer, pos: int) -> int {
  var i = pos
  while t[i] != 0 and t[i] != 10 { i += 1 }
  if t[i] == 10 { i += 1 }
  return i
}
function mgx_indent(t: pointer, start: int) -> pointer {
  var i = start
  while t[i] == 32 or t[i] == 9 { i += 1 }
  return sslice(t, start, i)
}
# the text from byte `at` on, without copying it
function data_at(t: pointer, at: int) -> pointer { return offset(t, at) }
# a var's declaration from its name to its end (`count: int = 0`), and the comment after it on its
# line when nothing else is there (two vars on one line keep their own text each)
function mgx_field(t: pointer, a: int, b: int) -> pointer {
  var out = sslice(t, a, b)
  var i = b
  while t[i] == 32 or t[i] == 9 { i += 1 }
  if t[i] == 35 {
    var e = i
    while t[e] != 0 and t[e] != 10 { e += 1 }
    out = out + sslice(t, b, e)
  }
  return mgx_rtrim(out)
}
function mgx_rtrim(s: pointer) -> pointer {
  var n = slen(s)
  while n > 0 and (s[n - 1] == 10 or s[n - 1] == 13 or s[n - 1] == 32) { n -= 1 }
  return sslice(s, 0, n)
}
# the brace that closes the first block after `at` (strings skipped), or -1
function mgx_close(t: pointer, at: int) -> int {
  var i = at
  while t[i] != 0 and t[i] != '{' { i += 1 }
  var depth = 0
  var instr = false
  while t[i] != 0 {
    let c = t[i]
    if instr {
      if c == 92 { i += 1 } else if c == 34 { instr = false }
    } else if c == 34 { instr = true }
    else if c == '{' { depth += 1 }
    else if c == '}' {
      depth -= 1
      if depth == 0 { return i }
    }
    i += 1
  }
  return -1
}
# a directory as the plans spell a path (relative to here, `..` resolved), with a trailing '/'
function mgx_dir(d: pointer) -> pointer {
  var p = mgx_norm(d)
  if len(p) == 0 or (p == ".") { return "" }
  if p[len(p) - 1] != '/' { p = p + "/" }
  return p
}
function mgx_norm(p0: pointer) -> pointer {
  var p = p0
  let cwd = getenv_or("PWD", "")
  if len(cwd) > 0 and s_starts(p, cwd + "/") { p = p[len(cwd) + 1 .. len(p)] }
  let abs = len(p) > 0 and p[0] == '/'
  let segs = new []pointer
  var n = 0
  var a = 0
  for i in 0 .. len(p) + 1 {
    if i == len(p) or p[i] == '/' {
      let seg = p[a .. i]
      if (seg == "..") and n > 0 and not (segs[n - 1] == "..") { n -= 1 }
      else if not (seg == ".") and not (seg == "") {
        if n < len(segs) { segs[n] = seg } else { push(segs, seg) }
        n += 1
      }
      a = i + 1
    }
  }
  var out = ""
  if abs { out = "/" }
  for i in 0 .. n {
    if i > 0 { out = out + "/" }
    out = out + segs[i]
  }
  return out
}
# may it write this file: it is under one of the programs' directories (a package the program only
# imports - installed, or a neighbour's - is not the migration's to change)
function mgx_mine(path: pointer) -> bool {
  let p = mgx_norm(path)
  for i in 0 .. len(mgx_roots) {
    if mgx_roots[i] == "" and not (p[0] == '/') and not s_starts(p, "../") { return true }
    if not (mgx_roots[i] == "") and s_starts(p, mgx_roots[i]) { return true }
  }
  return false
}
function cmd_migrate() -> int {
  if arg_count() < 3 or not (arg(2) == "state") {
    err("usage: ludic migrate state [file|dir...] [--runtime] [--unsafe] [--dry-run]\n")
    return 2
  }
  let srcs = new []pointer
  mgx_roots = new []pointer
  var runtime = ""
  var dry = false
  var ai = 3
  while ai < arg_count() {
    let a = arg(ai)
    if a == "--runtime" {
      runtime = " --migrate-runtime"
      push(mgx_roots, "runtime/")
    }
    else if a == "--unsafe" { g_unsafe_build = true }
    else if a == "--dry-run" { dry = true }
    else if a[0] == '-' { err(`ludic migrate: unknown option {a}\n`);  return 2 }
    else if shq(`test -d {sh_single(a)}`) {
      let found = test_files_in(a)
      for k in 0 .. len(found) { push(srcs, found[k]) }
      push(mgx_roots, mgx_dir(a))
    }
    else {
      push(srcs, a)
      push(mgx_roots, mgx_dir(dir_of_path(a)))
    }
    ai += 1
  }
  if len(srcs) == 0 {
    let entry = find_entry("")
    if entry == "" { return no_entry() }
    push(srcs, entry)
    push(mgx_roots, mgx_dir(dir_of_path(entry)))
  }
  ensure_ludicc()
  # every program's plan from the source as it is, then one pass over the files: programs that
  # share a file (a package, a test's fakes) agree about it, and none sees another's half-done work
  let all = new []pointer
  for i in 0 .. len(srcs) {
    let plan = tmp_path(`migrate{string(i)}.plan`)
    shell(`rm -f {plan}`)
    shq(`{ludicc()} --headless{unsafe_flag()}{runtime} --migrate-state {plan} {sh_single(srcs[i])} > {tmp_path("migrate.err")} 2>&1`)
    let text = read_file(plan)
    if text == null {
      err(`ludic migrate: the compiler wrote no plan for {srcs[i]}:\n`)
      err(capture(`grep -i error {tmp_path("migrate.err")} | head -5`))
      return 1
    }
    let ls = split_lines(text)
    for k in 0 .. len(ls) { push(all, ls[k]) }
  }
  return mgx_apply(mgx_unique(all), dry)
}
# the lines once each, in order of text (a merge sort: a render3d plan is thousands of lines)
function mgx_unique(xs: []pointer) -> []pointer {
  let tmp = new []pointer
  for i in 0 .. len(xs) { push(tmp, "") }
  mgx_sort(xs, 0, len(xs), tmp)
  let out = new []pointer
  for i in 0 .. len(xs) {
    if i == 0 or not (xs[i] == xs[i - 1]) { push(out, xs[i]) }
  }
  return out
}
function mgx_sort(xs: []pointer, lo: int, hi: int, tmp: []pointer) -> void {
  if hi - lo < 2 { return }
  let mid = (lo + hi) / 2
  mgx_sort(xs, lo, mid, tmp)
  mgx_sort(xs, mid, hi, tmp)
  var i = lo
  var j = mid
  var k = lo
  while k < hi {
    if j >= hi or (i < mid and not s_less(xs[j], xs[i])) {
      tmp[k] = xs[i]
      i += 1
    } else {
      tmp[k] = xs[j]
      j += 1
    }
    k += 1
  }
  for m in lo .. hi { xs[m] = tmp[m] }
}
function mgx_apply(lines: []pointer, dry: bool) -> int {
  mgx_file = new []pointer;  mgx_pos = new []int;  mgx_len = new []int;  mgx_text = new []pointer
  mgx_texts = new []pointer;  mgx_names = new []pointer
  let vfile = new []pointer
  let vpos = new []int
  let vpos2 = new []int
  let vstate = new []pointer
  let sname = new []pointer
  let sat = new []pointer             # where each is declared: file:pos
  let sfile = new []pointer
  let spos = new []int
  let sexp = new []int
  let smore = new []int                # 1: the state is already declared, and gains fields
  let reports = new []pointer
  let foreign = new []pointer          # files a plan would change that the migration may not
  let lfile = new []pointer
  let lpos = new []int
  for i in 0 .. len(lines) {
    let w = dp_words(lines[i])
    if len(w) == 0 { continue }
    var at = ""
    if (w[0] == "VAR") or (w[0] == "REF") or (w[0] == "INS") or (w[0] == "CLOSE") or (w[0] == "LET") { at = w[1] }
    if (w[0] == "STATE") and len(w) > 2 { at = w[2] }
    if len(at) > 0 and mgx_roots != null and not mgx_mine(at) {
      var seen = false
      for k in 0 .. len(foreign) { if foreign[k] == at { seen = true } }
      if not seen { push(foreign, at) }
      continue
    }
    if w[0] == "VAR" and len(w) == 6 {
      push(vfile, w[1])
      push(vpos, s_to_int(w[2]))
      push(vpos2, s_to_int(w[3]))
      push(vstate, w[5])                 # the state by where it is declared, not by its name
    } else if w[0] == "STATE" and len(w) >= 7 {
      # one state from several programs' plans: exported if any of them needs it outside
      # one state from several programs' plans, which saw different files of it: declared where one
      # is declared already, else at the first of its vars any of them saw; exported if any needs it
      let at = w[6]
      let more = len(w) >= 8
      var known = -1
      for k in 0 .. len(sat) { if sat[k] == at { known = k } }
      if known >= 0 {
        if s_to_int(w[4]) == 1 { sexp[known] = 1 }
        let earlier = s_less(w[2], sfile[known]) or (w[2] == sfile[known] and s_to_int(w[3]) < spos[known])
        if smore[known] == 0 and (more or earlier) {
          sfile[known] = w[2]
          spos[known] = s_to_int(w[3])
          if more { smore[known] = 1 }
        }
        continue
      }
      push(sname, w[1])
      push(sat, at)
      push(sfile, w[2])
      push(spos, s_to_int(w[3]))
      push(sexp, s_to_int(w[4]))
      if more { push(smore, 1) } else { push(smore, 0) }
    } else if (w[0] == "REF") and len(w) == 5 {
      mgx_edit(w[1], s_to_int(w[2]), s_to_int(w[3]), mgx_unesc(w[4]))
    } else if (w[0] == "INS") and len(w) == 4 {
      mgx_edit(w[1], s_to_int(w[2]), 0, mgx_unesc(w[3]))
    } else if (w[0] == "CLOSE") and len(w) == 4 {
      let t = mgx_text_of(w[1])
      var k = s_to_int(w[2])
      while t[k] != 0 and t[k] != ')' { k += 1 }
      mgx_edit(w[1], k, 0, mgx_unesc(w[3]))
    } else if w[0] == "LET" and len(w) == 3 {
      push(lfile, w[1])
      push(lpos, s_to_int(w[2]))
    } else if w[0] == "?" { push(reports, lines[i]) }
  }
  # a var nothing writes is a let - unless another program's plan saw it written
  var lets = 0
  for l in 0 .. len(lfile) {
    var moved = false
    for v in 0 .. len(vfile) { if vfile[v] == lfile[l] and vpos[v] == lpos[l] { moved = true } }
    if not moved {
      let t = mgx_text_of(lfile[l])
      var k = lpos[l] - 3
      let ls = mgx_line_start(t, lpos[l])
      while k >= ls and not (t[k] == 'v' and t[k + 1] == 'a' and t[k + 2] == 'r') { k -= 1 }
      if k >= ls {
        mgx_edit(lfile[l], k, 3, "let")
        lets += 1
      }
    }
  }
  # a change the programs need in a file it may not write (a package's, a neighbour's): refused,
  # whole - that code is migrated where it lives, or given here too
  if len(foreign) > 0 {
    err("ludic migrate: these files would have to change too, and are not under a program or directory given; nothing was changed:\n")
    for f in 0 .. len(foreign) { err(`  {foreign[f]}\n`) }
    err("migrate them first (a package: in its own checkout), or name their directory as well\n")
    return 1
  }
  # a var whose state would be declared in a file it may not write: refused, whole
  for v in 0 .. len(vfile) {
    var ok = false
    for k in 0 .. len(sat) { if sat[k] == vstate[v] { ok = true } }
    if not ok {
      err(`ludic migrate: {vfile[v]} has a var for the state of {vstate[v]}, which is declared where this migration may not write; nothing was changed\n`)
      return 1
    }
  }
  # the vars: each line goes, and the first of each state's becomes the state
  let del_file = new []pointer
  let del_a = new []int
  let del_b = new []int
  for v in 0 .. len(vfile) {
    let t = mgx_text_of(vfile[v])
    let a = mgx_line_start(t, vpos[v])
    let b = mgx_line_end(t, vpos2[v])
    push(del_file, vfile[v])
    push(del_a, a)
    push(del_b, b)
  }
  # the fields in the order they were declared: by file, then by place
  let order = new []int
  for v in 0 .. len(vfile) { push(order, v) }
  var oa = 1
  while oa < len(order) {
    let x = order[oa]
    var ob = oa - 1
    while ob >= 0 and (s_less(vfile[x], vfile[order[ob]]) or (vfile[x] == vfile[order[ob]] and vpos[x] < vpos[order[ob]])) {
      order[ob + 1] = order[ob]
      ob -= 1
    }
    order[ob + 1] = x
    oa += 1
  }
  for s in 0 .. len(sname) {
    var block = ""
    var at = -1
    var indent = ""
    if smore[s] == 1 {
      # the fields go in before the declaration's closing brace
      let t = mgx_text_of(sfile[s])
      indent = mgx_indent(t, mgx_line_start(t, spos[s]))
      var ff = ""
      for ov in 0 .. len(order) {
        let v = order[ov]
        if vstate[v] == sat[s] { ff = ff + `{indent}  {mgx_field(mgx_text_of(vfile[v]), vpos[v], vpos2[v])}\n` }
      }
      let close = mgx_close(t, spos[s])
      if close >= 0 { mgx_edit(sfile[s], mgx_line_start(t, close), 0, ff) }
      continue
    }
    for ov in 0 .. len(order) {
      let v = order[ov]
      if vstate[v] == sat[s] {
        let t = mgx_text_of(vfile[v])
        if vfile[v] == sfile[s] and vpos[v] == spos[s] {
          at = v
          indent = mgx_indent(t, del_a[v])
        }
        block = block + `{indent}  {mgx_field(t, vpos[v], vpos2[v])}\n`
      }
    }
    var head = ""
    if sexp[s] == 1 { head = "export " }
    if at >= 0 { mgx_edit(vfile[at], del_a[at], 0, `{indent}{head}state {sname[s]} {{\n{block}{indent}}}\n`) }
  }
  for d in 0 .. len(del_file) { mgx_edit(del_file[d], del_a[d], del_b[d] - del_a[d], "") }
  for e in 0 .. len(mgx_file) { mgx_text_of(mgx_file[e]) }    # every file with an edit, read
  # an edit inside a line that goes is dropped with it
  var files = 0
  var edits = 0
  for f in 0 .. len(mgx_names) {
    let path = mgx_names[f]
    var t = mgx_texts[f]
    let idx = new []int
    for e in 0 .. len(mgx_file) {
      if mgx_file[e] == path {
        var inside = false
        for d in 0 .. len(del_file) {
          if del_file[d] == path and mgx_len[e] < del_b[d] - del_a[d] and mgx_pos[e] > del_a[d] and mgx_pos[e] < del_b[d] { inside = true }
        }
        if not inside { push(idx, e) }
      }
    }
    # in the order of their places; at one place, what is put there before what is taken out
    var a = 1
    while a < len(idx) {
      let x = idx[a]
      var b = a - 1
      while b >= 0 and (mgx_pos[idx[b]] > mgx_pos[x] or (mgx_pos[idx[b]] == mgx_pos[x] and mgx_len[idx[b]] > mgx_len[x])) {
        idx[b + 1] = idx[b]
        b -= 1
      }
      idx[b + 1] = x
      a += 1
    }
    if len(idx) == 0 { continue }
    files += 1
    if dry { continue }
    let out = file_open(path, "wb")
    if out == null {
      err(`ludic migrate: cannot write {path}\n`)
      continue
    }
    var cur = 0
    for k in 0 .. len(idx) {
      let e = idx[k]
      # an edit made twice (a generic's instances share its source) is made once
      var dup = false
      if k > 0 {
        let p = idx[k - 1]
        dup = mgx_pos[p] == mgx_pos[e] and mgx_len[p] == mgx_len[e] and mgx_text[p] == mgx_text[e]
      }
      if not dup and mgx_pos[e] >= cur {
        if mgx_pos[e] > cur { file_write(out, data_at(t, cur), mgx_pos[e] - cur) }
        let txt = mgx_text[e]
        if slen(txt) > 0 { file_write(out, txt, slen(txt)) }
        cur = mgx_pos[e] + mgx_len[e]
        edits += 1
      }
    }
    let n = slen(t)
    if n > cur { file_write(out, data_at(t, cur), n - cur) }
    file_close(out)
  }
  print(`migrate: {string(len(vfile))} vars into {string(len(sname))} states, {string(lets)} into lets; {string(edits)} edits in {string(files)} files`)
  for r in 0 .. len(reports) { print(`  by hand: {sslice(reports[r], 2, slen(reports[r]))}`) }
  return 0
}