diff --git a/README.md b/README.md index 1dcbb172..53aeb2c3 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,7 @@ cached in a content-addressed store: ```bash ludic add git.workshopsoft.io/user/pkg # resolve, fetch, link into ludic_modules/ ludic get # install from package.ludic, write the lock +ludic remove git.workshopsoft.io/user/pkg # the inverse of add ludic verify # check locked packages against the store ``` diff --git a/changes/pkg-remove-get-json.md b/changes/pkg-remove-get-json.md new file mode 100644 index 00000000..eda7f107 --- /dev/null +++ b/changes/pkg-remove-get-json.md @@ -0,0 +1,10 @@ +bump: minor +type: feat +**`ludic remove ` and `ludic get --json`.** `remove` is the inverse of `add`: the `require` +leaves `package.ludic`, the lock keeps what the remaining requires still reach (read from the store's +copies of each package's manifest, no network), and each package leaving the lock loses the +`ludic_modules/` link `add` made - never the shared store entry. A module that is not required is +refused; source still importing a removed package is a warning. `ludic get --json` prints what the run +changed in the lock as one JSON object on stdout - `{"added", "removed", "changed", "unchanged"}`, an +entry as the lock records it (`name`, `version`, `hash`, `kind`, `provides`), a change as `{name, from, +to, from_hash, to_hash}` - with the resolver's own lines on stderr. diff --git a/docs/PACKAGES.md b/docs/PACKAGES.md index 219ebf18..41a25cd9 100644 --- a/docs/PACKAGES.md +++ b/docs/PACKAGES.md @@ -19,7 +19,8 @@ This is the v1 implementation of the direction decided in issue #63. ``` ludic add [@version] add a dependency to package.ludic, then resolve + fetch + link -ludic get resolve every dependency in package.ludic, link them, write the lock +ludic remove the inverse of add: drop the require, what only it locked, its links +ludic get [--json] resolve every dependency in package.ludic, link them, write the lock ludic update [module] bump a dependency (or all) to its latest published version, then relock ludic verify check every locked package against the store by content hash ludic vendor copy the resolved packages into ./vendor for hermetic/offline builds @@ -29,6 +30,23 @@ ludic vendor copy the resolved packages into ./vendor for herm minimum. All the install commands print the resolved build list and write `package.lock.ludic`. +`ludic get --json` prints what the run changed in the lock as one JSON object on stdout (the +resolver's lines go to stderr): `{"added": [...], "removed": [...], "changed": [...], "unchanged": n}`, +an added or removed entry being the lock's line - `{"name", "version", "hash", "kind", "provides"}` - +and a changed one `{"name", "from", "to", "from_hash", "to_hash"}` (the versions, and the content +hashes beside them). The lock is compared before and after, in memory. + +`ludic remove ` takes the `require` line out of `package.ludic` and re-reads the dependency +graph from the store's own copy of each locked package's manifest - no network - so the lock keeps +exactly what the remaining requires still reach: a package another one still requires stays locked +(at the version it had; `ludic get` settles versions), and what only the removed one brought in +leaves the lock with it. Each package leaving the lock loses its `ludic_modules/` link - the link +`add` made, never the store entry it points at, which other projects share; anything there that is +not such a link is left and reported, as is a copy under `vendor/`. A module `package.ludic` does +not require is refused (exit 1). Source that still imports a removed package is a warning, not a +failure - the caller may be about to delete it. If the store has no copy of a locked package, only +the named module leaves the lock, and it says so. + ## The manifest — `package.ludic` A line-oriented manifest. `#` starts a comment; strings are double-quoted. diff --git a/tools/ludic-cli/dev.ludic b/tools/ludic-cli/dev.ludic index 18867334..7bdf6dfd 100644 --- a/tools/ludic-cli/dev.ludic +++ b/tools/ludic-cli/dev.ludic @@ -39,6 +39,7 @@ program LudicDev { import "assets.ludic" import "release.ludic" import "pkg.ludic" + import "pkg_lock.ludic" import "pkg_test.ludic" function usage() -> void { diff --git a/tools/ludic-cli/main.ludic b/tools/ludic-cli/main.ludic index db413115..5c6663f9 100644 --- a/tools/ludic-cli/main.ludic +++ b/tools/ludic-cli/main.ludic @@ -25,6 +25,8 @@ program Ludic { import "split.ludic" import "project.ludic" import "pkg.ludic" + import "pkg_lock.ludic" + import "pkg_remove.ludic" import "assets.ludic" import "packignore.ludic" import "pack.ludic" @@ -59,7 +61,9 @@ program Ludic { print("") print("packages:") print(" add [@version] add a dependency, then resolve + fetch + link") - print(" get resolve every package.ludic dependency and write the lock") + print(" remove drop a dependency: its require, what only it locked, its ludic_modules/ link") + print(" get [--json] resolve every package.ludic dependency and write the lock") + print(" (--json: what changed in the lock, as a JSON object on stdout)") print(" update [module] bump a dependency (or all) to its latest published version") print(" verify check every locked package against the store by content hash") print(" vendor copy the resolved packages into ./vendor for offline builds") @@ -124,6 +128,7 @@ program Ludic { if (cmd == "add") { return cmd_pkg_add() } if (cmd == "get") { return cmd_pkg_get() } + if (cmd == "remove") { return cmd_pkg_remove() } if (cmd == "update") { return cmd_pkg_update() } if (cmd == "verify") { return cmd_pkg_verify() } if (cmd == "vendor") { return cmd_pkg_vendor() } diff --git a/tools/ludic-cli/pkg.ludic b/tools/ludic-cli/pkg.ludic index 22e92efb..ecce3938 100644 --- a/tools/ludic-cli/pkg.ludic +++ b/tools/ludic-cli/pkg.ludic @@ -1,5 +1,5 @@ # pkg.ludic — the Ludic package manager (issue #63), the `ludic add`/`get`/ -# `update`/`verify`/`vendor` commands. +# `update`/`verify`/`vendor` commands (`remove` and `get --json`: pkg_remove.ludic, pkg_lock.ludic). # # It realises the v1 direction decided in the RFC: # @@ -484,7 +484,7 @@ function write_lock(sels: []Manifest) -> bool { function do_install(root: Manifest) -> int { let sels = resolve(root) if len(sels) == 0 { - print("no dependencies to resolve") + pkg_say("no dependencies to resolve") write_lock(sels) return 0 } @@ -505,11 +505,11 @@ function do_install(root: Manifest) -> int { if slen(h) == 0 { err(`ludic: failed to snapshot {m.module}@{m.ver}\n`); return 1 } m.hash = `sha256:{h}` link_module(m.module, h) - print(` {m.module} {m.ver} ({m.kind}) sha256:{sslice(h, 0, 12)}…`) + pkg_say(` {m.module} {m.ver} ({m.kind}) sha256:{sslice(h, 0, 12)}…`) i += 1 } if not write_lock(sels) { err("ludic: cannot write package.lock.ludic\n"); return 1 } - print(`resolved {string(len(sels))} package(s) — see package.lock.ludic; linked under ludic_modules/`) + pkg_say(`resolved {string(len(sels))} package(s) — see package.lock.ludic; linked under ludic_modules/`) return 0 } @@ -596,12 +596,19 @@ function cmd_pkg_add() -> int { return do_install(read_root_manifest()) } -# ludic get — resolve + fetch + link every dependency in package.ludic, write lock +# ludic get [--json] — resolve + fetch + link every dependency in package.ludic, write lock. --json +# says what changed in the lock as one JSON object on stdout (lock_diff_json), the rest on stderr. function cmd_pkg_get() -> int { let txt = read_file("package.ludic") if txt == null { err("ludic: no package.ludic in the current directory (ludic add to start one)\n"); return 1 } - print("resolving dependencies (MVS)…") - return do_install(parse_manifest(txt)) + let json = pkg_has_flag("--json") + let before = read_lock_or_empty() + g_pkg_err = json + pkg_say("resolving dependencies (MVS)…") + let rc = do_install(parse_manifest(txt)) + if rc != 0 or not json { return rc } + out(lock_diff_json(before, read_lock_or_empty())) + return 0 } # ludic update [module] — bump a dep (or all) to its latest published version, relock diff --git a/tools/ludic-cli/pkg_lock.ludic b/tools/ludic-cli/pkg_lock.ludic new file mode 100644 index 00000000..686ca0f5 --- /dev/null +++ b/tools/ludic-cli/pkg_lock.ludic @@ -0,0 +1,116 @@ +# pkg_lock.ludic — the package manager's answers for editors and tools: `ludic get --json` (what a +# get changed in package.lock.ludic, as one JSON object on stdout) and `ludic remove `, the +# inverse of `ludic add`. Neither touches the network: the diff is of the lock before and after, and +# a removal re-reads the dependency graph from the store's own copies of each package.ludic. + +# the human lines of add / get / update: on stdout, or on stderr under --json +var g_pkg_err: bool = false +function pkg_say(line: pointer) -> void { + if g_pkg_err { err(line + nl()) } else { print(line) } +} +function pkg_has_flag(flag: pointer) -> bool { + var ai = 2 + while ai < arg_count() { + if arg(ai) == flag { return true } + ai += 1 + } + return false +} + +# ---- JSON out -------------------------------------------------------------------- + +function pkg_hex(d: int) -> pointer { + let b = bytes(2) + b[0] = '0' + d + if d > 9 { b[0] = 'a' + d - 10 } + b[1] = 0 + return b +} +function pkg_jesc(c: int) -> pointer { + if c == '"' { return "\\\"" } + if c == 92 { return "\\\\" } + if c == '\n' { return "\\n" } + if c == '\t' { return "\\t" } + if c == '\r' { return "\\r" } + return "\\u00" + pkg_hex(c >> 4) + pkg_hex(c & 15) +} +# `s` as a JSON string, quotes included +function pkg_jq(s: pointer) -> pointer { + var o = "\"" + let n = slen(s) + var start = 0 + var i = 0 + while i < n { + let c = s[i] & 255 + if c == '"' or c == 92 or c < 32 { + o = o + sslice(s, start, i) + pkg_jesc(c) + start = i + 1 + } + i += 1 + } + return o + sslice(s, start, n) + "\"" +} +# a lock line as the lock records it +function lock_entry_json(m: Manifest) -> pointer { + var o = `{{"name": {pkg_jq(m.module)}, "version": {pkg_jq(m.ver)}, "hash": {pkg_jq(m.hash)}, "kind": {pkg_jq(m.kind)}, "provides": [` + var p = 0 + while p < len(m.provides) { + if p > 0 { o = o + ", " } + o = o + pkg_jq(m.provides[p]) + p += 1 + } + return o + "]}" +} +function lock_index(ms: []Manifest, module: pointer) -> int { + var i = 0 + while i < len(ms) { + if ms[i].module == module { return i } + i += 1 + } + return -1 +} +function json_list(items: []pointer) -> pointer { + if len(items) == 0 { return "[]" } + var o = "[" + var i = 0 + while i < len(items) { + if i > 0 { o = o + "," } + o = o + nl() + " " + items[i] + i += 1 + } + return o + nl() + " ]" +} +# what changed between two locks: {"added", "removed", "changed", "unchanged"}. A module whose version +# or content hash moved is changed; `from` / `to` are its versions, the hashes beside them. +function lock_diff_json(before: []Manifest, after: []Manifest) -> pointer { + let added = new []pointer + let removed = new []pointer + let changed = new []pointer + var same = 0 + var i = 0 + while i < len(after) { + let m = after[i] + let at = lock_index(before, m.module) + if at < 0 { push(added, lock_entry_json(m)) } + else { + let o = before[at] + if o.ver == m.ver and o.hash == m.hash { same += 1 } + else { push(changed, `{{"name": {pkg_jq(m.module)}, "from": {pkg_jq(o.ver)}, "to": {pkg_jq(m.ver)}, "from_hash": {pkg_jq(o.hash)}, "to_hash": {pkg_jq(m.hash)}}}`) } + } + i += 1 + } + i = 0 + while i < len(before) { + if lock_index(after, before[i].module) < 0 { push(removed, lock_entry_json(before[i])) } + i += 1 + } + var o = "{" + nl() + ` "added": {json_list(added)},` + nl() + o = o + ` "removed": {json_list(removed)},` + nl() + o = o + ` "changed": {json_list(changed)},` + nl() + return o + ` "unchanged": {string(same)}` + nl() + "}" + nl() +} +function read_lock_or_empty() -> []Manifest { + let t = read_file("package.lock.ludic") + if t == null { return new []Manifest } + return parse_lock(t) +} diff --git a/tools/ludic-cli/pkg_remove.ludic b/tools/ludic-cli/pkg_remove.ludic new file mode 100644 index 00000000..f90c83cc --- /dev/null +++ b/tools/ludic-cli/pkg_remove.ludic @@ -0,0 +1,134 @@ +# pkg_remove.ludic — `ludic remove `, the inverse of `ludic add`: the `require` line leaves +# package.ludic, and the lock keeps exactly what the remaining requires still reach. The graph is +# read from the store's copies of each locked package.ludic, so nothing is fetched; what the removal +# leaves unreachable leaves the lock and its ludic_modules/ link (the link `add` made, never the store +# entry it points at, which other projects share). Source still importing a removed package is a +# warning: the caller may be about to delete it. + +# package.ludic's text with every `require "" ...` line dropped +function drop_require(txt: pointer, module: pointer) -> pointer { + var out = "" + let n = slen(txt) + var i = 0 + while i < n { + let line = line_at(txt, i) + i = i + slen(line) + 1 + let ts = tok_line(line) + if not (len(ts) >= 2 and ts[0] == "require" and ts[1] == module) { out = out + line + nl() } + } + return out +} +function requires(m: Manifest, module: pointer) -> bool { + var i = 0 + while i < len(m.deps) { + if m.deps[i].module == module { return true } + i += 1 + } + return false +} + +var g_graph_unknown: pointer = "" # a locked package the store has no copy of, when there is one + +# the locked entries the root's requires still reach, through the store's package.ludic files +function lock_reachable(root: Manifest, locked: []Manifest) -> []Manifest { + let seen = new []pointer + let todo = new []pointer + var d = 0 + while d < len(root.deps) { + push(todo, root.deps[d].module) + d += 1 + } + while len(todo) > 0 { + let m = List.pop(todo) + let at = lock_index(locked, m) + if find_mod(seen, m) < 0 and at >= 0 { + push(seen, m) + let dir = `{store_root()}{strip_prefix(locked[at].hash, "sha256:")}` + if not shq(`test -d "{dir}"`) { g_graph_unknown = m } + else { + let mt = read_file(`{dir}/package.ludic`) + if mt != null { + let dm = parse_manifest(mt) + var k = 0 + while k < len(dm.deps) { + push(todo, dm.deps[k].module) + k += 1 + } + } + } + } + } + let keep = new []Manifest + var i = 0 + while i < len(locked) { + if find_mod(seen, locked[i].module) >= 0 { push(keep, locked[i]) } + i += 1 + } + return keep +} + +# take down the ludic_modules/ view `add` linked, and the directories it leaves empty +function unlink_module(module: pointer) -> void { + let link = `ludic_modules/{module}` + if shq(`test -L "{link}"`) { + shell(`rm -f "{link}"; d=$(dirname "{link}"); while [ "$d" != ludic_modules ] && [ "$d" != . ] && rmdir "$d" 2>/dev/null; do d=$(dirname "$d"); done; rmdir ludic_modules 2>/dev/null`) + } else if shq(`test -e "{link}"`) { + err(`ludic remove: warning: {link} is not a link 'ludic add' made; left as it is\n`) + } + if shq(`test -e "vendor/{module}"`) { err(`ludic remove: vendor/{module} is still there (run 'ludic vendor' again)\n`) } +} + +# the project's own files that still import `module` +function warn_importers(module: pointer) -> void { + let hits = split_lines(capture(`grep -rlF --include='*.ludic' --exclude-dir=ludic_modules --exclude-dir=vendor --exclude-dir=build --exclude-dir=.git -e 'import "{module}/' -e 'import "{module}"' . | sed 's|^\\./||' | LC_ALL=C sort`)) + if len(hits) == 0 { return } + err(`ludic remove: warning: {module} is still imported by:\n`) + var i = 0 + while i < len(hits) { + err(` {hits[i]}\n`) + i += 1 + } +} + +# ludic remove — drop a dependency: the require, what only it brought into the lock, the links +function cmd_pkg_remove() -> int { + if arg_count() < 3 { err("usage: ludic remove \n"); return 1 } + let module = split_spec(arg(2))[0] + let txt = read_file("package.ludic") + if txt == null { err("ludic remove: no package.ludic in the current directory\n"); return 1 } + let locked = read_lock_or_empty() + if not requires(parse_manifest(txt), module) { + var why = "" + if lock_index(locked, module) >= 0 { why = " (the lock has it because another package requires it)" } + err(`ludic remove: package.ludic does not require {module}{why}\n`) + return 1 + } + if not write_file("package.ludic", drop_require(txt, module)) { err("ludic remove: cannot write package.ludic\n"); return 1 } + print(`ludic remove: {module} is no longer required`) + if file_exists("package.lock.ludic") { + var keep = lock_reachable(read_root_manifest(), locked) + if g_graph_unknown != "" { + err(`ludic remove: the store has no copy of {g_graph_unknown}, so only {module} leaves the lock; 'ludic get' settles the rest\n`) + keep = new []Manifest + var j = 0 + while j < len(locked) { + if not (locked[j].module == module) { push(keep, locked[j]) } + j += 1 + } + } + if not write_lock(keep) { err("ludic remove: cannot write package.lock.ludic\n"); return 1 } + var i = 0 + while i < len(locked) { + let m = locked[i] + if lock_index(keep, m.module) < 0 { + unlink_module(m.module) + print(` removed {m.module} {m.ver}`) + if not (m.module == module) { warn_importers(m.module) } + } + i += 1 + } + if lock_index(keep, module) >= 0 { print(` {module} stays locked: another package still requires it`) } + } + warn_importers(module) + return 0 +} diff --git a/tools/ludic-cli/pkg_test.ludic b/tools/ludic-cli/pkg_test.ludic index 67e19381..0fe81d64 100644 --- a/tools/ludic-cli/pkg_test.ludic +++ b/tools/ludic-cli/pkg_test.ludic @@ -12,6 +12,13 @@ function pt_write(path: pointer, body: pointer) -> void { write_file(path, body) } +# a file's text, "" when it is not there +function pt_read(path: pointer) -> pointer { + let t = read_file(path) + if t == null { return "" } + return t +} + # commit the current tree of `dir` and tag it `tag` function pt_commit_tag(dir: pointer, tag: pointer) -> void { shell(`git -C {dir} add -A`) @@ -170,6 +177,38 @@ function cmd_test_pkg() -> int { ok("ludic vendor copies the resolved packages into ./vendor") } else { bad2("ludic vendor failed", capture_line(`tail -1 {work}/vendor.out`)) } + # ---- for editors (R9): get --json says what changed in the lock; remove undoes add ---- + let proj5 = `{work}/proj5` + pt_write(`{proj5}/package.ludic`, `package "app"` + nl() + `version "0.0.0"` + nl() + `require "example.test/greeter" "1.0.0"` + nl()) + pt_write(`{proj5}/src/app.ludic`, "program App {\n import \"example.test/greeter/greet.ludic\"\n}\n") + shell(`( cd {proj5} && {envp} {root}/bin/ludic get --json > {work}/gj.json 2> {work}/gj.err )`) + let gj = pt_read(`{work}/gj.json`) + if s_contains(gj, `"name": "example.test/greeter", "version": "1.0.0"`) and s_contains(gj, `"name": "example.test/util", "version": "1.0.0"`) and s_contains(gj, `"unchanged": 0`) and shq(`python3 -m json.tool {work}/gj.json > /dev/null`) and not s_contains(gj, "resolving") { + ok("ludic get --json: a fresh lock is two added packages, one JSON object on stdout") + } else { bad2("ludic get --json (fresh)", gj) } + pt_write(`{proj5}/package.ludic`, `package "app"` + nl() + `version "0.0.0"` + nl() + `require "example.test/greeter" "1.0.0"` + nl() + `require "example.test/util" "1.2.0"` + nl()) + let gj2 = capture(`( cd {proj5} && {envp} {root}/bin/ludic get --json 2>/dev/null )`) + if s_contains(gj2, `"added": []`) and s_contains(gj2, `"name": "example.test/util", "from": "1.0.0", "to": "1.2.0"`) and s_contains(gj2, `"unchanged": 1`) { + ok("ludic get --json: a raised minimum is a change from one version to the other") + } else { bad2("ludic get --json (changed)", gj2) } + if not shq(`( cd {proj5} && {root}/bin/ludic remove example.test/nothere > {work}/rm0.out 2>&1 )`) and shq(`grep -q 'does not require' {work}/rm0.out`) { + ok("ludic remove refuses a module package.ludic does not require") + } else { bad2("ludic remove of an unrequired module", capture_line(`cat {work}/rm0.out`)) } + let store_n = capture_line(`ls {store} | wc -l`) + shell(`( cd {proj5} && LUDIC_STORE={store} {root}/bin/ludic remove example.test/util > {work}/rm1.out 2>&1 )`) + let lk1 = pt_read(`{proj5}/package.lock.ludic`) + if not s_contains(pt_read(`{proj5}/package.ludic`), "example.test/util") and s_contains(lk1, "example.test/util") and file_exists(`{proj5}/ludic_modules/example.test/util`) { + ok("ludic remove keeps what another package still requires locked and linked") + } else { bad2("ludic remove dropped a package still required", capture_line(`cat {work}/rm1.out`)) } + shell(`( cd {proj5} && LUDIC_STORE={store} {root}/bin/ludic remove example.test/greeter > {work}/rm2.out 2>&1 )`) + let lk2 = pt_read(`{proj5}/package.lock.ludic`) + if not s_contains(lk2, "example.test/") and not file_exists(`{proj5}/ludic_modules`) and capture_line(`ls {store} | wc -l`) == store_n { + ok("ludic remove drops the require, what only it locked and its links, and leaves the shared store alone") + } else { bad2("ludic remove of the last require", capture_line(`cat {work}/rm2.out`)) } + if shq(`grep -q 'still imported by' {work}/rm2.out`) and shq(`grep -q 'src/app.ludic' {work}/rm2.out`) { + ok("ludic remove warns about source still importing what it removed") + } else { bad("ludic remove said nothing about src/app.ludic importing the package") } + # ---- package-declarable engine-system + namespace (issue #62) ------------- # A source package registers a compile-time engine system (@EngineSystem) and a # Foo.* namespace (@Namespace) with no compiler edit; a consumer game imports it diff --git a/tools/ludic-cli/scripts.ludic b/tools/ludic-cli/scripts.ludic index 9d184864..910351c5 100644 --- a/tools/ludic-cli/scripts.ludic +++ b/tools/ludic-cli/scripts.ludic @@ -12,7 +12,7 @@ # the built-in commands a hook can wrap (every user command that does work) function is_hookable(cmd: pointer) -> bool { - return (cmd == "build") or (cmd == "run") or (cmd == "test") or (cmd == "deps") or (cmd == "schema") or (cmd == "migrate") or (cmd == "bundle") or (cmd == "pack") or (cmd == "clean") or (cmd == "fmt") or (cmd == "get") or (cmd == "add") or (cmd == "update") or (cmd == "verify") or (cmd == "vendor") or (cmd == "assets") or (cmd == "build-lib") + return (cmd == "build") or (cmd == "run") or (cmd == "test") or (cmd == "deps") or (cmd == "schema") or (cmd == "migrate") or (cmd == "bundle") or (cmd == "pack") or (cmd == "clean") or (cmd == "fmt") or (cmd == "get") or (cmd == "add") or (cmd == "remove") or (cmd == "update") or (cmd == "verify") or (cmd == "vendor") or (cmd == "assets") or (cmd == "build-lib") } # the directory this `ludic` binary lives in, with no trailing slash