SashegDev
874b6f20c2
Основа — Horizon (MIT). История апстрима не переносится: этот репозиторий ведёт свою.
752 lines
38 KiB
Python
752 lines
38 KiB
Python
#!/usr/bin/env python3
|
|
"""Port a new Minecraft release to Horizon: detect it, prepare it, diff it against the pinned one
|
|
(docs/version-ports.md).
|
|
|
|
version_port.py detect [--snapshots] what Mojang released that Horizon doesn't pin
|
|
version_port.py prepare VERSION download, verify, run the data generator, read its facts,
|
|
index its classes (tools/class_index.py)
|
|
version_port.py diff OLD NEW facts, packets, registries, blocks, items, commands,
|
|
data packs, assets, lang and classes, with the Horizon
|
|
code each changed class concerns: diff-OLD-NEW/diff.json
|
|
version_port.py regen NEW [--base REF] a port/NEW worktree pinned to NEW with every generated table
|
|
regenerated (tools/port_regen.py): diff-OLD-NEW/regen.json
|
|
version_port.py harnesses NEW [--baseline S] horizon-prepare and every harness in the candidate, each pass,
|
|
oracle (no longer compiles) or fail, and newly failing against
|
|
a run on the pinned version (tools/port_harness.py)
|
|
version_port.py report OLD NEW [--results S] the ranked port report: diff-OLD-NEW/report.md and .json
|
|
version_port.py publish OLD NEW [--yes] the report's work items as GitHub issues (tools/port_publish.py);
|
|
a dry run without --yes
|
|
version_port.py port NEW [--old OLD] prepare both, diff, regen, report
|
|
version_port.py gc [--keep VERSION...] keep the pinned, the newest two others and --keep
|
|
|
|
The downloads are for development only: they go to the version store (~/code/horizon-parity/versions,
|
|
HORIZON_VERSIONS overrides it), never into the repository, and nothing from them is shipped. Candidate
|
|
worktrees go to ~/code/horizon-parity/ports/NEW (HORIZON_PORTS overrides it). The pinned
|
|
version is not downloaded: its files are read where tools/lab.py prepared them (.local/).
|
|
|
|
detect exits 0 when nothing is new, 10 for a new release and 11 for a new snapshot (with --snapshots).
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.request
|
|
import zipfile
|
|
|
|
try:
|
|
from . import class_index, lab, port_harness, port_publish, port_regen, port_report, vanilla_map
|
|
except ImportError:
|
|
import class_index
|
|
import lab
|
|
import port_harness
|
|
import port_publish
|
|
import port_regen
|
|
import port_report
|
|
import vanilla_map
|
|
|
|
MANIFEST_URL = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"
|
|
STORE = Path(os.environ.get("HORIZON_VERSIONS", Path.home() / "code" / "horizon-parity" / "versions"))
|
|
PORTS = Path(os.environ.get("HORIZON_PORTS", STORE.parent / "ports"))
|
|
STATES = ["handshake", "status", "login", "configuration", "play"]
|
|
# How many changed JSON paths one changed file keeps in diff.json.
|
|
PATH_LIMIT = 50
|
|
|
|
|
|
def safe_version(version):
|
|
if not re.fullmatch(r"[A-Za-z0-9_.-]+", version) or version in {".", ".."}:
|
|
raise ValueError(f"Unsafe version identifier: {version!r}")
|
|
return version
|
|
|
|
|
|
def fetch_json(url, sha1=None):
|
|
request = urllib.request.Request(url, headers={"User-Agent": "horizon/0.1"})
|
|
with urllib.request.urlopen(request, timeout=60) as response:
|
|
data = response.read()
|
|
if sha1 is not None and hashlib.sha1(data).hexdigest() != sha1:
|
|
raise ValueError(f"{url}: sha1 mismatch")
|
|
return data
|
|
|
|
|
|
def pinned():
|
|
return json.loads(lab.LOCK.read_text(encoding="utf-8"))["minecraft"]["version"]
|
|
|
|
|
|
# --- detect ------------------------------------------------------------------------------------------------
|
|
|
|
def news(manifest, pin, seen, snapshots=False):
|
|
"""[(kind, version)] Mojang lists that are neither pinned nor seen before, newest kind first."""
|
|
found = []
|
|
for kind in ["release"] + (["snapshot"] if snapshots else []):
|
|
latest = manifest["latest"][kind]
|
|
if latest != pin and latest != seen.get(kind):
|
|
# The latest snapshot is the latest release right after a release: report it once.
|
|
if kind == "snapshot" and latest == manifest["latest"]["release"]:
|
|
continue
|
|
found.append((kind, latest))
|
|
return found
|
|
|
|
|
|
def detect(args):
|
|
manifest = json.loads(fetch_json(MANIFEST_URL))
|
|
state_path = STORE / "detect-state.json"
|
|
seen = json.loads(state_path.read_text()) if state_path.exists() else {}
|
|
found = news(manifest, pinned(), seen, args.snapshots)
|
|
if not found:
|
|
print(f"Nothing new: latest release {manifest['latest']['release']}, Horizon pins {pinned()}")
|
|
return 0
|
|
for kind, version in found:
|
|
print(f"New {kind}: {version} (Horizon pins {pinned()})")
|
|
seen[kind] = version
|
|
if not args.dry_run:
|
|
lab.write_json(state_path, seen)
|
|
return 10 if found[0][0] == "release" else 11
|
|
|
|
|
|
# --- prepare -----------------------------------------------------------------------------------------------
|
|
|
|
def version_dir(version):
|
|
return STORE / safe_version(version)
|
|
|
|
|
|
def paths(version):
|
|
"""The prepared files of a version: {client, server, server_game, reports, version_json}."""
|
|
record = version_dir(version) / "prepared.json"
|
|
if not record.exists():
|
|
raise SystemExit(f"{version} is not prepared; run version_port.py prepare {version}")
|
|
return {key: Path(value) for key, value in json.loads(record.read_text())["paths"].items()}
|
|
|
|
|
|
def jar_facts(client_jar):
|
|
"""The facts the game states about itself in the client jar's version.json."""
|
|
with zipfile.ZipFile(client_jar) as archive:
|
|
info = json.loads(archive.read("version.json"))
|
|
pack = info.get("pack_version", {})
|
|
return {
|
|
"id": info["id"],
|
|
"world_version": info["world_version"],
|
|
"protocol_version": info["protocol_version"],
|
|
"resource_pack": f"{pack.get('resource_major')}.{pack.get('resource_minor')}",
|
|
"data_pack": f"{pack.get('data_major')}.{pack.get('data_minor')}",
|
|
"java_version": info["java_version"],
|
|
"stable": info.get("stable"),
|
|
}
|
|
|
|
|
|
def unpack_game(server, destination, version):
|
|
"""The server download is a bundler: take its game jar only (the libraries stay inside it)."""
|
|
with zipfile.ZipFile(server) as archive:
|
|
if "META-INF/versions.list" not in archive.namelist():
|
|
shutil.copyfile(server, destination)
|
|
return destination
|
|
entries = list(lab.bundle_entries(archive, "META-INF/versions.list"))
|
|
if len(entries) != 1 or entries[0][1] != version:
|
|
raise ValueError(f"Expected one bundled server for {version}")
|
|
checksum, _, relative = entries[0]
|
|
lab.extract_verified(archive, f"META-INF/versions/{relative}", destination, checksum)
|
|
return destination
|
|
|
|
|
|
def java_command(major):
|
|
java = Path(os.environ["JAVA_HOME"]) / "bin" / "java" if os.environ.get("JAVA_HOME") else shutil.which("java")
|
|
if not java:
|
|
raise SystemExit("Java not found: set JAVA_HOME or put java on the PATH")
|
|
result = subprocess.run([str(java), "-version"], capture_output=True, text=True)
|
|
found = re.search(r'version "(\d+)', result.stderr + result.stdout)
|
|
if not found or int(found[1]) < major:
|
|
raise SystemExit(f"This version needs Java {major}; {java} is {found[1] if found else 'unknown'}. "
|
|
f"Install JDK {major} (a fleet toolchain change) and rerun.")
|
|
return str(java)
|
|
|
|
|
|
def run_datagen(server, reports, major):
|
|
"""vanilla's data generator, --reports only. The bundler unpacks its libraries into a scratch folder."""
|
|
scratch = Path(tempfile.mkdtemp(prefix=".datagen-", dir=STORE))
|
|
out = scratch / "out"
|
|
try:
|
|
log = scratch / "datagen.log"
|
|
with log.open("w") as stream:
|
|
result = subprocess.run([java_command(major), "-DbundlerMainClass=net.minecraft.data.Main", "-jar",
|
|
str(server), "--reports", "--output", str(out)],
|
|
cwd=scratch, stdout=stream, stderr=subprocess.STDOUT)
|
|
if result.returncode:
|
|
raise SystemExit(f"The data generator failed:\n{log.read_text()[-3000:]}")
|
|
if reports.exists():
|
|
shutil.rmtree(reports)
|
|
shutil.move(str(out / "reports"), reports)
|
|
finally:
|
|
shutil.rmtree(scratch, ignore_errors=True)
|
|
|
|
|
|
def prepare_pinned(version, directory):
|
|
"""The pinned version's files are already in .local: record where, don't copy them."""
|
|
game = lab.LOCAL / "minecraft" / version
|
|
found = {
|
|
"client": game / "jars" / "client.jar",
|
|
"server": game / "jars" / "server.jar",
|
|
"server_game": game / "jars" / "server-game.jar",
|
|
"reports": lab.LOCAL / "datagen" / "out" / "reports",
|
|
"version_json": game / "manifest.json",
|
|
}
|
|
missing = [str(p) for p in found.values() if not p.exists()]
|
|
if missing:
|
|
raise SystemExit("The pinned version is not prepared in .local (tools/lab.py prepare, horizon-prepare): "
|
|
+ ", ".join(missing))
|
|
return {key: path.resolve() for key, path in found.items()}
|
|
|
|
|
|
def prepare_download(version, directory):
|
|
manifest = json.loads(fetch_json(MANIFEST_URL))
|
|
entry = next((v for v in manifest["versions"] if v["id"] == version), None)
|
|
if entry is None:
|
|
raise SystemExit(f"Mojang's manifest has no version {version}")
|
|
version_json = directory / "version.json"
|
|
if not (version_json.exists() and lab.digest(version_json, "sha1") == entry["sha1"]):
|
|
version_json.write_bytes(fetch_json(entry["url"], entry["sha1"]))
|
|
meta = json.loads(version_json.read_text())
|
|
if "client_mappings" in meta["downloads"]:
|
|
raise SystemExit(f"{version} is obfuscated; the pipeline handles unobfuscated releases (26.1 and later)")
|
|
jars = directory / "jars"
|
|
found = {"version_json": version_json}
|
|
for side in ["client", "server"]:
|
|
download = meta["downloads"][side]
|
|
print(f"Fetching the {version} {side} jar ({download['size'] // 1_000_000} MB)", flush=True)
|
|
found[side] = lab.download(download["url"], jars / f"{side}.jar", download["sha1"])
|
|
found["server_game"] = unpack_game(found["server"], jars / "server-game.jar", version)
|
|
found["reports"] = directory / "datagen" / "reports"
|
|
if not found["reports"].exists():
|
|
print(f"Running the {version} data generator", flush=True)
|
|
run_datagen(found["server"], found["reports"], meta["javaVersion"]["majorVersion"])
|
|
return found
|
|
|
|
|
|
def prepare(args):
|
|
version = safe_version(args.version)
|
|
directory = version_dir(version)
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
found = (prepare_pinned if version == pinned() and not args.download else prepare_download)(version, directory)
|
|
facts = jar_facts(found["client"])
|
|
if facts["id"] != version:
|
|
raise ValueError(f"The client jar says it is {facts['id']}, not {version}")
|
|
lab.write_json(directory / "facts.json", facts)
|
|
classes = class_index.index_jars([found["client"], found["server_game"]], jobs=args.jobs)
|
|
(directory / "classes.json").write_text(json.dumps(classes, separators=(",", ":")))
|
|
print(f"Indexed {len(classes)} classes, {sum(len(c['methods']) for c in classes.values())} methods")
|
|
lab.write_json(directory / "prepared.json", {
|
|
"version": version, "pinned": version == pinned() and not args.download,
|
|
"paths": {key: str(path) for key, path in found.items()},
|
|
})
|
|
print(json.dumps(facts, indent=2))
|
|
return 0
|
|
|
|
|
|
# --- diff --------------------------------------------------------------------------------------------------
|
|
|
|
def clip(value, limit=200):
|
|
text = json.dumps(value, sort_keys=True)
|
|
return value if len(text) <= limit else text[:limit] + "..."
|
|
|
|
|
|
def json_diff(old, new, path="", limit=PATH_LIMIT):
|
|
"""[(path, old, new)] where two JSON values differ; lists of different lengths count as one change."""
|
|
changes = []
|
|
|
|
def walk(a, b, at):
|
|
if len(changes) >= limit or a == b:
|
|
return
|
|
if isinstance(a, dict) and isinstance(b, dict):
|
|
for key in sorted(a.keys() | b.keys()):
|
|
walk(a.get(key), b.get(key), f"{at}/{key}")
|
|
elif isinstance(a, list) and isinstance(b, list) and len(a) == len(b):
|
|
for index, (x, y) in enumerate(zip(a, b)):
|
|
walk(x, y, f"{at}[{index}]")
|
|
else:
|
|
changes.append((at or "/", clip(a), clip(b)))
|
|
|
|
walk(old, new, path)
|
|
return changes
|
|
|
|
|
|
def keyed_diff(old, new):
|
|
"""Added, removed and common keys of two mappings."""
|
|
return sorted(new.keys() - old.keys()), sorted(old.keys() - new.keys()), sorted(old.keys() & new.keys())
|
|
|
|
|
|
def diff_facts(old, new):
|
|
return {key: [old.get(key), new.get(key)] for key in sorted(old.keys() | new.keys())
|
|
if key != "id" and old.get(key) != new.get(key)}
|
|
|
|
|
|
def diff_packets(old, new):
|
|
"""Per state and direction: packets added, removed and renumbered (the ids are on the wire)."""
|
|
result = {}
|
|
for state in sorted(old.keys() | new.keys(), key=lambda s: (STATES.index(s) if s in STATES else 99, s)):
|
|
for direction in sorted(old.get(state, {}).keys() | new.get(state, {}).keys()):
|
|
a = {name: e["protocol_id"] for name, e in old.get(state, {}).get(direction, {}).items()}
|
|
b = {name: e["protocol_id"] for name, e in new.get(state, {}).get(direction, {}).items()}
|
|
added, removed, common = keyed_diff(a, b)
|
|
renumbered = {name: [a[name], b[name]] for name in common if a[name] != b[name]}
|
|
if added or removed or renumbered:
|
|
result[f"{state}/{direction}"] = {
|
|
"added": {name: b[name] for name in added}, "removed": {name: a[name] for name in removed},
|
|
"renumbered": renumbered, "count": [len(a), len(b)]}
|
|
return result
|
|
|
|
|
|
def diff_registries(old, new):
|
|
"""Registries added and removed; per registry its entries added and removed, and how many common
|
|
entries changed protocol id (the network sends those ids)."""
|
|
added, removed, common = keyed_diff(old, new)
|
|
result = {"added": {name: sorted(new[name].get("entries", {})) for name in added}, "removed": removed,
|
|
"changed": {}}
|
|
for name in common:
|
|
a, b = old[name].get("entries", {}), new[name].get("entries", {})
|
|
entries_added, entries_removed, shared = keyed_diff(a, b)
|
|
shifted = [e for e in shared if a[e].get("protocol_id") != b[e].get("protocol_id")]
|
|
change = {}
|
|
if entries_added:
|
|
change["added"] = entries_added
|
|
if entries_removed:
|
|
change["removed"] = entries_removed
|
|
if shifted:
|
|
change["ids_shifted"] = len(shifted)
|
|
if old[name].get("default") != new[name].get("default"):
|
|
change["default"] = [old[name].get("default"), new[name].get("default")]
|
|
if change:
|
|
result["changed"][name] = change
|
|
return result
|
|
|
|
|
|
def diff_blocks(old, new):
|
|
"""Blocks added and removed; per block property changes, state counts and default state; and the
|
|
global block-state ids (the network and the chunk palette use them)."""
|
|
added, removed, common = keyed_diff(old, new)
|
|
# A field one report has for no block at all is a change of the report's format, not of every block.
|
|
fields = [{key for block in side.values() for key in block} for side in (old, new)]
|
|
format_change = sorted(fields[0] ^ fields[1])
|
|
|
|
def default(block):
|
|
return next((s.get("properties", {}) for s in block["states"] if s.get("default")), {})
|
|
|
|
def first_id(block):
|
|
return min(s["id"] for s in block["states"])
|
|
|
|
changed = {}
|
|
for name in common:
|
|
a, b = old[name], new[name]
|
|
change = {}
|
|
if a.get("properties", {}) != b.get("properties", {}):
|
|
change["properties"] = json_diff(a.get("properties", {}), b.get("properties", {}))
|
|
if len(a["states"]) != len(b["states"]):
|
|
change["states"] = [len(a["states"]), len(b["states"])]
|
|
if default(a) != default(b):
|
|
change["default"] = [default(a), default(b)]
|
|
if a.get("definition") != b.get("definition") and "definition" not in format_change:
|
|
change["definition"] = json_diff(a.get("definition"), b.get("definition"))
|
|
if change:
|
|
changed[name] = change
|
|
shifted = [name for name in common if first_id(old[name]) != first_id(new[name])]
|
|
first = min(shifted, key=lambda n: first_id(new[n]), default=None)
|
|
return {"added": added, "removed": removed, "changed": changed, "format": format_change,
|
|
"state_count": [sum(len(b["states"]) for b in old.values()), sum(len(b["states"]) for b in new.values())],
|
|
"state_ids_shifted": len(shifted), "first_shifted": first}
|
|
|
|
|
|
def diff_items(old, new):
|
|
"""Items added and removed; per item the default components added, removed and changed."""
|
|
added, removed, common = keyed_diff(old, new)
|
|
changed = {}
|
|
for name in common:
|
|
a, b = old[name], new[name]
|
|
components_added, components_removed, shared = keyed_diff(a, b)
|
|
values = {c: json_diff(a[c], b[c], limit=10) for c in shared if a[c] != b[c]}
|
|
if components_added or components_removed or values:
|
|
changed[name] = {"added": components_added, "removed": components_removed, "changed": values}
|
|
components = {}
|
|
for change in changed.values():
|
|
for label in ("added", "removed", "changed"):
|
|
for component in change[label]:
|
|
components.setdefault(component, {"added": 0, "removed": 0, "changed": 0})[label] += 1
|
|
return {"added": added, "removed": removed, "changed": changed, "components": components}
|
|
|
|
|
|
def command_nodes(tree):
|
|
"""Every node of a commands report by its path ("execute as targets"), with what defines it."""
|
|
nodes = {}
|
|
|
|
def walk(node, path):
|
|
for name, child in node.get("children", {}).items():
|
|
at = f"{path} {name}".strip()
|
|
nodes[at] = {k: child[k] for k in ("type", "parser", "properties", "executable", "redirect", "permissions")
|
|
if k in child}
|
|
walk(child, at)
|
|
|
|
walk(tree, "")
|
|
return nodes
|
|
|
|
|
|
def diff_commands(old, new):
|
|
a, b = command_nodes(old), command_nodes(new)
|
|
added, removed, common = keyed_diff(a, b)
|
|
return {"added": added, "removed": removed,
|
|
"changed": {path: json_diff(a[path], b[path], limit=10) for path in common if a[path] != b[path]}}
|
|
|
|
|
|
def jar_entries(jar, prefix):
|
|
"""{path: crc} of a jar's files under prefix."""
|
|
with zipfile.ZipFile(jar) as archive:
|
|
return {i.filename: i.CRC for i in archive.infolist() if i.filename.startswith(prefix) and not i.is_dir()}
|
|
|
|
|
|
def content_kind(path):
|
|
"""data/minecraft/worldgen/biome/x.json -> worldgen/biome; assets/minecraft/models/block/x.json -> models/block."""
|
|
parts = path.split("/")[2:-1]
|
|
if parts and parts[0] in {"worldgen", "tags"} and len(parts) > 1:
|
|
return "/".join(parts[:2])
|
|
return parts[0] if parts else "(root)"
|
|
|
|
|
|
def diff_contents(old_jar, new_jar, prefix):
|
|
"""Files under prefix added, removed and changed (by CRC), by kind; changed JSON files with their paths."""
|
|
a, b = jar_entries(old_jar, prefix), jar_entries(new_jar, prefix)
|
|
added, removed, common = keyed_diff(a, b)
|
|
changed = [p for p in common if a[p] != b[p]]
|
|
kinds = {}
|
|
for label, names in [("added", added), ("removed", removed), ("changed", changed)]:
|
|
for name in names:
|
|
kinds.setdefault(content_kind(name), {"added": [], "removed": [], "changed": []})[label].append(name)
|
|
details = {}
|
|
with zipfile.ZipFile(old_jar) as left, zipfile.ZipFile(new_jar) as right:
|
|
for name in changed:
|
|
if name.endswith(".json") and not name.endswith("/lang/en_us.json"):
|
|
try:
|
|
details[name] = json_diff(json.loads(left.read(name)), json.loads(right.read(name)))
|
|
except ValueError:
|
|
pass
|
|
return {"kinds": dict(sorted(kinds.items())), "json": details,
|
|
"counts": {"added": len(added), "removed": len(removed), "changed": len(changed)}}
|
|
|
|
|
|
def diff_lang(old_jar, new_jar, name="assets/minecraft/lang/en_us.json"):
|
|
with zipfile.ZipFile(old_jar) as left, zipfile.ZipFile(new_jar) as right:
|
|
a, b = json.loads(left.read(name)), json.loads(right.read(name))
|
|
added, removed, common = keyed_diff(a, b)
|
|
return {"added": added, "removed": removed, "changed": [k for k in common if a[k] != b[k]]}
|
|
|
|
|
|
def read_reports(reports):
|
|
items = {}
|
|
for path in sorted((reports / "minecraft" / "components" / "item").glob("*.json")):
|
|
items[f"minecraft:{path.stem}"] = json.loads(path.read_text())["components"]
|
|
return {
|
|
"packets": json.loads((reports / "packets.json").read_text()),
|
|
"registries": json.loads((reports / "registries.json").read_text()),
|
|
"blocks": json.loads((reports / "blocks.json").read_text()),
|
|
"commands": json.loads((reports / "commands.json").read_text()),
|
|
"items": items,
|
|
}
|
|
|
|
|
|
def class_names(old, new, moved):
|
|
"""Every class name of either version, but a moved class only under its new name, so that its simple
|
|
name stays unambiguous."""
|
|
return sorted(set(new) | (set(old) - set(moved)))
|
|
|
|
|
|
def diff_classes(result, mapping):
|
|
"""The class diff, each changed or removed class with the Horizon files and harnesses that concern it."""
|
|
horizon = {}
|
|
for name in result["changed"]:
|
|
entry = vanilla_map.lookup(mapping, name)
|
|
if entry["rust"] or entry["harnesses"]:
|
|
horizon[name] = entry
|
|
for name in result["removed"]:
|
|
entry = vanilla_map.lookup(mapping, name)
|
|
if entry["rust"] or entry["harnesses"]:
|
|
horizon[name] = entry
|
|
result["horizon"] = horizon
|
|
return result
|
|
|
|
|
|
def diff_versions(old_paths, new_paths, old_facts, new_facts, classes=None, mapping=None):
|
|
a, b = read_reports(old_paths["reports"]), read_reports(new_paths["reports"])
|
|
classes = diff_classes(classes, mapping) if classes is not None else None
|
|
return {
|
|
"facts": diff_facts(old_facts, new_facts),
|
|
"packets": diff_packets(a["packets"], b["packets"]),
|
|
"registries": diff_registries(a["registries"], b["registries"]),
|
|
"blocks": diff_blocks(a["blocks"], b["blocks"]),
|
|
"items": diff_items(a["items"], b["items"]),
|
|
"commands": diff_commands(a["commands"], b["commands"]),
|
|
"data": diff_contents(old_paths["client"], new_paths["client"], "data/minecraft/"),
|
|
"assets": diff_contents(old_paths["client"], new_paths["client"], "assets/minecraft/"),
|
|
"lang": diff_lang(old_paths["client"], new_paths["client"]),
|
|
"classes": classes,
|
|
}
|
|
|
|
|
|
def summary(old, new, result):
|
|
"""A few lines a person reads first."""
|
|
lines = [f"Minecraft {old} -> {new}"]
|
|
for key, (a, b) in result["facts"].items():
|
|
lines.append(f" {key}: {a} -> {b}")
|
|
for flow, change in result["packets"].items():
|
|
lines.append(f" packets {flow}: {change['count'][0]} -> {change['count'][1]}, +{len(change['added'])} "
|
|
f"-{len(change['removed'])}, {len(change['renumbered'])} renumbered")
|
|
registries = result["registries"]
|
|
lines.append(f" registries: +{len(registries['added'])} -{len(registries['removed'])}, "
|
|
f"{len(registries['changed'])} changed")
|
|
for name, change in registries["changed"].items():
|
|
lines.append(f" {name}: +{len(change.get('added', []))} -{len(change.get('removed', []))}"
|
|
+ (f", {change['ids_shifted']} ids shifted" if change.get("ids_shifted") else ""))
|
|
blocks = result["blocks"]
|
|
lines.append(f" blocks: +{len(blocks['added'])} -{len(blocks['removed'])}, {len(blocks['changed'])} changed, "
|
|
f"states {blocks['state_count'][0]} -> {blocks['state_count'][1]}, "
|
|
f"{blocks['state_ids_shifted']} blocks' state ids shifted")
|
|
if blocks["format"]:
|
|
lines.append(f" report format: fields {', '.join(blocks['format'])} appear or disappear")
|
|
items = result["items"]
|
|
lines.append(f" items: +{len(items['added'])} -{len(items['removed'])}, {len(items['changed'])} with "
|
|
f"changed default components")
|
|
ranked = sorted(items["components"].items(), key=lambda kv: -sum(kv[1].values()))
|
|
for component, counts in ranked[:10]:
|
|
lines.append(f" {component}: " + ", ".join(f"{k} on {n}" for k, n in counts.items() if n))
|
|
commands = result["commands"]
|
|
lines.append(f" command nodes: +{len(commands['added'])} -{len(commands['removed'])}, "
|
|
f"{len(commands['changed'])} changed")
|
|
for key in ["data", "assets"]:
|
|
counts = result[key]["counts"]
|
|
lines.append(f" {key} files: +{counts['added']} -{counts['removed']} ~{counts['changed']}")
|
|
lang = result["lang"]
|
|
lines.append(f" en_us keys: +{len(lang['added'])} -{len(lang['removed'])} ~{len(lang['changed'])}")
|
|
classes = result.get("classes")
|
|
if classes:
|
|
methods = sum(class_index.changed_methods(c) for c in classes["changed"].values())
|
|
lines.append(f" classes: +{len(classes['added'])} -{len(classes['removed'])}, {len(classes['moved'])} moved, "
|
|
f"{len(classes['changed'])} changed ({methods} methods); {len(classes['horizon'])} of the "
|
|
f"changed or removed ones are ported or proved by Horizon")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def diff_dir(old, new):
|
|
return STORE / f"diff-{safe_version(old)}-{safe_version(new)}"
|
|
|
|
|
|
def load(version, name):
|
|
return json.loads((version_dir(version) / name).read_text())
|
|
|
|
|
|
def diff(args):
|
|
directory = diff_dir(args.old, args.new)
|
|
old_classes, new_classes = load(args.old, "classes.json"), load(args.new, "classes.json")
|
|
classes = class_index.diff(old_classes, new_classes)
|
|
mapping = vanilla_map.build(class_names(old_classes, new_classes, classes["moved"]),
|
|
dependencies=vanilla_map.recorded_dependencies(vanilla_map.default_shared()))
|
|
lab.write_json(directory / "vanilla_map.json", mapping)
|
|
result = diff_versions(paths(args.old), paths(args.new), load(args.old, "facts.json"), load(args.new, "facts.json"),
|
|
classes, mapping)
|
|
lab.write_json(directory / "diff.json", result)
|
|
print(summary(args.old, args.new, result))
|
|
print(f"\n{directory / 'diff.json'}")
|
|
return 0
|
|
|
|
|
|
# --- regen -------------------------------------------------------------------------------------------------
|
|
|
|
def regen(args):
|
|
version = safe_version(args.new)
|
|
result = port_regen.regen(version, paths(version), version_dir(version), args.base,
|
|
Path(args.worktree) if args.worktree else PORTS / version, args.only)
|
|
lab.write_json(diff_dir(result["old"], version) / "regen.json", result)
|
|
for name, entry in result["generators"].items():
|
|
stat = ", ".join(f"{p} +{a} -{r}" for p, (a, r) in entry.get("stat", {}).items())
|
|
print(f"{name}: {entry['status']}" + (f" ({stat})" if stat else ""))
|
|
print(f"\n{result['candidate']} at {result['commit'][:10]}; {diff_dir(result['old'], version) / 'regen.json'}")
|
|
return 1 if any(e["status"] == "failed" for e in result["generators"].values()) else 0
|
|
|
|
|
|
# --- harnesses ---------------------------------------------------------------------------------------------
|
|
|
|
def harnesses(args):
|
|
version, old = safe_version(args.new), args.old or pinned()
|
|
directory = diff_dir(old, version)
|
|
previous = json.loads((directory / "harnesses.json").read_text()) if args.no_run else None
|
|
result = port_harness.harnesses(version, version_dir(version),
|
|
Path(args.worktree) if args.worktree else PORTS / version,
|
|
args.baseline, args.names or None, args.timeout, previous)
|
|
lab.write_json(directory / "harnesses.json", result)
|
|
port_harness.keep_logs(result, directory / "harness-logs")
|
|
counts = result["summary"]
|
|
print(f"horizon-prepare: {result['prepare']['status']}")
|
|
print(f"harnesses: {counts['pass']} pass, {counts['oracle']} oracle no longer compiles, {counts['fail']} fail; "
|
|
f"{counts['new']} newly failing, {counts['known']} failing on the baseline too")
|
|
print(f"{directory / 'harnesses.json'}")
|
|
return 0
|
|
|
|
|
|
# --- report ------------------------------------------------------------------------------------------------
|
|
|
|
def harness_results(state_path):
|
|
"""{harness: "pass" | "fail"} from tools/parity.py's state; a harness fails if one of its shards does."""
|
|
found = {}
|
|
for name, record in json.loads(Path(state_path).read_text()).get("harnesses", {}).items():
|
|
harness = name.split("#")[0]
|
|
found[harness] = "fail" if record.get("status") != "pass" or found.get(harness) == "fail" else "pass"
|
|
return found
|
|
|
|
|
|
def report(args):
|
|
directory = diff_dir(args.old, args.new)
|
|
if not (directory / "diff.json").exists():
|
|
raise SystemExit(f"No diff yet; run version_port.py diff {args.old} {args.new}")
|
|
result = json.loads((directory / "diff.json").read_text())
|
|
mapping = json.loads((directory / "vanilla_map.json").read_text())
|
|
results = harness_results(args.results) if args.results else None
|
|
regenerated, ran = directory / "regen.json", directory / "harnesses.json"
|
|
regenerated = json.loads(regenerated.read_text()) if regenerated.exists() else None
|
|
ran = json.loads(ran.read_text()) if ran.exists() and not args.results else None
|
|
built = port_report.build(result, mapping, args.old, args.new, load(pinned(), "facts.json"), results,
|
|
regen=regenerated, harnesses=ran)
|
|
lab.write_json(directory / "report.json", built)
|
|
(directory / "report.md").write_text(port_report.markdown(built))
|
|
print("\n".join(port_report.markdown(built).splitlines()[:args.lines]))
|
|
print(f"\n{directory / 'report.md'} ({len(built['items'])} items)")
|
|
return 0
|
|
|
|
|
|
# --- publish -----------------------------------------------------------------------------------------------
|
|
|
|
def publish(args):
|
|
directory = diff_dir(args.old, args.new)
|
|
if not (directory / "report.json").exists():
|
|
raise SystemExit(f"No report yet; run version_port.py report {args.old} {args.new}")
|
|
built = json.loads((directory / "report.json").read_text())
|
|
result = json.loads((directory / "diff.json").read_text())
|
|
issues = port_publish.open_version_port_issues() if args.yes or args.check_github else {}
|
|
actions = port_publish.plan(built, result, directory, issues)
|
|
creates = sum(1 for a in actions if a[0] == "create")
|
|
if not args.yes:
|
|
for action, name, text, number in actions:
|
|
print(f"would {action}{f' #{number}' if number else ''}: {name}")
|
|
for action, name, text, number in actions[:args.show]:
|
|
print(f"\n--- {name}\n{text}")
|
|
missing = port_publish.missing_labels(args.new) if args.check_github else [port_publish.LABEL,
|
|
f"port-{args.new}"]
|
|
print(f"\nDry run: {creates} issues to create, {len(actions) - creates} to update"
|
|
f"{'' if args.check_github else ' (open issues not checked: --check-github)'}; labels to create: "
|
|
f"{', '.join(missing) or 'none'}; then one Kira card for the tracking issue. Rerun with --yes to do it.")
|
|
return 0
|
|
urls = port_publish.apply(actions, built, issues)
|
|
tracking = urls[actions[-1][1]]
|
|
card = port_publish.kira_card(args.new, tracking)
|
|
print(f"Tracking issue {tracking}" + (f"; Kira card {card}" if card else "; Kira card already there"))
|
|
return 0
|
|
|
|
|
|
def port(args):
|
|
"""prepare both, diff, regen, report."""
|
|
old = args.old or pinned()
|
|
for version in (old, args.new):
|
|
prepare(argparse.Namespace(version=version, download=False, jobs=args.jobs))
|
|
diff(argparse.Namespace(old=old, new=args.new))
|
|
if not args.no_regen:
|
|
regen(argparse.Namespace(new=args.new, base=args.base, worktree=None, only=None))
|
|
return report(argparse.Namespace(old=old, new=args.new, results=args.results, lines=args.lines))
|
|
|
|
|
|
# --- gc ----------------------------------------------------------------------------------------------------
|
|
|
|
def to_remove(entries, pin, keep):
|
|
"""Version store entries to delete: every version but the pinned one, --keep and the newest two others;
|
|
diffs whose versions are gone; scratch folders. entries: [(name, mtime, is_version)]."""
|
|
versions = sorted(((m, n) for n, m, is_version in entries if is_version and n not in keep | {pin}), reverse=True)
|
|
kept = keep | {pin} | {n for _, n in versions[:2]}
|
|
remove = []
|
|
for name, _, is_version in entries:
|
|
if name.startswith("."):
|
|
remove.append(name)
|
|
elif is_version and name not in kept:
|
|
remove.append(name)
|
|
elif name.startswith("diff-") and not any(name == f"diff-{a}-{b}" for a in kept for b in kept):
|
|
remove.append(name)
|
|
return sorted(remove)
|
|
|
|
|
|
def gc(args):
|
|
if not STORE.exists():
|
|
return 0
|
|
entries = [(p.name, p.stat().st_mtime, (p / "prepared.json").exists()) for p in STORE.iterdir() if p.is_dir()]
|
|
for name in to_remove(entries, pinned(), set(args.keep)):
|
|
print(f"Removing {STORE / name}")
|
|
if not args.dry_run:
|
|
shutil.rmtree(STORE / name)
|
|
return 0
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
command = commands.add_parser("detect", help="what Mojang released that Horizon doesn't pin")
|
|
command.add_argument("--snapshots", action="store_true")
|
|
command.add_argument("--dry-run", action="store_true", help="don't remember what was found")
|
|
command = commands.add_parser("prepare", help="download, verify and run the data generator")
|
|
command.add_argument("version")
|
|
command.add_argument("--download", action="store_true", help="download even the pinned version")
|
|
command.add_argument("-j", "--jobs", type=int, default=int(os.environ.get("FLEET_LOAD_JOBS", 0)) or None,
|
|
help="class indexing processes (default: $FLEET_LOAD_JOBS, else the CPU count)")
|
|
command = commands.add_parser("diff", help="diff two prepared versions")
|
|
command.add_argument("old")
|
|
command.add_argument("new")
|
|
command = commands.add_parser("regen", help="a candidate worktree pinned to NEW, every table regenerated")
|
|
command.add_argument("new")
|
|
command.add_argument("--base", default="origin/main", help="what port/NEW starts from (default origin/main)")
|
|
command.add_argument("--worktree", help="where (default: $HORIZON_PORTS/NEW)")
|
|
command.add_argument("--only", nargs="*", help="these generators only (names in tools/port_regen.py)")
|
|
command = commands.add_parser("harnesses", help="horizon-prepare and every harness in the candidate")
|
|
command.add_argument("new")
|
|
command.add_argument("names", nargs="*", help="these harnesses only (default: all)")
|
|
command.add_argument("--old", help="the version the port starts from (default: the pinned one)")
|
|
command.add_argument("--worktree", help="the candidate (default: $HORIZON_PORTS/NEW)")
|
|
command.add_argument("--baseline", help="tools/parity.py's state.json from a run on the pinned version: what "
|
|
"fails there too is not newly failing")
|
|
command.add_argument("--timeout", type=int, help="per harness, seconds (tools/parity.py --timeout)")
|
|
command.add_argument("--no-run", action="store_true", help="classify the last run again (another --baseline)")
|
|
command = commands.add_parser("report", help="the ranked port report of a diff")
|
|
command.add_argument("old")
|
|
command.add_argument("new")
|
|
command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW (default: "
|
|
"harnesses.json, when version_port.py harnesses ran)")
|
|
command.add_argument("--lines", type=int, default=60, help="how much of report.md to print")
|
|
command = commands.add_parser("publish", help="the report's work items as GitHub issues (dry run by default)")
|
|
command.add_argument("old")
|
|
command.add_argument("new")
|
|
command.add_argument("--yes", action="store_true", help="create and update the issues, labels and Kira card")
|
|
command.add_argument("--check-github", action="store_true",
|
|
help="dry run: read the open issues and labels to tell creates from updates")
|
|
command.add_argument("--show", type=int, default=1, help="dry run: print this many issue bodies")
|
|
command = commands.add_parser("port", help="prepare, diff and report in one go")
|
|
command.add_argument("new")
|
|
command.add_argument("--old", help="default: the pinned version")
|
|
command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW")
|
|
command.add_argument("--lines", type=int, default=60)
|
|
command.add_argument("--base", default="origin/main", help="what the candidate starts from")
|
|
command.add_argument("--no-regen", action="store_true", help="diff and report only (snapshots)")
|
|
command.add_argument("-j", "--jobs", type=int, default=int(os.environ.get("FLEET_LOAD_JOBS", 0)) or None)
|
|
command = commands.add_parser("gc", help="delete versions and scratch the store doesn't need")
|
|
command.add_argument("--keep", nargs="*", default=[])
|
|
command.add_argument("--dry-run", action="store_true")
|
|
args = parser.parse_args()
|
|
STORE.mkdir(parents=True, exist_ok=True)
|
|
return {"detect": detect, "prepare": prepare, "diff": diff, "regen": regen, "harnesses": harnesses,
|
|
"report": report, "publish": publish, "port": port, "gc": gc}[args.command](args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|