RustNotMC/tools/ci/nightly_set.py

#!/usr/bin/env python3
"""Trusted reviewers pick the pull requests a Nightly holds (docs/release-flow.md "Nightly channel").

    nightly_set.py command     one `/nightly …` comment (nightly-command.yml): checks who asked, updates the set,
                               starts the build and answers in the thread

A comment by Ludovic or by a login in .github/trusted-reviewers.txt, on any issue or pull request:

    /nightly 2640 2642     the set becomes these pull requests, merged onto main in this order; builds now
    /nightly add 2650      adds one, or takes its newest head again; builds now
    /nightly remove 2640   drops one; builds now. Ludovic's remove also keeps reviewers from adding it back
    /nightly clear         empties the set: the Nightly is the proven union or main again; builds now
    /nightly status        what the set holds
    /nightly dry-run 2640  builds main + these and signs with a throwaway key; publishes nothing, the set stays

The set (set.json on the nightly-state branch) keeps each pull request at the head it had when someone asked for
it; the scheduled nightly takes their newest heads and drops the merged and closed ones (nightly.py pick).
builds.jsonl there records every build's set, for the stats. Nothing here runs a pull request's code: its commits
are fetched and merged with `git merge-tree --write-tree`, which writes git objects and never a working tree, so no
file of theirs reaches the disk and no hook, filter or script of theirs runs.
"""
from __future__ import annotations

import copy
import datetime as dt
import json
import os
import re
import subprocess
import sys
import tempfile
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable

OWNER = "ludovic111"
REVIEWERS = ".github/trusted-reviewers.txt"
CODEOWNERS = ".github/CODEOWNERS"
# What CODEOWNERS guards today, refused even if the file moved: what builds, signs and checks releases.
ALWAYS_GUARDED = ["/.github/", "/tools/ci/", "/crates/horizon-release-sign/", "/crates/horizon-server/src/update.rs",
                  "/launcher/src/backend.rs", "/launcher/links.json"]
STATE_BRANCH = "nightly-state"
BUILD_PREFIX = "nightly/build-"
MAX_PRS = 30
MAX_BUILDS_PER_DAY = 12
BOT = ("github-actions[bot]", "41898283+github-actions[bot]@users.noreply.github.com")
# Git never runs anything of the merged commits; these make sure nothing of the runner's either.
SAFE_GIT = ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", "-c", "merge.renormalize=false"]
NUMBER = re.compile(r"^#?(\d{1,6}),?$")
LOGIN = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$")
VERBS = ("add", "remove", "clear", "status", "dry-run")
USAGE = ("Use `/nightly 2640 2642`, `/nightly add 2650`, `/nightly remove 2640`, `/nightly clear`, "
         "`/nightly status` or `/nightly dry-run 2640`.")


class Refused(Exception):
    """A command that changes nothing; its text is the answer."""


@dataclass
class Command:
    verb: str  # set, add, remove, clear, status, dry-run
    prs: list[int] = field(default_factory=list)

    def text(self) -> str:
        words = ["/nightly"] + ([] if self.verb == "set" else [self.verb]) + [str(n) for n in self.prs]
        return " ".join(words)


def safe(word: str) -> str:
    """A word of a comment, fit to quote back: no markdown, no mentions."""
    return re.sub(r"[^\w#:.-]", "", word)[:40] or "?"


def parse(body: str) -> Command | None:
    """The command on a comment's first line; None when it isn't one. Raises Refused when it is one, but wrong."""
    line = next((line.strip() for line in body.splitlines() if line.strip()), "")
    words = line.split()
    if not words or words[0].lower() != "/nightly":
        return None
    rest = words[1:]
    verb = "status" if not rest else "set"
    if rest and rest[0].lower() in VERBS:
        verb = rest.pop(0).lower()
    prs: list[int] = []
    for word in rest:
        m = NUMBER.match(word)
        if not m:
            raise Refused(f"`{safe(word)}` isn't a pull request number. {USAGE}")
        if (n := int(m.group(1))) not in prs:
            prs.append(n)
    if verb in ("clear", "status") and prs:
        raise Refused(f"`/nightly {verb}` takes no numbers. {USAGE}")
    if verb not in ("clear", "status") and not prs:
        raise Refused(f"`/nightly {'' if verb == 'set' else verb + ' '}` needs pull request numbers. {USAGE}")
    if len(prs) > MAX_PRS:
        raise Refused(f"A Nightly holds at most {MAX_PRS} pull requests; that's {len(prs)}.")
    return Command(verb, prs)


@dataclass
class Reviewers:
    logins: set[str]  # lowercase
    paused: bool


def read_reviewers(text: str) -> Reviewers:
    """The list file: one login per line, `#` comments, and a `paused` line that stops every reviewer."""
    logins, paused = set(), False
    for raw in text.splitlines():
        line = raw.split("#", 1)[0].strip()
        if not line:
            continue
        words = line.split()
        if len(words) != 1:  # not a login: ignored rather than half-read
            continue
        word = words[0]
        if word.lower() == "paused":
            paused = True
        elif LOGIN.match(word):
            logins.add(word.lower())
    return Reviewers(logins, paused)


def is_owner(login: str) -> bool:
    return login.lower() == OWNER


def authorize(login: str, reviewers: Reviewers) -> None:
    """Ludovic always; a listed reviewer unless the list is paused; nobody else."""
    if is_owner(login):
        return
    if login.lower() not in reviewers.logins:
        raise Refused(f"Only Ludovic and the trusted reviewers listed in `{REVIEWERS}` can pick the Nightly.")
    if reviewers.paused:
        raise Refused(f"`/nightly` is paused: Ludovic stopped it in `{REVIEWERS}`.")


def guarded_patterns(codeowners: str) -> list[str]:
    """The paths CODEOWNERS gives to Ludovic, plus ALWAYS_GUARDED."""
    found = [line.split()[0] for line in codeowners.splitlines() if line.strip().startswith("/")]
    return sorted(set(found + ALWAYS_GUARDED))


def guarded(path: str, patterns: list[str]) -> bool:
    p = "/" + path.lstrip("/")
    return any(p.startswith(pat) if pat.endswith("/") else p == pat for pat in patterns)


@dataclass
class PullRequest:
    number: int
    state: str  # open, closed, merged
    head: str
    title: str
    author: str
    base: str
    files: list[str]


def check(pr: PullRequest | None, number: int, patterns: list[str]) -> str | None:
    """Why `pr` can't go in a Nightly, or None."""
    if pr is None:
        return f"#{number} isn't a pull request here"
    if pr.state != "open":
        return f"#{number} is {pr.state}"
    if pr.base != "main":
        return f"#{number} targets `{safe(pr.base)}`, not main"
    bad = [f for f in pr.files if guarded(f, patterns)]
    if bad:
        more = f" and {len(bad) - 1} more" if len(bad) > 1 else ""
        return f"#{number} changes `{bad[0]}`{more}, which only Ludovic merges (CODEOWNERS)"
    return None


def entry(pr: PullRequest, asked_by: str, now: str) -> dict:
    return {"number": pr.number, "head": pr.head, "title": pr.title, "author": pr.author, "asked_by": asked_by,
            "asked_at": now}


@dataclass
class Result:
    state: dict
    answer: str
    build: bool
    dry_run: list[dict] | None = None  # the set a dry run builds; the saved set stays


def plain(text: str) -> str:
    """A title fit for a comment or the notes: no mention, no markdown that spans lines."""
    return text.replace("@", "@\u200b").replace("\n", " ").strip()[:200]


def describe(prs: list[dict], heading: str = "The set") -> str:
    if not prs:
        return "The set is empty: the Nightly is the newest proven union, or main."
    lines = [f"{heading} ({len(prs)}), merged onto main in this order:"]
    lines += [f"- #{e['number']} {plain(e['title'])} (by {e['author']}, at {e['head'][:7]}, asked by "
              f"{e['asked_by']})" for e in prs]
    return "\n".join(lines)


def apply(state: dict, cmd: Command, login: str, now: str, lookup: Callable[[int], PullRequest | None],
          patterns: list[str]) -> Result:
    """What `cmd` by `login` makes of `state` (set.json). Pure but for `lookup`; raises Refused."""
    st = copy.deepcopy(state)
    prs: list[dict] = st.setdefault("prs", [])
    blocked: list[int] = st.setdefault("blocked", [])
    owner = is_owner(login)
    if cmd.verb == "status":
        extra = f"\nOnly Ludovic can add these back: {', '.join(f'#{n}' for n in blocked)}." if blocked else ""
        return Result(state, describe(prs) + extra, False)
    if cmd.verb == "clear":
        st["prs"] = []
    elif cmd.verb == "remove":
        absent = [n for n in cmd.prs if n not in [e["number"] for e in prs]]
        if absent and len(absent) == len(cmd.prs) and not owner:
            raise Refused(f"{', '.join(f'#{n}' for n in absent)} isn't in the set. " + describe(prs))
        st["prs"] = [e for e in prs if e["number"] not in cmd.prs]
        if owner:
            st["blocked"] = blocked + [n for n in cmd.prs if n not in blocked]
    else:  # set, add, dry-run
        if not owner and (held := [n for n in cmd.prs if n in blocked]):
            raise Refused(f"Ludovic took {', '.join(f'#{n}' for n in held)} out of the Nightly; only he can put "
                          "it back.")
        problems, asked = [], []
        for n in cmd.prs:
            pr = lookup(n)
            if why := check(pr, n, patterns):
                problems.append(why)
            else:
                asked.append(entry(pr, login, now))
        if problems:
            raise Refused("Nothing changed: " + "; ".join(problems) + ".")
        if cmd.verb == "add":
            fresh = {e["number"]: e for e in asked}
            new = [fresh.pop(e["number"], e) for e in prs] + list(fresh.values())  # keeps its place, newest head
        else:
            new = asked
        if len(new) > MAX_PRS:
            raise Refused(f"A Nightly holds at most {MAX_PRS} pull requests; that would be {len(new)}.")
        if cmd.verb == "dry-run":
            return Result(state, describe(new, "Dry run, the saved set stays as it is. It builds"), True,
                          dry_run=new)
        if owner:
            st["blocked"] = [n for n in blocked if n not in cmd.prs]
        st["prs"] = new
    return Result(st, describe(st["prs"]), True)


def log_command(state: dict, cmd: Command, login: str, now: str, issue: str, url: str) -> dict:
    st = copy.deepcopy(state)
    st["log"] = (st.get("log", []) + [{"at": now, "by": login, "command": cmd.text(), "issue": issue,
                                       "url": url}])[-50:]
    return st


# --- git -------------------------------------------------------------------------------------------------------


def git(*args: str, cwd: str | None = None, env: dict | None = None, check: bool = True,
        input: str | None = None) -> subprocess.CompletedProcess:
    return subprocess.run(["git", *SAFE_GIT, *args], cwd=cwd, env={**os.environ, **(env or {})}, check=check,
                          capture_output=True, text=True, input=input)


def out(*args: str, cwd: str | None = None, env: dict | None = None) -> str:
    return git(*args, cwd=cwd, env=env).stdout.strip()


def identity(date: str) -> dict:
    """The bot's name, and a fixed date so the same inputs give the same commit."""
    name, email = BOT
    return {"GIT_AUTHOR_NAME": name, "GIT_AUTHOR_EMAIL": email, "GIT_COMMITTER_NAME": name,
            "GIT_COMMITTER_EMAIL": email, "GIT_AUTHOR_DATE": date, "GIT_COMMITTER_DATE": date}


def merge_tree(a: str, b: str, cwd: str | None = None) -> tuple[str | None, list[str]]:
    """The merged tree of commits a and b, or None and the conflicted paths. Objects only, no working tree."""
    r = git("merge-tree", "--write-tree", "--name-only", "--no-messages", a, b, cwd=cwd, check=False)
    if r.returncode == 0:
        return r.stdout.split()[0], []
    if r.returncode == 1:
        return None, [line for line in r.stdout.splitlines()[1:] if line.strip()]
    raise RuntimeError(f"git merge-tree {a[:9]} {b[:9]}: {r.stderr.strip()}")


def commit_tree(tree: str, parents: list[str], message: str, date: str, cwd: str | None = None) -> str:
    args = ["commit-tree", tree]
    for p in parents:
        args += ["-p", p]
    return out(*args, "-m", message, cwd=cwd, env=identity(date))


@dataclass
class Conflict(Exception):
    number: int
    other: int | None  # the earlier pull request it conflicts with; None: main; 0: only with several together
    files: list[str]

    def text(self) -> str:
        where = ", ".join(f"`{f}`" for f in self.files[:8]) + (" and more" if len(self.files) > 8 else "")
        if self.other is None:
            return (f"#{self.number} conflicts with main in {where}: it needs main merged into it first.")
        if self.other == 0:
            return (f"#{self.number} conflicts with the pull requests before it, taken together, in {where}.")
        return (f"#{self.other} and #{self.number} conflict in {where}. Make an integration pull request that "
                "combines them (in your fork) and pick that one instead.")


def changed_files(base: str, head: str, cwd: str | None = None) -> list[str]:
    """What `head` changes since it left `base`, renames' old paths included."""
    since = out("merge-base", base, head, cwd=cwd)
    names = out("diff", "--name-status", "--no-renames", since, head, cwd=cwd)
    return [line.split("\t", 1)[1] for line in names.splitlines() if "\t" in line]


def culprit(main: str, earlier: list[tuple[int, str]], head: str, date: str, cwd: str | None) -> int | None:
    if merge_tree(main, head, cwd)[0] is None:
        return None
    for n, h in earlier:
        tree, _ = merge_tree(main, h, cwd)
        if tree is None:
            continue
        if merge_tree(commit_tree(tree, [main, h], "probe", date, cwd), head, cwd)[0] is None:
            return n
    return 0


def assemble(main: str, heads: list[tuple[int, str]], cwd: str | None = None) -> str:
    """main, then each pull request's head merged in order: the commit a Nightly builds. Raises Conflict."""
    date = out("show", "-s", "--format=%cI", main, cwd=cwd)
    current = main
    for i, (n, head) in enumerate(heads):
        tree, files = merge_tree(current, head, cwd)
        if tree is None:
            raise Conflict(n, culprit(main, heads[:i], head, date, cwd), files)
        current = commit_tree(tree, [current, head], f"Nightly: merge #{n} at {head[:12]}", date, cwd)
    return current


def fetch_heads(heads: list[tuple[int, str]], cwd: str | None = None) -> list[int]:
    """Fetches each head from origin; returns the numbers whose head is gone (force-pushed away and collected)."""
    shas = [h for _, h in heads]
    if shas and git("fetch", "--quiet", "--no-tags", "origin", *shas, cwd=cwd, check=False).returncode == 0:
        return []
    gone = []
    for n, h in heads:
        if git("cat-file", "-e", f"{h}^{{commit}}", cwd=cwd, check=False).returncode == 0:
            continue
        git("fetch", "--quiet", "--no-tags", "origin", h, cwd=cwd, check=False)
        if git("cat-file", "-e", f"{h}^{{commit}}", cwd=cwd, check=False).returncode != 0:
            gone.append(n)
    return gone


def auth(token: str) -> list[str]:
    """A `git -c` for pushing with the job's token, never written to a file or a log."""
    import base64
    basic = base64.b64encode(f"x-access-token:{token}".encode()).decode()
    os.environ.setdefault("GIT_TERMINAL_PROMPT", "0")
    return ["-c", f"http.https://github.com/.extraheader=AUTHORIZATION: basic {basic}"]


def push(sha: str, ref: str, token: str, cwd: str | None = None, force: bool = False) -> bool:
    spec = f"{'+' if force else ''}{sha}:refs/heads/{ref}"
    return git(*auth(token), "push", "--quiet", "--no-verify", "origin", spec, cwd=cwd, check=False).returncode == 0


def builds_today(today: str, cwd: str | None = None) -> list[str]:
    listed = out("ls-remote", "origin", f"refs/heads/{BUILD_PREFIX}{today}-*", cwd=cwd)
    return [line.split("refs/heads/", 1)[1] for line in listed.splitlines() if "refs/heads/" in line]


def next_build(today: str, existing: list[str]) -> str:
    taken = [int(m.group(1)) for b in existing if (m := re.search(r"-(\d+)$", b))]
    return f"{BUILD_PREFIX}{today}-{max(taken, default=0) + 1}"


# --- the state branch ------------------------------------------------------------------------------------------


def load_state(cwd: str | None = None) -> tuple[dict, str, str | None]:
    """set.json, builds.jsonl and the branch's head (None before the first command)."""
    if git("fetch", "--quiet", "--no-tags", "origin", f"+refs/heads/{STATE_BRANCH}:refs/nightly/state", cwd=cwd,
           check=False).returncode != 0:
        return {}, "", None
    head = out("rev-parse", "refs/nightly/state", cwd=cwd)
    state = json.loads(git("show", f"{head}:set.json", cwd=cwd, check=False).stdout or "{}")
    builds = git("show", f"{head}:builds.jsonl", cwd=cwd, check=False).stdout
    return state, builds, head


def write_state(state: dict, builds: str, parent: str | None, message: str, cwd: str | None = None) -> str:
    """A commit of the state branch, made in a private index: the checkout isn't touched."""
    with tempfile.TemporaryDirectory() as tmp:
        env = {"GIT_INDEX_FILE": str(Path(tmp) / "index")}
        for name, text in (("set.json", json.dumps(state, indent=2) + "\n"), ("builds.jsonl", builds),
                           ("README.md", STATE_README)):
            blob = git("hash-object", "-w", "--stdin", cwd=cwd, input=text).stdout.strip()
            git("update-index", "--add", "--cacheinfo", f"100644,{blob},{name}", cwd=cwd, env=env)
        tree = out("write-tree", cwd=cwd, env=env)
    now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds")
    return commit_tree(tree, [parent] if parent else [], message, now, cwd)


STATE_README = """# nightly-state

Written by tools/ci/nightly_set.py and tools/ci/nightly.py, never by hand. `set.json`: the pull requests the trusted
reviewers picked for the Nightly (docs/release-flow.md "Nightly channel"), each at the head it was asked at.
`builds.jsonl`: one line per Nightly build with its set.
"""


def update_state(change: Callable[[dict, str], tuple[dict, str, str] | None], token: str,
                 cwd: str | None = None, tries: int = 5):
    """Applies change(state, builds) -> (state, builds, message), or None for no change, and pushes it; when another
    run pushed first, reads the branch again and applies it again. Returns the last state."""
    for _ in range(tries):
        state, builds, head = load_state(cwd)
        changed = change(state, builds)
        if changed is None:
            return state
        new, new_builds, message = changed
        if push(write_state(new, new_builds, head, message, cwd), STATE_BRANCH, token, cwd):
            return new
        time.sleep(2)
    raise RuntimeError(f"could not update {STATE_BRANCH} after {tries} tries")


# --- GitHub ----------------------------------------------------------------------------------------------------


def gh(*args: str, check: bool = True) -> str:
    r = subprocess.run(["gh", *args], capture_output=True, text=True)
    if check and r.returncode != 0:
        raise RuntimeError(f"gh {args[0]} {args[1] if len(args) > 1 else ''}: {r.stderr.strip()[:300]}")
    return r.stdout.strip() if r.returncode == 0 else ""


def lookup(repo: str) -> Callable[[int], PullRequest | None]:
    def get(n: int) -> PullRequest | None:
        raw = gh("api", f"repos/{repo}/pulls/{n}", check=False)
        if not raw:
            return None
        p = json.loads(raw)
        files = gh("api", "--paginate", f"repos/{repo}/pulls/{n}/files", "--jq",
                   ".[] | .filename, (.previous_filename // empty)").splitlines()
        state = "merged" if p.get("merged_at") else p["state"]
        return PullRequest(n, state, p["head"]["sha"], p["title"], p["user"]["login"], p["base"]["ref"], files)
    return get


def answer(repo: str, issue: str, text: str) -> None:
    gh("api", f"repos/{repo}/issues/{issue}/comments", "-f", f"body=🤖 Nightly: {text}")


def dispatch(repo: str, issue: str, request: str, dry_run: list[dict] | None) -> str:
    """Starts `nightly build` for this request; returns its run's URL once GitHub lists it."""
    args = ["workflow", "run", "nightly-build.yml", "--repo", repo, "--ref", "main", "-f", f"issue={issue}",
            "-f", f"request={request}", "-f", "force=true"]
    if dry_run is not None:
        args += ["-f", "dry_run=true", "-f", "prs=" + " ".join(f"{e['number']}@{e['head']}" for e in dry_run)]
    gh(*args)
    for _ in range(15):
        time.sleep(4)
        runs = json.loads(gh("run", "list", "--repo", repo, "--workflow", "nightly-build.yml", "--event",
                             "workflow_dispatch", "--limit", "20", "--json", "displayTitle,url", check=False) or "[]")
        for r in runs:
            if request in r["displayTitle"]:
                return r["url"]
    return f"https://github.com/{repo}/actions/workflows/nightly-build.yml"


def command() -> None:
    """nightly-command.yml: the comment arrives in env vars, never in a shell line."""
    repo, issue = os.environ["GITHUB_REPOSITORY"], os.environ["ISSUE"]
    login, body, url = os.environ["AUTHOR"], os.environ.get("BODY", ""), os.environ.get("COMMENT_URL", "")
    token = os.environ["GH_TOKEN"]
    try:
        cmd = parse(body)
    except Refused as e:
        answer(repo, issue, str(e))
        return
    if cmd is None:
        return
    now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds")
    try:
        reviewers = Path(REVIEWERS).read_text() if Path(REVIEWERS).exists() else ""
        authorize(login, read_reviewers(reviewers))
        patterns = guarded_patterns(Path(CODEOWNERS).read_text() if Path(CODEOWNERS).exists() else "")
        if cmd.verb != "status":
            today = now[:10]
            done = builds_today(today)
            if len(done) >= MAX_BUILDS_PER_DAY:
                raise Refused(f"{len(done)} Nightly builds today already, the most a day takes: the next scheduled "
                              "one (20:17 UTC) builds the set.")
        results: list[Result] = []

        def change(state: dict, builds: str):
            result = apply(state, cmd, login, now, lookup(repo), patterns)
            results.append(result)
            if result.state is state:  # status, dry run: nothing to save
                return None
            return log_command(result.state, cmd, login, now, issue, url), builds, f"{cmd.text()} by {login}"

        update_state(change, token)
        result = results[-1]
    except Refused as e:
        answer(repo, issue, str(e))
        return
    text = result.answer
    if result.build:
        run = dispatch(repo, issue, f"{issue}-{int(time.time())}", result.dry_run)
        text += f"\n\nBuilding: {run}. I'll answer here when it's {'done' if result.dry_run else 'published'}."
    answer(repo, issue, text)


def main() -> None:
    if sys.argv[1:] == ["command"]:
        command()
    else:
        sys.exit(__doc__)


if __name__ == "__main__":
    main()