From 3374c676084acde83ef37f6a38943fbd6708583f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 22:38:39 +0000 Subject: [PATCH 1/7] CI: shared files checksummed, release gate, shared crash-log and signing-key scripts Shared with java-llama.cpp, BitcoinAddressFinder, srcmorph and streambuffer (listed in .github/shared-files.sha256, checked by the new shared-files job, which fails on a copy changed in one repository alone and warns on a sibling whose copy differs): - .github/buildcheck/{workflow,releasegate,sharedfiles}.py + unit tests (stdlib-only Python), check-release-gate.py and check-shared-files.py; - print-crash-logs.sh (replaces the crash-log steps pasted into every test job) and verify-signing-key.sh (the body of the verify-signing-key job); - the files that were already kept identical by hand, now in the manifest. The release gate: every job of publish.yml gates both publish jobs unless .github/release-gate-exemptions.txt names it with a reason. It found vmlens gating nothing; vmlens and shared-files now gate both publish jobs. Also: java-version literals replaced by env.JAVA_VERSION; CLAUDE.md and CHANGELOG describe the job. Verified: build-check unit tests, check-release-gate.py, check-shared-files.py, actionlint, reuse lint. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AytmJF9faEiQEVt6eetQS2 --- .github/buildcheck/__init__.py | 14 ++ .github/buildcheck/releasegate.py | 60 +++++++ .github/buildcheck/sharedfiles.py | 137 +++++++++++++++ .github/buildcheck/tests/__init__.py | 3 + .github/buildcheck/tests/helpers.py | 26 +++ .github/buildcheck/tests/test_releasegate.py | 61 +++++++ .github/buildcheck/tests/test_sharedfiles.py | 112 +++++++++++++ .github/buildcheck/tests/test_workflow.py | 70 ++++++++ .github/buildcheck/workflow.py | 120 +++++++++++++ .github/check-release-gate.py | 33 ++++ .github/check-shared-files.py | 20 +++ .github/print-crash-logs.sh | 48 ++++++ .github/release-gate-exemptions.txt | 16 ++ .github/shared-files.sha256 | 29 ++++ .github/verify-bytecode-version.sh | 5 +- .github/verify-signing-key.sh | 100 +++++++++++ .github/workflows/publish.yml | 167 +++++-------------- .gitignore | 3 + CHANGELOG.md | 7 + CLAUDE.md | 16 +- 20 files changed, 919 insertions(+), 128 deletions(-) create mode 100644 .github/buildcheck/__init__.py create mode 100644 .github/buildcheck/releasegate.py create mode 100644 .github/buildcheck/sharedfiles.py create mode 100644 .github/buildcheck/tests/__init__.py create mode 100644 .github/buildcheck/tests/helpers.py create mode 100644 .github/buildcheck/tests/test_releasegate.py create mode 100644 .github/buildcheck/tests/test_sharedfiles.py create mode 100644 .github/buildcheck/tests/test_workflow.py create mode 100644 .github/buildcheck/workflow.py create mode 100755 .github/check-release-gate.py create mode 100755 .github/check-shared-files.py create mode 100755 .github/print-crash-logs.sh create mode 100644 .github/release-gate-exemptions.txt create mode 100644 .github/shared-files.sha256 create mode 100755 .github/verify-signing-key.sh diff --git a/.github/buildcheck/__init__.py b/.github/buildcheck/__init__.py new file mode 100644 index 00000000..e49ad24b --- /dev/null +++ b/.github/buildcheck/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Build checks as a library, so buildcheck/tests can test them. Standard library only. + +Two kinds of module live here. workflow.py, releasegate.py and sharedfiles.py (with their tests +and the check-*.py entry points next to this package) are kept BYTE-IDENTICAL in java-llama.cpp, +BitcoinAddressFinder, srcmorph and streambuffer: each repository lists them in +.github/shared-files.sha256, which its `shared-files` job checks (see sharedfiles.py). Every other +module is the repository's own. + +Run the tests from the repository root: + python3 -m unittest discover -s .github/buildcheck/tests -t .github +""" diff --git a/.github/buildcheck/releasegate.py b/.github/buildcheck/releasegate.py new file mode 100644 index 00000000..729e2150 --- /dev/null +++ b/.github/buildcheck/releasegate.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Every job of publish.yml gates both publish jobs, unless the repository lists it in +.github/release-gate-exemptions.txt with a reason. + +A job that nothing waits for can go red and a release still ships -- the natives-build jobs that +`package` once forgot to wait for, and the aarch64 fat jars that were signed and attached for +releases without any job launching them, were both of that shape. So "not gating" has to be a +decision written down here, not the default a new job gets by being forgotten in two `needs:` lists. + +The check runs both ways: a job outside the gates and outside the exemptions fails, and so does an +exemption that names no job or a job that gates both publish jobs after all (a stale +exemption would hide the next job of that name). +""" + +from . import workflow + +GATES = ("publish-snapshot", "publish-release") + +# Each repository lists its own exemptions here, one `: ` per line (# comments). +EXEMPTIONS_FILE = ".github/release-gate-exemptions.txt" + + +def read_exemptions(text): + """The exemptions file: job -> reason. A line without a reason is an error -- the point of the + file is that every job allowed to stay red says why.""" + exemptions = {} + for number, line in enumerate(text.splitlines(), 1): + line = line.strip() + if not line or line.startswith("#"): + continue + job, _, reason = line.partition(":") + if not reason.strip() or not job.strip(): + raise ValueError(f"line {number}: expected `: `, got {line!r}") + if job.strip() in exemptions: + raise ValueError(f"line {number}: {job.strip()} is listed twice") + exemptions[job.strip()] = reason.strip() + return exemptions + + +def check(jobs, non_gating, gates=GATES): + missing = [g for g in gates if g not in jobs] + if missing: + return [f"publish.yml has no job {g}" for g in missing] + closures = [workflow.closure(jobs, g) for g in gates] + failures = [] + for name in jobs: + gated = [g for g, c in zip(gates, closures) if name in c] + if name in non_gating: + if len(gated) == len(gates): + failures.append(f"release-gate-exemptions.txt lists {name}, which gates {', '.join(gates)} " + f"-- remove the stale exemption") + elif len(gated) != len(gates): + ungated = [g for g in gates if g not in gated] + failures.append(f"publish.yml: {', '.join(ungated)} does not wait for {name} -- add it to the " + f"needs, or to release-gate-exemptions.txt with the reason it may stay red") + failures += [f"release-gate-exemptions.txt lists {name}, which is no job of publish.yml" + for name in sorted(set(non_gating) - set(jobs))] + return failures diff --git a/.github/buildcheck/sharedfiles.py b/.github/buildcheck/sharedfiles.py new file mode 100644 index 00000000..1c771ee4 --- /dev/null +++ b/.github/buildcheck/sharedfiles.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""The files kept byte-identical across java-llama.cpp, BitcoinAddressFinder, srcmorph and +streambuffer, checked by every repository's own `shared-files` job. + +Each repository lists the shared files it carries, with their SHA-256, in +.github/shared-files.sha256 -- `sha256sum` format, so `sha256sum -c` reads it as well. The job + * fails when a listed file is missing or its content no longer matches its line: a shared file + was edited here alone. Edit it in every repository that lists it, then update each manifest + (`check-shared-files.py --write` rewrites the hashes of this repository's lines); + * warns when another repository's manifest (its default branch) lists the same file with a + different hash: one side of a sync is not merged yet. A warning and not a failure, because a + sync is one change per repository and lands in four steps. +A file is matched across repositories by its path, or, when the other side has no such path, by +its file name if that is unique there (lombok.config lives at llama/lombok.config in the +java-llama.cpp reactor and at the root elsewhere). +""" + +import hashlib +import os +import re +import sys +import urllib.request + +REPOS = ("java-llama.cpp", "BitcoinAddressFinder", "srcmorph", "streambuffer") +OWNER = "bernardladenthin" +MANIFEST = ".github/shared-files.sha256" +RAW_URL = "https://raw.githubusercontent.com/{owner}/{repo}/HEAD/" + MANIFEST +LINE = re.compile(r"^([0-9a-f]{64}) [ *](.+)$") + + +def parse(text): + """path -> sha256 of a manifest. Comment and blank lines are skipped; anything else that is + not a `sha256sum` line is an error, and so is a path listed twice.""" + entries = {} + for number, line in enumerate(text.splitlines(), 1): + if not line.strip() or line.startswith("#"): + continue + match = LINE.match(line) + if not match: + raise ValueError(f"{MANIFEST} line {number}: not ` `: {line!r}") + if match.group(2) in entries: + raise ValueError(f"{MANIFEST} line {number}: {match.group(2)} is listed twice") + entries[match.group(2)] = match.group(1) + return entries + + +def sha256(path): + with open(path, "rb") as f: + return hashlib.sha256(f.read()).hexdigest() + + +def verify(root, entries): + """Failures: every listed file exists and still has its listed hash.""" + failures = [] + for path, digest in entries.items(): + full = os.path.join(root, path) + if not os.path.isfile(full): + failures.append(f"{path} is listed in {MANIFEST} but does not exist") + elif sha256(full) != digest: + failures.append(f"{path} differs from its line in {MANIFEST}: a shared file was changed in " + f"this repository alone -- change every copy, then update every manifest") + return failures + + +def rewrite(text, root): + """The manifest with every hash recomputed from the files; comments and order kept.""" + out = [] + for line in text.splitlines(): + match = LINE.match(line) + out.append(f"{sha256(os.path.join(root, match.group(2)))} {match.group(2)}" if match else line) + return "\n".join(out) + "\n" + + +def counterpart(path, other): + """The path `other` (another manifest) lists for our `path`: the same path, else a unique + file of the same name, else None.""" + if path in other: + return path + same_name = [p for p in other if os.path.basename(p) == os.path.basename(path)] + return same_name[0] if len(same_name) == 1 else None + + +def compare(own, others): + """Warnings for every shared file another repository lists with a different hash.""" + warnings = [] + for repo, entries in sorted(others.items()): + for path, digest in own.items(): + theirs = counterpart(path, entries) + if theirs is not None and entries[theirs] != digest: + warnings.append(f"{path} differs from {repo}'s {theirs}: finish or redo the sync") + return warnings + + +def current_repo(root): + name = os.environ.get("GITHUB_REPOSITORY", "").split("/")[-1] + return name or os.path.basename(os.path.abspath(root)) + + +def fetch(repo): + with urllib.request.urlopen(RAW_URL.format(owner=OWNER, repo=repo), timeout=20) as response: + return response.read().decode("utf-8") + + +def main(argv, root, fetcher=fetch, out=sys.stdout, err=sys.stderr): + """check-shared-files.py [--write | --offline]""" + path = os.path.join(root, MANIFEST) + with open(path, encoding="utf-8") as f: + text = f.read() + if argv[1:] == ["--write"]: + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write(rewrite(text, root)) + print(f"rewrote the hashes of {MANIFEST}", file=out) + return 0 + if argv[1:] not in ([], ["--offline"]): + print(main.__doc__, file=err) + return 2 + own = parse(text) + failures = verify(root, own) + for failure in failures: + print(f"::error::{failure}", file=err) + print(f"{len(own)} shared files, {len(failures)} changed here alone", file=out) + if argv[1:] == ["--offline"]: + return 1 if failures else 0 + others = {} + for repo in REPOS: + if repo == current_repo(root): + continue + try: + others[repo] = parse(fetcher(repo)) + except Exception as e: # noqa: BLE001 -- a sibling that cannot be read is a warning, never a red + print(f"::warning::could not read {repo}'s {MANIFEST}: {e}", file=err) + for warning in compare(own, others): + print(f"::warning::{warning}", file=err) + print(f"compared with {', '.join(sorted(others)) or 'no other repository'}", file=out) + return 1 if failures else 0 diff --git a/.github/buildcheck/tests/__init__.py b/.github/buildcheck/tests/__init__.py new file mode 100644 index 00000000..4833295f --- /dev/null +++ b/.github/buildcheck/tests/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 diff --git a/.github/buildcheck/tests/helpers.py b/.github/buildcheck/tests/helpers.py new file mode 100644 index 00000000..375a8128 --- /dev/null +++ b/.github/buildcheck/tests/helpers.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Shared fixtures of the buildcheck tests.""" + +import os + +REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + + +def workflow_text(jobs): + """A workflow with the given jobs, each (name, needs, body lines).""" + lines = ["name: t", "on:", " push:", "jobs:"] + for name, needs, body in jobs: + lines.append(f" {name}:") + if needs is not None: + lines.append(f" needs: {needs}") + lines.append(" runs-on: ubuntu-latest") + lines.append(" steps:") + lines += body + return "\n".join(lines) + "\n" + + +def upload(artifact, path="x/"): + return [" - uses: actions/upload-artifact@v7", " with:", f" name: {artifact}", + f" path: {path}"] diff --git a/.github/buildcheck/tests/test_releasegate.py b/.github/buildcheck/tests/test_releasegate.py new file mode 100644 index 00000000..19fe7049 --- /dev/null +++ b/.github/buildcheck/tests/test_releasegate.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 + +import os +import unittest + +from buildcheck import releasegate, workflow +from buildcheck.tests.helpers import REPO, workflow_text + +GATES = ("pub-a", "pub-b") + + +def jobs(*extra): + return workflow.parse(workflow_text([("test", None, []), ("build", "test", []), *extra, + ("pub-a", "[build]", []), ("pub-b", "[build]", [])])) + + +class ReleaseGateTest(unittest.TestCase): + + def check(self, js, non_gating): + return releasegate.check(js, dict.fromkeys(non_gating, "reason"), GATES) + + def test_transitively_gating_jobs_pass(self): + self.assertEqual(self.check(jobs(), ["pub-a", "pub-b"]), []) + + def test_a_job_nothing_waits_for_fails(self): + self.assertEqual(self.check(jobs(("lint", None, [])), ["pub-a", "pub-b"]), + ["publish.yml: pub-a, pub-b does not wait for lint -- add it to the needs, or to " + "release-gate-exemptions.txt with the reason it may stay red"]) + + def test_gating_only_one_publish_job_fails(self): + js = workflow.parse(workflow_text([("lint", None, []), ("pub-a", "[lint]", []), ("pub-b", None, [])])) + self.assertEqual(len(self.check(js, ["pub-a", "pub-b"])), 1) + + def test_a_listed_job_may_stay_outside(self): + self.assertEqual(self.check(jobs(("lint", None, [])), ["pub-a", "pub-b", "lint"]), []) + + def test_a_stale_exemption_fails(self): + self.assertIn("remove the stale exemption", self.check(jobs(), ["pub-a", "pub-b", "build"])[0]) + self.assertEqual(self.check(jobs(), ["pub-a", "pub-b", "gone"]), + ["release-gate-exemptions.txt lists gone, which is no job of publish.yml"]) + + def test_reads_the_exemptions_file(self): + self.assertEqual(releasegate.read_exemptions("# c\n\na: why: because\n b :x\n"), + {"a": "why: because", "b": "x"}) + + def test_an_exemption_without_a_reason_is_an_error(self): + for text in ("a\n", "a:\n", "a: \n", ": x\n", "a: x\na: y\n"): + with self.subTest(text=text), self.assertRaises(ValueError): + releasegate.read_exemptions(text) + + def test_the_repository_passes(self): + with open(os.path.join(REPO, ".github", "workflows", "publish.yml"), encoding="utf-8") as f: + jobs = workflow.parse(f.read()) + with open(os.path.join(REPO, releasegate.EXEMPTIONS_FILE), encoding="utf-8") as f: + self.assertEqual(releasegate.check(jobs, releasegate.read_exemptions(f.read())), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/buildcheck/tests/test_sharedfiles.py b/.github/buildcheck/tests/test_sharedfiles.py new file mode 100644 index 00000000..e78d610a --- /dev/null +++ b/.github/buildcheck/tests/test_sharedfiles.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 + +import hashlib +import io +import os +import tempfile +import unittest + +from buildcheck import sharedfiles +from buildcheck.tests.helpers import REPO + +A = hashlib.sha256(b"a").hexdigest() +B = hashlib.sha256(b"b").hexdigest() + + +class ParseTest(unittest.TestCase): + + def test_reads_sha256sum_lines_and_skips_comments(self): + text = f"# header\n\n{A} x/one.sh\n{B} *two.py\n" + self.assertEqual(sharedfiles.parse(text), {"x/one.sh": A, "two.py": B}) + + def test_rejects_malformed_and_duplicate_lines(self): + for text in ("nonsense\n", f"{A[:-1]} x\n", f"{A} x\n{B} x\n"): + with self.subTest(text=text), self.assertRaises(ValueError): + sharedfiles.parse(text) + + +class TreeTest(unittest.TestCase): + + def setUp(self): + self.root = tempfile.mkdtemp() + os.makedirs(os.path.join(self.root, ".github")) + with open(os.path.join(self.root, "one.sh"), "wb") as f: + f.write(b"a") + + def manifest(self, text): + with open(os.path.join(self.root, sharedfiles.MANIFEST), "w", encoding="utf-8") as f: + f.write(text) + + def run_main(self, *args, others=None): + out, err = io.StringIO(), io.StringIO() + others = others or {} + + def fetcher(repo): + if repo not in others: + raise OSError("unreachable") + return others[repo] + code = sharedfiles.main(["x", *args], self.root, fetcher, out, err) + return code, err.getvalue() + + def test_an_unchanged_file_passes(self): + self.manifest(f"# h\n{A} one.sh\n") + self.assertEqual(self.run_main("--offline"), (0, "")) + + def test_a_file_changed_alone_or_missing_fails(self): + self.manifest(f"{B} one.sh\n{A} gone.sh\n") + code, err = self.run_main("--offline") + self.assertEqual(code, 1) + self.assertIn("one.sh differs", err) + self.assertIn("gone.sh is listed", err) + + def test_write_recomputes_the_hashes_and_keeps_the_comments(self): + self.manifest(f"# keep me\n{B} one.sh\n") + self.assertEqual(self.run_main("--write")[0], 0) + with open(os.path.join(self.root, sharedfiles.MANIFEST), encoding="utf-8") as f: + self.assertEqual(f.read(), f"# keep me\n{A} one.sh\n") + + def test_sibling_differences_are_warnings_not_failures(self): + self.manifest(f"{A} one.sh\n") + others = {"srcmorph": f"{B} one.sh\n", "streambuffer": f"{A} one.sh\n"} + code, err = self.run_main(others=others) + self.assertEqual(code, 0) + self.assertIn("::warning::one.sh differs from srcmorph's one.sh", err) + self.assertNotIn("streambuffer's", err) + self.assertIn("could not read BitcoinAddressFinder", err) + + def test_unknown_arguments(self): + self.manifest("") + self.assertEqual(self.run_main("--nope")[0], 2) + + +class CompareTest(unittest.TestCase): + + def test_matches_by_path_then_by_a_unique_file_name(self): + own = {"llama/lombok.config": A, "a/__init__.py": A} + self.assertEqual(sharedfiles.compare(own, {"r": {"lombok.config": B}}), + ["llama/lombok.config differs from r's lombok.config: finish or redo the sync"]) + # two candidates of the same name: no guess + self.assertEqual(sharedfiles.compare(own, {"r": {"b/__init__.py": B, "c/__init__.py": B}}), []) + # a file the other repository does not share is no difference + self.assertEqual(sharedfiles.compare(own, {"r": {}}), []) + + +class RepositoryTest(unittest.TestCase): + + def test_this_repository_matches_its_manifest(self): + with open(os.path.join(REPO, sharedfiles.MANIFEST), encoding="utf-8") as f: + entries = sharedfiles.parse(f.read()) + self.assertEqual(sharedfiles.verify(REPO, entries), []) + + def test_the_shared_build_checks_list_themselves(self): + with open(os.path.join(REPO, sharedfiles.MANIFEST), encoding="utf-8") as f: + entries = sharedfiles.parse(f.read()) + for path in (".github/buildcheck/sharedfiles.py", ".github/check-shared-files.py", + ".github/buildcheck/tests/test_sharedfiles.py"): + self.assertIn(path, entries) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/buildcheck/tests/test_workflow.py b/.github/buildcheck/tests/test_workflow.py new file mode 100644 index 00000000..b9bf48d8 --- /dev/null +++ b/.github/buildcheck/tests/test_workflow.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 + +import os +import unittest + +from buildcheck import workflow +from buildcheck.tests.helpers import REPO, upload, workflow_text + + +class ParseTest(unittest.TestCase): + + def test_reads_every_needs_shape(self): + text = workflow_text([ + ("a", None, []), + ("b", "a", []), + ("c", "[a, b]", []), + ("d", "", []), + ]).replace(" needs: \n", " needs:\n - a\n - c # why\n") + jobs = workflow.parse(text) + self.assertEqual(list(jobs), ["a", "b", "c", "d"]) + self.assertEqual(jobs["a"].needs, []) + self.assertEqual(jobs["b"].needs, ["a"]) + self.assertEqual(jobs["c"].needs, ["a", "b"]) + self.assertEqual(jobs["d"].needs, ["a", "c"]) + + def test_keys_outside_jobs_are_not_jobs(self): + jobs = workflow.parse(workflow_text([("a", None, [])]) + "env:\n X: 1\n") + self.assertEqual(list(jobs), ["a"]) + + def test_unknown_need_is_an_error(self): + with self.assertRaisesRegex(ValueError, "needs nope"): + workflow.parse(workflow_text([("a", "nope", [])])) + + def test_unreadable_needs_is_an_error_not_a_guess(self): + with self.assertRaisesRegex(ValueError, "cannot read needs"): + workflow.parse(workflow_text([("a", None, []), ("b", "${{ fromJSON(x) }}", [])])) + + def test_uploads_reads_the_artifact_name_not_the_step_name(self): + body = [" - name: Upload something", " if: always()", + " uses: actions/upload-artifact@v7"] + upload("natives-x")[1:] \ + + [" - run: echo name: fake"] + upload("second") + job = workflow.parse(workflow_text([("a", None, body)]))["a"] + self.assertEqual(job.uploads(), ["natives-x", "second"]) + + def test_closure_is_transitive_and_excludes_the_job(self): + jobs = workflow.parse(workflow_text([("a", None, []), ("b", "a", []), ("c", "[b]", []), ("x", None, [])])) + self.assertEqual(workflow.closure(jobs, "c"), {"a", "b"}) + self.assertEqual(workflow.closure(jobs, "a"), set()) + + +class RepositoryWorkflowsTest(unittest.TestCase): + + def test_every_workflow_of_the_repository_parses(self): + folder = os.path.join(REPO, ".github", "workflows") + for name in sorted(os.listdir(folder)): + with self.subTest(name): + with open(os.path.join(folder, name), encoding="utf-8") as f: + self.assertTrue(workflow.parse(f.read())) + + def test_publish_has_both_publish_jobs_and_they_wait_for_something(self): + with open(os.path.join(REPO, ".github", "workflows", "publish.yml"), encoding="utf-8") as f: + jobs = workflow.parse(f.read()) + for gate in ("publish-snapshot", "publish-release"): + self.assertGreater(len(workflow.closure(jobs, gate)), 3, gate) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/buildcheck/workflow.py b/.github/buildcheck/workflow.py new file mode 100644 index 00000000..de7055a6 --- /dev/null +++ b/.github/buildcheck/workflow.py @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""The part of a GitHub Actions workflow the checks need: its jobs, their `needs`, the artifacts +they upload and the text of their steps. + +Deliberately not a YAML parser. The runners' Python has no PyYAML guaranteed, and the checks +need only the job graph, which the workflow writes in two fixed shapes: jobs at two spaces of +indentation below `jobs:`, and `needs:` at four, either inline (`needs: a` / `needs: [a, b]`) or +as a block list (`- a` at six). A workflow written any other way is reported by `parse`, never +misread silently: every job must be found, and a `needs:` that matches neither shape fails. +""" + +import re + +JOB = re.compile(r"^ ([A-Za-z0-9_-]+):\s*(#.*)?$") +NEEDS = re.compile(r"^ needs:\s*(.*?)\s*(#.*)?$") +NEEDS_ITEM = re.compile(r"^ - ([A-Za-z0-9_-]+)\s*(#.*)?$") +STEP = re.compile(r"^ - ") + + +class Job: + """One job: its id, the jobs it needs, and its lines (the job header included).""" + + def __init__(self, name): + self.name = name + self.needs = [] + self.lines = [] + + @property + def text(self): + return "\n".join(self.lines) + + def steps(self): + """The job's steps, each as a list of lines (a step starts at ` - `).""" + out = [] + for line in self.lines: + if STEP.match(line): + out.append([line]) + elif out: + out[-1].append(line) + return out + + def uploads(self): + """The names of the artifacts the job uploads (upload-artifact's `with: name:`).""" + names = [] + for step in self.steps(): + if not any("actions/upload-artifact@" in line for line in step): + continue + with_indent = None + for line in step: + stripped = line.lstrip() + indent = len(line) - len(stripped) + if stripped.startswith("with:"): + with_indent = indent + elif with_indent is not None and indent > with_indent and stripped.startswith("name:"): + names.append(stripped[len("name:"):].strip().strip("'\"")) + break + elif with_indent is not None and indent <= with_indent: + with_indent = None + return names + + +def parse(text): + """Jobs by id, in file order. Raises ValueError on a `needs:` in a shape it does not read.""" + jobs = {} + in_jobs = False + current = None + block_needs = False + for number, line in enumerate(text.splitlines(), 1): + if line and not line[0].isspace() and not line.startswith("#"): + in_jobs = line.rstrip() == "jobs:" + current = None + continue + if not in_jobs: + continue + match = JOB.match(line) + if match: + current = jobs[match.group(1)] = Job(match.group(1)) + current.lines.append(line) + block_needs = False + continue + if current is None: + continue + current.lines.append(line) + match = NEEDS.match(line) + if match: + value = match.group(1) + if value == "": + block_needs = True + elif value.startswith("[") and value.endswith("]"): + current.needs += [n.strip() for n in value[1:-1].split(",") if n.strip()] + elif re.fullmatch(r"[A-Za-z0-9_-]+", value): + current.needs.append(value) + else: + raise ValueError(f"line {number}: cannot read needs of job {current.name}: {value!r}") + continue + if block_needs: + item = NEEDS_ITEM.match(line) + if item: + current.needs.append(item.group(1)) + elif line.strip() and not line.lstrip().startswith("#"): + block_needs = False + for job in jobs.values(): + for need in job.needs: + if need not in jobs: + raise ValueError(f"job {job.name} needs {need}, which is no job of this workflow") + return jobs + + +def closure(jobs, name): + """Every job `name` waits for, directly or through another job (itself excluded).""" + seen = set() + todo = list(jobs[name].needs) + while todo: + need = todo.pop() + if need not in seen: + seen.add(need) + todo += jobs[need].needs + return seen diff --git a/.github/check-release-gate.py b/.github/check-release-gate.py new file mode 100755 index 00000000..447256fe --- /dev/null +++ b/.github/check-release-gate.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Fail when a job of publish.yml gates neither publish job without a written reason. + +See buildcheck/releasegate.py; the reasons are in .github/release-gate-exemptions.txt. +Usage (from anywhere): check-release-gate.py +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from buildcheck import releasegate, workflow # noqa: E402 + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def main(): + with open(os.path.join(ROOT, ".github", "workflows", "publish.yml"), encoding="utf-8") as f: + jobs = workflow.parse(f.read()) + with open(os.path.join(ROOT, releasegate.EXEMPTIONS_FILE), encoding="utf-8") as f: + exemptions = releasegate.read_exemptions(f.read()) + failures = releasegate.check(jobs, exemptions) + for failure in failures: + print(f"::error::{failure}", file=sys.stderr) + print(f"{len(jobs)} jobs, {len(exemptions)} exempt, {len(failures)} violations") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/check-shared-files.py b/.github/check-shared-files.py new file mode 100755 index 00000000..47359ee6 --- /dev/null +++ b/.github/check-shared-files.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Check the files this repository shares byte-identically with its sibling repositories. + +See buildcheck/sharedfiles.py; the list is .github/shared-files.sha256. +Usage: check-shared-files.py verify the local copies, compare with the siblings (warnings) + check-shared-files.py --offline verify the local copies only + check-shared-files.py --write recompute the hashes of the listed files after a sync +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from buildcheck import sharedfiles # noqa: E402 + +if __name__ == "__main__": + sys.exit(sharedfiles.main(sys.argv, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) diff --git a/.github/print-crash-logs.sh b/.github/print-crash-logs.sh new file mode 100755 index 00000000..faa409ed --- /dev/null +++ b/.github/print-crash-logs.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 + +# Cross-repo shared script -- kept BYTE-IDENTICAL in java-llama.cpp, BitcoinAddressFinder, +# srcmorph and streambuffer (listed in each repo's .github/shared-files.sha256). The "Print crash +# logs" step of workspace/policies/ci-test-diagnostics.md section 3.1: echoes the JVM crash logs +# of a failed test job into the job log, because an uploaded artifact is unreadable from anywhere +# that cannot fetch from Azure Blob Storage (a phone, a restricted network, an agent sandbox). +# +# Usage: print-crash-logs.sh [...] (default: the current directory) +# For each module directory: /hs_err_pid*.log (first 200 lines -- the diagnostic core is at +# the top, the tail is thread dumps and the memory map) and +# /target/surefire-reports/*.dumpstream|*.dump (whole; they are small). +# +# Always exits 0: it runs under `if: failure()` and must never replace the job's real failure. +shopt -s nullglob +[ "$#" -gt 0 ] || set -- . +found=0 +for dir in "$@"; do + for f in "$dir"/hs_err_pid*.log; do + found=1 + echo "===== $f (first 200 lines; full file in the uploaded artifact) =====" + sed -n '1,200p' "$f" + done + for f in "$dir"/target/surefire-reports/*.dumpstream "$dir"/target/surefire-reports/*.dump; do + found=1 + echo "===== $f =====" + cat "$f" + done +done +if [ "$found" = 0 ]; then + # Worded defensively on purpose (policy section 3): the step fires on EVERY job failure, and + # an ordinary assertion failure writes no crash log. Never assert an abort from a missing file. + echo "No hs_err_pid*.log and no surefire dump/dumpstream was written (looked in: $*)." + echo + echo "For an ordinary test failure that is EXPECTED, not a finding: this step runs on" + echo "any job failure, and an assertion failure, a timeout or a compile error writes no" + echo "crash log. Read the surefire output above for the real cause." + echo + echo "It points at a JVM-level abort only if the log ALSO shows a fork ending abnormally" + echo "-- 'The forked VM terminated without properly saying goodbye', or an exit with no" + echo "test results. In that case the abort bypassed the JVM error handler (a native" + echo "exit()/terminate() rather than a raised signal), which is why no file was written." +fi +exit 0 diff --git a/.github/release-gate-exemptions.txt b/.github/release-gate-exemptions.txt new file mode 100644 index 00000000..21aa0d9b --- /dev/null +++ b/.github/release-gate-exemptions.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: Apache-2.0 +# +# Jobs of publish.yml that do not gate both publish jobs, each with the reason it may stay red. +# Every other job must gate both; check-release-gate.py (shared-files job) fails on a job that does +# not, and on a line here that names no job or a job that gates both after all. +# Format: : +publish-snapshot: a publish job itself +publish-release: a publish job itself +check-snapshot: publish-snapshot's own precondition (the version is a SNAPSHOT) +check-tag: publish-release's own precondition (the version matches the tag) +github-snapshot: runs after publishing; attaches the assets to the rolling snapshot release +github-release: runs after publishing; attaches the signed assets to the tag release +verify-signing-key: red by design where the secret is withheld (forks, other contributors' branches); the publish jobs sign with the same key and fail on it themselves +verify-signing-key-gradle: same as verify-signing-key, for the Gradle/BouncyCastle signing path diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 new file mode 100644 index 00000000..b90e0ea9 --- /dev/null +++ b/.github/shared-files.sha256 @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: Apache-2.0 +# +# The files this repository keeps BYTE-IDENTICAL with java-llama.cpp, BitcoinAddressFinder, +# srcmorph and streambuffer (each lists the ones it carries). Checked by the `shared-files` job of +# .github/workflows/publish.yml (.github/check-shared-files.py, see .github/buildcheck/sharedfiles.py): +# a copy changed here alone fails it, a copy another repository lists with a different hash warns. +# To change a shared file: change it in every repository listing it, then run +# `python3 .github/check-shared-files.py --write` in each. `sha256sum -c` reads this file too. +82171de97c6621b4c1f3e9cd66afd49990f6d31331857423b37f2b9a2f35fbc0 .github/buildcheck/__init__.py +3d1a9f2e93709941261a9655bc19bf50d5becefd8f6a6c2f5564096a757996bb .github/buildcheck/workflow.py +5a70eee1bb8af180edfd2a4bae61be61fac15b409f5df6a8bb9869425d7fc381 .github/buildcheck/releasegate.py +86545e924c95b69a16f4b9f759a38779b441aa04d8b635a52eb029a196e39a11 .github/buildcheck/sharedfiles.py +0059120aab539c6ce8055675ae0d3b040097f80320051e40385072f68845ddb0 .github/buildcheck/tests/__init__.py +561efd3f26d7728e475c8ce83c10b4761a6703b585b7b2a7ab052df10f73166c .github/buildcheck/tests/helpers.py +4985f25fa177bd79a79335742adfd0da0431e101124b4e7e214906b56ae903de .github/buildcheck/tests/test_workflow.py +cbe504ad081ba31ec16e11c11f879c64179ca7741bf0b1d1318a7fc969ed67dd .github/buildcheck/tests/test_releasegate.py +7cbef54c09c6612503da7bd3733b4ab51c6f34666ef062885a7586e075a907de .github/buildcheck/tests/test_sharedfiles.py +7fe208dd33184b4543328f97b339c95e23c408b120e6bfa6095de4182aeb9a19 .github/check-release-gate.py +1604ccdea83998814a3c1f98f746272222039867b0c7d692355e61495c1c648b .github/check-shared-files.py +7da53fdb9537ccf7d3f3740be31f295bacb914fe186304b9ed8a65fd3d3d413e .github/print-crash-logs.sh +c8c02dc39aeacc154fad74071443c6513666aa94ca9f9c5fab49c5f39dea7ee6 .github/verify-signing-key.sh +e0b2a4f331eae326b61c71f78d6272d06d99140a85dedb7710f93402de862466 .github/verify-bytecode-version.sh +ab45f5c102b47dd16c325d4d9c283d158ba90c05f484eac45b2767885c4462f9 .github/signing-selftest/build.gradle.kts +9b2ea5b5ff8d48607e26e4e211ad6d496f7660e71c84e42caaa82b84f7001710 .github/signing-selftest/settings.gradle.kts +7232b092d3ba49b97bee7b539aaf6ee4c698e86bd3d4dd256e8ae2f85f653ee9 .github/ISSUE_TEMPLATE/bug_report.md +0f08122e597f93dbbdc9c80e88984b4bf4738951d5902813df3d4640cdb11bac .github/ISSUE_TEMPLATE/feature_request.md +ebfcc0adf59f5858bbe4dc077c906304a197f72a55256f0d5aac669bee5e871f .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/verify-bytecode-version.sh b/.github/verify-bytecode-version.sh index a7a39546..384f3c96 100755 --- a/.github/verify-bytecode-version.sh +++ b/.github/verify-bytecode-version.sh @@ -5,8 +5,9 @@ # SPDX-License-Identifier: MIT OR Apache-2.0 # Cross-repo shared script — kept BYTE-IDENTICAL in java-llama.cpp, srcmorph, -# BitcoinAddressFinder and streambuffer (sync any edit to all four, and to the checksum table in -# workspace/crossrepostatus.md). Fails when a built jar contains a class file newer than the Java +# BitcoinAddressFinder and streambuffer (listed in each repo's .github/shared-files.sha256, which +# the `shared-files` job checks: sync any edit to all four, then `check-shared-files.py --write`). +# Fails when a built jar contains a class file newer than the Java # release the artifact claims to support. # # Why: `maven.compiler.release` governs only the code WE compile. A dependency compiled for a newer diff --git a/.github/verify-signing-key.sh b/.github/verify-signing-key.sh new file mode 100755 index 00000000..8940490b --- /dev/null +++ b/.github/verify-signing-key.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 + +# Cross-repo shared script -- kept BYTE-IDENTICAL in java-llama.cpp, BitcoinAddressFinder, +# srcmorph and streambuffer (listed in each repo's .github/shared-files.sha256). The body of the +# `verify-signing-key` job: reproduces what maven-gpg-plugin does at deploy time, so a bad or +# expired key or a wrong passphrase is caught in seconds instead of failing the publish stage. +# +# Input: GPG_PRIVATE_KEY (armored secret key) and GPG_PASSPHRASE in the environment. +# +# SECURITY: prints no secret material. The key is imported into an ephemeral keyring via stdin; +# only PUBLIC key metadata is printed (key id, fingerprint, owner UID, algorithm, created/expiry, +# all of which live on public keyservers); the passphrase is validated by producing and verifying +# a throwaway signature, reaches gpg on fd 3 only (never argv, never a log line), and is +# additionally `::add-mask::`ed. `set -x` must never be enabled here. +set -euo pipefail # NOTE: deliberately NO `set -x` — it would echo the passphrase. + +if [ -z "${GPG_PRIVATE_KEY:-}" ]; then + echo "::error::GPG_PRIVATE_KEY is empty for this run. Either the secret is not set, or it is scoped to a different environment/branch than 'maven-central' on this ref. Nothing to verify." + exit 1 +fi +# Defensive: even though we never print it, register the passphrase as a +# masked value so any accidental echo downstream is redacted. +if [ -n "${GPG_PASSPHRASE:-}" ]; then echo "::add-mask::${GPG_PASSPHRASE}"; fi + +echo "gpg: $(gpg --version | head -n1)" + +# Ephemeral, private keyring; removed on exit. +export GNUPGHOME="$(mktemp -d)" +chmod 700 "$GNUPGHOME" +cleanup() { gpgconf --kill gpg-agent >/dev/null 2>&1 || true; rm -rf "$GNUPGHOME"; } +trap cleanup EXIT + +echo "== Import private key into an ephemeral keyring (key via stdin, never argv) ==" +printf '%s\n' "$GPG_PRIVATE_KEY" | gpg --batch --import + +COLONS="$(gpg --list-secret-keys --with-colons --fixed-list-mode)" +SECCOUNT="$(printf '%s\n' "$COLONS" | awk -F: '$1=="sec"{n++} END{print n+0}')" +echo "Secret keys imported: $SECCOUNT" +if [ "$SECCOUNT" -lt 1 ]; then + echo "::error::No secret key was imported — GPG_PRIVATE_KEY is not a valid armored secret key (check that the secret contains the full -----BEGIN PGP PRIVATE KEY BLOCK----- with intact newlines)." + exit 1 +fi + +KEYID="$(printf '%s\n' "$COLONS" | awk -F: '$1=="sec"{print $5; exit}')" +ALGO="$(printf '%s\n' "$COLONS" | awk -F: '$1=="sec"{print $4; exit}')" +CREATED="$(printf '%s\n' "$COLONS"| awk -F: '$1=="sec"{print $6; exit}')" +EXPIRES="$(printf '%s\n' "$COLONS"| awk -F: '$1=="sec"{print $7; exit}')" +FPR="$(printf '%s\n' "$COLONS" | awk -F: '$1=="fpr"{print $10; exit}')" + +echo "== PUBLIC key metadata ==" +echo " Key ID (long): $KEYID" +echo " Fingerprint: $FPR" +echo " Pubkey algo id: $ALGO" +echo " Created (UTC): $(date -u -d "@$CREATED" 2>/dev/null || echo "$CREATED")" +echo " Owner UID(s):" +printf '%s\n' "$COLONS" | awk -F: '$1=="uid"{print " - " $10}' + +# --- Expiration gate --- +NOW="$(date -u +%s)" +if [ -n "$EXPIRES" ]; then + echo " Expires (UTC): $(date -u -d "@$EXPIRES" 2>/dev/null || echo "$EXPIRES")" + if [ "$EXPIRES" -le "$NOW" ]; then + echo "::error::Signing key is EXPIRED — Maven Central will reject its signatures. Extend the key's expiry and update the GPG_PRIVATE_KEY secret." + exit 1 + fi + echo " Days to expiry: $(( (EXPIRES - NOW) / 86400 ))" + if [ "$(( (EXPIRES - NOW) / 86400 ))" -lt 30 ]; then + echo "::warning::Signing key expires in under 30 days — plan to rotate it." + fi +else + echo " Expires (UTC): never" +fi + +# --- Signing-capability gate --- +if printf '%s\n' "$COLONS" | awk -F: '($1=="sec"||$1=="ssb"){print $12}' | grep -q 's'; then + echo " Signing capability: present" +else + echo "::error::No signing-capable (sub)key found — this key cannot produce release signatures." + exit 1 +fi + +# --- Passphrase unlock + sign + verify roundtrip (the exact failure mode) --- +# Passphrase on fd 3 only. Payload is a throwaway nonce; only the signature's +# validity (exit codes) matters — no secret is ever emitted. +echo "== Passphrase unlock + detached-sign + verify self-test ==" +WORK="$(mktemp -d)" +printf '%s' "ai-index signing-selftest" > "$WORK/payload.txt" +gpg --batch --yes --pinentry-mode loopback --passphrase-fd 3 \ + --local-user "$KEYID" \ + --detach-sign --armor --output "$WORK/payload.txt.asc" "$WORK/payload.txt" \ + 3<<<"${GPG_PASSPHRASE:-}" +echo " Signature produced: $(wc -c < "$WORK/payload.txt.asc") armored bytes" +gpg --batch --verify "$WORK/payload.txt.asc" "$WORK/payload.txt" +rm -rf "$WORK" + +echo "RESULT: OK — key imports, is not expired, is signing-capable, and the passphrase successfully unlocked it to produce a VALID signature. maven-gpg-plugin will be able to sign with this key/passphrase." diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9fa21e0f..94800191 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -48,6 +48,9 @@ concurrency: permissions: contents: read +env: + JAVA_VERSION: '21' + jobs: # --------------------------------------------------------------------------- # Start gate — single cancellable abort window before the pipeline starts. @@ -61,6 +64,28 @@ jobs: steps: - run: echo "Start gate elapsed — proceeding with pipeline." + # --------------------------------------------------------------------------- + # Files kept byte-identical with the sibling repositories (java-llama.cpp, + # BitcoinAddressFinder, srcmorph, streambuffer), listed in .github/shared-files.sha256. + # A copy changed here alone fails; one the siblings list with another hash warns. + # Also runs the tests of the shared build checks and the release-gate check. This + # job is itself identical in all four repositories. + # --------------------------------------------------------------------------- + shared-files: + name: Shared files + build checks + needs: startgate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Shared files match .github/shared-files.sha256 (siblings compared, warnings only) + run: python3 .github/check-shared-files.py + - name: Build-check tests + run: python3 -m unittest discover -s .github/buildcheck/tests -t .github + - name: Release gate (every job gates both publish jobs, or release-gate-exemptions.txt says why) + run: python3 .github/check-release-gate.py + # --------------------------------------------------------------------------- # GPG signing-key preflight (standalone, no `needs:` — runs in parallel at the # very start on every trigger). Reproduces what maven-gpg-plugin does at deploy @@ -85,94 +110,15 @@ jobs: runs-on: ubuntu-latest environment: maven-central steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false - name: Import key + run sign/verify self-test (prints only PUBLIC metadata) - shell: bash env: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - run: | - set -euo pipefail # NOTE: deliberately NO `set -x` — it would echo the passphrase. - - if [ -z "${GPG_PRIVATE_KEY:-}" ]; then - echo "::error::GPG_PRIVATE_KEY is empty for this run. Either the secret is not set, or it is scoped to a different environment/branch than 'maven-central' on this ref. Nothing to verify." - exit 1 - fi - # Defensive: even though we never print it, register the passphrase as a - # masked value so any accidental echo downstream is redacted. - if [ -n "${GPG_PASSPHRASE:-}" ]; then echo "::add-mask::${GPG_PASSPHRASE}"; fi - - echo "gpg: $(gpg --version | head -n1)" - - # Ephemeral, private keyring; removed on exit. - export GNUPGHOME="$(mktemp -d)" - chmod 700 "$GNUPGHOME" - cleanup() { gpgconf --kill gpg-agent >/dev/null 2>&1 || true; rm -rf "$GNUPGHOME"; } - trap cleanup EXIT - - echo "== Import private key into an ephemeral keyring (key via stdin, never argv) ==" - printf '%s\n' "$GPG_PRIVATE_KEY" | gpg --batch --import - - COLONS="$(gpg --list-secret-keys --with-colons --fixed-list-mode)" - SECCOUNT="$(printf '%s\n' "$COLONS" | awk -F: '$1=="sec"{n++} END{print n+0}')" - echo "Secret keys imported: $SECCOUNT" - if [ "$SECCOUNT" -lt 1 ]; then - echo "::error::No secret key was imported — GPG_PRIVATE_KEY is not a valid armored secret key (check that the secret contains the full -----BEGIN PGP PRIVATE KEY BLOCK----- with intact newlines)." - exit 1 - fi - - KEYID="$(printf '%s\n' "$COLONS" | awk -F: '$1=="sec"{print $5; exit}')" - ALGO="$(printf '%s\n' "$COLONS" | awk -F: '$1=="sec"{print $4; exit}')" - CREATED="$(printf '%s\n' "$COLONS"| awk -F: '$1=="sec"{print $6; exit}')" - EXPIRES="$(printf '%s\n' "$COLONS"| awk -F: '$1=="sec"{print $7; exit}')" - FPR="$(printf '%s\n' "$COLONS" | awk -F: '$1=="fpr"{print $10; exit}')" - - echo "== PUBLIC key metadata ==" - echo " Key ID (long): $KEYID" - echo " Fingerprint: $FPR" - echo " Pubkey algo id: $ALGO" - echo " Created (UTC): $(date -u -d "@$CREATED" 2>/dev/null || echo "$CREATED")" - echo " Owner UID(s):" - printf '%s\n' "$COLONS" | awk -F: '$1=="uid"{print " - " $10}' - - # --- Expiration gate --- - NOW="$(date -u +%s)" - if [ -n "$EXPIRES" ]; then - echo " Expires (UTC): $(date -u -d "@$EXPIRES" 2>/dev/null || echo "$EXPIRES")" - if [ "$EXPIRES" -le "$NOW" ]; then - echo "::error::Signing key is EXPIRED — Maven Central will reject its signatures. Extend the key's expiry and update the GPG_PRIVATE_KEY secret." - exit 1 - fi - echo " Days to expiry: $(( (EXPIRES - NOW) / 86400 ))" - if [ "$(( (EXPIRES - NOW) / 86400 ))" -lt 30 ]; then - echo "::warning::Signing key expires in under 30 days — plan to rotate it." - fi - else - echo " Expires (UTC): never" - fi - - # --- Signing-capability gate --- - if printf '%s\n' "$COLONS" | awk -F: '($1=="sec"||$1=="ssb"){print $12}' | grep -q 's'; then - echo " Signing capability: present" - else - echo "::error::No signing-capable (sub)key found — this key cannot produce release signatures." - exit 1 - fi - - # --- Passphrase unlock + sign + verify roundtrip (the exact failure mode) --- - # Passphrase on fd 3 only. Payload is a throwaway nonce; only the signature's - # validity (exit codes) matters — no secret is ever emitted. - echo "== Passphrase unlock + detached-sign + verify self-test ==" - WORK="$(mktemp -d)" - printf '%s' "ai-index signing-selftest" > "$WORK/payload.txt" - gpg --batch --yes --pinentry-mode loopback --passphrase-fd 3 \ - --local-user "$KEYID" \ - --detach-sign --armor --output "$WORK/payload.txt.asc" "$WORK/payload.txt" \ - 3<<<"${GPG_PASSPHRASE:-}" - echo " Signature produced: $(wc -c < "$WORK/payload.txt.asc") armored bytes" - gpg --batch --verify "$WORK/payload.txt.asc" "$WORK/payload.txt" - rm -rf "$WORK" - - echo "RESULT: OK — key imports, is not expired, is signing-capable, and the passphrase successfully unlocked it to produce a VALID signature. maven-gpg-plugin will be able to sign with this key/passphrase." + # Shared, byte-identical in all four repos; the security notes are in the script. + run: bash .github/verify-signing-key.sh # --------------------------------------------------------------------------- # GPG signing-key preflight — GRADLE / BouncyCastle path. @@ -202,7 +148,7 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-java@v6 with: - java-version: '21' + java-version: ${{ env.JAVA_VERSION }} distribution: temurin - uses: gradle/actions/setup-gradle@v6 with: @@ -243,7 +189,7 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-java@v6 with: - java-version: '21' + java-version: ${{ env.JAVA_VERSION }} distribution: temurin - name: Spotless check (fail fast on format violations) run: mvn -B --no-transfer-progress spotless:check @@ -264,7 +210,7 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-java@v6 with: - java-version: '21' + java-version: ${{ env.JAVA_VERSION }} distribution: temurin cache: maven - name: Build @@ -291,7 +237,7 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-java@v6 with: - java-version: '21' + java-version: ${{ env.JAVA_VERSION }} distribution: temurin - uses: actions/download-artifact@v8 with: { name: jars, path: jars/ } @@ -335,40 +281,11 @@ jobs: - name: Memory after tests if: always() run: free -h - # A forked test JVM that aborts leaves an hs_err_pid log and a surefire - # dumpstream -- both otherwise ONLY inside the artifact uploaded below, - # which is unreachable from anywhere that cannot fetch from Azure Blob - # (a phone, a restricted network, an agent sandbox). Echo them here so the - # aborting frame is readable from the run page itself. See - # ../workspace/policies/ci-test-diagnostics.md section 3.1. + # Echo the crash logs into the job log (workspace/policies/ci-test-diagnostics.md 3.1). - name: Print crash logs (on failure) if: failure() shell: bash - run: | - shopt -s nullglob - found=0 - for f in hs_err_pid*.log; do - found=1 - echo "===== $f (first 200 lines; full file in the uploaded artifact) =====" - sed -n '1,200p' "$f" - done - for f in target/surefire-reports/*.dumpstream target/surefire-reports/*.dump; do - found=1 - echo "===== $f =====" - cat "$f" - done - if [ "$found" = 0 ]; then - echo "No hs_err_pid*.log and no surefire dump/dumpstream was written." - echo - echo "For an ordinary test failure that is EXPECTED, not a finding: this step runs on" - echo "any job failure, and an assertion failure, a timeout or a compile error writes no" - echo "crash log. Read the surefire output above for the real cause." - echo - echo "It points at a JVM-level abort only if the log ALSO shows a fork ending abnormally" - echo "-- 'The forked VM terminated without properly saying goodbye', or an exit with no" - echo "test results. In that case the abort bypassed the JVM error handler (a native" - echo "exit()/terminate() rather than a raised signal), which is why no file was written." - fi + run: bash .github/print-crash-logs.sh . - name: Upload crash & surefire dumps if: failure() uses: actions/upload-artifact@v7 @@ -390,7 +307,7 @@ jobs: steps: - uses: actions/checkout@v7 - uses: actions/setup-java@v6 - with: { java-version: '21', distribution: temurin, cache: maven } + with: { java-version: '${{ env.JAVA_VERSION }}', distribution: temurin, cache: maven } - name: Test under vmlens run: mvn --batch-mode --no-transfer-progress -Pvmlens test - uses: actions/upload-artifact@v7 @@ -409,7 +326,7 @@ jobs: steps: - uses: actions/checkout@v7 - uses: actions/setup-java@v6 - with: { java-version: '21', distribution: temurin, cache: maven } + with: { java-version: '${{ env.JAVA_VERSION }}', distribution: temurin, cache: maven } - uses: actions/download-artifact@v8 with: { name: jacoco-report, path: target/site/jacoco/ } continue-on-error: true @@ -480,7 +397,7 @@ jobs: publish-snapshot: name: Publish Snapshot to Central - needs: [check-snapshot, code-style, smoke-jar] + needs: [check-snapshot, code-style, smoke-jar, shared-files, vmlens] if: needs.check-snapshot.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central @@ -488,7 +405,7 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-java@v6 with: - java-version: '21' + java-version: ${{ env.JAVA_VERSION }} distribution: temurin cache: maven server-id: central @@ -597,7 +514,7 @@ jobs: publish-release: name: Publish Release to Central - needs: [check-tag, code-style, smoke-jar] + needs: [check-tag, code-style, smoke-jar, shared-files, vmlens] if: needs.check-tag.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central @@ -607,7 +524,7 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-java@v6 with: - java-version: '21' + java-version: ${{ env.JAVA_VERSION }} distribution: temurin cache: maven server-id: central diff --git a/.gitignore b/.gitignore index b4f1f368..5de12ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ src/test/jpf/jpf-output.log # Netbeans files nbactions.xml + +# Python bytecode of the .github/buildcheck checks +__pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 063938d7..59543652 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Cancellable pipeline start-gate via a `startgate` GitHub Environment with configurable wait timer. ### Changed +- **CI: shared files and the release gate are checked.** The files kept byte-identical with the sibling + repositories are listed with their SHA-256 in `.github/shared-files.sha256`; a new `shared-files` job + fails on a copy changed here alone and warns on a sibling's differing copy. The same job runs the + shared build-check library's tests and `check-release-gate.py`: every job must gate both publish + jobs unless `.github/release-gate-exemptions.txt` says why (`vmlens` now gates). The crash-log step + and the signing-key preflight are shared scripts (`print-crash-logs.sh`, `verify-signing-key.sh`) + instead of copies pasted into the workflow. - Build and test tooling bumped to latest stable in step with the sibling repos: spotless 3.10.2 → 3.10.3, palantir-java-format 2.98.0 → 2.99.0, NullAway 0.14.1 → 0.14.2, archunit-junit5 1.5.0 → 1.5.1. - Build plugins bumped to latest stable: `com.diffplug.spotless:spotless-maven-plugin` 3.9.0 → 3.10.0, diff --git a/CLAUDE.md b/CLAUDE.md index 7ec3b176..6df385db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -209,7 +209,7 @@ its deliverable is the plain library jar, not a fat jar — but it runs the same dependency cannot change that quietly. **The gate: `.github/verify-bytecode-version.sh`.** Kept **byte-identical** across java-llama.cpp / -BitcoinAddressFinder / streambuffer / srcmorph (checksum table in `workspace/crossrepostatus.md`). +BitcoinAddressFinder / streambuffer / srcmorph (listed in `.github/shared-files.sha256`, checked by the `shared-files` job). It opens every `.class` in every jar it is given and fails on any whose class-file major version exceeds `--max-major`: @@ -240,6 +240,20 @@ backstop — added because it previously had none. Convention + the `excludedSco enforcer default gotcha are in [`../workspace/policies/dependency-convergence-pinning.md`](../workspace/policies/dependency-convergence-pinning.md). +## Shared files and the release gate (`shared-files` job) + +Files kept byte-identical with java-llama.cpp, BitcoinAddressFinder, srcmorph and streambuffer are +listed with their SHA-256 in **`.github/shared-files.sha256`** — the reference for what must stay +equal. The `shared-files` job of `publish.yml` (identical in all four repositories, gating both +publish jobs) fails when a listed file changed here alone and warns when another repository's +default branch lists it with a different hash. To change a shared file, change every copy, then run +`python3 .github/check-shared-files.py --write` in each repository. The shared build-check library +(`.github/buildcheck/`, stdlib-only Python with unit tests: `python3 -m unittest discover -s +.github/buildcheck/tests -t .github`) also runs **`check-release-gate.py`**: every job must gate both +publish jobs unless `.github/release-gate-exemptions.txt` names it with a reason. Details and the +reasoning (copies with a checksum rather than a shared actions repository): +[`../workspace/crossrepostatus.md`](../workspace/crossrepostatus.md), "Cross-repo byte-identical files". + ## Open TODOs Open TODOs for this repo live in [`TODO.md`](TODO.md). Cross-repo status From 3bbf956a505172947031190f1fe912c82eccc447 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 06:34:01 +0000 Subject: [PATCH 2/7] CI: shared-files checks the workflow jobs kept identical across the repositories An entry .github/workflows/publish.yml# in .github/shared-files.sha256 hashes one job of the workflow (its header and body, not the comment lines before the next job). startgate, shared-files, verify-signing-key, check-snapshot and check-tag are identical in all four publish.yml files, and verify-signing-key-gradle, github-snapshot and github-release in the three Maven-only ones; they were identical by convention only and are now checked like files, without moving them into a reusable workflow. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AytmJF9faEiQEVt6eetQS2 --- .github/buildcheck/sharedfiles.py | 50 +++++++++++++--- .github/buildcheck/tests/test_sharedfiles.py | 61 +++++++++++++++++++- .github/shared-files.sha256 | 16 ++++- CHANGELOG.md | 4 ++ CLAUDE.md | 6 +- 5 files changed, 122 insertions(+), 15 deletions(-) diff --git a/.github/buildcheck/sharedfiles.py b/.github/buildcheck/sharedfiles.py index 1c771ee4..5caf6cbe 100644 --- a/.github/buildcheck/sharedfiles.py +++ b/.github/buildcheck/sharedfiles.py @@ -5,7 +5,11 @@ streambuffer, checked by every repository's own `shared-files` job. Each repository lists the shared files it carries, with their SHA-256, in -.github/shared-files.sha256 -- `sha256sum` format, so `sha256sum -c` reads it as well. The job +.github/shared-files.sha256 -- `sha256sum` format. An entry can also name ONE JOB of a workflow, +`.github/workflows/publish.yml#startgate`: jobs such as `startgate` or `check-tag` are kept identical +inside four otherwise different workflows, and a job entry hashes just that job's text (its header +and body, without the comment lines between it and the next job, which belong to the next one). +`sha256sum -c` reads the file entries only. The job * fails when a listed file is missing or its content no longer matches its line: a shared file was edited here alone. Edit it in every repository that lists it, then update each manifest (`check-shared-files.py --write` rewrites the hashes of this repository's lines); @@ -23,6 +27,8 @@ import sys import urllib.request +from . import workflow + REPOS = ("java-llama.cpp", "BitcoinAddressFinder", "srcmorph", "streambuffer") OWNER = "bernardladenthin" MANIFEST = ".github/shared-files.sha256" @@ -46,19 +52,39 @@ def parse(text): return entries -def sha256(path): - with open(path, "rb") as f: - return hashlib.sha256(f.read()).hexdigest() +def content(root, entry): + """The bytes an entry stands for: a file, or for `#` that one job's text. + None when the file or the job does not exist.""" + path, _, job = entry.partition("#") + full = os.path.join(root, path) + if not os.path.isfile(full): + return None + with open(full, "rb") as f: + data = f.read() + if not job: + return data + jobs = workflow.parse(data.decode("utf-8")) + if job not in jobs: + return None + lines = jobs[job].lines + while lines and (not lines[-1].strip() or lines[-1].startswith(" #")): + lines = lines[:-1] + return ("\n".join(lines) + "\n").encode("utf-8") + + +def sha256(root, entry): + data = content(root, entry) + return None if data is None else hashlib.sha256(data).hexdigest() def verify(root, entries): - """Failures: every listed file exists and still has its listed hash.""" + """Failures: every listed file (or job) exists and still has its listed hash.""" failures = [] for path, digest in entries.items(): - full = os.path.join(root, path) - if not os.path.isfile(full): + actual = sha256(root, path) + if actual is None: failures.append(f"{path} is listed in {MANIFEST} but does not exist") - elif sha256(full) != digest: + elif actual != digest: failures.append(f"{path} differs from its line in {MANIFEST}: a shared file was changed in " f"this repository alone -- change every copy, then update every manifest") return failures @@ -69,7 +95,13 @@ def rewrite(text, root): out = [] for line in text.splitlines(): match = LINE.match(line) - out.append(f"{sha256(os.path.join(root, match.group(2)))} {match.group(2)}" if match else line) + if not match: + out.append(line) + continue + digest = sha256(root, match.group(2)) + if digest is None: + raise ValueError(f"{match.group(2)} is listed in {MANIFEST} but does not exist") + out.append(f"{digest} {match.group(2)}") return "\n".join(out) + "\n" diff --git a/.github/buildcheck/tests/test_sharedfiles.py b/.github/buildcheck/tests/test_sharedfiles.py index e78d610a..bf09e9c4 100644 --- a/.github/buildcheck/tests/test_sharedfiles.py +++ b/.github/buildcheck/tests/test_sharedfiles.py @@ -81,6 +81,64 @@ def test_unknown_arguments(self): self.assertEqual(self.run_main("--nope")[0], 2) +WORKFLOW = """name: t +on: + push: +jobs: + first: + runs-on: ubuntu-latest + steps: + - run: echo one + + # a comment about the second job belongs to the second job + second: + needs: first + runs-on: ubuntu-latest + steps: + - run: echo two +""" +FIRST = " first:\n runs-on: ubuntu-latest\n steps:\n - run: echo one\n" + + +class JobEntryTest(unittest.TestCase): + + def setUp(self): + self.root = tempfile.mkdtemp() + os.makedirs(os.path.join(self.root, ".github", "workflows")) + self.write(WORKFLOW) + + def write(self, text): + with open(os.path.join(self.root, ".github/workflows/w.yml"), "w", encoding="utf-8") as f: + f.write(text) + + def test_a_job_entry_hashes_the_job_alone(self): + self.assertEqual(sharedfiles.content(self.root, ".github/workflows/w.yml#first"), FIRST.encode()) + self.assertEqual(sharedfiles.sha256(self.root, ".github/workflows/w.yml#first"), + hashlib.sha256(FIRST.encode()).hexdigest()) + + def test_the_comment_before_the_next_job_is_not_part_of_the_job(self): + self.write(WORKFLOW.replace("belongs to the second job", "was reworded")) + self.assertEqual(sharedfiles.content(self.root, ".github/workflows/w.yml#first"), FIRST.encode()) + + def test_a_change_inside_the_job_is_seen_and_one_elsewhere_is_not(self): + entry = ".github/workflows/w.yml#first" + entries = {entry: sharedfiles.sha256(self.root, entry)} + self.write(WORKFLOW.replace("echo two", "echo three")) + self.assertEqual(sharedfiles.verify(self.root, entries), []) + self.write(WORKFLOW.replace("echo one", "echo changed")) + self.assertIn("differs from its line", sharedfiles.verify(self.root, entries)[0]) + + def test_a_missing_job_or_workflow_is_reported(self): + entries = {".github/workflows/w.yml#gone": A, ".github/workflows/x.yml#first": A} + failures = sharedfiles.verify(self.root, entries) + self.assertEqual(len(failures), 2) + self.assertTrue(all("does not exist" in f for f in failures), failures) + + def test_write_refuses_an_entry_that_does_not_exist(self): + with self.assertRaises(ValueError): + sharedfiles.rewrite(f"{A} .github/workflows/w.yml#gone\n", self.root) + + class CompareTest(unittest.TestCase): def test_matches_by_path_then_by_a_unique_file_name(self): @@ -104,7 +162,8 @@ def test_the_shared_build_checks_list_themselves(self): with open(os.path.join(REPO, sharedfiles.MANIFEST), encoding="utf-8") as f: entries = sharedfiles.parse(f.read()) for path in (".github/buildcheck/sharedfiles.py", ".github/check-shared-files.py", - ".github/buildcheck/tests/test_sharedfiles.py"): + ".github/buildcheck/tests/test_sharedfiles.py", + ".github/workflows/publish.yml#shared-files"): self.assertIn(path, entries) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index b90e0ea9..0bbab40b 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -7,16 +7,18 @@ # .github/workflows/publish.yml (.github/check-shared-files.py, see .github/buildcheck/sharedfiles.py): # a copy changed here alone fails it, a copy another repository lists with a different hash warns. # To change a shared file: change it in every repository listing it, then run -# `python3 .github/check-shared-files.py --write` in each. `sha256sum -c` reads this file too. +# `python3 .github/check-shared-files.py --write` in each. An entry `#` stands for one +# job of a workflow (kept identical inside otherwise different workflows); `sha256sum -c` reads the +# file entries only. 82171de97c6621b4c1f3e9cd66afd49990f6d31331857423b37f2b9a2f35fbc0 .github/buildcheck/__init__.py 3d1a9f2e93709941261a9655bc19bf50d5becefd8f6a6c2f5564096a757996bb .github/buildcheck/workflow.py 5a70eee1bb8af180edfd2a4bae61be61fac15b409f5df6a8bb9869425d7fc381 .github/buildcheck/releasegate.py -86545e924c95b69a16f4b9f759a38779b441aa04d8b635a52eb029a196e39a11 .github/buildcheck/sharedfiles.py +95285b6e9fdd30a71e97aff236e4a23d42101bc182955cae7ca1e4993939321a .github/buildcheck/sharedfiles.py 0059120aab539c6ce8055675ae0d3b040097f80320051e40385072f68845ddb0 .github/buildcheck/tests/__init__.py 561efd3f26d7728e475c8ce83c10b4761a6703b585b7b2a7ab052df10f73166c .github/buildcheck/tests/helpers.py 4985f25fa177bd79a79335742adfd0da0431e101124b4e7e214906b56ae903de .github/buildcheck/tests/test_workflow.py cbe504ad081ba31ec16e11c11f879c64179ca7741bf0b1d1318a7fc969ed67dd .github/buildcheck/tests/test_releasegate.py -7cbef54c09c6612503da7bd3733b4ab51c6f34666ef062885a7586e075a907de .github/buildcheck/tests/test_sharedfiles.py +f040b63c801e1d8b1b27b0fe8ae733d6fb5cf1b166e79470a6c3e3282a1bcb50 .github/buildcheck/tests/test_sharedfiles.py 7fe208dd33184b4543328f97b339c95e23c408b120e6bfa6095de4182aeb9a19 .github/check-release-gate.py 1604ccdea83998814a3c1f98f746272222039867b0c7d692355e61495c1c648b .github/check-shared-files.py 7da53fdb9537ccf7d3f3740be31f295bacb914fe186304b9ed8a65fd3d3d413e .github/print-crash-logs.sh @@ -27,3 +29,11 @@ ab45f5c102b47dd16c325d4d9c283d158ba90c05f484eac45b2767885c4462f9 .github/signin 7232b092d3ba49b97bee7b539aaf6ee4c698e86bd3d4dd256e8ae2f85f653ee9 .github/ISSUE_TEMPLATE/bug_report.md 0f08122e597f93dbbdc9c80e88984b4bf4738951d5902813df3d4640cdb11bac .github/ISSUE_TEMPLATE/feature_request.md ebfcc0adf59f5858bbe4dc077c906304a197f72a55256f0d5aac669bee5e871f .github/PULL_REQUEST_TEMPLATE.md +f26d05f25a154b84d8f281bb92857811749e2225cdbc167362de930c48b532d1 .github/workflows/publish.yml#startgate +14d4f71b0f60f64913413fbabbafd786672215f5baa5dc2f97c0877565b373a2 .github/workflows/publish.yml#shared-files +fcb6cd0e2a71205b918f545f2680d378c790b774b9852129c5f9c7f3f8eb9860 .github/workflows/publish.yml#verify-signing-key +09230b04c5f14bdda50a0c862152ff197a656cbc3fd0d110899c5f8150f8aa9a .github/workflows/publish.yml#check-snapshot +51e987d59efd9887b43e345edc0e0f3db21686879e3ca6ea9f0fdbc493d37121 .github/workflows/publish.yml#check-tag +3d028534dc2369e45d48272a71708c0c327eca998bec0dc4a80fe9c5a5a81e99 .github/workflows/publish.yml#verify-signing-key-gradle +de7ae553e6ac327094bcedf3acacd8b0c9e25607f9556c99419abf1dde405493 .github/workflows/publish.yml#github-snapshot +4232da9e0a0742e21b78c971642d99805b7365a025771f48fa26118849500ee9 .github/workflows/publish.yml#github-release diff --git a/CHANGELOG.md b/CHANGELOG.md index 59543652..627f6e29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 jobs unless `.github/release-gate-exemptions.txt` says why (`vmlens` now gates). The crash-log step and the signing-key preflight are shared scripts (`print-crash-logs.sh`, `verify-signing-key.sh`) instead of copies pasted into the workflow. +- **Workflow jobs kept identical across the repositories are checked too**: a + `.github/shared-files.sha256` entry `.github/workflows/publish.yml#` hashes one job (`startgate`, + `shared-files`, `verify-signing-key`, `check-snapshot`, `check-tag`, and where present + `verify-signing-key-gradle`, `github-snapshot`, `github-release`). - Build and test tooling bumped to latest stable in step with the sibling repos: spotless 3.10.2 → 3.10.3, palantir-java-format 2.98.0 → 2.99.0, NullAway 0.14.1 → 0.14.2, archunit-junit5 1.5.0 → 1.5.1. - Build plugins bumped to latest stable: `com.diffplug.spotless:spotless-maven-plugin` 3.9.0 → 3.10.0, diff --git a/CLAUDE.md b/CLAUDE.md index 6df385db..d0028bb9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -244,8 +244,10 @@ enforcer default gotcha are in Files kept byte-identical with java-llama.cpp, BitcoinAddressFinder, srcmorph and streambuffer are listed with their SHA-256 in **`.github/shared-files.sha256`** — the reference for what must stay -equal. The `shared-files` job of `publish.yml` (identical in all four repositories, gating both -publish jobs) fails when a listed file changed here alone and warns when another repository's +equal. An entry `.github/workflows/publish.yml#` stands for one job of the workflow: the jobs kept +identical across the repositories (`startgate`, `shared-files`, `verify-signing-key`, `check-snapshot`, +`check-tag`, `verify-signing-key-gradle`, `github-snapshot`, `github-release`) are checked like files. +The `shared-files` job of `publish.yml` (gating both publish jobs) fails when a listed file changed here alone and warns when another repository's default branch lists it with a different hash. To change a shared file, change every copy, then run `python3 .github/check-shared-files.py --write` in each repository. The shared build-check library (`.github/buildcheck/`, stdlib-only Python with unit tests: `python3 -m unittest discover -s From bbff845bb3c71a3525efb855a500455d7a8d39d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 06:37:29 +0000 Subject: [PATCH 3/7] CI: warn where a Maven dependency or plugin differs from a sibling repository check-versions.py (shared build-check library, run in the shared-files job) compares every groupId:artifactId the POMs use -- plugins, dependencies, annotation-processor paths and the Spotless formatter version, ${...} resolved -- with the default branches of the three sibling repositories and warns per difference. Dependabot bumps each repository on its own; this is where the drift now shows instead of in a hand-kept table. Warnings only: a bump lands in four pull requests. The repositories' own net.ladenthin artifacts are left out. All 33 coordinates the four repositories share agree today. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AytmJF9faEiQEVt6eetQS2 --- .github/buildcheck/__init__.py | 2 +- .github/buildcheck/tests/test_versions.py | 120 +++++++++++++++++++ .github/buildcheck/versions.py | 139 ++++++++++++++++++++++ .github/check-versions.py | 16 +++ .github/shared-files.sha256 | 7 +- .github/workflows/publish.yml | 2 + CHANGELOG.md | 3 + CLAUDE.md | 5 +- 8 files changed, 290 insertions(+), 4 deletions(-) create mode 100644 .github/buildcheck/tests/test_versions.py create mode 100644 .github/buildcheck/versions.py create mode 100755 .github/check-versions.py diff --git a/.github/buildcheck/__init__.py b/.github/buildcheck/__init__.py index e49ad24b..cd57e0d9 100644 --- a/.github/buildcheck/__init__.py +++ b/.github/buildcheck/__init__.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT OR Apache-2.0 """Build checks as a library, so buildcheck/tests can test them. Standard library only. -Two kinds of module live here. workflow.py, releasegate.py and sharedfiles.py (with their tests +Two kinds of module live here. workflow.py, releasegate.py, sharedfiles.py and versions.py (with their tests and the check-*.py entry points next to this package) are kept BYTE-IDENTICAL in java-llama.cpp, BitcoinAddressFinder, srcmorph and streambuffer: each repository lists them in .github/shared-files.sha256, which its `shared-files` job checks (see sharedfiles.py). Every other diff --git a/.github/buildcheck/tests/test_versions.py b/.github/buildcheck/tests/test_versions.py new file mode 100644 index 00000000..b2f03ca0 --- /dev/null +++ b/.github/buildcheck/tests/test_versions.py @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 + +import io +import os +import unittest +from unittest import mock + +from buildcheck import versions +from buildcheck.tests.helpers import REPO + + +def pom(body, properties=""): + return (f'{properties}' + f'{body}') + + +PARENT = pom("", "6.1.3${junit.version}2.99.0") +CHILD = pom(""" + org.junit.jupiterjunit-jupiter + ${junit.version} + anested${nested} + aunresolved${nowhere} + amanaged + net.ladenthinllama5.2.0 + +maven-jar-plugin3.5.0 + com.google.errorprone + error_prone_core2.50.0 + + com.diffplug.spotlessspotless-maven-plugin3.10.3 + ${palantir} + """) + + +class CoordinatesTest(unittest.TestCase): + + def test_reads_plugins_dependencies_and_processor_paths_with_properties_resolved(self): + self.assertEqual(versions.coordinates([PARENT, CHILD]), { + "org.junit.jupiter:junit-jupiter": {"6.1.3"}, + "a:nested": {"6.1.3"}, + "org.apache.maven.plugins:maven-jar-plugin": {"3.5.0"}, + "com.google.errorprone:error_prone_core": {"2.50.0"}, + "com.diffplug.spotless:spotless-maven-plugin": {"3.10.3"}, + "com.palantir.javaformat:palantir-java-format": {"2.99.0"}, + }) + + def test_a_repository_using_two_versions_of_one_coordinate_keeps_both(self): + other = pom('org.junit.jupiter' + 'junit-jupiter6.0.0') + self.assertEqual(versions.coordinates([PARENT, CHILD, other])["org.junit.jupiter:junit-jupiter"], + {"6.0.0", "6.1.3"}) + + +class CompareTest(unittest.TestCase): + + def test_only_coordinates_both_use_and_only_differences(self): + own = {"j:u": {"1"}, "s:p": {"2"}, "only:here": {"3"}} + others = {"r1": {"j:u": {"1"}, "s:p": {"9"}}, "r2": {"only:there": {"4"}}} + self.assertEqual(versions.compare(own, others), + ["s:p is 2 here and 9 in r1: bump it in every repository"]) + + +class MainTest(unittest.TestCase): + + def run_main(self, siblings): + out, err = io.StringIO(), io.StringIO() + + def fetcher(repo, path): + if repo not in siblings: + raise OSError("unreachable") + return siblings[repo] + with mock.patch.object(versions, "current_repo", return_value="streambuffer"), \ + mock.patch.object(versions, "read_local", return_value=[PARENT, CHILD]): + code = versions.main(["x"], "/nowhere", fetcher, out, err) + return code, out.getvalue(), err.getvalue() + + def test_differences_are_warnings_and_never_fail(self): + bumped = pom('org.junit.jupiter' + 'junit-jupiter6.1.4') + code, out, err = self.run_main({"srcmorph": bumped, "BitcoinAddressFinder": CHILD.replace( + "${junit.version}", "6.1.3").replace("${nested}", "6.1.3")}) + self.assertEqual(code, 0) + self.assertIn("::warning::org.junit.jupiter:junit-jupiter is 6.1.3 here and 6.1.4 in srcmorph", err) + self.assertIn("could not read java-llama.cpp's pom.xml", err) + self.assertNotIn("BitcoinAddressFinder:", err) + self.assertIn("1 version differences", out) + + def test_a_missing_pom_skips_that_file_not_the_repository(self): + out, err = io.StringIO(), io.StringIO() + bumped = pom('org.junit.jupiter' + 'junit-jupiter6.1.4') + + def fetcher(repo, path): + if repo == "srcmorph" and path == "pom.xml": + return bumped + raise OSError("404") + with mock.patch.object(versions, "current_repo", return_value="streambuffer"), \ + mock.patch.object(versions, "read_local", return_value=[PARENT, CHILD]): + self.assertEqual(versions.main(["x"], "/nowhere", fetcher, out, err), 0) + self.assertIn("could not read srcmorph's srcmorph/pom.xml", err.getvalue()) + self.assertIn("6.1.4 in srcmorph", err.getvalue()) + + def test_arguments_are_refused(self): + self.assertEqual(versions.main(["x", "--offline"], REPO, err=io.StringIO()), 2) + + +class RepositoryTest(unittest.TestCase): + + def test_this_repository_s_poms_are_listed_and_readable(self): + repo = versions.current_repo(REPO) + if repo not in versions.POMS: + self.skipTest(f"checked out as {repo}, not under its repository name") + found = versions.coordinates(versions.read_local(REPO, repo)) + self.assertTrue(found, "no versioned coordinate found -- the POM list in versions.POMS is stale") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/buildcheck/versions.py b/.github/buildcheck/versions.py new file mode 100644 index 00000000..2935a761 --- /dev/null +++ b/.github/buildcheck/versions.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""The Maven dependencies and plugins java-llama.cpp, BitcoinAddressFinder, srcmorph and streambuffer +have in common, compared across the repositories: a warning per coordinate whose version differs. + +The four repositories pin the same build tooling (JUnit, Error Prone, NullAway, Spotless, SpotBugs, +JaCoCo, the Maven plugins, ...), and Dependabot bumps each repository on its own, so they drift one +pull request at a time. This check makes the drift visible in every repository's `shared-files` +job. It compares what the POMs actually USE -- every `groupId:artifactId` of a plugin, dependency or +annotation-processor path with its version, plus the formatter version Spotless is configured with, +`${property}` references resolved -- not property +names, which differ between repositories and include values that must differ (the Java release). + +Warnings, never failures: a bump lands in four pull requests, and the first one merged would +otherwise turn the other three repositories red. The repository's own artifacts +(`net.ladenthin:*`) are left out: a sibling depends on the last RELEASE of java-llama.cpp while +java-llama.cpp itself is at the next snapshot. +""" + +import re +import sys +import urllib.request +import xml.etree.ElementTree as ET + +from .sharedfiles import OWNER, REPOS, current_repo + +NS = "{http://maven.apache.org/POM/4.0.0}" +POMS = { + "java-llama.cpp": ("pom.xml", "llama/pom.xml", "llama-langchain4j/pom.xml", "llama-kotlin/pom.xml", + "llama-platform/pom.xml", "llama-atmosphere-agent/pom.xml"), + "BitcoinAddressFinder": ("pom.xml",), + "srcmorph": ("pom.xml", "srcmorph/pom.xml", "srcmorph-cli/pom.xml", "srcmorph-maven-plugin/pom.xml"), + "streambuffer": ("pom.xml",), +} +OWN_GROUP = "net.ladenthin" +DEFAULT_PLUGIN_GROUP = "org.apache.maven.plugins" +VERSIONED = ("plugin", "dependency", "path", "annotationProcessorPath") +# Versions a plugin's names without coordinates: Spotless takes its formatter as +# . The versions plugin cannot see those, which is how palantir fell +# behind unnoticed more than once (workspace crossrepostatus, "Tool versions"). +CONFIGURED = {"palantirJavaFormat": "com.palantir.javaformat:palantir-java-format"} +RAW_URL = "https://raw.githubusercontent.com/{owner}/{repo}/HEAD/{path}" +PROPERTY = re.compile(r"\$\{([^}]+)\}") + + +def coordinates(pom_texts): + """groupId:artifactId -> set of versions used across the POMs of one repository. Properties + are looked up across all of them (a child module uses its parent's), up to a few levels deep; + a version that does not resolve to a literal is left out.""" + roots = [ET.fromstring(text) for text in pom_texts] + props = {} + for root in roots: + for element in root.findall(NS + "properties/*"): + props[element.tag[len(NS):]] = (element.text or "").strip() + + def resolve(value): + for _ in range(5): + match = PROPERTY.fullmatch(value) + if not match or match.group(1) not in props: + break + value = props[match.group(1)] + return None if "${" in value else value + + found = {} + for root in roots: + for element in root.iter(): + tag = element.tag[len(NS):] + if tag in CONFIGURED: + version = resolve((element.findtext(NS + "version") or "").strip()) + if version: + found.setdefault(CONFIGURED[tag], set()).add(version) + continue + if tag not in VERSIONED: + continue + artifact, version = element.findtext(NS + "artifactId"), element.findtext(NS + "version") + if not artifact or not version: + continue + group = (element.findtext(NS + "groupId") or DEFAULT_PLUGIN_GROUP).strip() + version = resolve(version.strip()) + if version and group != OWN_GROUP: + found.setdefault(f"{group}:{artifact.strip()}", set()).add(version) + return found + + +def compare(own, others): + """Warnings for every coordinate another repository uses in a different version.""" + warnings = [] + for repo, theirs in sorted(others.items()): + for coordinate in sorted(set(own) & set(theirs)): + if own[coordinate] != theirs[coordinate]: + warnings.append(f"{coordinate} is {', '.join(sorted(own[coordinate]))} here and " + f"{', '.join(sorted(theirs[coordinate]))} in {repo}: bump it in every repository") + return warnings + + +def fetch(repo, path): + with urllib.request.urlopen(RAW_URL.format(owner=OWNER, repo=repo, path=path), timeout=20) as response: + return response.read().decode("utf-8") + + +def read_local(root, repo): + texts = [] + for path in POMS[repo]: + with open(f"{root}/{path}", encoding="utf-8") as f: + texts.append(f.read()) + return texts + + +def main(argv, root, fetcher=fetch, out=sys.stdout, err=sys.stderr): + """check-versions.py -- compare with the siblings' default branches (warnings only)""" + if argv[1:]: + print(main.__doc__, file=err) + return 2 + repo = current_repo(root) + if repo not in POMS: + print(f"::error::{repo} is not one of {', '.join(REPOS)}", file=err) + return 2 + own = coordinates(read_local(root, repo)) + others = {} + for sibling in REPOS: + if sibling == repo: + continue + texts = [] + for path in POMS[sibling]: + try: + texts.append(fetcher(sibling, path)) + except Exception as e: # noqa: BLE001 -- a sibling that cannot be read is a warning, never a red + print(f"::warning::could not read {sibling}'s {path}: {e}", file=err) + if texts: + others[sibling] = coordinates(texts) + warnings = compare(own, others) + for warning in warnings: + print(f"::warning::{warning}", file=err) + shared = set(own).intersection(*others.values()) if others else set() + print(f"{len(own)} versioned coordinates here, {len(shared)} used by every repository read, " + f"{len(warnings)} version differences (compared with {', '.join(sorted(others)) or 'no other repository'})", + file=out) + return 0 diff --git a/.github/check-versions.py b/.github/check-versions.py new file mode 100755 index 00000000..69745035 --- /dev/null +++ b/.github/check-versions.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Warn about Maven dependencies and plugins whose version differs between this repository and its +siblings. See buildcheck/versions.py. Usage: check-versions.py +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from buildcheck import versions # noqa: E402 + +if __name__ == "__main__": + sys.exit(versions.main(sys.argv, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 0bbab40b..b70b700a 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -10,7 +10,7 @@ # `python3 .github/check-shared-files.py --write` in each. An entry `#` stands for one # job of a workflow (kept identical inside otherwise different workflows); `sha256sum -c` reads the # file entries only. -82171de97c6621b4c1f3e9cd66afd49990f6d31331857423b37f2b9a2f35fbc0 .github/buildcheck/__init__.py +e43a52dd303a363512b43b13b2b941cba16ac4f039617c6e53810dd17eb4a4ca .github/buildcheck/__init__.py 3d1a9f2e93709941261a9655bc19bf50d5becefd8f6a6c2f5564096a757996bb .github/buildcheck/workflow.py 5a70eee1bb8af180edfd2a4bae61be61fac15b409f5df6a8bb9869425d7fc381 .github/buildcheck/releasegate.py 95285b6e9fdd30a71e97aff236e4a23d42101bc182955cae7ca1e4993939321a .github/buildcheck/sharedfiles.py @@ -21,6 +21,9 @@ cbe504ad081ba31ec16e11c11f879c64179ca7741bf0b1d1318a7fc969ed67dd .github/buildc f040b63c801e1d8b1b27b0fe8ae733d6fb5cf1b166e79470a6c3e3282a1bcb50 .github/buildcheck/tests/test_sharedfiles.py 7fe208dd33184b4543328f97b339c95e23c408b120e6bfa6095de4182aeb9a19 .github/check-release-gate.py 1604ccdea83998814a3c1f98f746272222039867b0c7d692355e61495c1c648b .github/check-shared-files.py +e5f1679dc7151b2676b2f587860bbaa17e1b89b44a5a4e9703eb3499248c4a06 .github/buildcheck/versions.py +d454d33b61089b5f7bd6d439e3e5b58e01e7b01eeba6b4c2f7ed3360bbb158f0 .github/buildcheck/tests/test_versions.py +0c78f43dd181f079d35edde7cbf25bb22eac92c907845d8dc8eec78e812aa097 .github/check-versions.py 7da53fdb9537ccf7d3f3740be31f295bacb914fe186304b9ed8a65fd3d3d413e .github/print-crash-logs.sh c8c02dc39aeacc154fad74071443c6513666aa94ca9f9c5fab49c5f39dea7ee6 .github/verify-signing-key.sh e0b2a4f331eae326b61c71f78d6272d06d99140a85dedb7710f93402de862466 .github/verify-bytecode-version.sh @@ -30,7 +33,7 @@ ab45f5c102b47dd16c325d4d9c283d158ba90c05f484eac45b2767885c4462f9 .github/signin 0f08122e597f93dbbdc9c80e88984b4bf4738951d5902813df3d4640cdb11bac .github/ISSUE_TEMPLATE/feature_request.md ebfcc0adf59f5858bbe4dc077c906304a197f72a55256f0d5aac669bee5e871f .github/PULL_REQUEST_TEMPLATE.md f26d05f25a154b84d8f281bb92857811749e2225cdbc167362de930c48b532d1 .github/workflows/publish.yml#startgate -14d4f71b0f60f64913413fbabbafd786672215f5baa5dc2f97c0877565b373a2 .github/workflows/publish.yml#shared-files +068c3b1d9db88437927e5005eead1e1952fc611450dca705a213fd946e4547b4 .github/workflows/publish.yml#shared-files fcb6cd0e2a71205b918f545f2680d378c790b774b9852129c5f9c7f3f8eb9860 .github/workflows/publish.yml#verify-signing-key 09230b04c5f14bdda50a0c862152ff197a656cbc3fd0d110899c5f8150f8aa9a .github/workflows/publish.yml#check-snapshot 51e987d59efd9887b43e345edc0e0f3db21686879e3ca6ea9f0fdbc493d37121 .github/workflows/publish.yml#check-tag diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 94800191..c90002f1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -85,6 +85,8 @@ jobs: run: python3 -m unittest discover -s .github/buildcheck/tests -t .github - name: Release gate (every job gates both publish jobs, or release-gate-exemptions.txt says why) run: python3 .github/check-release-gate.py + - name: Maven versions (warns where a dependency or plugin differs from a sibling repository) + run: python3 .github/check-versions.py # --------------------------------------------------------------------------- # GPG signing-key preflight (standalone, no `needs:` — runs in parallel at the diff --git a/CHANGELOG.md b/CHANGELOG.md index 627f6e29..176d02fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `.github/shared-files.sha256` entry `.github/workflows/publish.yml#` hashes one job (`startgate`, `shared-files`, `verify-signing-key`, `check-snapshot`, `check-tag`, and where present `verify-signing-key-gradle`, `github-snapshot`, `github-release`). +- **Maven versions are compared with the sibling repositories**: `check-versions.py` (in the + `shared-files` job) warns where a dependency or plugin -- incl. annotation-processor paths and the + Spotless formatter version -- is used in another version than in a sibling's default branch. - Build and test tooling bumped to latest stable in step with the sibling repos: spotless 3.10.2 → 3.10.3, palantir-java-format 2.98.0 → 2.99.0, NullAway 0.14.1 → 0.14.2, archunit-junit5 1.5.0 → 1.5.1. - Build plugins bumped to latest stable: `com.diffplug.spotless:spotless-maven-plugin` 3.9.0 → 3.10.0, diff --git a/CLAUDE.md b/CLAUDE.md index d0028bb9..e90d0d45 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -252,7 +252,10 @@ default branch lists it with a different hash. To change a shared file, change e `python3 .github/check-shared-files.py --write` in each repository. The shared build-check library (`.github/buildcheck/`, stdlib-only Python with unit tests: `python3 -m unittest discover -s .github/buildcheck/tests -t .github`) also runs **`check-release-gate.py`**: every job must gate both -publish jobs unless `.github/release-gate-exemptions.txt` names it with a reason. Details and the +publish jobs unless `.github/release-gate-exemptions.txt` names it with a reason, and +**`check-versions.py`**, which **warns** where a Maven dependency or plugin (incl. the Spotless +formatter version) is used here in another version than in a sibling repository -- Dependabot bumps +each repository on its own, so this is where the drift shows. Details and the reasoning (copies with a checksum rather than a shared actions repository): [`../workspace/crossrepostatus.md`](../workspace/crossrepostatus.md), "Cross-repo byte-identical files". From 3e08a4fcbca28de2017ebc681a1c017a7632c234 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 07:08:51 +0000 Subject: [PATCH 4/7] CI: bash -n over every bash run script of the workflows and actions The shared-files job now parses every run: script of .github/workflows and the composite actions that runs in bash (shell decided as the runner does: step shell, job and workflow defaults, else PowerShell on a Windows runner and bash elsewhere). A broken script -- such as a lost line continuation that leaves a line starting with || -- fails within minutes instead of in the job that runs it. New shared buildcheck module runscripts.py with tests and the check-run-scripts.py CLI, listed in the shared-files manifest of all four repositories. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AytmJF9faEiQEVt6eetQS2 --- .github/buildcheck/__init__.py | 10 +- .github/buildcheck/runscripts.py | 273 ++++++++++++++++++++ .github/buildcheck/tests/test_runscripts.py | 127 +++++++++ .github/check-run-scripts.py | 16 ++ .github/shared-files.sha256 | 7 +- .github/workflows/publish.yml | 2 + CHANGELOG.md | 3 + CLAUDE.md | 4 +- 8 files changed, 434 insertions(+), 8 deletions(-) create mode 100644 .github/buildcheck/runscripts.py create mode 100644 .github/buildcheck/tests/test_runscripts.py create mode 100755 .github/check-run-scripts.py diff --git a/.github/buildcheck/__init__.py b/.github/buildcheck/__init__.py index cd57e0d9..add4f0f5 100644 --- a/.github/buildcheck/__init__.py +++ b/.github/buildcheck/__init__.py @@ -3,11 +3,11 @@ # SPDX-License-Identifier: MIT OR Apache-2.0 """Build checks as a library, so buildcheck/tests can test them. Standard library only. -Two kinds of module live here. workflow.py, releasegate.py, sharedfiles.py and versions.py (with their tests -and the check-*.py entry points next to this package) are kept BYTE-IDENTICAL in java-llama.cpp, -BitcoinAddressFinder, srcmorph and streambuffer: each repository lists them in -.github/shared-files.sha256, which its `shared-files` job checks (see sharedfiles.py). Every other -module is the repository's own. +Two kinds of module live here. workflow.py, releasegate.py, sharedfiles.py, versions.py and +runscripts.py (with their tests and the check-*.py entry points next to this package) are kept +BYTE-IDENTICAL in java-llama.cpp, BitcoinAddressFinder, srcmorph and streambuffer: each repository +lists them in .github/shared-files.sha256, which its `shared-files` job checks (see sharedfiles.py). +Every other module is the repository's own. Run the tests from the repository root: python3 -m unittest discover -s .github/buildcheck/tests -t .github diff --git a/.github/buildcheck/runscripts.py b/.github/buildcheck/runscripts.py new file mode 100644 index 00000000..ea685058 --- /dev/null +++ b/.github/buildcheck/runscripts.py @@ -0,0 +1,273 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Every `run:` script of the workflows and composite actions that runs in bash, parsed with +`bash -n` -- so a broken script fails the `shared-files` job in the first minutes of a run instead +of the job that executes it, possibly hours later or only on a release path. + +The case this exists for: a cleanup lost the trailing backslash of six continued lines, leaving a +line that starts with `||` or `-e` -- in a step only a publish run or a native build reaches. Neither +actionlint (it parses scripts only when shellcheck is installed) nor a review saw it; `bash -n` sees +every one of them. + +Which scripts are bash is decided the way the runner decides it: the step's `shell:`, else the job's +and then the workflow's `defaults.run.shell`, else the runner's default -- PowerShell on a Windows +runner, bash everywhere else. A runner chosen by an expression (a matrix, a workflow input) counts +as Windows only when the step's `if:` says so; a step without `shell:` on such a job runs on every +OS of the matrix and has to be valid bash anyway. Scripts in `sh`, `bash -el {0}` or a bash given by +path are bash; pwsh, powershell, cmd and python are skipped. + +Like workflow.py this reads the two-space layout the workflows are written in, not general YAML. +The run value is taken as YAML defines it: a literal block (`|`) line for line, a folded block +(`>`) and a plain scalar folded into one line, a single-quoted scalar unquoted. A double-quoted +scalar is skipped (its escapes are not worth a YAML parser). Every `${{ ... }}` expression becomes +a word, which is what the runner substitutes before bash sees the script. +""" + +import glob +import os +import re +import shutil +import subprocess +import sys + +RUN = re.compile(r"^(?P\s*(?:- )?)run:(?:\s+(?P.*?))?\s*$") +BLOCK_HEADER = re.compile(r"^(?P