diff --git a/.github/scripts/list_new_security_advisories.py b/.github/scripts/list_new_security_advisories.py deleted file mode 100755 index 420982604..000000000 --- a/.github/scripts/list_new_security_advisories.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 -"""List repository security advisories in triage or draft not yet posted. - -Run by .github/workflows/security-advisory-notify.yml, which posts the result -with roostorg/discord-bot. Writes PAYLOAD_FILE with one repository_advisory -payload per line, holding only the GHSA ID, state, severity and link, and -updates STATE_FILE to include them. The workflow saves the state only after -posting succeeds. - -Workflow logs and caches on this repository are public, so the script never -prints anything about an advisory, and the state holds keyed hashes of GHSA -IDs, never the IDs. - -Environment: - GITHUB_REPOSITORY owner/repo whose advisories are listed - ADVISORY_READ_TOKEN token that can read unpublished advisories - STATE_KEY secret key for the state hashes (the webhook URL) - STATE_FILE JSON file of advisories already posted - PAYLOAD_FILE where to write the payloads to post - GITHUB_OUTPUT set by Actions; receives post= and changed= - -Local check with advisories from a file instead of the API (refused under -GitHub Actions): - STATE_FILE=state.json PAYLOAD_FILE=payload.jsonl \\ - python3 list_new_security_advisories.py --advisories-file advisories.json -""" - -from __future__ import annotations - -import argparse -import hashlib -import hmac -import json -import os -import re -import sys -import urllib.error -import urllib.request -from collections.abc import Iterable - -STATES = ("triage", "draft") -SEVERITIES = {"critical", "high", "medium", "low"} -GHSA_ID = re.compile(r"^GHSA(-[23456789cfghjmpqrvwx]{4}){3}$") -LINK_NEXT = re.compile(r'<([^>]+)>;\s*rel="next"') -LOCAL_KEY = "local" - - -class ListError(Exception): - """An error whose message is safe to print in a public log.""" - - -def state_digest(key: str, ghsa_id: str) -> str: - return hmac.new(key.encode(), ghsa_id.encode(), hashlib.sha256).hexdigest() - - -def load_state(path: str) -> set[str]: - try: - with open(path, encoding="utf-8") as f: - data = json.load(f) - except FileNotFoundError: - return set() - except (OSError, ValueError) as e: - raise ListError("state file is unreadable") from e - posted = data.get("posted") if isinstance(data, dict) else None - if not isinstance(posted, list): - raise ListError("state file has an unexpected format") - return {d for d in posted if isinstance(d, str)} - - -def write_file(path: str, text: str) -> None: - os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) - tmp = f"{path}.tmp" - with open(tmp, "w", encoding="utf-8") as f: - f.write(text) - os.replace(tmp, path) - - -def save_state(path: str, posted: Iterable[str]) -> None: - write_file(path, json.dumps({"version": 1, "posted": sorted(posted)})) - - -def fetch_advisories(api_url: str, repo: str, token: str) -> list[dict]: - advisories: list[dict] = [] - for state in STATES: - url: str | None = ( - f"{api_url}/repos/{repo}/security-advisories" - f"?state={state}&per_page=100&sort=created&direction=asc" - ) - while url: - request = urllib.request.Request( - url, - headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {token}", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": "security-advisory-notify", - }, - ) - try: - with urllib.request.urlopen(request, timeout=30) as response: - page = json.load(response) - link = response.headers.get("Link", "") - except urllib.error.HTTPError as e: - e.close() - raise ListError( - f"listing {state} advisories failed with HTTP {e.code}" - ) from None - except (urllib.error.URLError, TimeoutError, ValueError): - raise ListError(f"listing {state} advisories failed") from None - if not isinstance(page, list): - raise ListError(f"listing {state} advisories returned non-list") - advisories.extend(a for a in page if isinstance(a, dict)) - match = LINK_NEXT.search(link) - url = match.group(1) if match else None - return advisories - - -def payload(advisory: dict, server_url: str, repo: str) -> dict: - """The repository_advisory payload for discord-bot, with only what it posts.""" - ghsa_id = advisory["ghsa_id"] - state = advisory.get("state") - severity = advisory.get("severity") - return { - "repository_advisory": { - "ghsa_id": ghsa_id, - "state": state if state in STATES else None, - "severity": severity if severity in SEVERITIES else None, - "html_url": f"{server_url}/{repo}/security/advisories/{ghsa_id}", - } - } - - -def select_new( - advisories: list[dict], posted: set[str], key: str -) -> tuple[list[dict], set[str]]: - """Return advisories not yet posted, oldest first, and the state after - posting them. The state keeps only advisories still in triage or draft, so - it stays small.""" - current: dict[str, dict] = {} - for advisory in advisories: - ghsa_id = advisory.get("ghsa_id") - if isinstance(ghsa_id, str) and GHSA_ID.match(ghsa_id): - current.setdefault(ghsa_id, advisory) - digests = {ghsa_id: state_digest(key, ghsa_id) for ghsa_id in current} - new = sorted( - (a for i, a in current.items() if digests[i] not in posted), - key=lambda a: (str(a.get("created_at") or ""), a["ghsa_id"]), - ) - return new, set(digests.values()) - - -def write_output(**values: bool) -> None: - path = os.environ.get("GITHUB_OUTPUT") - if path: - with open(path, "a", encoding="utf-8") as f: - f.writelines( - f"{name}={'true' if value else 'false'}\n" - for name, value in values.items() - ) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - parser.add_argument( - "--advisories-file", - help="read advisories from this JSON list instead of the API (local runs only)", - ) - args = parser.parse_args(argv) - - if os.environ.get("GITHUB_ACTIONS") == "true" and args.advisories_file: - print("--advisories-file is for local runs only", file=sys.stderr) - return 2 - state_file = os.environ.get("STATE_FILE") - payload_file = os.environ.get("PAYLOAD_FILE") - if not state_file or not payload_file: - print("set STATE_FILE and PAYLOAD_FILE", file=sys.stderr) - return 2 - - repo = os.environ.get("GITHUB_REPOSITORY", "roostorg/coop") - server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com") - api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com") - key = os.environ.get("STATE_KEY", "") - token = os.environ.get("ADVISORY_READ_TOKEN", "") - if args.advisories_file: - key = key or LOCAL_KEY - elif not key or not token: - print("set STATE_KEY and ADVISORY_READ_TOKEN", file=sys.stderr) - return 2 - - try: - posted = load_state(state_file) - if args.advisories_file: - with open(args.advisories_file, encoding="utf-8") as f: - advisories = json.load(f) - else: - advisories = fetch_advisories(api_url, repo, token) - new, new_state = select_new(advisories, posted, key) - except ListError as e: - print(f"::error::{e}", file=sys.stderr) - return 1 - - write_file( - payload_file, - "".join(json.dumps(payload(a, server_url, repo)) + "\n" for a in new), - ) - changed = new_state != posted - if changed: - save_state(state_file, new_state) - write_output(post=bool(new), changed=changed) - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except Exception as e: # noqa: BLE001 - # A traceback could include advisory data. - print(f"::error::unexpected {type(e).__name__}", file=sys.stderr) - sys.exit(1) diff --git a/.github/scripts/test_list_new_security_advisories.py b/.github/scripts/test_list_new_security_advisories.py deleted file mode 100644 index bae3bfcd5..000000000 --- a/.github/scripts/test_list_new_security_advisories.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Tests for list_new_security_advisories.py. - -Run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' -""" - -from __future__ import annotations - -import contextlib -import io -import json -import os -import sys -import tempfile -import threading -import unittest -import unittest.mock as mock -from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import ClassVar -from urllib.parse import parse_qs, urlparse - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import list_new_security_advisories as lister - -SECRET_TEXT = "SECRET-DESCRIPTION-DO-NOT-LEAK" - - -def advisory(ghsa_id, state="triage", created="2026-10-01T00:00:00Z", **extra): - return { - "ghsa_id": ghsa_id, - "state": state, - "summary": SECRET_TEXT, - "severity": "high", - "created_at": created, - "description": SECRET_TEXT, - "html_url": f"https://github.com/roostorg/coop/security/advisories/{ghsa_id}", - "credits": [{"login": "reporter"}], - "vulnerabilities": [{"package": {"name": SECRET_TEXT}}], - **extra, - } - - -A1 = "GHSA-2222-3333-4444" -A2 = "GHSA-cfgh-jmpq-rvwx" -A3 = "GHSA-5678-9cfg-hjmp" - - -class FakeGitHub(BaseHTTPRequestHandler): - """Serves the advisories API, two per page.""" - - advisories: ClassVar[dict[str, list[dict]]] = {} - - def log_message(self, *args): - pass - - def do_GET(self): - url = urlparse(self.path) - query = parse_qs(url.query) - if url.path != "/repos/roostorg/coop/security-advisories": - self.send_response(404) - self.end_headers() - return - if self.headers.get("Authorization") != "Bearer test-token": - self.send_response(401) - self.end_headers() - return - items = self.advisories.get(query["state"][0], []) - start = int(query.get("after", ["0"])[0]) - self.send_response(200) - if start + 2 < len(items): - host = f"http://{self.headers['Host']}" - nxt = f"{host}{url.path}?state={query['state'][0]}&after={start + 2}" - self.send_header("Link", f'<{nxt}>; rel="next"') - self.end_headers() - self.wfile.write(json.dumps(items[start : start + 2]).encode()) - - -class ListerTest(unittest.TestCase): - def setUp(self): - FakeGitHub.advisories = {} - self.server = HTTPServer(("127.0.0.1", 0), FakeGitHub) - threading.Thread(target=self.server.serve_forever, daemon=True).start() - self.tmp = tempfile.TemporaryDirectory() - self.state_file = os.path.join(self.tmp.name, "state", "state.json") - self.payload_file = os.path.join(self.tmp.name, "state", "advisories.jsonl") - self.output_file = os.path.join(self.tmp.name, "output") - self.env = { - "GITHUB_ACTIONS": "true", - "GITHUB_REPOSITORY": "roostorg/coop", - "GITHUB_SERVER_URL": "https://github.com", - "GITHUB_API_URL": f"http://127.0.0.1:{self.server.server_port}", - "GITHUB_OUTPUT": self.output_file, - "ADVISORY_READ_TOKEN": "test-token", - "STATE_KEY": "https://discord.com/api/webhooks/1/secret", - "STATE_FILE": self.state_file, - "PAYLOAD_FILE": self.payload_file, - } - - def tearDown(self): - self.server.shutdown() - self.server.server_close() - self.tmp.cleanup() - - def run_ci(self, env=None, argv=()): - """Run main() as the workflow does; return (exit code, stdout+stderr).""" - out = io.StringIO() - with ( - mock.patch.dict(os.environ, env or self.env, clear=True), - contextlib.redirect_stdout(out), - contextlib.redirect_stderr(out), - ): - code = lister.main(list(argv)) - return code, out.getvalue() - - def outputs(self): - with open(self.output_file) as f: - lines = f.read().splitlines() - return dict(line.split("=", 1) for line in lines[-2:]) - - def payloads(self): - with open(self.payload_file) as f: - return [json.loads(line) for line in f] - - def posted_ids(self): - return [p["repository_advisory"]["ghsa_id"] for p in self.payloads()] - - def test_lists_triage_and_draft_oldest_first_and_prints_nothing(self): - FakeGitHub.advisories = { - "triage": [ - advisory(A2, created="2026-10-03T00:00:00Z"), - advisory(A1, created="2026-10-01T00:00:00Z"), - advisory(A3, created="2026-10-04T00:00:00Z"), - ], - "draft": [advisory("GHSA-wxwx-wxwx-wxwx", "draft", "2026-10-02T00:00:00Z")], - } - self.assertEqual(self.run_ci(), (0, "")) - self.assertEqual(self.posted_ids(), [A1, "GHSA-wxwx-wxwx-wxwx", A2, A3]) - self.assertEqual(self.outputs(), {"post": "true", "changed": "true"}) - - def test_lists_nothing_already_posted(self): - FakeGitHub.advisories = {"triage": [advisory(A1)]} - self.run_ci() - self.assertEqual(self.run_ci(), (0, "")) - self.assertEqual(self.payloads(), []) - self.assertEqual(self.outputs(), {"post": "false", "changed": "false"}) - - def test_payload_contains_only_id_state_severity_and_link(self): - FakeGitHub.advisories = {"triage": [advisory(A1)]} - self.run_ci() - self.assertEqual( - self.payloads(), - [ - { - "repository_advisory": { - "ghsa_id": A1, - "state": "triage", - "severity": "high", - "html_url": f"https://github.com/roostorg/coop/security/advisories/{A1}", - } - } - ], - ) - - def test_state_file_does_not_contain_ghsa_ids(self): - FakeGitHub.advisories = {"triage": [advisory(A1)]} - self.run_ci() - with open(self.state_file) as f: - state = f.read() - self.assertNotIn(A1, state) - self.assertNotIn(A1.lower(), state) - - def test_advisory_moving_from_triage_to_draft_is_not_listed_again(self): - FakeGitHub.advisories = {"triage": [advisory(A1)]} - self.run_ci() - FakeGitHub.advisories = {"draft": [advisory(A1, "draft")]} - self.run_ci() - self.assertEqual(self.payloads(), []) - - def test_state_drops_advisories_no_longer_in_triage_or_draft(self): - FakeGitHub.advisories = {"triage": [advisory(A1), advisory(A2)]} - self.run_ci() - FakeGitHub.advisories = {"triage": [advisory(A2)]} - self.run_ci() - self.assertEqual(self.outputs(), {"post": "false", "changed": "true"}) - with open(self.state_file) as f: - self.assertEqual(len(json.load(f)["posted"]), 1) - - def test_api_error_fails_without_touching_state_or_payload(self): - code, logs = self.run_ci({**self.env, "ADVISORY_READ_TOKEN": "wrong"}) - self.assertEqual(code, 1) - self.assertEqual( - logs, "::error::listing triage advisories failed with HTTP 401\n" - ) - self.assertFalse(os.path.exists(self.state_file)) - self.assertFalse(os.path.exists(self.payload_file)) - - def test_changing_the_key_lists_open_advisories_again(self): - FakeGitHub.advisories = {"triage": [advisory(A1)]} - self.run_ci() - self.run_ci({**self.env, "STATE_KEY": "https://discord.com/api/webhooks/2/x"}) - self.assertEqual(self.posted_ids(), [A1]) - - def test_ignores_malformed_ids_and_severities(self): - FakeGitHub.advisories = { - "triage": [ - advisory("not-a-ghsa"), - advisory("GHSA-aaaa-bbbb-cccc"), - advisory(A1, severity="@everyone"), - ] - } - self.run_ci() - self.assertEqual(self.posted_ids(), [A1]) - self.assertIsNone(self.payloads()[0]["repository_advisory"]["severity"]) - - def test_requires_token_and_key(self): - for missing in ("STATE_KEY", "ADVISORY_READ_TOKEN"): - env = {k: v for k, v in self.env.items() if k != missing} - code, _ = self.run_ci(env) - self.assertEqual(code, 2) - - def test_advisories_file_is_refused_in_actions(self): - code, _ = self.run_ci(argv=["--advisories-file", "x.json"]) - self.assertEqual(code, 2) - - def test_local_run_from_advisories_file(self): - fixture = os.path.join(self.tmp.name, "advisories.json") - with open(fixture, "w") as f: - json.dump([advisory(A1)], f) - env = {"STATE_FILE": self.state_file, "PAYLOAD_FILE": self.payload_file} - self.assertEqual(self.run_ci(env, ["--advisories-file", fixture]), (0, "")) - self.assertEqual(self.posted_ids(), [A1]) - with open(self.payload_file) as f: - self.assertNotIn(SECRET_TEXT, f.read()) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/workflows/security-advisory-notify.yml b/.github/workflows/security-advisory-notify.yml index 39a12ce3c..577b96adb 100644 --- a/.github/workflows/security-advisory-notify.yml +++ b/.github/workflows/security-advisory-notify.yml @@ -4,14 +4,14 @@ # the REST API. # # Secrets of the security-advisory-notify environment, which only main can -# deploy to (the job skips silently when either is missing): +# deploy to (the action skips silently when either is missing): # SECURITY_DISCORD_WEBHOOK webhook URL of the private security channel # ADVISORY_READ_TOKEN fine-grained token, this repository only, # "Repository security advisories: Read-only" # # Logs, step summaries and caches on this repository are readable by anyone, -# so nothing about an advisory is printed and the "already posted" state -# cached between runs holds HMACs of GHSA IDs keyed by the webhook URL. +# so the action prints nothing about an advisory and the "already posted" +# state cached between runs holds HMACs of GHSA IDs keyed by the webhook URL. name: Security advisory notifications @@ -35,49 +35,9 @@ jobs: environment: security-advisory-notify runs-on: ubuntu-latest timeout-minutes: 10 - env: - CONFIGURED: ${{ secrets.SECURITY_DISCORD_WEBHOOK != '' && secrets.ADVISORY_READ_TOKEN != '' }} steps: - - name: Checkout advisory script - if: env.CONFIGURED == 'true' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - sparse-checkout: .github/scripts/list_new_security_advisories.py - sparse-checkout-cone-mode: false - - - name: Restore posted-advisory state - if: env.CONFIGURED == 'true' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/security-advisory-notify/state.json - key: security-advisory-notify-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: security-advisory-notify- - - - name: List new advisories - id: list - if: env.CONFIGURED == 'true' - env: - ADVISORY_READ_TOKEN: ${{ secrets.ADVISORY_READ_TOKEN }} - STATE_KEY: ${{ secrets.SECURITY_DISCORD_WEBHOOK }} - STATE_FILE: ${{ runner.temp }}/security-advisory-notify/state.json - PAYLOAD_FILE: ${{ runner.temp }}/security-advisory-notify/advisories.jsonl - run: python3 .github/scripts/list_new_security_advisories.py - - name: Post new advisories - if: env.CONFIGURED == 'true' && steps.list.outputs.post == 'true' - uses: roostorg/discord-bot@ec1c56411627b77bdcc6e47fca9ab8d817e739ca + uses: roostorg/discord-bot/security-advisories@ec1c56411627b77bdcc6e47fca9ab8d817e739ca with: webhook-url: ${{ secrets.SECURITY_DISCORD_WEBHOOK }} - event: repository_advisory - payload-path: ${{ runner.temp }}/security-advisory-notify/advisories.jsonl - - # Only after posting succeeds, so a failed post is retried next run. If a - # post fails partway through a batch, the advisories posted before it are - # posted again on the next run: a duplicate rather than a missed alert. - - name: Save posted-advisory state - if: env.CONFIGURED == 'true' && steps.list.outputs.changed == 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/security-advisory-notify/state.json - key: security-advisory-notify-${{ github.run_id }}-${{ github.run_attempt }} + advisory-read-token: ${{ secrets.ADVISORY_READ_TOKEN }}