#!/usr/bin/env python3 """Check the self-review declaration of agent-assisted pull requests. A pull request is agent-assisted when any of its commits carries an ``Assisted-by:`` trailer (or Claude Code's ``Co-Authored-By: ... ``), when its body carries an agent marker (an ``Assisted-by:`` line or Claude Code's "Generated with [Claude Code]" footer), or when the "AI-assisted" box of the pull request template is ticked. A trailer or body marker without the ticked box is only noted. An agent-assisted pull request must carry a ``## Self-review`` section in its body with a ``Reviewed:`` line naming the full 40-character lowercase SHA of the current head commit. A missing section, a missing or malformed SHA, or a SHA other than the head are the only errors. The other lines of ``SELF_REVIEW_FORMAT`` are expected from agents but optional for the check; when present but malformed they are only noted: - ``Base:`` a non-empty ref, - ``Standards: hard, judgement`` with numbers, - ``Spec: open (source: ...)`` with a number and a non-empty source. Open findings do not fail the check. Pull requests by dependabot and titles starting with ``Release`` are skipped. The check does not verify the review itself; see ``docs/adr/0001-self-declared-sha-bound-review-gate.md`` and ``docs/develop/ai.md``. Only the Python standard library is used, so CI needs no project environment. Usage:: python scripts/check_pr_self_review.py --body-file body.md --commits-file commits.jsonl \\ --head-sha --title "" --author <login> ``commits.jsonl`` holds one JSON-encoded commit message per line, e.g. from ``gh api --paginate repos/<owner>/<repo>/pulls/<n>/commits --jq '.[].commit.message | @json'``. With ``--report-file`` the check also writes a Markdown report for the pull request comment and the job summary: how to fix a failing check, or a short note that the check passes. The report starts with ``COMMENT_MARKER``, by which the workflow finds and updates its one comment. """ from __future__ import annotations import argparse import json import re import sys from dataclasses import dataclass, field from pathlib import Path SELF_REVIEW_FORMAT = """\ ## Self-review Reviewed: <full 40-character SHA of the reviewed HEAD commit> Base: <base branch or commit> Standards: <n> hard, <m> judgement Spec: <k> open (source: <#issue | .scratch/<branch>/plan.md | none>) """ COMMENT_MARKER = "<!-- ai-self-review -->" DOCS_URL = ( "https://github.com/Akkudoktor-EOS/EOS/blob/main/docs/develop/ai.md" "#self-review-and-the-pull-request-check" ) _HTML_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL) _ASSISTED_BY = re.compile(r"^Assisted-by:[ \t]*\S", re.IGNORECASE | re.MULTILINE) _ANTHROPIC_CO_AUTHOR = re.compile( r"^Co-Authored-By:.*<noreply@anthropic\.com>", re.IGNORECASE | re.MULTILINE ) # Footers that agents write into pull request bodies on their own. Add a tool's footer here # once it is known; do not guess. _AGENT_BODY_FOOTERS = (re.compile(r"Generated with \[Claude Code\]", re.IGNORECASE),) _AI_CHECKBOX = re.compile(r"^[ \t]*[-*][ \t]*\[([ xX])\][ \t]*AI-assisted\b", re.MULTILINE) _SELF_REVIEW_HEADING = re.compile(r"^##[ \t]+Self-review[ \t]*$", re.MULTILINE) _NEXT_HEADING = re.compile(r"^#{1,2}[ \t]+\S", re.MULTILINE) _REVIEWED = re.compile(r"^Reviewed:[ \t]*(\S+)[ \t]*$", re.MULTILINE) _BASE = re.compile(r"^Base:[ \t]*(\S.*?)[ \t]*$", re.MULTILINE) _STANDARDS = re.compile(r"^Standards:[ \t]*\d+[ \t]+hard,[ \t]*\d+[ \t]+judgement[ \t]*$", re.MULTILINE) _SPEC = re.compile(r"^Spec:[ \t]*\d+[ \t]+open[ \t]+\(source:[ \t]*[^)\s][^)]*\)[ \t]*$", re.MULTILINE) _OPTIONAL_LINES = ( ("Base", _BASE, "Base: <ref>"), ("Standards", _STANDARDS, "Standards: <n> hard, <m> judgement"), ("Spec", _SPEC, "Spec: <k> open (source: ...)"), ) _FULL_SHA = re.compile(r"^[0-9a-f]{40}$") _SKIPPED_AUTHORS = re.compile(r"^dependabot(\[bot\])?$", re.IGNORECASE) _SKIPPED_TITLES = re.compile(r"^Release") @dataclass class CheckResult: """Outcome of the self-review check of one pull request. Attributes: agent_assisted: Whether the pull request counts as agent-assisted. skipped_reason: Why the check was skipped, or an empty string. errors: Problems that fail the check; empty when the check passes. notes: Problems reported without failing the check. """ agent_assisted: bool = False skipped_reason: str = "" errors: list[str] = field(default_factory=list) notes: list[str] = field(default_factory=list) @property def passed(self) -> bool: """Return whether the check passed.""" return not self.errors def normalize_newlines(text: str) -> str: """Convert Windows and old Mac line endings to ``\\n``. GitHub's web editor often saves pull request bodies with ``\\r\\n``; the line patterns anchor at ``$`` and would not match a line that ends in ``\\r``. Args: text: Text with any line endings. Returns: The text with ``\\n`` line endings only. """ return text.replace("\r\n", "\n").replace("\r", "\n") def strip_html_comments(text: str) -> str: """Remove HTML comments, so template hints never count as content. Args: text: Markdown text, e.g. a pull request body. Returns: The text without ``<!-- ... -->`` comments. """ return _HTML_COMMENT.sub("", text) def has_agent_trailer(message: str) -> bool: """Return whether a commit message carries an agent trailer. Args: message: The full commit message. Returns: True for an ``Assisted-by:`` trailer or an Anthropic ``Co-Authored-By:`` trailer. """ return bool(_ASSISTED_BY.search(message) or _ANTHROPIC_CO_AUTHOR.search(message)) def has_agent_body_marker(body: str) -> bool: """Return whether a pull request body carries an agent marker. Args: body: The pull request body without HTML comments. Returns: True for an ``Assisted-by:`` line or a known agent footer such as "Generated with [Claude Code]". """ return bool(_ASSISTED_BY.search(body)) or any( footer.search(body) for footer in _AGENT_BODY_FOOTERS ) def is_ai_box_ticked(body: str) -> bool: """Return whether the "AI-assisted" box of the pull request template is ticked. Args: body: The pull request body without HTML comments. Returns: True when the box is ticked. """ match = _AI_CHECKBOX.search(body) return bool(match and match.group(1) in "xX") def extract_self_review(body: str) -> str | None: """Return the ``## Self-review`` section of a pull request body. Args: body: The pull request body without HTML comments. Returns: The section text up to the next level-one or level-two heading, or None if the body has no such section. """ heading = _SELF_REVIEW_HEADING.search(body) if heading is None: return None rest = body[heading.end() :] next_heading = _NEXT_HEADING.search(rest) return rest[: next_heading.start()] if next_heading else rest def check_self_review_section(section: str, head_sha: str) -> tuple[list[str], list[str]]: """Check the fields of a self-review section. Only the ``Reviewed:`` SHA is required. The other lines are optional; a present but malformed optional line is only noted. Args: section: The section text below the ``## Self-review`` heading. head_sha: The full SHA of the pull request head commit. Returns: The errors (missing, malformed or stale SHA) and the notes (malformed optional lines). """ errors: list[str] = [] notes: list[str] = [] reviewed = _REVIEWED.search(section) if reviewed is None: errors.append("The self-review has no 'Reviewed: <sha>' line.") elif not _FULL_SHA.match(reviewed.group(1)): errors.append( f"'Reviewed: {reviewed.group(1)}' is not a full 40-character lowercase commit SHA." ) elif reviewed.group(1) != head_sha.lower(): errors.append( f"The self-review is for {reviewed.group(1)}, but the head of the pull request is" f" {head_sha}. Rerun the self-review and update the pull request body." ) for key, pattern, expected in _OPTIONAL_LINES: present = re.search(rf"(?m)^{key}:", section) is not None if present and pattern.search(section) is None: notes.append(f"The '{key}:' line does not have the form '{expected}'.") return errors, notes def check_pull_request( body: str, commit_messages: list[str], head_sha: str, title: str, author: str ) -> CheckResult: """Check the self-review declaration of one pull request. Args: body: The pull request body (Markdown). commit_messages: The full messages of all commits of the pull request. head_sha: The full SHA of the pull request head commit. title: The pull request title. author: The login of the pull request author. Returns: The check result. """ result = CheckResult() if _SKIPPED_AUTHORS.match(author): result.skipped_reason = f"pull request by {author}" return result if _SKIPPED_TITLES.match(title): result.skipped_reason = "release pull request" return result body = strip_html_comments(normalize_newlines(body)) trailer = any(has_agent_trailer(normalize_newlines(message)) for message in commit_messages) body_marker = has_agent_body_marker(body) box_ticked = is_ai_box_ticked(body) result.agent_assisted = trailer or body_marker or box_ticked if not result.agent_assisted: return result if trailer and not box_ticked: result.notes.append( "Commits carry an agent trailer (Assisted-by: or Co-Authored-By: Claude), but the" " 'AI-assisted' box in the pull request body is not ticked." ) if body_marker and not box_ticked: result.notes.append( "The pull request body carries an agent marker (Assisted-by: or 'Generated with" " [Claude Code]'), but the 'AI-assisted' box is not ticked." ) section = extract_self_review(body) if section is None: result.errors.append("The pull request body has no '## Self-review' section.") else: errors, notes = check_self_review_section(section, head_sha) result.errors.extend(errors) result.notes.extend(notes) return result def read_commit_messages(path: Path) -> list[str]: """Read commit messages from a JSON Lines file. Args: path: File with one JSON-encoded commit message per line. Returns: The commit messages. Raises: ValueError: If a line is not a JSON string. """ messages: list[str] = [] for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): if not line.strip(): continue message = json.loads(line) if not isinstance(message, str): raise ValueError(f"{path}:{number}: expected a JSON string, got {type(message).__name__}") messages.append(message) return messages def _fenced(text: str, language: str = "") -> str: """Return text as a fenced code block that the text cannot close early. Pull request authors control parts of the error messages (e.g. the ``Reviewed:`` value). In a code block they cannot mention users or inject Markdown, as long as the fence is longer than any backtick run in the text. Args: text: The block content. language: Optional info string of the block. Returns: The fenced block. """ longest_run = max((len(run) for run in re.findall(r"`+", text)), default=0) fence = "`" * max(3, longest_run + 1) return f"{fence}{language}\n{text.rstrip()}\n{fence}" def render_report(result: CheckResult, head_sha: str) -> str: """Render the Markdown report for the pull request comment and the job summary. Args: result: The check result. head_sha: The full SHA of the pull request head commit. Returns: The report, starting with ``COMMENT_MARKER``. """ notes = "" if result.notes: listed = "\n".join(f"- {note}" for note in result.notes) notes = f"\nNotes (they do not fail the check):\n\n{_fenced(listed)}\n" if result.passed: if result.skipped_reason: detail = f"The check does not apply: {result.skipped_reason}." elif result.agent_assisted: detail = f"The self-review declaration is up to date for `{head_sha}`." else: detail = "The pull request is not agent-assisted; there is nothing to check." return f"{COMMENT_MARKER}\n### ✅ AI self-review check passed\n\n{detail}\n{notes}" errors = "\n".join(f"- {error}" for error in result.errors) return ( f"{COMMENT_MARKER}\n" "### ❌ AI self-review check failed\n\n" "This pull request counts as agent-assisted (an agent trailer in a commit, an agent marker" " in the body, or the ticked \"AI-assisted\" box), but its self-review declaration is" f" missing, incomplete or out of date for the head commit `{head_sha}`:\n\n" f"{_fenced(errors)}\n" f"{notes}\n" "**How to fix**\n\n" "1. Let the agent rerun the self-review on the head you are going to push and rewrite the" " section in `.scratch/<branch>/pr.md`. Do not just edit the SHA. If the review leads to" " fixes, commit them and review the new head.\n" "2. Tick the \"AI-assisted\" box in `pr.md`, if not done yet.\n" "3. Push the new commits, if any, and right after the push replace the body, either" " with `gh pr edit <number> --repo Akkudoktor-EOS/EOS --body-file .scratch/<branch>/pr.md`," " or by hand: open the" " \"...\" menu of the pull request description on GitHub, choose \"Edit\", replace the" " whole text with the content of `pr.md` and click \"Update comment\". Editing the body" " reruns the check, and this comment is updated.\n\n" "Repeat this before every later push.\n\n" "Expected section at the end of the body; the check requires the `Reviewed:` line, the" " other lines are expected from agents:\n\n" f"{_fenced(SELF_REVIEW_FORMAT, 'markdown')}\n\n" f"See [Self-review and the pull request check]({DOCS_URL}) in `docs/develop/ai.md`.\n" ) def main(argv: list[str] | None = None) -> int: """Run the check from the command line. Args: argv: Command line arguments; defaults to ``sys.argv[1:]``. Returns: 0 when the check passes or does not apply, 1 when it fails. """ parser = argparse.ArgumentParser( description="Check the self-review declaration of agent-assisted pull requests." ) parser.add_argument("--body-file", type=Path, required=True, help="pull request body") parser.add_argument( "--commits-file", type=Path, required=True, help="JSON Lines file of commit messages" ) parser.add_argument("--head-sha", required=True, help="full SHA of the head commit") parser.add_argument("--title", default="", help="pull request title") parser.add_argument("--author", default="", help="login of the pull request author") parser.add_argument( "--report-file", type=Path, help="write the Markdown report for the comment and summary" ) args = parser.parse_args(argv) result = check_pull_request( body=args.body_file.read_text(encoding="utf-8"), commit_messages=read_commit_messages(args.commits_file), head_sha=args.head_sha, title=args.title, author=args.author, ) if args.report_file is not None: args.report_file.write_text(render_report(result, args.head_sha), encoding="utf-8") if result.skipped_reason: print(f"Self-review check skipped: {result.skipped_reason}.") return 0 if not result.agent_assisted: print( "Not agent-assisted: no agent trailer in the commits, no agent marker in the body," " 'AI-assisted' box not ticked." ) return 0 for note in result.notes: print(f"::warning::{note}") if result.passed: print("Agent-assisted pull request with an up-to-date self-review declaration.") return 0 print("Agent-assisted pull request: the self-review declaration is incomplete.") for error in result.errors: print(f"::error::{error}") print("\nExpected section in the pull request body (see docs/develop/ai.md):\n") print(SELF_REVIEW_FORMAT) return 1 if __name__ == "__main__": sys.exit(main())