"""Tests for the AI agent development support in ``ai/``.""" import json import re import stat import sys import time import warnings from collections.abc import Callable from pathlib import Path from typing import Any, Protocol import git import pytest DIR_PROJECT_ROOT = Path(__file__).parent.parent if str(DIR_PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(DIR_PROJECT_ROOT)) from scripts import check_pr_self_review as self_review_check # noqa: E402 from scripts import update_mattpocock_skills as skills_updater # noqa: E402 class KeyValueCache(Protocol): """The part of ``pytest.Cache`` used here.""" def get(self, key: str, default: Any) -> Any: """Return the cached value for a key, or ``default``.""" ... def set(self, key: str, value: object) -> None: """Store a JSON-serializable value for a key.""" ... class TestMattpocockSkills: """Vendored mattpocock engineering skills in ``ai/skills/engineering``.""" # Key and time to live of the upstream hashes in the pytest cache. UPSTREAM_CACHE_KEY = "ai/mattpocock-skills/upstream-hashes" UPSTREAM_CACHE_TTL_SECONDS = 24 * 60 * 60 # Files of the fake upstream repository, by repository path. UPSTREAM_FILES = { "LICENSE": b"MIT", "README.md": b"readme", "skills/engineering/tdd/SKILL.md": b"tdd", "skills/engineering/wizard/template.sh": b"#!/bin/sh\n", "skills/productivity/grilling/SKILL.md": b"grilling", } # The vendored subset, by path in the local skills directory. VENDORED_FILES = { "LICENSE": b"MIT", "tdd/SKILL.md": b"tdd", "wizard/template.sh": b"#!/bin/sh\n", } @staticmethod def read_files(directory: Path) -> dict[str, bytes]: """Return all files below a directory, keyed by relative POSIX path.""" return { path.relative_to(directory).as_posix(): path.read_bytes() for path in directory.rglob("*") if path.is_file() } @staticmethod def fetch_upstream_hashes() -> skills_updater.SkillHashes: """Fetch the upstream hashes from the default upstream repository.""" with skills_updater.upstream_clone() as repo: return skills_updater.upstream_hashes(repo) @classmethod def cached_upstream_hashes( cls, cache: KeyValueCache, fetch: Callable[[], skills_updater.SkillHashes], now: float, ) -> skills_updater.SkillHashes: """Return the upstream hashes, fetched at most once per time to live. Args: cache: Cache that keeps the hashes between test runs, usually the pytest cache. fetch: Fetches the upstream hashes. now: Current time in seconds since the epoch. Returns: The cached hashes if they are younger than the time to live and were fetched from the same upstream, otherwise freshly fetched ones. Raises: UpstreamSkillsError: If the hashes must be fetched and that fails. Nothing is cached then. """ upstream = f"{skills_updater.UPSTREAM_URL}@{skills_updater.UPSTREAM_DEFAULT_REF}" entry = cache.get(cls.UPSTREAM_CACHE_KEY, None) if ( isinstance(entry, dict) and entry.get("upstream") == upstream and isinstance(entry.get("fetched_at"), (int, float)) and 0 <= now - entry["fetched_at"] < cls.UPSTREAM_CACHE_TTL_SECONDS and isinstance(entry.get("hashes"), dict) ): return entry["hashes"] hashes = fetch() cache.set( cls.UPSTREAM_CACHE_KEY, {"upstream": upstream, "fetched_at": now, "hashes": hashes} ) return hashes @pytest.fixture def memory_cache(self) -> KeyValueCache: """Return an in-memory stand-in for the pytest cache.""" class MemoryCache(dict[str, Any]): def set(self, key: str, value: object) -> None: self[key] = value return MemoryCache() @pytest.fixture def skills_dir(self, tmp_path: Path) -> Path: """Return a not yet existing local skills directory. A subdirectory keeps the files other fixtures put into ``tmp_path`` (e.g. the EOS configuration) out of the skill files. """ return tmp_path / "engineering" @pytest.fixture def upstream_url(self, tmp_path: Path) -> str: """Create a local upstream repository and return its ``file://`` URL.""" repo_dir = tmp_path / "upstream" repo = git.Repo.init(repo_dir, initial_branch="main") with repo.config_writer() as config: config.set_value("uploadpack", "allowFilter", "true") for repo_path, content in self.UPSTREAM_FILES.items(): path = repo_dir / repo_path path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content) if path.suffix == ".sh": path.chmod(0o755) repo.index.add(list(self.UPSTREAM_FILES)) actor = git.Actor("Upstream", "upstream@example.com") repo.index.commit("Add skills", author=actor, committer=actor) repo.close() return repo_dir.as_uri() def test_engineering_skills_match_upstream(self, pytestconfig: pytest.Config) -> None: """Warn if the vendored skills differ from the upstream repository. Compares git blob hashes from a blobless clone; no skill files are downloaded. The upstream hashes are kept in the pytest cache and fetched at most once a day; ``pytest --cache-clear`` forces a fetch. """ if pytestconfig.cache is None: pytest.skip("Needs the pytest cache provider") try: upstream = self.cached_upstream_hashes( pytestconfig.cache, self.fetch_upstream_hashes, time.time() ) except skills_updater.UpstreamSkillsError as exc: pytest.skip(f"Upstream skills not reachable: {exc}") local = skills_updater.local_hashes(skills_updater.DEFAULT_TARGET_DIR) diff = skills_updater.diff_skills(local, upstream) if not diff.is_empty(): warnings.warn( f"ai/skills/engineering differs from {skills_updater.UPSTREAM_REPOSITORY}" f"@{skills_updater.UPSTREAM_DEFAULT_REF}:\n{diff.summary()}\n" f"Run '{skills_updater.UPDATE_COMMAND}' to update.", UserWarning, stacklevel=1, ) def test_cached_upstream_hashes(self, memory_cache: KeyValueCache) -> None: """Upstream hashes are fetched once per time to live and reused in between.""" fetched: list[skills_updater.SkillHashes] = [] def fetch() -> skills_updater.SkillHashes: fetched.append({"LICENSE": f"hash{len(fetched)}"}) return fetched[-1] ttl = self.UPSTREAM_CACHE_TTL_SECONDS assert self.cached_upstream_hashes(memory_cache, fetch, 1000.0) == {"LICENSE": "hash0"} assert self.cached_upstream_hashes(memory_cache, fetch, 1000.0 + ttl - 1) == { "LICENSE": "hash0" } assert len(fetched) == 1 assert self.cached_upstream_hashes(memory_cache, fetch, 1000.0 + ttl) == { "LICENSE": "hash1" } assert len(fetched) == 2 @pytest.mark.parametrize( "entry", [ {"upstream": "https://example.com/other.git@main", "fetched_at": 1000.0}, {"fetched_at": 2000.0}, # from the future {"fetched_at": "yesterday"}, "garbage", ], ) def test_cached_upstream_hashes_refetches_unusable_entry( self, memory_cache: KeyValueCache, entry: Any ) -> None: """Entries of another upstream, with a bad timestamp or malformed are not used.""" upstream = f"{skills_updater.UPSTREAM_URL}@{skills_updater.UPSTREAM_DEFAULT_REF}" if isinstance(entry, dict): entry = {"upstream": upstream, "hashes": {"LICENSE": "stale"}, **entry} memory_cache.set(self.UPSTREAM_CACHE_KEY, entry) hashes = self.cached_upstream_hashes(memory_cache, lambda: {"LICENSE": "fresh"}, 1000.0) assert hashes == {"LICENSE": "fresh"} assert memory_cache.get(self.UPSTREAM_CACHE_KEY, None)["hashes"] == hashes def test_cached_upstream_hashes_does_not_cache_errors( self, memory_cache: KeyValueCache ) -> None: """A failed fetch is raised and leaves the cache empty, so the next run retries.""" def fail() -> skills_updater.SkillHashes: raise skills_updater.UpstreamSkillsError("offline") with pytest.raises(skills_updater.UpstreamSkillsError): self.cached_upstream_hashes(memory_cache, fail, 1000.0) assert memory_cache.get(self.UPSTREAM_CACHE_KEY, None) is None def test_upstream_hashes_match_local_hashes(self, upstream_url: str, skills_dir: Path) -> None: """Hashes from the upstream trees equal the hashes of the same local files.""" skills_dir.mkdir() for relative_path, content in self.VENDORED_FILES.items(): (skills_dir / relative_path).parent.mkdir(parents=True, exist_ok=True) (skills_dir / relative_path).write_bytes(content) with skills_updater.upstream_clone(upstream_url) as repo: upstream = skills_updater.upstream_hashes(repo) assert upstream == skills_updater.local_hashes(skills_dir) assert set(upstream) == set(self.VENDORED_FILES) def test_upstream_clone_unknown_ref(self, upstream_url: str) -> None: """An unknown branch or tag is reported as upstream error.""" with pytest.raises(skills_updater.UpstreamSkillsError, match="Cannot clone"): with skills_updater.upstream_clone(upstream_url, "no-such-branch"): pass def test_local_hashes_missing_directory(self, skills_dir: Path) -> None: """A missing local directory has no files.""" assert skills_updater.local_hashes(skills_dir) == {} def test_diff_skills(self) -> None: """Added, removed and changed files are reported sorted by path.""" local = {"same.md": "1", "changed.md": "2", "removed.md": "3"} upstream = {"same.md": "1", "changed.md": "4", "added.md": "5"} diff = skills_updater.diff_skills(local, upstream) assert diff == skills_updater.SkillsDiff( added=["added.md"], removed=["removed.md"], changed=["changed.md"] ) assert not diff.is_empty() assert skills_updater.diff_skills(upstream, upstream).is_empty() def test_main_check_reports_difference( self, upstream_url: str, skills_dir: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: """``--check`` exits with 1 and neither fetches nor writes files.""" def fail_update(*args: object) -> None: raise AssertionError("--check must not update") monkeypatch.setattr(skills_updater, "update_skills", fail_update) status = skills_updater.main(["--check", "--url", upstream_url, "--target", str(skills_dir)]) assert status == 1 assert not skills_dir.exists() assert skills_updater.UPDATE_COMMAND in capsys.readouterr().out def test_main_updates(self, upstream_url: str, skills_dir: Path) -> None: """An update makes the local directory identical to upstream.""" (skills_dir / "stale").mkdir(parents=True) (skills_dir / "stale" / "SKILL.md").write_bytes(b"stale") (skills_dir / "LICENSE").write_bytes(b"old license") args = ["--url", upstream_url, "--target", str(skills_dir)] assert skills_updater.main(args) == 0 assert self.read_files(skills_dir) == self.VENDORED_FILES assert not (skills_dir / "stale").exists() assert (skills_dir / "wizard" / "template.sh").stat().st_mode & stat.S_IXUSR assert skills_updater.main(["--check", *args]) == 0 def test_main_upstream_error(self, tmp_path: Path, skills_dir: Path) -> None: """An unreachable upstream exits with 2.""" missing_url = (tmp_path / "missing").as_uri() assert skills_updater.main(["--url", missing_url, "--target", str(skills_dir)]) == 2 class TestPrSelfReviewCheck: """CI check of the self-review declaration in ``scripts/check_pr_self_review.py``.""" HEAD_SHA = "0123456789abcdef0123456789abcdef01234567" OTHER_SHA = "fedcba9876543210fedcba9876543210fedcba98" AGENT_COMMIT = "feat: add x\n\nAssisted-by: Claude Code (claude-opus-5-5)\n" CLAUDE_COMMIT = "fix: y\n\nCo-Authored-By: Claude Opus 5.5 \n" HUMAN_COMMIT = "fix: z\n\nCo-Authored-By: Jane Doe \n" @classmethod def body(cls, ticked: bool = True, review: str | None = None) -> str: """Return a pull request body following the template.""" box = "x" if ticked else " " if review is None: review = ( f"Reviewed: {cls.HEAD_SHA}\n" "Base: main\n" "Standards: 0 hard, 2 judgement\n" "Spec: 1 open (source: .scratch/chore/ai_qa/plan.md)\n\n" "- judgement: possible Feature Envy, left as is\n" ) return ( "## Summary\n\nSomething.\n\n" f"- [{box}] AI-assisted (an agent wrote or co-wrote part of this PR)\n" "- [x] I reviewed the complete diff myself (author review)\n\n" "\n" f"## Self-review\n\n{review}" ) def check( self, body: str, commits: list[str], title: str = "feat: x", author: str = "someone" ) -> self_review_check.CheckResult: """Run the check for the head SHA.""" return self_review_check.check_pull_request(body, commits, self.HEAD_SHA, title, author) def test_not_agent_assisted_passes(self) -> None: """Human-only commits and an unticked box need no self-review.""" result = self.check("## Summary\n\n- [ ] AI-assisted\n", [self.HUMAN_COMMIT]) assert not result.agent_assisted assert result.passed @pytest.mark.parametrize("commit", [AGENT_COMMIT, CLAUDE_COMMIT]) def test_complete_self_review_passes(self, commit: str) -> None: """A ticked box and a fresh, complete self-review pass, open findings included.""" result = self.check(self.body(), [self.HUMAN_COMMIT, commit]) assert result.agent_assisted assert result.errors == [] def test_ticked_box_without_trailer_needs_self_review(self) -> None: """A ticked box alone makes the pull request agent-assisted.""" result = self.check(self.body(review="nothing\n").replace("## Self-review", "## Notes"), []) assert result.agent_assisted assert result.errors == ["The pull request body has no '## Self-review' section."] def test_trailer_without_ticked_box_is_noted(self) -> None: """An agent trailer with an unticked box passes with a note.""" result = self.check(self.body(ticked=False), [self.AGENT_COMMIT]) assert result.passed assert len(result.notes) == 1 assert "box" in result.notes[0] @pytest.mark.parametrize( "marker", [ "Assisted-by: OpenCode (qwen3-coder)", "🤖 Generated with [Claude Code](https://claude.com/claude-code)", ], ) def test_body_marker_makes_agent_assisted(self, marker: str) -> None: """An agent marker in the body counts; an unticked box is noted.""" ticked = self.check(self.body() + f"\n{marker}\n", [self.HUMAN_COMMIT]) unticked = self.check(self.body(ticked=False) + f"\n{marker}\n", [self.HUMAN_COMMIT]) assert ticked.agent_assisted assert ticked.errors == [] assert ticked.notes == [] assert unticked.agent_assisted assert unticked.passed assert len(unticked.notes) == 1 assert "agent marker" in unticked.notes[0] def test_marker_without_section_fails(self) -> None: """An agent marker makes the section with the SHA required, ticked box or not.""" body = "- [ ] AI-assisted\n\nAssisted-by: OpenCode (qwen3-coder)\n" result = self.check(body, [self.HUMAN_COMMIT]) assert "The pull request body has no '## Self-review' section." in result.errors def test_body_marker_in_html_comment_does_not_count(self) -> None: """A marker inside a template comment is not a signal.""" body = "- [ ] AI-assisted\n\n" result = self.check(body, [self.HUMAN_COMMIT]) assert not result.agent_assisted def test_stale_self_review_fails(self) -> None: """A self-review for another commit than the head fails.""" body = self.body().replace(self.HEAD_SHA, self.OTHER_SHA) result = self.check(body, [self.AGENT_COMMIT]) assert len(result.errors) == 1 assert self.OTHER_SHA in result.errors[0] assert self.HEAD_SHA in result.errors[0] def test_abbreviated_sha_fails(self) -> None: """An abbreviated SHA is rejected, even if it is a prefix of the head.""" body = self.body().replace(self.HEAD_SHA, self.HEAD_SHA[:12]) result = self.check(body, [self.AGENT_COMMIT]) assert len(result.errors) == 1 assert "full 40-character" in result.errors[0] def test_missing_sha_fails(self) -> None: """A section without the Reviewed line fails; missing optional lines are not reported.""" result = self.check(self.body(review="- looks fine\n"), [self.AGENT_COMMIT]) assert result.errors == ["The self-review has no 'Reviewed: ' line."] assert result.notes == [] def test_only_sha_passes(self) -> None: """The Reviewed line alone is enough to pass.""" result = self.check(self.body(review=f"Reviewed: {self.HEAD_SHA}\n"), [self.AGENT_COMMIT]) assert result.passed assert result.notes == [] def test_uppercase_sha_fails(self) -> None: """A full SHA in uppercase is malformed.""" body = self.body().replace(self.HEAD_SHA, self.HEAD_SHA.upper()) result = self.check(body, [self.AGENT_COMMIT]) assert len(result.errors) == 1 assert "lowercase" in result.errors[0] @pytest.mark.parametrize( ("good", "bad", "key"), [ ("Standards: 0 hard, 2 judgement", "Standards: two hard, 2 judgement", "Standards:"), ("Standards: 0 hard, 2 judgement", "Standards: 0 hard", "Standards:"), ( "Spec: 1 open (source: .scratch/chore/ai_qa/plan.md)", "Spec: 1 open (source: )", "Spec:", ), ("Spec: 1 open (source: .scratch/chore/ai_qa/plan.md)", "Spec: 1 open", "Spec:"), ("Base: main", "Base:", "Base:"), ], ids=["standards-word", "standards-short", "spec-empty-source", "spec-no-source", "base"], ) def test_malformed_optional_lines_are_noted(self, good: str, bad: str, key: str) -> None: """A present but malformed optional line passes with a note.""" body = self.body() assert good in body result = self.check(body.replace(good, bad), [self.AGENT_COMMIT]) assert result.passed assert len(result.notes) == 1 assert f"'{key}" in result.notes[0] def test_section_inside_html_comment_does_not_count(self) -> None: """An example section in a template comment is not a declaration.""" body = self.body().replace("## Self-review", "\n" result = self.check(body, [self.AGENT_COMMIT]) assert result.errors == ["The pull request body has no '## Self-review' section."] def test_section_ends_at_next_heading(self) -> None: """Fields after the next heading do not belong to the self-review.""" body = self.body(review=f"Reviewed: {self.HEAD_SHA}\n\n## Other\n\nBase: main\n") result = self.check(body, [self.AGENT_COMMIT]) assert result.passed assert result.notes == [] def test_windows_line_endings_pass(self) -> None: """A body and commits saved with CRLF line endings are checked like LF ones.""" body = self.body().replace("\n", "\r\n") commit = self.AGENT_COMMIT.replace("\n", "\r\n") result = self.check(body, [commit]) assert result.agent_assisted assert result.errors == [] assert result.notes == [] def test_report_lists_notes(self) -> None: """Notes appear in the report of a passing and of a failing check.""" passing = self.check(self.body(ticked=False), [self.AGENT_COMMIT]) failing = self.check( self.body(ticked=False).replace(self.HEAD_SHA, self.OTHER_SHA), [self.AGENT_COMMIT] ) for result in (passing, failing): report = self_review_check.render_report(result, self.HEAD_SHA) assert "do not fail the check" in report assert result.notes[0] in report @pytest.mark.parametrize( ("title", "author"), [("chore: bump x", "dependabot[bot]"), ("Release 1.2.3", "someone")], ) def test_skipped_pull_requests(self, title: str, author: str) -> None: """Dependabot and release pull requests are not checked.""" result = self.check("", [self.AGENT_COMMIT], title=title, author=author) assert result.skipped_reason assert result.passed def test_main_exit_codes(self, tmp_path: Path) -> None: """The command line exits 0 for a valid and 1 for a stale declaration.""" body_file = tmp_path / "body.md" commits_file = tmp_path / "commits.jsonl" body_file.write_text(self.body(), encoding="utf-8") commits_file.write_text( "\n".join(json.dumps(m) for m in [self.HUMAN_COMMIT, self.AGENT_COMMIT]) + "\n", encoding="utf-8", ) args = ["--body-file", str(body_file), "--commits-file", str(commits_file)] assert self_review_check.main([*args, "--head-sha", self.HEAD_SHA]) == 0 assert self_review_check.main([*args, "--head-sha", self.OTHER_SHA]) == 1 def test_report_for_failure_explains_the_fix(self) -> None: """A failing check renders the marked comment with the errors and how to fix them.""" body = self.body().replace(self.HEAD_SHA, self.OTHER_SHA) result = self.check(body, [self.AGENT_COMMIT]) report = self_review_check.render_report(result, self.HEAD_SHA) assert report.startswith(self_review_check.COMMENT_MARKER + "\n") assert "check failed" in report assert f"- {result.errors[0]}" in report assert "How to fix" in report assert "gh pr edit" in report assert "Reviewed: None: """Backticks from the body cannot close the code block and inject Markdown.""" body = self.body().replace(self.HEAD_SHA, "````\n@someone") result = self.check(body, [self.AGENT_COMMIT]) report = self_review_check.render_report(result, self.HEAD_SHA) fence_match = re.search(r"(?m)^(`{4,})$", report) assert fence_match is not None fence = fence_match.group(1) block = report[fence_match.end() : report.index("\n" + fence + "\n", fence_match.end())] assert "@someone" not in report.replace(block, "") assert "````" in block @pytest.mark.parametrize( ("body", "commits", "title", "expected"), [ ("BODY", ["AGENT"], "feat: x", "up to date for"), ("- [ ] AI-assisted\n", ["HUMAN"], "feat: x", "not agent-assisted"), ("", ["AGENT"], "Release 1.0.0", "does not apply"), ], ) def test_report_for_passing_check( self, body: str, commits: list[str], title: str, expected: str ) -> None: """A passing check renders a short marked note.""" messages = {"AGENT": self.AGENT_COMMIT, "HUMAN": self.HUMAN_COMMIT} result = self.check( self.body() if body == "BODY" else body, [messages[c] for c in commits], title=title ) report = self_review_check.render_report(result, self.HEAD_SHA) assert report.startswith(self_review_check.COMMENT_MARKER + "\n") assert "check passed" in report assert expected in report def test_main_writes_report_file(self, tmp_path: Path) -> None: """``--report-file`` receives the report for both outcomes.""" body_file = tmp_path / "body.md" commits_file = tmp_path / "commits.jsonl" report_file = tmp_path / "report.md" body_file.write_text(self.body(), encoding="utf-8") commits_file.write_text(json.dumps(self.AGENT_COMMIT) + "\n", encoding="utf-8") args = ["--body-file", str(body_file), "--commits-file", str(commits_file)] assert self_review_check.main( [*args, "--head-sha", self.HEAD_SHA, "--report-file", str(report_file)] ) == 0 assert "check passed" in report_file.read_text(encoding="utf-8") assert self_review_check.main( [*args, "--head-sha", self.OTHER_SHA, "--report-file", str(report_file)] ) == 1 assert "check failed" in report_file.read_text(encoding="utf-8") def test_workflow_uses_the_script_marker(self) -> None: """The workflow finds its comment by the same marker the script writes.""" workflow = (DIR_PROJECT_ROOT / ".github" / "workflows" / "ai-self-review.yml").read_text( encoding="utf-8" ) assert f'COMMENT_MARKER: "{self_review_check.COMMENT_MARKER}"' in workflow def test_report_for_failure_links_the_docs(self) -> None: """The failure comment links the docs and covers the push.""" result = self.check(self.body(review="- nothing\n"), [self.AGENT_COMMIT]) report = self_review_check.render_report(result, self.HEAD_SHA) assert f"]({self_review_check.DOCS_URL})" in report assert "right after the push" in report assert "before every later push" in report assert "gh pr edit" in report assert "by hand" in report def test_read_commit_messages_rejects_non_strings(self, tmp_path: Path) -> None: """A commits file line that is not a JSON string is an error.""" commits_file = tmp_path / "commits.jsonl" commits_file.write_text('{"message": "x"}\n', encoding="utf-8") with pytest.raises(ValueError, match="expected a JSON string"): self_review_check.read_commit_messages(commits_file)