Move the code and terraform audits into the reviews plugin
Copy the standalone code-review and terraform-review skills into
plugins/reviews as audit-code and audit-terraform. The rename separates the
automated, linter-driven audits from the guided review-pr walkthrough that
already lived here.
Resolve bundled script paths through ${SKILL_DIR}, exported in a new step 0.
CLAUDE_PLUGIN_ROOT is not set in the Bash tool environment, so the obvious
substitution would have expanded to nothing and broken every collection
script invocation.
Replace the PLAN and DESIGN docs with READMEs written from the current
SKILL.md and scripts. The old docs had drifted badly: they named semgrep
where the code calls opengrep, scoped five review agents where there are
now eight, and predated Lua, PowerShell, and GitHub Actions support.
Add CONSISTENCY_NORMS to the audit-terraform agent inputs. The collection
script writes consistency_norms.json and the agent prompt declares it, but
SKILL.md never listed it, leaving the variable unsubstituted.
Drop the --ingest-verdicts instruction from both skills. review_stats.py
parses no arguments, so the ref-mode verdict template it told users to feed
back could never be read.
Point audit-terraform's smoke test at README.md and resolve its fixture
paths relative to the test file rather than an absolute home directory.
Tests: 197 passing (audit-code), 106 passing (audit-terraform).
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
"""Tests for scripts/log-run.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
_SPEC = importlib.util.spec_from_file_location(
|
||||
"log_run",
|
||||
Path(__file__).parent.parent / "scripts" / "log-run.py",
|
||||
)
|
||||
log_run = importlib.util.module_from_spec(_SPEC)
|
||||
_SPEC.loader.exec_module(log_run)
|
||||
|
||||
|
||||
def _write_usage(tmp_path: Path, usage: dict) -> Path:
|
||||
p = tmp_path / "usage.json"
|
||||
p.write_text(json.dumps(usage), encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def _read_log(log: Path) -> list[dict]:
|
||||
return [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def test_writes_one_row_per_agent_with_finding_counts(tmp_path: Path) -> None:
|
||||
output = tmp_path / "out"
|
||||
output.mkdir()
|
||||
(output / "findings-security-triage-reviewer.json").write_text(
|
||||
json.dumps({"findings": [{"file": "a.py", "line": 1, "rule_id": "B1"},
|
||||
{"file": "b.py", "line": 2, "rule_id": "B2"}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(output / "findings-walkthrough-reviewer.json").write_text(
|
||||
json.dumps({"overview": "summary", "files": []}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
log = tmp_path / "runs.jsonl"
|
||||
usage = _write_usage(tmp_path, {
|
||||
"security-triage-reviewer": {"model": "sonnet", "input_tokens": 100,
|
||||
"output_tokens": 50, "duration_ms": 1234},
|
||||
"walkthrough-reviewer": {"model": "sonnet", "input_tokens": 200,
|
||||
"output_tokens": 80, "duration_ms": 4321},
|
||||
})
|
||||
|
||||
rc = log_run.main([
|
||||
"--output-dir", str(output), "--run-id", "abc123",
|
||||
"--repo", "/tmp/repo", "--mode", "local",
|
||||
"--log-path", str(log), "--usage-json", str(usage),
|
||||
])
|
||||
assert rc == 0
|
||||
|
||||
rows = _read_log(log)
|
||||
assert len(rows) == 2
|
||||
|
||||
by_agent = {r["agent"]: r for r in rows}
|
||||
assert by_agent["security-triage-reviewer"]["finding_count"] == 2
|
||||
assert by_agent["security-triage-reviewer"]["input_tokens"] == 100
|
||||
assert by_agent["security-triage-reviewer"]["model"] == "sonnet"
|
||||
assert by_agent["walkthrough-reviewer"]["finding_count"] == 0
|
||||
assert by_agent["walkthrough-reviewer"]["duration_ms"] == 4321
|
||||
|
||||
|
||||
def test_missing_findings_file_counts_zero(tmp_path: Path) -> None:
|
||||
output = tmp_path / "out"
|
||||
output.mkdir()
|
||||
log = tmp_path / "runs.jsonl"
|
||||
usage = _write_usage(tmp_path, {
|
||||
"phantom-reviewer": {"model": "sonnet", "input_tokens": 0,
|
||||
"output_tokens": 0, "duration_ms": 0},
|
||||
})
|
||||
|
||||
rc = log_run.main([
|
||||
"--output-dir", str(output), "--run-id", "x", "--repo", "/r",
|
||||
"--mode", "ref", "--log-path", str(log), "--usage-json", str(usage),
|
||||
])
|
||||
assert rc == 0
|
||||
rows = _read_log(log)
|
||||
assert rows[0]["finding_count"] == 0
|
||||
|
||||
|
||||
def test_malformed_findings_file_counts_zero(tmp_path: Path) -> None:
|
||||
output = tmp_path / "out"
|
||||
output.mkdir()
|
||||
(output / "findings-broken-reviewer.json").write_text("{not json", encoding="utf-8")
|
||||
log = tmp_path / "runs.jsonl"
|
||||
usage = _write_usage(tmp_path, {
|
||||
"broken-reviewer": {"model": "haiku", "input_tokens": 10,
|
||||
"output_tokens": 5, "duration_ms": 100},
|
||||
})
|
||||
|
||||
rc = log_run.main([
|
||||
"--output-dir", str(output), "--run-id", "x", "--repo", "/r",
|
||||
"--mode", "local", "--log-path", str(log), "--usage-json", str(usage),
|
||||
])
|
||||
assert rc == 0
|
||||
rows = _read_log(log)
|
||||
assert rows[0]["finding_count"] == 0
|
||||
|
||||
|
||||
def test_findings_as_bare_list_is_counted(tmp_path: Path) -> None:
|
||||
output = tmp_path / "out"
|
||||
output.mkdir()
|
||||
(output / "findings-x-reviewer.json").write_text(
|
||||
json.dumps([{"rule_id": "A"}, {"rule_id": "B"}, {"rule_id": "C"}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
log = tmp_path / "runs.jsonl"
|
||||
usage = _write_usage(tmp_path, {
|
||||
"x-reviewer": {"model": "sonnet", "input_tokens": 1,
|
||||
"output_tokens": 1, "duration_ms": 1},
|
||||
})
|
||||
|
||||
rc = log_run.main([
|
||||
"--output-dir", str(output), "--run-id", "x", "--repo", "/r",
|
||||
"--mode", "local", "--log-path", str(log), "--usage-json", str(usage),
|
||||
])
|
||||
assert rc == 0
|
||||
rows = _read_log(log)
|
||||
assert rows[0]["finding_count"] == 3
|
||||
|
||||
|
||||
def test_log_path_parent_is_created(tmp_path: Path) -> None:
|
||||
output = tmp_path / "out"
|
||||
output.mkdir()
|
||||
log = tmp_path / "nested" / "deeper" / "runs.jsonl"
|
||||
usage = _write_usage(tmp_path, {
|
||||
"x-reviewer": {"model": "sonnet", "input_tokens": 0,
|
||||
"output_tokens": 0, "duration_ms": 0},
|
||||
})
|
||||
|
||||
rc = log_run.main([
|
||||
"--output-dir", str(output), "--run-id", "x", "--repo", "/r",
|
||||
"--mode", "local", "--log-path", str(log), "--usage-json", str(usage),
|
||||
])
|
||||
assert rc == 0
|
||||
assert log.exists()
|
||||
Reference in New Issue
Block a user