"""Tests for scripts/log-run.py.""" from __future__ import annotations import importlib.util import json import os import subprocess import sys from pathlib import Path _SPEC = importlib.util.spec_from_file_location( "log_run", Path(__file__).parent.parent / "scripts" / "log-run.py", ) log_run = importlib.util.module_from_spec(_SPEC) _SPEC.loader.exec_module(log_run) def _write_usage(tmp_path: Path, usage: dict) -> Path: p = tmp_path / "usage.json" p.write_text(json.dumps(usage), encoding="utf-8") return p def _read_log(log: Path) -> list[dict]: return [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines() if line.strip()] def test_writes_one_row_per_agent_with_finding_counts(tmp_path: Path) -> None: output = tmp_path / "out" output.mkdir() (output / "findings-security-triage-reviewer.json").write_text( json.dumps({"findings": [{"file": "a.py", "line": 1, "rule_id": "B1"}, {"file": "b.py", "line": 2, "rule_id": "B2"}]}), encoding="utf-8", ) (output / "findings-walkthrough-reviewer.json").write_text( json.dumps({"overview": "summary", "files": []}), encoding="utf-8", ) log = tmp_path / "runs.jsonl" usage = _write_usage(tmp_path, { "security-triage-reviewer": {"model": "sonnet", "input_tokens": 100, "output_tokens": 50, "duration_ms": 1234}, "walkthrough-reviewer": {"model": "sonnet", "input_tokens": 200, "output_tokens": 80, "duration_ms": 4321}, }) rc = log_run.main([ "--output-dir", str(output), "--run-id", "abc123", "--repo", "/tmp/repo", "--mode", "local", "--log-path", str(log), "--usage-json", str(usage), ]) assert rc == 0 rows = _read_log(log) assert len(rows) == 2 by_agent = {r["agent"]: r for r in rows} assert by_agent["security-triage-reviewer"]["finding_count"] == 2 assert by_agent["security-triage-reviewer"]["input_tokens"] == 100 assert by_agent["security-triage-reviewer"]["model"] == "sonnet" assert by_agent["walkthrough-reviewer"]["finding_count"] == 0 assert by_agent["walkthrough-reviewer"]["duration_ms"] == 4321 def test_missing_findings_file_counts_zero(tmp_path: Path) -> None: output = tmp_path / "out" output.mkdir() log = tmp_path / "runs.jsonl" usage = _write_usage(tmp_path, { "phantom-reviewer": {"model": "sonnet", "input_tokens": 0, "output_tokens": 0, "duration_ms": 0}, }) rc = log_run.main([ "--output-dir", str(output), "--run-id", "x", "--repo", "/r", "--mode", "ref", "--log-path", str(log), "--usage-json", str(usage), ]) assert rc == 0 rows = _read_log(log) assert rows[0]["finding_count"] == 0 def test_malformed_findings_file_counts_zero(tmp_path: Path) -> None: output = tmp_path / "out" output.mkdir() (output / "findings-broken-reviewer.json").write_text("{not json", encoding="utf-8") log = tmp_path / "runs.jsonl" usage = _write_usage(tmp_path, { "broken-reviewer": {"model": "haiku", "input_tokens": 10, "output_tokens": 5, "duration_ms": 100}, }) rc = log_run.main([ "--output-dir", str(output), "--run-id", "x", "--repo", "/r", "--mode", "local", "--log-path", str(log), "--usage-json", str(usage), ]) assert rc == 0 rows = _read_log(log) assert rows[0]["finding_count"] == 0 def test_findings_as_bare_list_is_counted(tmp_path: Path) -> None: output = tmp_path / "out" output.mkdir() (output / "findings-x-reviewer.json").write_text( json.dumps([{"rule_id": "A"}, {"rule_id": "B"}, {"rule_id": "C"}]), encoding="utf-8", ) log = tmp_path / "runs.jsonl" usage = _write_usage(tmp_path, { "x-reviewer": {"model": "sonnet", "input_tokens": 1, "output_tokens": 1, "duration_ms": 1}, }) rc = log_run.main([ "--output-dir", str(output), "--run-id", "x", "--repo", "/r", "--mode", "local", "--log-path", str(log), "--usage-json", str(usage), ]) assert rc == 0 rows = _read_log(log) assert rows[0]["finding_count"] == 3 def test_log_path_parent_is_created(tmp_path: Path) -> None: output = tmp_path / "out" output.mkdir() log = tmp_path / "nested" / "deeper" / "runs.jsonl" usage = _write_usage(tmp_path, { "x-reviewer": {"model": "sonnet", "input_tokens": 0, "output_tokens": 0, "duration_ms": 0}, }) rc = log_run.main([ "--output-dir", str(output), "--run-id", "x", "--repo", "/r", "--mode", "local", "--log-path", str(log), "--usage-json", str(usage), ]) assert rc == 0 assert log.exists() def test_runs_as_a_script_from_another_cwd(tmp_path: Path) -> None: log = tmp_path / "runs.jsonl" env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"} r = subprocess.run( [sys.executable, str(_SPEC.origin), "--output-dir", str(tmp_path), "--run-id", "x", "--repo", "/r", "--mode", "local", "--log-path", str(log), "--usage-json", "-"], input=json.dumps({"x-reviewer": {"model": "sonnet"}}), capture_output=True, text=True, cwd=tmp_path, env=env, check=False, ) assert r.returncode == 0, r.stderr assert _read_log(log)[0]["agent"] == "x-reviewer"