mroberts
37fc3fb291
audit-code install-tools.sh:
- Buffer the opengrep release JSON before grep -m1; curl died with (23)
under pipefail when grep quit early.
- Use ${m}: in the PowerShell block; $m: parsed as a scope-qualified var.
- On Arch, skip paru/yay when pacman -Q shows every package installed,
since --needed still invokes sudo.
- Add --check-only (fast, installs nothing, non-zero naming missing tools)
and --user-only (no system package managers, no sudo).
log-run.py (both skills): put the skill dir on sys.path so running it as
a script from any cwd no longer raises ModuleNotFoundError.
audit-terraform: move deps from requirements.txt into pyproject
dependency groups and add scripts/install-tools.sh (uv sync --group tools,
then check trivy, tflint, tofu, terragrunt, gh).
Both SKILL.md files gain a 0.5 Preflight step and call scripts through
uv run --project ${SKILL_DIR}. tools_unavailable is now a map of tool to
exact install command; audit-terraform skips trivy when absent and stops
with an install hint instead of crashing when tofu/terragrunt is missing.
111 lines
3.7 KiB
Python
111 lines
3.7 KiB
Python
"""Tests for scripts/log-run.py."""
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
_SPEC = importlib.util.spec_from_file_location(
|
|
"log_run",
|
|
Path(__file__).parent.parent / "scripts" / "log-run.py",
|
|
)
|
|
log_run = importlib.util.module_from_spec(_SPEC)
|
|
_SPEC.loader.exec_module(log_run)
|
|
|
|
|
|
def _write_usage(tmp_path: Path, usage: dict) -> Path:
|
|
p = tmp_path / "usage.json"
|
|
p.write_text(json.dumps(usage), encoding="utf-8")
|
|
return p
|
|
|
|
|
|
def _read_log(log: Path) -> list[dict]:
|
|
return [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
|
|
|
|
def test_writes_one_row_per_agent_with_finding_counts(tmp_path: Path) -> None:
|
|
output = tmp_path / "out"
|
|
output.mkdir()
|
|
(output / "findings-aws-bp-reviewer.json").write_text(
|
|
json.dumps({"findings": [{"resource": "r1", "control": "c1"}]}),
|
|
encoding="utf-8",
|
|
)
|
|
(output / "findings-walkthrough-reviewer.json").write_text(
|
|
json.dumps({"overview": "x", "plan_units": []}),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
log = tmp_path / "runs.jsonl"
|
|
usage = _write_usage(tmp_path, {
|
|
"aws-bp-reviewer": {"model": "sonnet", "input_tokens": 100,
|
|
"output_tokens": 50, "duration_ms": 1234},
|
|
"walkthrough-reviewer": {"model": "sonnet", "input_tokens": 200,
|
|
"output_tokens": 80, "duration_ms": 4321},
|
|
})
|
|
|
|
rc = log_run.main([
|
|
"--output-dir", str(output), "--run-id", "abc",
|
|
"--repo", "/tmp/repo", "--mode", "local",
|
|
"--log-path", str(log), "--usage-json", str(usage),
|
|
])
|
|
assert rc == 0
|
|
|
|
rows = _read_log(log)
|
|
assert len(rows) == 2
|
|
|
|
by_agent = {r["agent"]: r for r in rows}
|
|
assert by_agent["aws-bp-reviewer"]["finding_count"] == 1
|
|
assert by_agent["walkthrough-reviewer"]["finding_count"] == 0
|
|
|
|
|
|
def test_missing_findings_file_counts_zero(tmp_path: Path) -> None:
|
|
output = tmp_path / "out"
|
|
output.mkdir()
|
|
log = tmp_path / "runs.jsonl"
|
|
usage = _write_usage(tmp_path, {
|
|
"phantom-reviewer": {"model": "sonnet", "input_tokens": 0,
|
|
"output_tokens": 0, "duration_ms": 0},
|
|
})
|
|
|
|
rc = log_run.main([
|
|
"--output-dir", str(output), "--run-id", "x", "--repo", "/r",
|
|
"--mode", "ref", "--log-path", str(log), "--usage-json", str(usage),
|
|
])
|
|
assert rc == 0
|
|
assert _read_log(log)[0]["finding_count"] == 0
|
|
|
|
|
|
def test_malformed_findings_file_counts_zero(tmp_path: Path) -> None:
|
|
output = tmp_path / "out"
|
|
output.mkdir()
|
|
(output / "findings-broken-reviewer.json").write_text("{bad", encoding="utf-8")
|
|
log = tmp_path / "runs.jsonl"
|
|
usage = _write_usage(tmp_path, {
|
|
"broken-reviewer": {"model": "haiku", "input_tokens": 10,
|
|
"output_tokens": 5, "duration_ms": 100},
|
|
})
|
|
|
|
rc = log_run.main([
|
|
"--output-dir", str(output), "--run-id", "x", "--repo", "/r",
|
|
"--mode", "local", "--log-path", str(log), "--usage-json", str(usage),
|
|
])
|
|
assert rc == 0
|
|
assert _read_log(log)[0]["finding_count"] == 0
|
|
|
|
|
|
def test_runs_as_a_script_from_another_cwd(tmp_path: Path) -> None:
|
|
log = tmp_path / "runs.jsonl"
|
|
env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"}
|
|
r = subprocess.run(
|
|
[sys.executable, str(_SPEC.origin),
|
|
"--output-dir", str(tmp_path), "--run-id", "x", "--repo", "/r",
|
|
"--mode", "local", "--log-path", str(log), "--usage-json", "-"],
|
|
input=json.dumps({"x-reviewer": {"model": "sonnet"}}),
|
|
capture_output=True, text=True, cwd=tmp_path, env=env, check=False,
|
|
)
|
|
assert r.returncode == 0, r.stderr
|
|
assert _read_log(log)[0]["agent"] == "x-reviewer"
|