Move the code and terraform audits into the reviews plugin
Copy the standalone code-review and terraform-review skills into
plugins/reviews as audit-code and audit-terraform. The rename separates the
automated, linter-driven audits from the guided review-pr walkthrough that
already lived here.
Resolve bundled script paths through ${SKILL_DIR}, exported in a new step 0.
CLAUDE_PLUGIN_ROOT is not set in the Bash tool environment, so the obvious
substitution would have expanded to nothing and broken every collection
script invocation.
Replace the PLAN and DESIGN docs with READMEs written from the current
SKILL.md and scripts. The old docs had drifted badly: they named semgrep
where the code calls opengrep, scoped five review agents where there are
now eight, and predated Lua, PowerShell, and GitHub Actions support.
Add CONSISTENCY_NORMS to the audit-terraform agent inputs. The collection
script writes consistency_norms.json and the agent prompt declares it, but
SKILL.md never listed it, leaving the variable unsubstituted.
Drop the --ingest-verdicts instruction from both skills. review_stats.py
parses no arguments, so the ref-mode verdict template it told users to feed
back could never be read.
Point audit-terraform's smoke test at README.md and resolve its fixture
paths relative to the test file rather than an absolute home directory.
Tests: 197 passing (audit-code), 106 passing (audit-terraform).
This commit is contained in:
@@ -0,0 +1,421 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
_SKILL_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def _load_cli():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"collect_changes", _SKILL_ROOT / "scripts" / "collect-changes.py"
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def _stage_fixture(dest: Path) -> Path:
|
||||
src = _SKILL_ROOT / "tests" / "fixtures" / "tofu-sample"
|
||||
shutil.copytree(src, dest)
|
||||
return dest
|
||||
|
||||
|
||||
def _fake_subprocess(cmd, **kwargs):
|
||||
if cmd[:2] == ["git", "-C"]:
|
||||
sub = cmd[2:]
|
||||
else:
|
||||
sub = cmd
|
||||
|
||||
if "symbolic-ref" in sub:
|
||||
return MagicMock(returncode=0, stdout="refs/remotes/origin/main\n", stderr="")
|
||||
if "fetch" in sub:
|
||||
return MagicMock(returncode=0, stdout="", stderr="")
|
||||
if "rev-parse" in sub and "--verify" in sub:
|
||||
return MagicMock(returncode=0, stdout="abc123\n", stderr="")
|
||||
if cmd[:1] == ["git"] and "diff" in cmd:
|
||||
diff = (
|
||||
"diff --git a/main.tf b/main.tf\n"
|
||||
"index 1..2 100644\n"
|
||||
"--- a/main.tf\n"
|
||||
"+++ b/main.tf\n"
|
||||
"@@ -20,0 +21,1 @@\n"
|
||||
"+ bucket_prefix = \"x\"\n"
|
||||
"diff --git a/modules/widget/main.tf b/modules/widget/main.tf\n"
|
||||
"index 3..4 100644\n"
|
||||
"--- a/modules/widget/main.tf\n"
|
||||
"+++ b/modules/widget/main.tf\n"
|
||||
"@@ -1,0 +2,1 @@\n"
|
||||
"+ # touched\n"
|
||||
)
|
||||
return MagicMock(returncode=0, stdout=diff, stderr="")
|
||||
if cmd[0] == "tofu" and "init" in cmd:
|
||||
return MagicMock(returncode=0, stdout="initialized\n", stderr="")
|
||||
if cmd[0] == "tofu" and "plan" in cmd:
|
||||
plan = (
|
||||
"OpenTofu will perform the following actions:\n\n"
|
||||
" # null_resource.top will be updated in-place\n"
|
||||
" ~ resource \"null_resource\" \"top\" {}\n\n"
|
||||
" # module.widget_a.null_resource.thing will be updated in-place\n"
|
||||
" ~ resource \"null_resource\" \"thing\" {}\n\n"
|
||||
" # module.widget_b.null_resource.thing will be updated in-place\n"
|
||||
" ~ resource \"null_resource\" \"thing\" {}\n\n"
|
||||
"Plan: 0 to add, 3 to change, 0 to destroy.\n"
|
||||
)
|
||||
return MagicMock(returncode=0, stdout=plan, stderr="")
|
||||
if cmd[:3] == ["trivy", "config", "--quiet"]:
|
||||
trivy = {
|
||||
"SchemaVersion": 2,
|
||||
"Results": [
|
||||
{
|
||||
"Target": "main.tf",
|
||||
"Class": "config",
|
||||
"Type": "terraform",
|
||||
"Misconfigurations": [
|
||||
{
|
||||
"ID": "AVD-AWS-0089",
|
||||
"AVDID": "AVD-AWS-0089",
|
||||
"Title": "S3 bucket allows public ACL",
|
||||
"Description": "Buckets should not allow public ACLs.",
|
||||
"Message": "Bucket ACL allows public access.",
|
||||
"Severity": "HIGH",
|
||||
"CauseMetadata": {
|
||||
"Resource": "aws_s3_bucket.audit_logs",
|
||||
"StartLine": 21,
|
||||
"EndLine": 30,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
return MagicMock(returncode=0, stdout=json.dumps(trivy), stderr="")
|
||||
return MagicMock(returncode=0, stdout="", stderr="")
|
||||
|
||||
|
||||
def test_cli_happy_path(tmp_path):
|
||||
repo = _stage_fixture(tmp_path / "repo")
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
|
||||
with patch("subprocess.run", side_effect=_fake_subprocess):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--base", "main",
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
|
||||
assert rc == 0, (out_dir / "manifest.json").read_text()
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert manifest["mode"] == "local"
|
||||
assert manifest["base_ref"] == "main"
|
||||
assert set(manifest["changed_source_dirs"]) == {".", "modules/widget"}
|
||||
plan_dirs = {pu["plan_dir"] for pu in manifest["plan_units"]}
|
||||
assert plan_dirs == {"."}
|
||||
pu = manifest["plan_units"][0]
|
||||
assert pu["tool"] == "tofu"
|
||||
assert pu["plan"]["summary"] == "0 to add, 3 to change, 0 to destroy"
|
||||
catalog = manifest["catalog"]
|
||||
keys = {(e["source_dir"], e["local_address"]) for e in catalog}
|
||||
assert (".", "null_resource.top") in keys
|
||||
assert ("modules/widget", "null_resource.thing") in keys
|
||||
module_entries = [e for e in catalog if e["source_dir"] == "modules/widget"]
|
||||
for e in module_entries:
|
||||
plan_dirs = {i["plan_dir"] for i in e["instances"]}
|
||||
assert plan_dirs == {"."}, e
|
||||
addrs = {i["address_at_plan"] for i in e["instances"]}
|
||||
assert any(a.startswith("module.widget_a") for a in addrs)
|
||||
assert any(a.startswith("module.widget_b") for a in addrs)
|
||||
for e in module_entries:
|
||||
assert e["block_header"], f"missing block_header on {e}"
|
||||
assert e["evidence_line"], f"missing evidence_line on {e}"
|
||||
assert isinstance(e["key_attributes"], dict)
|
||||
assert "review_context" in e
|
||||
assert set(e["review_context"]) == {"variables", "locals", "related_blocks"}
|
||||
assert e["block_file"].endswith(".tf")
|
||||
assert e["block_start"] >= 1
|
||||
assert "block_text" not in e
|
||||
assert (out_dir / "trivy-findings.json").is_file()
|
||||
assert manifest["trivy_findings"] == [
|
||||
{
|
||||
"check_id": "AVD-AWS-0089",
|
||||
"title": "S3 bucket allows public ACL",
|
||||
"severity": "high",
|
||||
"message": "Bucket ACL allows public access.",
|
||||
"file": "main.tf",
|
||||
"start_line": 21,
|
||||
"end_line": 30,
|
||||
"resource_type": "aws_s3_bucket",
|
||||
"source": "trivy",
|
||||
}
|
||||
]
|
||||
refs = json.loads((out_dir / "reference_sets.json").read_text())
|
||||
assert "." in refs or "modules/widget" in refs
|
||||
|
||||
|
||||
def test_cli_writes_per_agent_slices(tmp_path):
|
||||
repo = _stage_fixture(tmp_path / "repo")
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
with patch("subprocess.run", side_effect=_fake_subprocess):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo), "--base", "main", "--head", "HEAD",
|
||||
"--output-dir", str(out_dir), "--mode", "local",
|
||||
])
|
||||
assert rc == 0
|
||||
full = json.loads((out_dir / "manifest.json").read_text())
|
||||
for agent in ("fsbp", "cis", "aws-bp", "consistency"):
|
||||
sliced = json.loads((out_dir / f"manifest-{agent}.json").read_text())
|
||||
assert sliced["base_ref"] == full["base_ref"]
|
||||
if agent in ("fsbp", "cis", "aws-bp"):
|
||||
for entry in sliced["catalog"]:
|
||||
assert entry["type"].startswith("aws_"), (
|
||||
f"non-aws type leaked into {agent}: {entry['type']}"
|
||||
)
|
||||
assert sliced["catalog"] == []
|
||||
assert sliced["trivy_findings"] == full["trivy_findings"]
|
||||
if agent == "consistency":
|
||||
assert sliced["changed_source_dirs"] == full["changed_source_dirs"]
|
||||
assert len(sliced["catalog"]) == len(full["catalog"])
|
||||
assert sliced["trivy_findings"] == full["trivy_findings"]
|
||||
|
||||
|
||||
def test_cli_preserves_terragrunt_only_change_context(tmp_path):
|
||||
repo = _stage_fixture(tmp_path / "repo")
|
||||
terragrunt_dir = repo / "live" / "prod" / "app"
|
||||
terragrunt_dir.mkdir(parents=True)
|
||||
(terragrunt_dir / "terragrunt.hcl").write_text("inputs = { instance_count = 2 }\n")
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
|
||||
def _fake(cmd, **kw):
|
||||
if cmd[:1] == ["git"] and "diff" in cmd:
|
||||
diff = (
|
||||
"diff --git a/live/prod/app/terragrunt.hcl b/live/prod/app/terragrunt.hcl\n"
|
||||
"index 1..2 100644\n"
|
||||
"--- a/live/prod/app/terragrunt.hcl\n"
|
||||
"+++ b/live/prod/app/terragrunt.hcl\n"
|
||||
"@@ -1 +1 @@\n"
|
||||
"-inputs = { instance_count = 1 }\n"
|
||||
"+inputs = { instance_count = 2 }\n"
|
||||
)
|
||||
return MagicMock(returncode=0, stdout=diff, stderr="")
|
||||
if cmd[0] == "terragrunt" and "init" in cmd:
|
||||
return MagicMock(returncode=0, stdout="initialized\n", stderr="")
|
||||
if cmd[0] == "terragrunt" and "plan" in cmd:
|
||||
plan = (
|
||||
"OpenTofu will perform the following actions:\n\n"
|
||||
"Plan: 0 to add, 0 to change, 0 to destroy.\n"
|
||||
)
|
||||
return MagicMock(returncode=0, stdout=plan, stderr="")
|
||||
return _fake_subprocess(cmd, **kw)
|
||||
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--base", "main",
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
|
||||
assert rc == 0, (out_dir / "manifest.json").read_text()
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert "live/prod/app" in manifest["changed_source_dirs"]
|
||||
assert manifest["catalog"] == []
|
||||
assert manifest["plan_units"] == [{
|
||||
"plan_dir": "live/prod/app",
|
||||
"tool": "terragrunt",
|
||||
"init": {"ok": True, "stdout_tail": "initialized\n", "stderr_tail": ""},
|
||||
"plan": {
|
||||
"ok": True,
|
||||
"stdout_path": str(out_dir / "plans" / "live_prod_app.txt"),
|
||||
"exit_code": 0,
|
||||
"summary": "0 to add, 0 to change, 0 to destroy",
|
||||
},
|
||||
"triggered_by": ["live/prod/app"],
|
||||
"terragrunt_changed": True,
|
||||
"changed_files": ["live/prod/app/terragrunt.hcl"],
|
||||
}]
|
||||
fsbp = json.loads((out_dir / "manifest-fsbp.json").read_text())
|
||||
assert fsbp["plan_units"] == [{
|
||||
"plan_dir": "live/prod/app",
|
||||
"tool": "terragrunt",
|
||||
"plan_ok": True,
|
||||
"summary": "0 to add, 0 to change, 0 to destroy",
|
||||
"terragrunt_changed": True,
|
||||
"changed_files": ["live/prod/app/terragrunt.hcl"],
|
||||
}]
|
||||
consistency = json.loads((out_dir / "manifest-consistency.json").read_text())
|
||||
assert consistency["plan_units"] == manifest["plan_units"]
|
||||
|
||||
|
||||
def test_cli_aborts_when_plan_fails(tmp_path):
|
||||
repo = _stage_fixture(tmp_path / "repo")
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
|
||||
def _fake(cmd, **kw):
|
||||
if cmd[0] == "tofu" and "plan" in cmd:
|
||||
return MagicMock(
|
||||
returncode=1,
|
||||
stdout="Error: syntax error in main.tf\n",
|
||||
stderr="",
|
||||
)
|
||||
return _fake_subprocess(cmd, **kw)
|
||||
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--base", "main",
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
|
||||
assert rc == 1
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert any("syntax error" in e for e in manifest["errors"])
|
||||
|
||||
|
||||
def test_cli_records_git_diff_failure_in_manifest(tmp_path):
|
||||
repo = _stage_fixture(tmp_path / "repo")
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
|
||||
def _fake(cmd, **kw):
|
||||
if cmd[:1] == ["git"] and "diff" in cmd:
|
||||
return MagicMock(returncode=128, stdout="",
|
||||
stderr="fatal: bad revision 'main...HEAD'\n")
|
||||
return _fake_subprocess(cmd, **kw)
|
||||
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--base", "main",
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
|
||||
assert rc == 1
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert any("bad revision" in e for e in manifest["errors"])
|
||||
|
||||
|
||||
def test_cli_auto_resolves_base_to_origin(tmp_path):
|
||||
"""With no --base, the CLI fetches and diffs against origin/<default>."""
|
||||
repo = _stage_fixture(tmp_path / "repo")
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
|
||||
captured_diff_base: list[str] = []
|
||||
|
||||
def _fake(cmd, **kw):
|
||||
if cmd[:1] == ["git"] and "diff" in cmd:
|
||||
for arg in cmd:
|
||||
if "..." in arg:
|
||||
captured_diff_base.append(arg)
|
||||
return _fake_subprocess(cmd, **kw)
|
||||
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
|
||||
assert rc == 0
|
||||
assert captured_diff_base, "git diff was never invoked"
|
||||
assert captured_diff_base[0].startswith("origin/main..."), (
|
||||
f"diff base should be origin/<default>, got: {captured_diff_base[0]}"
|
||||
)
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert manifest["base_ref"] == "origin/main"
|
||||
|
||||
|
||||
def test_cli_falls_back_to_local_branch_when_origin_missing(tmp_path):
|
||||
"""If origin/<default> doesn't exist (no remote, fetch fails), fall back."""
|
||||
repo = _stage_fixture(tmp_path / "repo")
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
|
||||
def _fake(cmd, **kw):
|
||||
if cmd[:2] == ["git", "-C"]:
|
||||
sub = cmd[2:]
|
||||
else:
|
||||
sub = cmd
|
||||
if "rev-parse" in sub and "--verify" in sub:
|
||||
return MagicMock(returncode=1, stdout="", stderr="fatal: unknown revision\n")
|
||||
return _fake_subprocess(cmd, **kw)
|
||||
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
|
||||
assert rc == 0
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert manifest["base_ref"] == "main"
|
||||
|
||||
|
||||
def _has_tofu() -> bool:
|
||||
from shutil import which
|
||||
return which("tofu") is not None
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.skipif(not _has_tofu(), reason="tofu not installed")
|
||||
def test_cli_real_tofu_plan(tmp_path, monkeypatch):
|
||||
repo = _stage_fixture(tmp_path / "repo")
|
||||
subprocess.run(["git", "init", "-q", str(repo)], check=True)
|
||||
subprocess.run(["git", "-C", str(repo), "checkout", "-qb", "main"], check=True)
|
||||
subprocess.run(["git", "-C", str(repo), "add", "-A"], check=True)
|
||||
subprocess.run(
|
||||
["git", "-C", str(repo), "-c", "user.email=t@t", "-c", "user.name=t",
|
||||
"commit", "-qm", "init"], check=True,
|
||||
)
|
||||
subprocess.run(["git", "-C", str(repo), "checkout", "-qb", "feature"], check=True)
|
||||
main_tf = repo / "main.tf"
|
||||
main_tf.write_text(main_tf.read_text() + "\n# touched\n")
|
||||
subprocess.run(["git", "-C", str(repo), "add", "-A"], check=True)
|
||||
subprocess.run(
|
||||
["git", "-C", str(repo), "-c", "user.email=t@t", "-c", "user.name=t",
|
||||
"commit", "-qm", "touch"], check=True,
|
||||
)
|
||||
|
||||
out_dir = tmp_path / "out"
|
||||
plugin_cache = tmp_path / "plugin-cache"
|
||||
plugin_cache.mkdir()
|
||||
monkeypatch.setenv("TF_PLUGIN_CACHE_DIR", str(plugin_cache))
|
||||
monkeypatch.setenv("TF_IN_AUTOMATION", "1")
|
||||
|
||||
mod = _load_cli()
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--base", "main",
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert manifest["plan_units"], (
|
||||
f"no plan unit recorded; errors={manifest.get('errors')}"
|
||||
)
|
||||
pu = manifest["plan_units"][0]
|
||||
assert pu["tool"] == "tofu"
|
||||
assert pu["init"]["ok"] is True, pu["init"]
|
||||
assert pu["plan"]["ok"] is True, pu["plan"]
|
||||
assert rc == 0
|
||||
Reference in New Issue
Block a user