mroberts
37fc3fb291
audit-code install-tools.sh:
- Buffer the opengrep release JSON before grep -m1; curl died with (23)
under pipefail when grep quit early.
- Use ${m}: in the PowerShell block; $m: parsed as a scope-qualified var.
- On Arch, skip paru/yay when pacman -Q shows every package installed,
since --needed still invokes sudo.
- Add --check-only (fast, installs nothing, non-zero naming missing tools)
and --user-only (no system package managers, no sudo).
log-run.py (both skills): put the skill dir on sys.path so running it as
a script from any cwd no longer raises ModuleNotFoundError.
audit-terraform: move deps from requirements.txt into pyproject
dependency groups and add scripts/install-tools.sh (uv sync --group tools,
then check trivy, tflint, tofu, terragrunt, gh).
Both SKILL.md files gain a 0.5 Preflight step and call scripts through
uv run --project ${SKILL_DIR}. tools_unavailable is now a map of tool to
exact install command; audit-terraform skips trivy when absent and stops
with an install hint instead of crashing when tofu/terragrunt is missing.
467 lines
17 KiB
Python
467 lines
17 KiB
Python
import importlib.util
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
import pytest
|
|
|
|
|
|
_SKILL_ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def _load_cli():
|
|
spec = importlib.util.spec_from_file_location(
|
|
"collect_changes", _SKILL_ROOT / "scripts" / "collect-changes.py"
|
|
)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
def _stage_fixture(dest: Path) -> Path:
|
|
src = _SKILL_ROOT / "tests" / "fixtures" / "tofu-sample"
|
|
shutil.copytree(src, dest)
|
|
return dest
|
|
|
|
|
|
def _fake_subprocess(cmd, **kwargs):
|
|
if cmd[:2] == ["git", "-C"]:
|
|
sub = cmd[2:]
|
|
else:
|
|
sub = cmd
|
|
|
|
if "symbolic-ref" in sub:
|
|
return MagicMock(returncode=0, stdout="refs/remotes/origin/main\n", stderr="")
|
|
if "fetch" in sub:
|
|
return MagicMock(returncode=0, stdout="", stderr="")
|
|
if "rev-parse" in sub and "--verify" in sub:
|
|
return MagicMock(returncode=0, stdout="abc123\n", stderr="")
|
|
if cmd[:1] == ["git"] and "diff" in cmd:
|
|
diff = (
|
|
"diff --git a/main.tf b/main.tf\n"
|
|
"index 1..2 100644\n"
|
|
"--- a/main.tf\n"
|
|
"+++ b/main.tf\n"
|
|
"@@ -20,0 +21,1 @@\n"
|
|
"+ bucket_prefix = \"x\"\n"
|
|
"diff --git a/modules/widget/main.tf b/modules/widget/main.tf\n"
|
|
"index 3..4 100644\n"
|
|
"--- a/modules/widget/main.tf\n"
|
|
"+++ b/modules/widget/main.tf\n"
|
|
"@@ -1,0 +2,1 @@\n"
|
|
"+ # touched\n"
|
|
)
|
|
return MagicMock(returncode=0, stdout=diff, stderr="")
|
|
if cmd[0] == "tofu" and "init" in cmd:
|
|
return MagicMock(returncode=0, stdout="initialized\n", stderr="")
|
|
if cmd[0] == "tofu" and "plan" in cmd:
|
|
plan = (
|
|
"OpenTofu will perform the following actions:\n\n"
|
|
" # null_resource.top will be updated in-place\n"
|
|
" ~ resource \"null_resource\" \"top\" {}\n\n"
|
|
" # module.widget_a.null_resource.thing will be updated in-place\n"
|
|
" ~ resource \"null_resource\" \"thing\" {}\n\n"
|
|
" # module.widget_b.null_resource.thing will be updated in-place\n"
|
|
" ~ resource \"null_resource\" \"thing\" {}\n\n"
|
|
"Plan: 0 to add, 3 to change, 0 to destroy.\n"
|
|
)
|
|
return MagicMock(returncode=0, stdout=plan, stderr="")
|
|
if cmd[:3] == ["trivy", "config", "--quiet"]:
|
|
trivy = {
|
|
"SchemaVersion": 2,
|
|
"Results": [
|
|
{
|
|
"Target": "main.tf",
|
|
"Class": "config",
|
|
"Type": "terraform",
|
|
"Misconfigurations": [
|
|
{
|
|
"ID": "AVD-AWS-0089",
|
|
"AVDID": "AVD-AWS-0089",
|
|
"Title": "S3 bucket allows public ACL",
|
|
"Description": "Buckets should not allow public ACLs.",
|
|
"Message": "Bucket ACL allows public access.",
|
|
"Severity": "HIGH",
|
|
"CauseMetadata": {
|
|
"Resource": "aws_s3_bucket.audit_logs",
|
|
"StartLine": 21,
|
|
"EndLine": 30,
|
|
},
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
return MagicMock(returncode=0, stdout=json.dumps(trivy), stderr="")
|
|
return MagicMock(returncode=0, stdout="", stderr="")
|
|
|
|
|
|
def test_cli_happy_path(tmp_path):
|
|
repo = _stage_fixture(tmp_path / "repo")
|
|
out_dir = tmp_path / "out"
|
|
mod = _load_cli()
|
|
|
|
with patch("subprocess.run", side_effect=_fake_subprocess):
|
|
rc = mod.main([
|
|
"--repo", str(repo),
|
|
"--base", "main",
|
|
"--head", "HEAD",
|
|
"--output-dir", str(out_dir),
|
|
"--mode", "local",
|
|
])
|
|
|
|
assert rc == 0, (out_dir / "manifest.json").read_text()
|
|
manifest = json.loads((out_dir / "manifest.json").read_text())
|
|
assert manifest["mode"] == "local"
|
|
assert manifest["base_ref"] == "main"
|
|
assert set(manifest["changed_source_dirs"]) == {".", "modules/widget"}
|
|
plan_dirs = {pu["plan_dir"] for pu in manifest["plan_units"]}
|
|
assert plan_dirs == {"."}
|
|
pu = manifest["plan_units"][0]
|
|
assert pu["tool"] == "tofu"
|
|
assert pu["plan"]["summary"] == "0 to add, 3 to change, 0 to destroy"
|
|
catalog = manifest["catalog"]
|
|
keys = {(e["source_dir"], e["local_address"]) for e in catalog}
|
|
assert (".", "null_resource.top") in keys
|
|
assert ("modules/widget", "null_resource.thing") in keys
|
|
module_entries = [e for e in catalog if e["source_dir"] == "modules/widget"]
|
|
for e in module_entries:
|
|
plan_dirs = {i["plan_dir"] for i in e["instances"]}
|
|
assert plan_dirs == {"."}, e
|
|
addrs = {i["address_at_plan"] for i in e["instances"]}
|
|
assert any(a.startswith("module.widget_a") for a in addrs)
|
|
assert any(a.startswith("module.widget_b") for a in addrs)
|
|
for e in module_entries:
|
|
assert e["block_header"], f"missing block_header on {e}"
|
|
assert e["evidence_line"], f"missing evidence_line on {e}"
|
|
assert isinstance(e["key_attributes"], dict)
|
|
assert "review_context" in e
|
|
assert set(e["review_context"]) == {"variables", "locals", "related_blocks"}
|
|
assert e["block_file"].endswith(".tf")
|
|
assert e["block_start"] >= 1
|
|
assert "block_text" not in e
|
|
assert (out_dir / "trivy-findings.json").is_file()
|
|
assert manifest["trivy_findings"] == [
|
|
{
|
|
"check_id": "AVD-AWS-0089",
|
|
"title": "S3 bucket allows public ACL",
|
|
"severity": "high",
|
|
"message": "Bucket ACL allows public access.",
|
|
"file": "main.tf",
|
|
"start_line": 21,
|
|
"end_line": 30,
|
|
"resource_type": "aws_s3_bucket",
|
|
"source": "trivy",
|
|
}
|
|
]
|
|
refs = json.loads((out_dir / "reference_sets.json").read_text())
|
|
assert "." in refs or "modules/widget" in refs
|
|
|
|
|
|
def test_cli_writes_per_agent_slices(tmp_path):
|
|
repo = _stage_fixture(tmp_path / "repo")
|
|
out_dir = tmp_path / "out"
|
|
mod = _load_cli()
|
|
with patch("subprocess.run", side_effect=_fake_subprocess):
|
|
rc = mod.main([
|
|
"--repo", str(repo), "--base", "main", "--head", "HEAD",
|
|
"--output-dir", str(out_dir), "--mode", "local",
|
|
])
|
|
assert rc == 0
|
|
full = json.loads((out_dir / "manifest.json").read_text())
|
|
for agent in ("fsbp", "cis", "aws-bp", "consistency"):
|
|
sliced = json.loads((out_dir / f"manifest-{agent}.json").read_text())
|
|
assert sliced["base_ref"] == full["base_ref"]
|
|
if agent in ("fsbp", "cis", "aws-bp"):
|
|
for entry in sliced["catalog"]:
|
|
assert entry["type"].startswith("aws_"), (
|
|
f"non-aws type leaked into {agent}: {entry['type']}"
|
|
)
|
|
assert sliced["catalog"] == []
|
|
assert sliced["trivy_findings"] == full["trivy_findings"]
|
|
if agent == "consistency":
|
|
assert sliced["changed_source_dirs"] == full["changed_source_dirs"]
|
|
assert len(sliced["catalog"]) == len(full["catalog"])
|
|
assert sliced["trivy_findings"] == full["trivy_findings"]
|
|
|
|
|
|
def test_cli_records_missing_scanners_with_install_command(tmp_path):
|
|
repo = _stage_fixture(tmp_path / "repo")
|
|
out_dir = tmp_path / "out"
|
|
mod = _load_cli()
|
|
ran: list[str] = []
|
|
|
|
def _fake(cmd, **kw):
|
|
ran.append(cmd[0])
|
|
return _fake_subprocess(cmd, **kw)
|
|
|
|
with patch("subprocess.run", side_effect=_fake), \
|
|
patch("shutil.which", side_effect=lambda t: None if t in ("trivy", "tflint") else f"/usr/bin/{t}"):
|
|
rc = mod.main([
|
|
"--repo", str(repo), "--base", "main", "--head", "HEAD",
|
|
"--output-dir", str(out_dir), "--mode", "local",
|
|
])
|
|
assert rc == 0
|
|
manifest = json.loads((out_dir / "manifest.json").read_text())
|
|
assert manifest["tools_unavailable"] == {
|
|
"trivy": "go install github.com/aquasecurity/trivy/cmd/trivy@latest",
|
|
"tflint": "go install github.com/terraform-linters/tflint@latest",
|
|
}
|
|
assert "trivy" not in ran
|
|
assert manifest["trivy_findings"] == []
|
|
|
|
|
|
|
|
def test_cli_stops_with_install_command_when_plan_tool_missing(tmp_path):
|
|
repo = _stage_fixture(tmp_path / "repo")
|
|
out_dir = tmp_path / "out"
|
|
mod = _load_cli()
|
|
with patch("subprocess.run", side_effect=_fake_subprocess), \
|
|
patch("shutil.which", side_effect=lambda t: None if t == "tofu" else f"/usr/bin/{t}"):
|
|
rc = mod.main([
|
|
"--repo", str(repo), "--base", "main", "--head", "HEAD",
|
|
"--output-dir", str(out_dir), "--mode", "local",
|
|
])
|
|
assert rc == 1
|
|
manifest = json.loads((out_dir / "manifest.json").read_text())
|
|
install = "go install github.com/opentofu/opentofu/cmd/tofu@latest"
|
|
assert manifest["tools_unavailable"] == {"tofu": install}
|
|
assert manifest["errors"] == [
|
|
f"tofu is not installed, so the changed units cannot be planned. Install it with `{install}` and re-run."
|
|
]
|
|
|
|
def test_cli_preserves_terragrunt_only_change_context(tmp_path):
|
|
repo = _stage_fixture(tmp_path / "repo")
|
|
terragrunt_dir = repo / "live" / "prod" / "app"
|
|
terragrunt_dir.mkdir(parents=True)
|
|
(terragrunt_dir / "terragrunt.hcl").write_text("inputs = { instance_count = 2 }\n")
|
|
out_dir = tmp_path / "out"
|
|
mod = _load_cli()
|
|
|
|
def _fake(cmd, **kw):
|
|
if cmd[:1] == ["git"] and "diff" in cmd:
|
|
diff = (
|
|
"diff --git a/live/prod/app/terragrunt.hcl b/live/prod/app/terragrunt.hcl\n"
|
|
"index 1..2 100644\n"
|
|
"--- a/live/prod/app/terragrunt.hcl\n"
|
|
"+++ b/live/prod/app/terragrunt.hcl\n"
|
|
"@@ -1 +1 @@\n"
|
|
"-inputs = { instance_count = 1 }\n"
|
|
"+inputs = { instance_count = 2 }\n"
|
|
)
|
|
return MagicMock(returncode=0, stdout=diff, stderr="")
|
|
if cmd[0] == "terragrunt" and "init" in cmd:
|
|
return MagicMock(returncode=0, stdout="initialized\n", stderr="")
|
|
if cmd[0] == "terragrunt" and "plan" in cmd:
|
|
plan = (
|
|
"OpenTofu will perform the following actions:\n\n"
|
|
"Plan: 0 to add, 0 to change, 0 to destroy.\n"
|
|
)
|
|
return MagicMock(returncode=0, stdout=plan, stderr="")
|
|
return _fake_subprocess(cmd, **kw)
|
|
|
|
with patch("subprocess.run", side_effect=_fake):
|
|
rc = mod.main([
|
|
"--repo", str(repo),
|
|
"--base", "main",
|
|
"--head", "HEAD",
|
|
"--output-dir", str(out_dir),
|
|
"--mode", "local",
|
|
])
|
|
|
|
assert rc == 0, (out_dir / "manifest.json").read_text()
|
|
manifest = json.loads((out_dir / "manifest.json").read_text())
|
|
assert "live/prod/app" in manifest["changed_source_dirs"]
|
|
assert manifest["catalog"] == []
|
|
assert manifest["plan_units"] == [{
|
|
"plan_dir": "live/prod/app",
|
|
"tool": "terragrunt",
|
|
"init": {"ok": True, "stdout_tail": "initialized\n", "stderr_tail": ""},
|
|
"plan": {
|
|
"ok": True,
|
|
"stdout_path": str(out_dir / "plans" / "live_prod_app.txt"),
|
|
"exit_code": 0,
|
|
"summary": "0 to add, 0 to change, 0 to destroy",
|
|
},
|
|
"triggered_by": ["live/prod/app"],
|
|
"terragrunt_changed": True,
|
|
"changed_files": ["live/prod/app/terragrunt.hcl"],
|
|
}]
|
|
fsbp = json.loads((out_dir / "manifest-fsbp.json").read_text())
|
|
assert fsbp["plan_units"] == [{
|
|
"plan_dir": "live/prod/app",
|
|
"tool": "terragrunt",
|
|
"plan_ok": True,
|
|
"summary": "0 to add, 0 to change, 0 to destroy",
|
|
"terragrunt_changed": True,
|
|
"changed_files": ["live/prod/app/terragrunt.hcl"],
|
|
}]
|
|
consistency = json.loads((out_dir / "manifest-consistency.json").read_text())
|
|
assert consistency["plan_units"] == manifest["plan_units"]
|
|
|
|
|
|
def test_cli_aborts_when_plan_fails(tmp_path):
|
|
repo = _stage_fixture(tmp_path / "repo")
|
|
out_dir = tmp_path / "out"
|
|
mod = _load_cli()
|
|
|
|
def _fake(cmd, **kw):
|
|
if cmd[0] == "tofu" and "plan" in cmd:
|
|
return MagicMock(
|
|
returncode=1,
|
|
stdout="Error: syntax error in main.tf\n",
|
|
stderr="",
|
|
)
|
|
return _fake_subprocess(cmd, **kw)
|
|
|
|
with patch("subprocess.run", side_effect=_fake):
|
|
rc = mod.main([
|
|
"--repo", str(repo),
|
|
"--base", "main",
|
|
"--head", "HEAD",
|
|
"--output-dir", str(out_dir),
|
|
"--mode", "local",
|
|
])
|
|
|
|
assert rc == 1
|
|
manifest = json.loads((out_dir / "manifest.json").read_text())
|
|
assert any("syntax error" in e for e in manifest["errors"])
|
|
|
|
|
|
def test_cli_records_git_diff_failure_in_manifest(tmp_path):
|
|
repo = _stage_fixture(tmp_path / "repo")
|
|
out_dir = tmp_path / "out"
|
|
mod = _load_cli()
|
|
|
|
def _fake(cmd, **kw):
|
|
if cmd[:1] == ["git"] and "diff" in cmd:
|
|
return MagicMock(returncode=128, stdout="",
|
|
stderr="fatal: bad revision 'main...HEAD'\n")
|
|
return _fake_subprocess(cmd, **kw)
|
|
|
|
with patch("subprocess.run", side_effect=_fake):
|
|
rc = mod.main([
|
|
"--repo", str(repo),
|
|
"--base", "main",
|
|
"--head", "HEAD",
|
|
"--output-dir", str(out_dir),
|
|
"--mode", "local",
|
|
])
|
|
|
|
assert rc == 1
|
|
manifest = json.loads((out_dir / "manifest.json").read_text())
|
|
assert any("bad revision" in e for e in manifest["errors"])
|
|
|
|
|
|
def test_cli_auto_resolves_base_to_origin(tmp_path):
|
|
"""With no --base, the CLI fetches and diffs against origin/<default>."""
|
|
repo = _stage_fixture(tmp_path / "repo")
|
|
out_dir = tmp_path / "out"
|
|
mod = _load_cli()
|
|
|
|
captured_diff_base: list[str] = []
|
|
|
|
def _fake(cmd, **kw):
|
|
if cmd[:1] == ["git"] and "diff" in cmd:
|
|
for arg in cmd:
|
|
if "..." in arg:
|
|
captured_diff_base.append(arg)
|
|
return _fake_subprocess(cmd, **kw)
|
|
|
|
with patch("subprocess.run", side_effect=_fake):
|
|
rc = mod.main([
|
|
"--repo", str(repo),
|
|
"--head", "HEAD",
|
|
"--output-dir", str(out_dir),
|
|
"--mode", "local",
|
|
])
|
|
|
|
assert rc == 0
|
|
assert captured_diff_base, "git diff was never invoked"
|
|
assert captured_diff_base[0].startswith("origin/main..."), (
|
|
f"diff base should be origin/<default>, got: {captured_diff_base[0]}"
|
|
)
|
|
manifest = json.loads((out_dir / "manifest.json").read_text())
|
|
assert manifest["base_ref"] == "origin/main"
|
|
|
|
|
|
def test_cli_falls_back_to_local_branch_when_origin_missing(tmp_path):
|
|
"""If origin/<default> doesn't exist (no remote, fetch fails), fall back."""
|
|
repo = _stage_fixture(tmp_path / "repo")
|
|
out_dir = tmp_path / "out"
|
|
mod = _load_cli()
|
|
|
|
def _fake(cmd, **kw):
|
|
if cmd[:2] == ["git", "-C"]:
|
|
sub = cmd[2:]
|
|
else:
|
|
sub = cmd
|
|
if "rev-parse" in sub and "--verify" in sub:
|
|
return MagicMock(returncode=1, stdout="", stderr="fatal: unknown revision\n")
|
|
return _fake_subprocess(cmd, **kw)
|
|
|
|
with patch("subprocess.run", side_effect=_fake):
|
|
rc = mod.main([
|
|
"--repo", str(repo),
|
|
"--head", "HEAD",
|
|
"--output-dir", str(out_dir),
|
|
"--mode", "local",
|
|
])
|
|
|
|
assert rc == 0
|
|
manifest = json.loads((out_dir / "manifest.json").read_text())
|
|
assert manifest["base_ref"] == "main"
|
|
|
|
|
|
def _has_tofu() -> bool:
|
|
from shutil import which
|
|
return which("tofu") is not None
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.skipif(not _has_tofu(), reason="tofu not installed")
|
|
def test_cli_real_tofu_plan(tmp_path, monkeypatch):
|
|
repo = _stage_fixture(tmp_path / "repo")
|
|
subprocess.run(["git", "init", "-q", str(repo)], check=True)
|
|
subprocess.run(["git", "-C", str(repo), "checkout", "-qb", "main"], check=True)
|
|
subprocess.run(["git", "-C", str(repo), "add", "-A"], check=True)
|
|
subprocess.run(
|
|
["git", "-C", str(repo), "-c", "user.email=t@t", "-c", "user.name=t",
|
|
"commit", "-qm", "init"], check=True,
|
|
)
|
|
subprocess.run(["git", "-C", str(repo), "checkout", "-qb", "feature"], check=True)
|
|
main_tf = repo / "main.tf"
|
|
main_tf.write_text(main_tf.read_text() + "\n# touched\n")
|
|
subprocess.run(["git", "-C", str(repo), "add", "-A"], check=True)
|
|
subprocess.run(
|
|
["git", "-C", str(repo), "-c", "user.email=t@t", "-c", "user.name=t",
|
|
"commit", "-qm", "touch"], check=True,
|
|
)
|
|
|
|
out_dir = tmp_path / "out"
|
|
plugin_cache = tmp_path / "plugin-cache"
|
|
plugin_cache.mkdir()
|
|
monkeypatch.setenv("TF_PLUGIN_CACHE_DIR", str(plugin_cache))
|
|
monkeypatch.setenv("TF_IN_AUTOMATION", "1")
|
|
|
|
mod = _load_cli()
|
|
rc = mod.main([
|
|
"--repo", str(repo),
|
|
"--base", "main",
|
|
"--head", "HEAD",
|
|
"--output-dir", str(out_dir),
|
|
"--mode", "local",
|
|
])
|
|
|
|
manifest = json.loads((out_dir / "manifest.json").read_text())
|
|
assert manifest["plan_units"], (
|
|
f"no plan unit recorded; errors={manifest.get('errors')}"
|
|
)
|
|
pu = manifest["plan_units"][0]
|
|
assert pu["tool"] == "tofu"
|
|
assert pu["init"]["ok"] is True, pu["init"]
|
|
assert pu["plan"]["ok"] is True, pu["plan"]
|
|
assert rc == 0
|