mroberts
37fc3fb291
audit-code install-tools.sh:
- Buffer the opengrep release JSON before grep -m1; curl died with (23)
under pipefail when grep quit early.
- Use ${m}: in the PowerShell block; $m: parsed as a scope-qualified var.
- On Arch, skip paru/yay when pacman -Q shows every package installed,
since --needed still invokes sudo.
- Add --check-only (fast, installs nothing, non-zero naming missing tools)
and --user-only (no system package managers, no sudo).
log-run.py (both skills): put the skill dir on sys.path so running it as
a script from any cwd no longer raises ModuleNotFoundError.
audit-terraform: move deps from requirements.txt into pyproject
dependency groups and add scripts/install-tools.sh (uv sync --group tools,
then check trivy, tflint, tofu, terragrunt, gh).
Both SKILL.md files gain a 0.5 Preflight step and call scripts through
uv run --project ${SKILL_DIR}. tools_unavailable is now a map of tool to
exact install command; audit-terraform skips trivy when absent and stops
with an install hint instead of crashing when tofu/terragrunt is missing.
343 lines
12 KiB
Python
Executable File
343 lines
12 KiB
Python
Executable File
"""collect-changes.py — produce a audit-terraform manifest.json."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import concurrent.futures as cf
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
_HERE = Path(__file__).resolve().parent
|
|
if str(_HERE.parent) not in sys.path:
|
|
sys.path.insert(0, str(_HERE.parent))
|
|
|
|
from scripts.catalog import build_catalog, DiffHit, PlanHit
|
|
from scripts.git_diff import (
|
|
changed_files,
|
|
changed_line_ranges,
|
|
is_terragrunt_file,
|
|
)
|
|
from scripts.hcl_diff import touched_resources
|
|
from scripts.manifest import Manifest, PlanResult, PlanUnit
|
|
from scripts.module_graph import build_module_graph
|
|
from scripts.scanners import _run_trivy_config, _run_tflint
|
|
from scripts.slicing import _slice_for_agent
|
|
from scripts.plan_output import parse_plan_resources
|
|
from scripts.plan_runner import detect_tool, run_init, run_plan
|
|
from scripts.reference_set import compute_consistency_norms, compute_reference_sets
|
|
from scripts.resolve_plan_units import resolve_plan_units
|
|
from scripts.source_lookup import find_block
|
|
|
|
|
|
_PLAN_CONCURRENCY = 8
|
|
|
|
_INSTALL_COMMANDS = {
|
|
"trivy": "go install github.com/aquasecurity/trivy/cmd/trivy@latest",
|
|
"tflint": "go install github.com/terraform-linters/tflint@latest",
|
|
"tofu": "go install github.com/opentofu/opentofu/cmd/tofu@latest",
|
|
"terragrunt": "go install github.com/gruntwork-io/terragrunt@latest",
|
|
}
|
|
|
|
|
|
def _resolve_default_branch(repo: Path) -> str:
|
|
try:
|
|
r = subprocess.run(
|
|
["git", "-C", str(repo), "symbolic-ref", "refs/remotes/origin/HEAD"],
|
|
capture_output=True, text=True, check=False,
|
|
)
|
|
if r.returncode == 0:
|
|
return r.stdout.strip().rsplit("/", 1)[-1]
|
|
except FileNotFoundError:
|
|
pass
|
|
for candidate in ("main", "master"):
|
|
r = subprocess.run(
|
|
["git", "-C", str(repo), "rev-parse", f"origin/{candidate}"],
|
|
capture_output=True, text=True, check=False,
|
|
)
|
|
if r.returncode == 0:
|
|
return candidate
|
|
return "main"
|
|
|
|
|
|
def _resolve_base_ref(repo: Path, branch: str) -> str:
|
|
"""Return the diff base for `branch`, fetching origin first so the
|
|
comparison is against the canonical remote tip rather than a stale local
|
|
branch. Falls back to the local branch name if origin isn't reachable.
|
|
"""
|
|
try:
|
|
subprocess.run(
|
|
["git", "-C", str(repo), "fetch", "--quiet", "--no-tags",
|
|
"origin", branch],
|
|
capture_output=True, text=True, check=False, timeout=60,
|
|
)
|
|
except (FileNotFoundError, subprocess.TimeoutExpired):
|
|
pass
|
|
check = subprocess.run(
|
|
["git", "-C", str(repo), "rev-parse", "--verify", "--quiet",
|
|
f"origin/{branch}"],
|
|
capture_output=True, text=True, check=False,
|
|
)
|
|
return f"origin/{branch}" if check.returncode == 0 else branch
|
|
|
|
|
|
def _safe_plan_filename(plan_dir: str) -> str:
|
|
"""Map a plan_dir like '.' or 'live/prod/app' to a stable, filename-safe stem.
|
|
|
|
'.' becomes 'root' (otherwise we'd produce '..txt' on disk).
|
|
"""
|
|
cleaned = plan_dir.replace("/", "_").strip(".")
|
|
return cleaned or "root"
|
|
|
|
|
|
def _run_one(
|
|
repo: Path,
|
|
plan_dir: str,
|
|
triggered_by: list[str],
|
|
changed_files_for_unit: list[str],
|
|
out_dir: Path,
|
|
) -> tuple[PlanUnit, list[PlanHit], str | None]:
|
|
full = repo / plan_dir
|
|
tool = detect_tool(full)
|
|
terragrunt_changed = any(
|
|
is_terragrunt_file(changed_file)
|
|
for changed_file in changed_files_for_unit
|
|
)
|
|
init_res = run_init(full, tool)
|
|
if not init_res.ok:
|
|
plan_res = PlanResult(ok=False, stdout_path="", exit_code=-1,
|
|
summary="init-failed")
|
|
err = f"init failed in {plan_dir}: {init_res.stderr_tail.strip()}"
|
|
return (
|
|
PlanUnit(plan_dir=plan_dir, tool=tool, init=init_res,
|
|
plan=plan_res, triggered_by=triggered_by,
|
|
terragrunt_changed=terragrunt_changed,
|
|
changed_files=changed_files_for_unit),
|
|
[], err,
|
|
)
|
|
|
|
stdout_path = out_dir / "plans" / f"{_safe_plan_filename(plan_dir)}.txt"
|
|
plan_res = run_plan(full, tool, stdout_path)
|
|
if not plan_res.ok:
|
|
captured = Path(plan_res.stdout_path).read_text()[-1000:].strip()
|
|
err = f"plan failed in {plan_dir} (exit {plan_res.exit_code}): {captured}"
|
|
return (
|
|
PlanUnit(plan_dir=plan_dir, tool=tool, init=init_res,
|
|
plan=plan_res, triggered_by=triggered_by,
|
|
terragrunt_changed=terragrunt_changed,
|
|
changed_files=changed_files_for_unit),
|
|
[], err,
|
|
)
|
|
|
|
parsed = parse_plan_resources(Path(plan_res.stdout_path).read_text())
|
|
hits = [
|
|
PlanHit(address=p.address, type=p.type, action=p.action) for p in parsed
|
|
]
|
|
return (
|
|
PlanUnit(plan_dir=plan_dir, tool=tool, init=init_res,
|
|
plan=plan_res, triggered_by=triggered_by,
|
|
terragrunt_changed=terragrunt_changed,
|
|
changed_files=changed_files_for_unit),
|
|
hits, None,
|
|
)
|
|
|
|
|
|
def _changed_files_for_plan_unit(
|
|
all_changed_files: list[str],
|
|
triggered_by: list[str],
|
|
) -> list[str]:
|
|
triggered_dirs = set(triggered_by)
|
|
return sorted(
|
|
changed_file
|
|
for changed_file in all_changed_files
|
|
if str(Path(changed_file).parent.as_posix()) in triggered_dirs
|
|
)
|
|
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--repo", required=True)
|
|
parser.add_argument("--base", default=None)
|
|
parser.add_argument("--head", default="HEAD")
|
|
parser.add_argument("--output-dir", required=True)
|
|
parser.add_argument("--mode", choices=("local", "ref"), required=True)
|
|
args = parser.parse_args(argv)
|
|
|
|
repo = Path(args.repo).resolve()
|
|
out_dir = Path(args.output_dir).resolve()
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
default_branch = _resolve_default_branch(repo)
|
|
base = args.base or _resolve_base_ref(repo, default_branch)
|
|
|
|
errors: list[str] = []
|
|
|
|
try:
|
|
files = changed_files(str(repo), base, args.head)
|
|
dirs = {str(Path(path).parent.as_posix()) or "." for path in files}
|
|
terragrunt_files = {path for path in files if is_terragrunt_file(path)}
|
|
except RuntimeError as e:
|
|
manifest = Manifest(
|
|
base_ref=base, head_ref=args.head, mode=args.mode,
|
|
default_branch=default_branch,
|
|
changed_source_dirs=[], plan_units=[], catalog=[],
|
|
trivy_findings=[],
|
|
module_graph={}, errors=[str(e)],
|
|
)
|
|
(out_dir / "manifest.json").write_text(manifest.to_json())
|
|
return 1
|
|
|
|
if not files:
|
|
manifest = Manifest(
|
|
base_ref=base, head_ref=args.head, mode=args.mode,
|
|
default_branch=default_branch,
|
|
changed_source_dirs=[], plan_units=[], catalog=[],
|
|
trivy_findings=[],
|
|
module_graph={}, errors=["no terraform/hcl files changed"],
|
|
)
|
|
(out_dir / "manifest.json").write_text(manifest.to_json())
|
|
return 0
|
|
|
|
module_graph = build_module_graph(repo)
|
|
plan_units_map, orphan_modules = resolve_plan_units(repo, dirs, module_graph)
|
|
for orphan in orphan_modules:
|
|
errors.append(f"module has no callsites; diff-only review: {orphan}")
|
|
|
|
plan_tools = sorted({detect_tool(repo / pd) for pd in plan_units_map})
|
|
tools_unavailable = {
|
|
t: _INSTALL_COMMANDS[t] for t in ("trivy", "tflint", *plan_tools)
|
|
if shutil.which(t) is None
|
|
}
|
|
missing_plan_tools = [t for t in plan_tools if t in tools_unavailable]
|
|
if missing_plan_tools:
|
|
manifest = Manifest(
|
|
base_ref=base, head_ref=args.head, mode=args.mode,
|
|
default_branch=default_branch,
|
|
changed_source_dirs=sorted(dirs), plan_units=[], catalog=[],
|
|
trivy_findings=[], module_graph=module_graph,
|
|
errors=[
|
|
f"{t} is not installed, so the changed units cannot be planned. "
|
|
f"Install it with `{tools_unavailable[t]}` and re-run."
|
|
for t in missing_plan_tools
|
|
],
|
|
tools_unavailable=tools_unavailable,
|
|
)
|
|
(out_dir / "manifest.json").write_text(manifest.to_json())
|
|
return 1
|
|
|
|
plan_hits: dict[str, list[PlanHit]] = {}
|
|
plan_unit_records: list[PlanUnit] = []
|
|
|
|
if plan_units_map:
|
|
with cf.ThreadPoolExecutor(max_workers=_PLAN_CONCURRENCY) as ex:
|
|
futures = {
|
|
ex.submit(
|
|
_run_one,
|
|
repo,
|
|
pd,
|
|
tb,
|
|
_changed_files_for_plan_unit(files, tb),
|
|
out_dir,
|
|
): pd
|
|
for pd, tb in plan_units_map.items()
|
|
}
|
|
for fut in cf.as_completed(futures):
|
|
record, hits, err = fut.result()
|
|
plan_unit_records.append(record)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
plan_hits[record.plan_dir] = hits
|
|
|
|
diff_hits: list[DiffHit] = []
|
|
for f in files:
|
|
if not f.endswith(".tf"):
|
|
continue
|
|
try:
|
|
ranges = changed_line_ranges(str(repo), base, args.head, f)
|
|
except RuntimeError as e:
|
|
errors.append(f"diff scan failed for {f}: {e}")
|
|
continue
|
|
if not ranges:
|
|
continue
|
|
blocks = touched_resources(repo / f, ranges)
|
|
src_dir = str(Path(f).parent.as_posix()) or "."
|
|
for b in blocks:
|
|
diff_hits.append(DiffHit(
|
|
source_dir=src_dir,
|
|
local_address=f"{b.type}.{b.name}",
|
|
type=b.type,
|
|
))
|
|
|
|
if terragrunt_files:
|
|
changed_plan_dirs = {pu.plan_dir for pu in plan_unit_records}
|
|
for terragrunt_file in sorted(terragrunt_files):
|
|
src_dir = str(Path(terragrunt_file).parent.as_posix()) or "."
|
|
if src_dir not in changed_plan_dirs:
|
|
errors.append(
|
|
f"terragrunt change has no planned unit context: {terragrunt_file}"
|
|
)
|
|
|
|
trivy_payload, trivy_findings, trivy_error = (
|
|
({}, [], None) if "trivy" in tools_unavailable else _run_trivy_config(repo, files)
|
|
)
|
|
(out_dir / "trivy-findings.json").write_text(json.dumps(trivy_payload, indent=2))
|
|
if trivy_error:
|
|
errors.append(trivy_error)
|
|
|
|
tflint_payload, tflint_findings, tflint_error = _run_tflint(repo, files)
|
|
(out_dir / "tflint-findings.json").write_text(json.dumps(tflint_payload, indent=2))
|
|
if tflint_error:
|
|
errors.append(tflint_error)
|
|
|
|
catalog = build_catalog(plan_hits, diff_hits, module_graph)
|
|
for entry in catalog:
|
|
loc = find_block(repo / entry.source_dir, entry.local_address)
|
|
if loc is None:
|
|
continue
|
|
entry.block_header = loc.header
|
|
entry.evidence_line = loc.evidence_line
|
|
entry.key_attributes = loc.key_attributes
|
|
entry.review_context = loc.review_context
|
|
entry.block_file = loc.file.name
|
|
entry.block_start = loc.start_line
|
|
entry.block_end = loc.end_line
|
|
ref_sets = compute_reference_sets(repo, dirs, module_graph)
|
|
(out_dir / "reference_sets.json").write_text(
|
|
json.dumps(ref_sets, indent=2)
|
|
)
|
|
consistency_norms = compute_consistency_norms(repo, ref_sets)
|
|
(out_dir / "consistency_norms.json").write_text(
|
|
json.dumps(consistency_norms, indent=2)
|
|
)
|
|
|
|
manifest = Manifest(
|
|
base_ref=base,
|
|
head_ref=args.head,
|
|
mode=args.mode,
|
|
default_branch=default_branch,
|
|
changed_source_dirs=sorted(dirs),
|
|
plan_units=sorted(plan_unit_records, key=lambda p: p.plan_dir),
|
|
catalog=catalog,
|
|
trivy_findings=trivy_findings,
|
|
tflint_findings=tflint_findings,
|
|
module_graph=module_graph,
|
|
errors=errors,
|
|
tools_unavailable=tools_unavailable,
|
|
)
|
|
(out_dir / "manifest.json").write_text(manifest.to_json())
|
|
manifest_dict = manifest.to_dict()
|
|
for agent in ("fsbp", "cis", "aws-bp", "consistency", "tf-hygiene", "walkthrough"):
|
|
sliced = _slice_for_agent(manifest_dict, agent)
|
|
(out_dir / f"manifest-{agent}.json").write_text(
|
|
json.dumps(sliced, indent=2)
|
|
)
|
|
|
|
return 1 if any(not pu.plan.ok for pu in plan_unit_records) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|