"""collect-changes.py — produce a audit-terraform manifest.json.""" from __future__ import annotations import argparse import concurrent.futures as cf import json import shutil import subprocess import sys from pathlib import Path _HERE = Path(__file__).resolve().parent if str(_HERE.parent) not in sys.path: sys.path.insert(0, str(_HERE.parent)) from scripts.catalog import build_catalog, DiffHit, PlanHit from scripts.git_diff import ( changed_files, changed_line_ranges, is_terragrunt_file, ) from scripts.hcl_diff import touched_resources from scripts.manifest import Manifest, PlanResult, PlanUnit from scripts.module_graph import build_module_graph from scripts.scanners import _run_trivy_config, _run_tflint from scripts.slicing import _slice_for_agent from scripts.plan_output import parse_plan_resources from scripts.plan_runner import detect_tool, run_init, run_plan from scripts.reference_set import compute_consistency_norms, compute_reference_sets from scripts.resolve_plan_units import resolve_plan_units from scripts.source_lookup import find_block _PLAN_CONCURRENCY = 8 _INSTALL_COMMANDS = { "trivy": "go install github.com/aquasecurity/trivy/cmd/trivy@latest", "tflint": "go install github.com/terraform-linters/tflint@latest", "tofu": "go install github.com/opentofu/opentofu/cmd/tofu@latest", "terragrunt": "go install github.com/gruntwork-io/terragrunt@latest", } def _resolve_default_branch(repo: Path) -> str: try: r = subprocess.run( ["git", "-C", str(repo), "symbolic-ref", "refs/remotes/origin/HEAD"], capture_output=True, text=True, check=False, ) if r.returncode == 0: return r.stdout.strip().rsplit("/", 1)[-1] except FileNotFoundError: pass for candidate in ("main", "master"): r = subprocess.run( ["git", "-C", str(repo), "rev-parse", f"origin/{candidate}"], capture_output=True, text=True, check=False, ) if r.returncode == 0: return candidate return "main" def _resolve_base_ref(repo: Path, branch: str) -> str: """Return the diff base for `branch`, fetching origin first so the comparison is against the canonical remote tip rather than a stale local branch. Falls back to the local branch name if origin isn't reachable. """ try: subprocess.run( ["git", "-C", str(repo), "fetch", "--quiet", "--no-tags", "origin", branch], capture_output=True, text=True, check=False, timeout=60, ) except (FileNotFoundError, subprocess.TimeoutExpired): pass check = subprocess.run( ["git", "-C", str(repo), "rev-parse", "--verify", "--quiet", f"origin/{branch}"], capture_output=True, text=True, check=False, ) return f"origin/{branch}" if check.returncode == 0 else branch def _safe_plan_filename(plan_dir: str) -> str: """Map a plan_dir like '.' or 'live/prod/app' to a stable, filename-safe stem. '.' becomes 'root' (otherwise we'd produce '..txt' on disk). """ cleaned = plan_dir.replace("/", "_").strip(".") return cleaned or "root" def _run_one( repo: Path, plan_dir: str, triggered_by: list[str], changed_files_for_unit: list[str], out_dir: Path, ) -> tuple[PlanUnit, list[PlanHit], str | None]: full = repo / plan_dir tool = detect_tool(full) terragrunt_changed = any( is_terragrunt_file(changed_file) for changed_file in changed_files_for_unit ) init_res = run_init(full, tool) if not init_res.ok: plan_res = PlanResult(ok=False, stdout_path="", exit_code=-1, summary="init-failed") err = f"init failed in {plan_dir}: {init_res.stderr_tail.strip()}" return ( PlanUnit(plan_dir=plan_dir, tool=tool, init=init_res, plan=plan_res, triggered_by=triggered_by, terragrunt_changed=terragrunt_changed, changed_files=changed_files_for_unit), [], err, ) stdout_path = out_dir / "plans" / f"{_safe_plan_filename(plan_dir)}.txt" plan_res = run_plan(full, tool, stdout_path) if not plan_res.ok: captured = Path(plan_res.stdout_path).read_text()[-1000:].strip() err = f"plan failed in {plan_dir} (exit {plan_res.exit_code}): {captured}" return ( PlanUnit(plan_dir=plan_dir, tool=tool, init=init_res, plan=plan_res, triggered_by=triggered_by, terragrunt_changed=terragrunt_changed, changed_files=changed_files_for_unit), [], err, ) parsed = parse_plan_resources(Path(plan_res.stdout_path).read_text()) hits = [ PlanHit(address=p.address, type=p.type, action=p.action) for p in parsed ] return ( PlanUnit(plan_dir=plan_dir, tool=tool, init=init_res, plan=plan_res, triggered_by=triggered_by, terragrunt_changed=terragrunt_changed, changed_files=changed_files_for_unit), hits, None, ) def _changed_files_for_plan_unit( all_changed_files: list[str], triggered_by: list[str], ) -> list[str]: triggered_dirs = set(triggered_by) return sorted( changed_file for changed_file in all_changed_files if str(Path(changed_file).parent.as_posix()) in triggered_dirs ) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument("--repo", required=True) parser.add_argument("--base", default=None) parser.add_argument("--head", default="HEAD") parser.add_argument("--output-dir", required=True) parser.add_argument("--mode", choices=("local", "ref"), required=True) args = parser.parse_args(argv) repo = Path(args.repo).resolve() out_dir = Path(args.output_dir).resolve() out_dir.mkdir(parents=True, exist_ok=True) default_branch = _resolve_default_branch(repo) base = args.base or _resolve_base_ref(repo, default_branch) errors: list[str] = [] try: files = changed_files(str(repo), base, args.head) dirs = {str(Path(path).parent.as_posix()) or "." for path in files} terragrunt_files = {path for path in files if is_terragrunt_file(path)} except RuntimeError as e: manifest = Manifest( base_ref=base, head_ref=args.head, mode=args.mode, default_branch=default_branch, changed_source_dirs=[], plan_units=[], catalog=[], trivy_findings=[], module_graph={}, errors=[str(e)], ) (out_dir / "manifest.json").write_text(manifest.to_json()) return 1 if not files: manifest = Manifest( base_ref=base, head_ref=args.head, mode=args.mode, default_branch=default_branch, changed_source_dirs=[], plan_units=[], catalog=[], trivy_findings=[], module_graph={}, errors=["no terraform/hcl files changed"], ) (out_dir / "manifest.json").write_text(manifest.to_json()) return 0 module_graph = build_module_graph(repo) plan_units_map, orphan_modules = resolve_plan_units(repo, dirs, module_graph) for orphan in orphan_modules: errors.append(f"module has no callsites; diff-only review: {orphan}") plan_tools = sorted({detect_tool(repo / pd) for pd in plan_units_map}) tools_unavailable = { t: _INSTALL_COMMANDS[t] for t in ("trivy", "tflint", *plan_tools) if shutil.which(t) is None } missing_plan_tools = [t for t in plan_tools if t in tools_unavailable] if missing_plan_tools: manifest = Manifest( base_ref=base, head_ref=args.head, mode=args.mode, default_branch=default_branch, changed_source_dirs=sorted(dirs), plan_units=[], catalog=[], trivy_findings=[], module_graph=module_graph, errors=[ f"{t} is not installed, so the changed units cannot be planned. " f"Install it with `{tools_unavailable[t]}` and re-run." for t in missing_plan_tools ], tools_unavailable=tools_unavailable, ) (out_dir / "manifest.json").write_text(manifest.to_json()) return 1 plan_hits: dict[str, list[PlanHit]] = {} plan_unit_records: list[PlanUnit] = [] if plan_units_map: with cf.ThreadPoolExecutor(max_workers=_PLAN_CONCURRENCY) as ex: futures = { ex.submit( _run_one, repo, pd, tb, _changed_files_for_plan_unit(files, tb), out_dir, ): pd for pd, tb in plan_units_map.items() } for fut in cf.as_completed(futures): record, hits, err = fut.result() plan_unit_records.append(record) if err: errors.append(err) else: plan_hits[record.plan_dir] = hits diff_hits: list[DiffHit] = [] for f in files: if not f.endswith(".tf"): continue try: ranges = changed_line_ranges(str(repo), base, args.head, f) except RuntimeError as e: errors.append(f"diff scan failed for {f}: {e}") continue if not ranges: continue blocks = touched_resources(repo / f, ranges) src_dir = str(Path(f).parent.as_posix()) or "." for b in blocks: diff_hits.append(DiffHit( source_dir=src_dir, local_address=f"{b.type}.{b.name}", type=b.type, )) if terragrunt_files: changed_plan_dirs = {pu.plan_dir for pu in plan_unit_records} for terragrunt_file in sorted(terragrunt_files): src_dir = str(Path(terragrunt_file).parent.as_posix()) or "." if src_dir not in changed_plan_dirs: errors.append( f"terragrunt change has no planned unit context: {terragrunt_file}" ) trivy_payload, trivy_findings, trivy_error = ( ({}, [], None) if "trivy" in tools_unavailable else _run_trivy_config(repo, files) ) (out_dir / "trivy-findings.json").write_text(json.dumps(trivy_payload, indent=2)) if trivy_error: errors.append(trivy_error) tflint_payload, tflint_findings, tflint_error = _run_tflint(repo, files) (out_dir / "tflint-findings.json").write_text(json.dumps(tflint_payload, indent=2)) if tflint_error: errors.append(tflint_error) catalog = build_catalog(plan_hits, diff_hits, module_graph) for entry in catalog: loc = find_block(repo / entry.source_dir, entry.local_address) if loc is None: continue entry.block_header = loc.header entry.evidence_line = loc.evidence_line entry.key_attributes = loc.key_attributes entry.review_context = loc.review_context entry.block_file = loc.file.name entry.block_start = loc.start_line entry.block_end = loc.end_line ref_sets = compute_reference_sets(repo, dirs, module_graph) (out_dir / "reference_sets.json").write_text( json.dumps(ref_sets, indent=2) ) consistency_norms = compute_consistency_norms(repo, ref_sets) (out_dir / "consistency_norms.json").write_text( json.dumps(consistency_norms, indent=2) ) manifest = Manifest( base_ref=base, head_ref=args.head, mode=args.mode, default_branch=default_branch, changed_source_dirs=sorted(dirs), plan_units=sorted(plan_unit_records, key=lambda p: p.plan_dir), catalog=catalog, trivy_findings=trivy_findings, tflint_findings=tflint_findings, module_graph=module_graph, errors=errors, tools_unavailable=tools_unavailable, ) (out_dir / "manifest.json").write_text(manifest.to_json()) manifest_dict = manifest.to_dict() for agent in ("fsbp", "cis", "aws-bp", "consistency", "tf-hygiene", "walkthrough"): sliced = _slice_for_agent(manifest_dict, agent) (out_dir / f"manifest-{agent}.json").write_text( json.dumps(sliced, indent=2) ) return 1 if any(not pu.plan.ok for pu in plan_unit_records) else 0 if __name__ == "__main__": sys.exit(main())