"""refresh-controls.py — populate data/controls/{fsbp,cis,meta}.json.""" from __future__ import annotations import argparse import json import sys from datetime import datetime, timezone, UTC from pathlib import Path import requests _HERE = Path(__file__).resolve().parent if str(_HERE.parent) not in sys.path: sys.path.insert(0, str(_HERE.parent)) from scripts.controls_schema import Control, ControlsFile from scripts.scrape_cis import CIS_URL, parse_cis_page from scripts.scrape_fsbp import FSBP_INDEX_URL, parse_fsbp_index _TIMEOUT = 30 _USER_AGENT = "audit-terraform-controls-refresh/1.0" def _fetch(url: str) -> str: r = requests.get(url, headers={"User-Agent": _USER_AGENT}, timeout=_TIMEOUT) r.raise_for_status() return r.text def _now() -> datetime: return datetime.now(UTC) def _build_fsbp() -> ControlsFile: html = _fetch(FSBP_INDEX_URL) rows = parse_fsbp_index(html, base_url=FSBP_INDEX_URL) controls = [ Control( control_id=f"FSBP {r.control_id}", title=r.title, severity=r.severity, resource_types=r.resource_types, requirement=r.requirement or r.title, source_url=r.detail_url, ) for r in rows ] return ControlsFile(source="fsbp", fetched_at=_now(), controls=controls) def _build_cis() -> ControlsFile: html = _fetch(CIS_URL) rows = parse_cis_page(html, source_url=CIS_URL) controls = [ Control( control_id=r.control_id, title=r.title, severity=r.severity, resource_types=r.resource_types, requirement=r.requirement, source_url=r.source_url, ) for r in rows ] return ControlsFile(source="cis", fetched_at=_now(), controls=controls) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument("--output-dir", default=str(_HERE.parent / "data" / "controls")) args = parser.parse_args(argv) out_dir = Path(args.output_dir).resolve() out_dir.mkdir(parents=True, exist_ok=True) fsbp = _build_fsbp() cis = _build_cis() (out_dir / "fsbp.json").write_text(fsbp.to_json()) (out_dir / "cis.json").write_text(cis.to_json()) meta = { "fsbp": {"url": FSBP_INDEX_URL, "fetched_at": fsbp.fetched_at.isoformat()}, "cis": {"url": CIS_URL, "fetched_at": cis.fetched_at.isoformat()}, } (out_dir / "meta.json").write_text(json.dumps(meta, indent=2)) print(f"Wrote {len(fsbp.controls)} FSBP controls, {len(cis.controls)} CIS controls " f"to {out_dir}") return 0 if __name__ == "__main__": sys.exit(main())