Move the code and terraform audits into the reviews plugin
Copy the standalone code-review and terraform-review skills into
plugins/reviews as audit-code and audit-terraform. The rename separates the
automated, linter-driven audits from the guided review-pr walkthrough that
already lived here.
Resolve bundled script paths through ${SKILL_DIR}, exported in a new step 0.
CLAUDE_PLUGIN_ROOT is not set in the Bash tool environment, so the obvious
substitution would have expanded to nothing and broken every collection
script invocation.
Replace the PLAN and DESIGN docs with READMEs written from the current
SKILL.md and scripts. The old docs had drifted badly: they named semgrep
where the code calls opengrep, scoped five review agents where there are
now eight, and predated Lua, PowerShell, and GitHub Actions support.
Add CONSISTENCY_NORMS to the audit-terraform agent inputs. The collection
script writes consistency_norms.json and the agent prompt declares it, but
SKILL.md never listed it, leaving the variable unsubstituted.
Drop the --ingest-verdicts instruction from both skills. review_stats.py
parses no arguments, so the ref-mode verdict template it told users to feed
back could never be read.
Point audit-terraform's smoke test at README.md and resolve its fixture
paths relative to the test file rather than an absolute home directory.
Tests: 197 passing (audit-code), 106 passing (audit-terraform).
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_root = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_root))
|
||||
@@ -0,0 +1,18 @@
|
||||
[
|
||||
{
|
||||
"message": "shellcheck reported issue in this script: SC2086:info:1:6: Double quote to prevent globbing and word splitting",
|
||||
"filepath": ".github/workflows/ci.yml",
|
||||
"line": 22,
|
||||
"column": 9,
|
||||
"kind": "shellcheck",
|
||||
"snippet": " run: echo $SOME_VAR"
|
||||
},
|
||||
{
|
||||
"message": "property \"foo\" is not defined in object type {}",
|
||||
"filepath": ".github/workflows/ci.yml",
|
||||
"line": 15,
|
||||
"column": 20,
|
||||
"kind": "expression",
|
||||
"snippet": " - run: echo ${{ foo }}"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"code": "...",
|
||||
"filename": "src/api.py",
|
||||
"issue_confidence": "HIGH",
|
||||
"issue_severity": "HIGH",
|
||||
"issue_text": "Possible SQL injection vector through string-based query construction.",
|
||||
"line_number": 45,
|
||||
"line_range": [45, 45],
|
||||
"more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html",
|
||||
"test_id": "B608",
|
||||
"test_name": "hardcoded_sql_expressions"
|
||||
},
|
||||
{
|
||||
"code": "...",
|
||||
"filename": "src/api.py",
|
||||
"issue_confidence": "MEDIUM",
|
||||
"issue_severity": "LOW",
|
||||
"issue_text": "Use of assert detected.",
|
||||
"line_number": 12,
|
||||
"line_range": [12, 12],
|
||||
"test_id": "B101",
|
||||
"test_name": "assert_used"
|
||||
}
|
||||
],
|
||||
"metrics": {"_totals": {"SEVERITY.HIGH": 1, "SEVERITY.LOW": 1}}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
Microsoft (R) Build Engine version 17.0
|
||||
Copyright (C) Microsoft Corporation. All rights reserved.
|
||||
|
||||
/repo/src/Program.cs(22,5): warning SCS0018: Path traversal: injection possible. [/repo/src/MyApp.csproj]
|
||||
/repo/src/Program.cs(45,12): error CS8602: Dereference of a possibly null reference. [/repo/src/MyApp.csproj]
|
||||
/repo/src/Program.cs(60,8): warning CA1834: Use 'StringBuilder.Append(char)' for single-character strings. [/repo/src/MyApp.csproj]
|
||||
|
||||
Build succeeded.
|
||||
1 Warning(s)
|
||||
0 Error(s)
|
||||
@@ -0,0 +1,33 @@
|
||||
[
|
||||
{
|
||||
"filePath": "/repo/src/auth.ts",
|
||||
"messages": [
|
||||
{
|
||||
"ruleId": "security/detect-eval-with-expression",
|
||||
"severity": 2,
|
||||
"message": "eval with non-literal expression",
|
||||
"line": 22,
|
||||
"column": 5,
|
||||
"endLine": 22,
|
||||
"endColumn": 35
|
||||
},
|
||||
{
|
||||
"ruleId": "@typescript-eslint/no-explicit-any",
|
||||
"severity": 1,
|
||||
"message": "Unexpected any. Specify a different type.",
|
||||
"line": 8,
|
||||
"column": 12,
|
||||
"endLine": 8,
|
||||
"endColumn": 15
|
||||
}
|
||||
],
|
||||
"errorCount": 1,
|
||||
"warningCount": 1
|
||||
},
|
||||
{
|
||||
"filePath": "/repo/src/utils.ts",
|
||||
"messages": [],
|
||||
"errorCount": 0,
|
||||
"warningCount": 0
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,20 @@
|
||||
[
|
||||
{
|
||||
"RuleID": "aws-access-token",
|
||||
"Description": "AWS Access Token",
|
||||
"StartLine": 12,
|
||||
"EndLine": 12,
|
||||
"File": "src/config.py",
|
||||
"Secret": "redacted",
|
||||
"Match": "aws_access_key_id assignment"
|
||||
},
|
||||
{
|
||||
"RuleID": "generic-api-key",
|
||||
"Description": "Generic API Key",
|
||||
"StartLine": 30,
|
||||
"EndLine": 30,
|
||||
"File": "src/api.py",
|
||||
"Secret": "redacted",
|
||||
"Match": "API key assignment"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,18 @@
|
||||
[
|
||||
{
|
||||
"RuleName": "InjectionRisk.InvokeExpression",
|
||||
"Severity": "Warning",
|
||||
"ScriptPath": "/repo/scripts/Deploy.ps1",
|
||||
"Line": 12,
|
||||
"EndLine": 12,
|
||||
"Message": "Possible script injection risk via the Invoke-Expression cmdlet."
|
||||
},
|
||||
{
|
||||
"RuleName": "InjectionRisk.AddScriptBlock",
|
||||
"Severity": "Warning",
|
||||
"ScriptPath": "/repo/scripts/Deploy.ps1",
|
||||
"Line": 33,
|
||||
"EndLine": 35,
|
||||
"Message": "Possible script injection risk via the AddScript method."
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,5 @@
|
||||
src/api.py:12: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment]
|
||||
src/api.py:45: error: Argument 1 has incompatible type "Any"; expected "str" [arg-type]
|
||||
src/utils.py:8: note: Possible overload variants:
|
||||
src/api.py:60: warning: Unused "type: ignore" comment [unused-ignore]
|
||||
Found 3 errors in 2 files (checked 5 source files)
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"version": "1.25.0",
|
||||
"results": [
|
||||
{
|
||||
"check_id": "python.lang.security.audit.dangerous-code.dangerous-code",
|
||||
"path": "src/api.py",
|
||||
"start": {"line": 12, "col": 5},
|
||||
"end": {"line": 12, "col": 30},
|
||||
"extra": {
|
||||
"severity": "ERROR",
|
||||
"message": "Detected use of dynamic code construction.",
|
||||
"metadata": {"cwe": ["CWE-94: Improper Control of Generation of Code ('Code Injection')"]}
|
||||
}
|
||||
},
|
||||
{
|
||||
"check_id": "javascript.lang.audit.dynamic-expression",
|
||||
"path": "src/auth.ts",
|
||||
"start": {"line": 22, "col": 5},
|
||||
"end": {"line": 22, "col": 35},
|
||||
"extra": {
|
||||
"severity": "WARNING",
|
||||
"message": "Dynamic expression with non-literal input",
|
||||
"metadata": {"cwe": ["CWE-95"]}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"source": {"path": "package-lock.json", "type": "lockfile"},
|
||||
"packages": [
|
||||
{
|
||||
"package": {"name": "axios", "version": "0.21.0", "ecosystem": "npm"},
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"id": "GHSA-cph5-m8f7-6c5x",
|
||||
"summary": "Axios is vulnerable to Server-Side Request Forgery"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"dependencies": [
|
||||
{
|
||||
"name": "pyyaml",
|
||||
"version": "5.4.1",
|
||||
"vulns": [
|
||||
{
|
||||
"id": "GHSA-8q59-q68h-6hv4",
|
||||
"fix_versions": ["6.0"],
|
||||
"description": "PyYAML vulnerable to arbitrary code generation"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "requests",
|
||||
"version": "2.31.0",
|
||||
"vulns": []
|
||||
}
|
||||
]
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
[
|
||||
{
|
||||
"RuleName": "PSAvoidUsingInvokeExpression",
|
||||
"Severity": "Warning",
|
||||
"ScriptPath": "/repo/scripts/Deploy.ps1",
|
||||
"Line": 12,
|
||||
"EndLine": 12,
|
||||
"Message": "Invoke-Expression is used. Please remove Invoke-Expression from script and find other options instead."
|
||||
},
|
||||
{
|
||||
"RuleName": "PSAvoidUsingConvertToSecureStringWithPlainText",
|
||||
"Severity": "Error",
|
||||
"ScriptPath": "/repo/scripts/Deploy.ps1",
|
||||
"Line": 20,
|
||||
"EndLine": 22,
|
||||
"Message": "File 'Deploy.ps1' uses ConvertTo-SecureString with plaintext."
|
||||
},
|
||||
{
|
||||
"RuleName": "PSAvoidUsingWriteHost",
|
||||
"Severity": "Warning",
|
||||
"ScriptPath": "/repo/module/Helpers.psm1",
|
||||
"Line": 5,
|
||||
"EndLine": 5,
|
||||
"Message": "File 'Helpers.psm1' uses Write-Host."
|
||||
},
|
||||
{
|
||||
"RuleName": "PSUseDeclaredVarsMoreThanAssignments",
|
||||
"Severity": "Information",
|
||||
"ScriptPath": "/repo/module/Helpers.psm1",
|
||||
"Line": 30,
|
||||
"EndLine": 30,
|
||||
"Message": "The variable 'unused' is assigned but never used."
|
||||
},
|
||||
{
|
||||
"RuleName": "TypeNotFound",
|
||||
"Severity": "ParseError",
|
||||
"ScriptPath": "/repo/module/Helpers.psm1",
|
||||
"Line": 41,
|
||||
"EndLine": 41,
|
||||
"Message": "Unable to find type [Foo]."
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,23 @@
|
||||
[
|
||||
{
|
||||
"code": "S608",
|
||||
"location": {"row": 45, "column": 12},
|
||||
"end_location": {"row": 45, "column": 80},
|
||||
"filename": "src/api.py",
|
||||
"message": "Possible SQL injection vector through string-based query construction"
|
||||
},
|
||||
{
|
||||
"code": "E501",
|
||||
"location": {"row": 30, "column": 1},
|
||||
"end_location": {"row": 30, "column": 105},
|
||||
"filename": "src/api.py",
|
||||
"message": "Line too long (104 > 88)"
|
||||
},
|
||||
{
|
||||
"code": "F401",
|
||||
"location": {"row": 3, "column": 1},
|
||||
"end_location": {"row": 3, "column": 20},
|
||||
"filename": "src/api.py",
|
||||
"message": "imported but unused"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,44 @@
|
||||
[
|
||||
{
|
||||
"filename": "src/game.lua",
|
||||
"primary_label": "undefined variable `undefined_var`",
|
||||
"secondary_labels": [],
|
||||
"notes": [],
|
||||
"code": {
|
||||
"name": "undefined_variable",
|
||||
"severity": "Error"
|
||||
},
|
||||
"span": {
|
||||
"start": {"line": 5, "character": 1},
|
||||
"end": {"line": 5, "character": 14}
|
||||
}
|
||||
},
|
||||
{
|
||||
"filename": "src/game.lua",
|
||||
"primary_label": "accessing undefined field `nonexistent` on type `table`",
|
||||
"secondary_labels": [],
|
||||
"notes": [],
|
||||
"code": {
|
||||
"name": "undefined_field",
|
||||
"severity": "Warning"
|
||||
},
|
||||
"span": {
|
||||
"start": {"line": 12, "character": 3},
|
||||
"end": {"line": 12, "character": 15}
|
||||
}
|
||||
},
|
||||
{
|
||||
"filename": "src/game.lua",
|
||||
"primary_label": "variable `_unused` is set but never used",
|
||||
"secondary_labels": [],
|
||||
"notes": [],
|
||||
"code": {
|
||||
"name": "unused_variable",
|
||||
"severity": "Note"
|
||||
},
|
||||
"span": {
|
||||
"start": {"line": 20, "character": 5},
|
||||
"end": {"line": 20, "character": 12}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,3 @@
|
||||
src/auth.ts(22,5): error TS2322: Type 'string' is not assignable to type 'number'.
|
||||
src/utils.ts(8,12): error TS2345: Argument of type 'unknown' is not assignable to parameter of type 'string'.
|
||||
src/index.ts(3,1): error TS6133: 'fs' is declared but its value is never read.
|
||||
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"version": "2.1.0",
|
||||
"runs": [
|
||||
{
|
||||
"tool": {
|
||||
"driver": {
|
||||
"name": "zizmor",
|
||||
"version": "1.0.0",
|
||||
"rules": [
|
||||
{
|
||||
"id": "unpinned-uses",
|
||||
"name": "Unpinned uses",
|
||||
"shortDescription": {"text": "Action ref is not pinned to a SHA digest"},
|
||||
"properties": {"tags": ["CWE-829"]}
|
||||
},
|
||||
{
|
||||
"id": "excessive-permissions",
|
||||
"name": "Excessive permissions",
|
||||
"shortDescription": {"text": "Workflow grants write-all permissions"},
|
||||
"properties": {"tags": []}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"ruleId": "unpinned-uses",
|
||||
"level": "warning",
|
||||
"message": {"text": "uses: actions/checkout@v4 is not pinned by digest"},
|
||||
"locations": [
|
||||
{
|
||||
"physicalLocation": {
|
||||
"artifactLocation": {"uri": ".github/workflows/ci.yml"},
|
||||
"region": {
|
||||
"startLine": 12,
|
||||
"startColumn": 9,
|
||||
"endLine": 12,
|
||||
"endColumn": 38
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "excessive-permissions",
|
||||
"level": "error",
|
||||
"message": {"text": "Workflow uses write-all permissions; scope to minimum required"},
|
||||
"locations": [
|
||||
{
|
||||
"physicalLocation": {
|
||||
"artifactLocation": {"uri": ".github/workflows/ci.yml"},
|
||||
"region": {
|
||||
"startLine": 5,
|
||||
"startColumn": 1,
|
||||
"endLine": 5,
|
||||
"endColumn": 30
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.actionlint import parse_actionlint_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_findings():
|
||||
payload = (_FIXTURES / "actionlint_sample.json").read_text()
|
||||
findings = parse_actionlint_output(payload)
|
||||
assert len(findings) == 2
|
||||
|
||||
|
||||
def test_shellcheck_finding_fields():
|
||||
payload = (_FIXTURES / "actionlint_sample.json").read_text()
|
||||
findings = parse_actionlint_output(payload)
|
||||
sc = next(f for f in findings if f.rule_id == "shellcheck")
|
||||
assert sc.tool == "actionlint"
|
||||
assert sc.severity == "high"
|
||||
assert sc.file == ".github/workflows/ci.yml"
|
||||
assert sc.line == 22
|
||||
assert sc.end_line == 22
|
||||
assert "SC2086" in sc.message
|
||||
|
||||
|
||||
def test_expression_finding_fields():
|
||||
payload = (_FIXTURES / "actionlint_sample.json").read_text()
|
||||
findings = parse_actionlint_output(payload)
|
||||
expr = next(f for f in findings if f.rule_id == "expression")
|
||||
assert expr.line == 15
|
||||
|
||||
|
||||
def test_handles_empty_array():
|
||||
findings = parse_actionlint_output("[]")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_handles_invalid_json():
|
||||
findings = parse_actionlint_output("not json")
|
||||
assert findings == []
|
||||
@@ -0,0 +1,28 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.bandit import parse_bandit_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_findings_with_severity_lowercase():
|
||||
payload = (_FIXTURES / "bandit_sample.json").read_text()
|
||||
findings = parse_bandit_output(payload)
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
assert "B608" in by_rule
|
||||
assert by_rule["B608"].severity == "high"
|
||||
assert by_rule["B608"].file == "src/api.py"
|
||||
assert by_rule["B608"].line == 45
|
||||
assert by_rule["B608"].end_line == 45
|
||||
assert "SQL injection" in by_rule["B608"].message
|
||||
|
||||
|
||||
def test_handles_empty_results():
|
||||
findings = parse_bandit_output('{"results": [], "metrics": {}}')
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_handles_invalid_json():
|
||||
findings = parse_bandit_output('not json')
|
||||
assert findings == []
|
||||
@@ -0,0 +1,34 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.dotnet import parse_dotnet_build_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_scs_as_high():
|
||||
txt = (_FIXTURES / "dotnet_build_sample.txt").read_text()
|
||||
findings = parse_dotnet_build_output(txt, repo_root="/repo")
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
assert "SCS0018" in by_rule
|
||||
assert by_rule["SCS0018"].severity == "high"
|
||||
assert by_rule["SCS0018"].file == "src/Program.cs"
|
||||
assert by_rule["SCS0018"].line == 22
|
||||
|
||||
|
||||
def test_parses_cs_error_as_medium():
|
||||
txt = (_FIXTURES / "dotnet_build_sample.txt").read_text()
|
||||
findings = parse_dotnet_build_output(txt, repo_root="/repo")
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
assert by_rule["CS8602"].severity == "medium"
|
||||
|
||||
|
||||
def test_parses_ca_warning_as_low():
|
||||
txt = (_FIXTURES / "dotnet_build_sample.txt").read_text()
|
||||
findings = parse_dotnet_build_output(txt, repo_root="/repo")
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
assert by_rule["CA1834"].severity == "low"
|
||||
|
||||
|
||||
def test_handles_empty():
|
||||
assert parse_dotnet_build_output("", repo_root="/repo") == []
|
||||
@@ -0,0 +1,41 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.eslint import parse_eslint_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_security_rule_as_high():
|
||||
payload = (_FIXTURES / "eslint_sample.json").read_text()
|
||||
findings = parse_eslint_output(payload, repo_root="/repo")
|
||||
rules = {f.rule_id: f for f in findings}
|
||||
assert "security/detect-eval-with-expression" in rules
|
||||
sec = rules["security/detect-eval-with-expression"]
|
||||
assert sec.severity == "high"
|
||||
assert sec.file == "src/auth.ts"
|
||||
assert sec.line == 22
|
||||
|
||||
|
||||
def test_warning_severity_low():
|
||||
payload = (_FIXTURES / "eslint_sample.json").read_text()
|
||||
findings = parse_eslint_output(payload, repo_root="/repo")
|
||||
rules = {f.rule_id: f for f in findings}
|
||||
assert rules["@typescript-eslint/no-explicit-any"].severity == "low"
|
||||
|
||||
|
||||
def test_non_security_error_severity_medium():
|
||||
payload = '[{"filePath": "/repo/a.js", "messages": [{"ruleId": "no-undef", "severity": 2, "message": "x", "line": 1, "column": 1, "endLine": 1, "endColumn": 5}]}]'
|
||||
findings = parse_eslint_output(payload, repo_root="/repo")
|
||||
assert findings[0].severity == "medium"
|
||||
|
||||
|
||||
def test_strips_repo_prefix():
|
||||
payload = (_FIXTURES / "eslint_sample.json").read_text()
|
||||
findings = parse_eslint_output(payload, repo_root="/repo")
|
||||
for f in findings:
|
||||
assert not f.file.startswith("/")
|
||||
|
||||
|
||||
def test_handles_invalid_json():
|
||||
assert parse_eslint_output("nope", repo_root="/repo") == []
|
||||
@@ -0,0 +1,25 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.gitleaks import parse_gitleaks_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_findings_as_critical():
|
||||
payload = (_FIXTURES / "gitleaks_sample.json").read_text()
|
||||
findings = parse_gitleaks_output(payload)
|
||||
rules = {f.rule_id for f in findings}
|
||||
assert "aws-access-token" in rules
|
||||
assert "generic-api-key" in rules
|
||||
for f in findings:
|
||||
assert f.severity == "critical"
|
||||
assert f.tool == "gitleaks"
|
||||
|
||||
|
||||
def test_handles_empty():
|
||||
assert parse_gitleaks_output("[]") == []
|
||||
|
||||
|
||||
def test_handles_invalid():
|
||||
assert parse_gitleaks_output("nope") == []
|
||||
@@ -0,0 +1,69 @@
|
||||
import json
|
||||
|
||||
from scripts.adapters.interrogate import parse_interrogate_output
|
||||
|
||||
|
||||
def test_parses_missing_public_docstring():
|
||||
payload = {
|
||||
"files": {
|
||||
"src/foo.py": {
|
||||
"missing": [
|
||||
{"name": "src/foo.py:my_public_fn", "type": "function", "lineno": 12, "private": False}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
findings = parse_interrogate_output(json.dumps(payload))
|
||||
assert len(findings) == 1
|
||||
f = findings[0]
|
||||
assert f.tool == "interrogate"
|
||||
assert f.file == "src/foo.py"
|
||||
assert f.line == 12
|
||||
assert "my_public_fn" in f.message
|
||||
assert f.rule_id == "interrogate:missing-docstring"
|
||||
|
||||
|
||||
def test_skips_private_symbol():
|
||||
payload = {
|
||||
"files": {
|
||||
"src/foo.py": {
|
||||
"missing": [
|
||||
{"name": "src/foo.py:_helper", "type": "function", "lineno": 30, "private": True}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
assert parse_interrogate_output(json.dumps(payload)) == []
|
||||
|
||||
|
||||
def test_skips_underscore_named_symbol_even_if_private_false():
|
||||
payload = {
|
||||
"files": {
|
||||
"src/foo.py": {
|
||||
"missing": [
|
||||
{"name": "src/foo.py:_internal", "type": "function", "lineno": 5, "private": False}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
assert parse_interrogate_output(json.dumps(payload)) == []
|
||||
|
||||
|
||||
def test_low_severity_always():
|
||||
payload = {
|
||||
"files": {
|
||||
"src/foo.py": {
|
||||
"missing": [
|
||||
{"name": "src/foo.py:pub_fn", "type": "function", "lineno": 1, "private": False}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
findings = parse_interrogate_output(json.dumps(payload))
|
||||
assert findings[0].severity == "low"
|
||||
|
||||
|
||||
def test_empty_or_malformed_input():
|
||||
assert parse_interrogate_output("") == []
|
||||
assert parse_interrogate_output("not json") == []
|
||||
assert parse_interrogate_output("{}") == []
|
||||
@@ -0,0 +1,47 @@
|
||||
import json
|
||||
|
||||
from scripts.adapters.jscpd import parse_jscpd_output
|
||||
|
||||
|
||||
def test_parses_small_clone_as_low():
|
||||
payload = {
|
||||
"duplicates": [
|
||||
{
|
||||
"firstFile": {"name": "src/a.ts", "start": 1, "end": 10},
|
||||
"secondFile": {"name": "src/b.ts", "start": 5, "end": 14},
|
||||
"lines": 10,
|
||||
}
|
||||
]
|
||||
}
|
||||
findings = parse_jscpd_output(json.dumps(payload))
|
||||
assert len(findings) == 1
|
||||
f = findings[0]
|
||||
assert f.tool == "jscpd"
|
||||
assert f.severity == "low"
|
||||
assert f.rule_id == "jscpd:clone-10lines"
|
||||
assert f.file == "src/a.ts"
|
||||
assert f.line == 1
|
||||
assert f.end_line == 10
|
||||
assert "src/b.ts" in f.message
|
||||
assert "10 lines" in f.message
|
||||
|
||||
|
||||
def test_parses_large_clone_as_medium():
|
||||
payload = {
|
||||
"duplicates": [
|
||||
{
|
||||
"firstFile": {"name": "src/a.ts", "start": 1, "end": 40},
|
||||
"secondFile": {"name": "src/b.ts", "start": 1, "end": 40},
|
||||
"lines": 40,
|
||||
}
|
||||
]
|
||||
}
|
||||
findings = parse_jscpd_output(json.dumps(payload))
|
||||
assert findings[0].severity == "medium"
|
||||
|
||||
|
||||
def test_empty_or_no_duplicates():
|
||||
assert parse_jscpd_output("") == []
|
||||
assert parse_jscpd_output("{}") == []
|
||||
assert parse_jscpd_output(json.dumps({"duplicates": []})) == []
|
||||
assert parse_jscpd_output("not json") == []
|
||||
@@ -0,0 +1,46 @@
|
||||
import json
|
||||
|
||||
from scripts.adapters.knip import parse_knip_output
|
||||
|
||||
|
||||
def test_parses_dead_export():
|
||||
payload = {
|
||||
"issues": [
|
||||
{
|
||||
"file": "src/foo.ts",
|
||||
"exports": [{"name": "unusedExport", "line": 12, "col": 1}],
|
||||
}
|
||||
]
|
||||
}
|
||||
findings = parse_knip_output(json.dumps(payload))
|
||||
assert len(findings) == 1
|
||||
f = findings[0]
|
||||
assert f.tool == "knip"
|
||||
assert f.rule_id == "knip:dead-export"
|
||||
assert f.severity == "medium"
|
||||
assert f.file == "src/foo.ts"
|
||||
assert f.line == 12
|
||||
assert "unusedExport" in f.message
|
||||
|
||||
|
||||
def test_multiple_exports():
|
||||
payload = {
|
||||
"issues": [
|
||||
{
|
||||
"file": "src/foo.ts",
|
||||
"exports": [
|
||||
{"name": "exportA", "line": 5, "col": 1},
|
||||
{"name": "exportB", "line": 10, "col": 1},
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
findings = parse_knip_output(json.dumps(payload))
|
||||
assert len(findings) == 2
|
||||
|
||||
|
||||
def test_empty_or_no_issues():
|
||||
assert parse_knip_output("") == []
|
||||
assert parse_knip_output("{}") == []
|
||||
assert parse_knip_output(json.dumps({"issues": []})) == []
|
||||
assert parse_knip_output("not json") == []
|
||||
@@ -0,0 +1,35 @@
|
||||
from scripts.adapters.lizard import parse_lizard_output
|
||||
|
||||
|
||||
def test_parses_high_ccn():
|
||||
row = "12,15,80,3,15,my_fn@10@src/foo.py\n"
|
||||
findings = parse_lizard_output(row)
|
||||
assert len(findings) == 1
|
||||
f = findings[0]
|
||||
assert f.tool == "lizard"
|
||||
assert f.severity == "medium"
|
||||
assert f.rule_id == "lizard:ccn=15"
|
||||
assert f.file == "src/foo.py"
|
||||
assert f.line == 10
|
||||
assert f.end_line == 10
|
||||
assert "my_fn" in f.message
|
||||
assert "15" in f.message
|
||||
|
||||
|
||||
def test_very_high_ccn_is_high():
|
||||
row = "30,25,200,5,40,complex_fn@5@src/bar.py\n"
|
||||
findings = parse_lizard_output(row)
|
||||
assert findings[0].severity == "high"
|
||||
|
||||
|
||||
def test_low_ccn_skipped():
|
||||
row = "5,3,30,1,6,simple@1@src/foo.py\n"
|
||||
assert parse_lizard_output(row) == []
|
||||
|
||||
|
||||
def test_malformed_row_skipped():
|
||||
assert parse_lizard_output("1,2,3\n") == []
|
||||
|
||||
|
||||
def test_empty_input():
|
||||
assert parse_lizard_output("") == []
|
||||
@@ -0,0 +1,43 @@
|
||||
from scripts.adapters.luac import parse_luac_output
|
||||
|
||||
|
||||
def test_parses_single_syntax_error():
|
||||
stderr = "luac: ./src/game.lua:8: 'end' expected (to close 'function' at line 1) near '<eof>'"
|
||||
findings = parse_luac_output(stderr, repo_root="")
|
||||
assert len(findings) == 1
|
||||
f = findings[0]
|
||||
assert f.tool == "luac"
|
||||
assert f.rule_id == "syntax-error"
|
||||
assert f.severity == "critical"
|
||||
assert f.line == 8
|
||||
assert f.end_line == 8
|
||||
assert "end" in f.message
|
||||
|
||||
|
||||
def test_parses_multiple_errors():
|
||||
stderr = (
|
||||
"luac: ./a.lua:3: unexpected symbol near '='\n"
|
||||
"luac: ./b.lua:17: 'end' expected near '<eof>'"
|
||||
)
|
||||
findings = parse_luac_output(stderr, repo_root="")
|
||||
assert len(findings) == 2
|
||||
files = {f.file for f in findings}
|
||||
assert "./a.lua" in files
|
||||
assert "./b.lua" in files
|
||||
|
||||
|
||||
def test_strips_repo_root_prefix():
|
||||
stderr = "luac: /home/user/myrepo/src/game.lua:5: unexpected symbol near '+'"
|
||||
findings = parse_luac_output(stderr, repo_root="/home/user/myrepo")
|
||||
assert findings[0].file == "src/game.lua"
|
||||
|
||||
|
||||
def test_empty_stderr_returns_empty():
|
||||
findings = parse_luac_output("", repo_root="")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_non_matching_lines_ignored():
|
||||
stderr = "warning: something else\nnot a luac error"
|
||||
findings = parse_luac_output(stderr, repo_root="")
|
||||
assert findings == []
|
||||
@@ -0,0 +1,32 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.mypy import parse_mypy_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_error_lines():
|
||||
txt = (_FIXTURES / "mypy_sample.txt").read_text()
|
||||
findings = parse_mypy_output(txt)
|
||||
by_line = {f.line: f for f in findings if f.file == "src/api.py"}
|
||||
assert 12 in by_line
|
||||
assert by_line[12].severity == "medium"
|
||||
assert by_line[12].rule_id == "assignment"
|
||||
|
||||
|
||||
def test_parses_warning_lines():
|
||||
txt = (_FIXTURES / "mypy_sample.txt").read_text()
|
||||
findings = parse_mypy_output(txt)
|
||||
warns = [f for f in findings if f.severity == "low"]
|
||||
assert any(f.line == 60 and f.rule_id == "unused-ignore" for f in warns)
|
||||
|
||||
|
||||
def test_skips_note_lines():
|
||||
txt = (_FIXTURES / "mypy_sample.txt").read_text()
|
||||
findings = parse_mypy_output(txt)
|
||||
assert not any(f.file == "src/utils.py" and f.line == 8 for f in findings)
|
||||
|
||||
|
||||
def test_handles_empty_input():
|
||||
assert parse_mypy_output("") == []
|
||||
@@ -0,0 +1,43 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.opengrep import parse_opengrep_output, _clean_rule_id
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_error_as_high():
|
||||
payload = (_FIXTURES / "opengrep_sample.json").read_text()
|
||||
findings = parse_opengrep_output(payload)
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
key = "python.lang.security.audit.dangerous-code.dangerous-code"
|
||||
assert key in by_rule
|
||||
assert by_rule[key].severity == "high"
|
||||
assert by_rule[key].cwe == "CWE-94"
|
||||
assert by_rule[key].line == 12
|
||||
|
||||
|
||||
def test_parses_warning_as_medium():
|
||||
payload = (_FIXTURES / "opengrep_sample.json").read_text()
|
||||
findings = parse_opengrep_output(payload)
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
key = "javascript.lang.audit.dynamic-expression"
|
||||
assert by_rule[key].severity == "medium"
|
||||
|
||||
|
||||
def test_handles_empty():
|
||||
assert parse_opengrep_output('{"results": []}') == []
|
||||
|
||||
|
||||
def test_handles_invalid():
|
||||
assert parse_opengrep_output("nope") == []
|
||||
|
||||
|
||||
def test_clean_rule_id_strips_local_rules_path_prefix():
|
||||
mangled = "home.mroberts..claude.plugins.reviews.skills.audit-code.scripts.rules.lua-os-execute"
|
||||
assert _clean_rule_id(mangled) == "lua-os-execute"
|
||||
|
||||
|
||||
def test_clean_rule_id_leaves_registry_ids_untouched():
|
||||
registry_id = "python.lang.security.audit.dangerous-code.dangerous-code"
|
||||
assert _clean_rule_id(registry_id) == registry_id
|
||||
@@ -0,0 +1,26 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.osv_scanner import parse_osv_scanner_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_emits_finding_per_vuln():
|
||||
payload = (_FIXTURES / "osv_scanner_sample.json").read_text()
|
||||
findings = parse_osv_scanner_output(payload)
|
||||
assert len(findings) == 1
|
||||
f = findings[0]
|
||||
assert f.rule_id == "GHSA-cph5-m8f7-6c5x"
|
||||
assert f.tool == "osv-scanner"
|
||||
assert f.severity == "high"
|
||||
assert "axios" in f.message
|
||||
assert f.file == "package-lock.json"
|
||||
|
||||
|
||||
def test_handles_empty():
|
||||
assert parse_osv_scanner_output('{"results": []}') == []
|
||||
|
||||
|
||||
def test_handles_invalid():
|
||||
assert parse_osv_scanner_output("nope") == []
|
||||
@@ -0,0 +1,27 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.pip_audit import parse_pip_audit_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_emits_finding_per_vuln():
|
||||
payload = (_FIXTURES / "pip_audit_sample.json").read_text()
|
||||
findings = parse_pip_audit_output(payload, manifest_path="requirements.txt")
|
||||
assert len(findings) == 1
|
||||
f = findings[0]
|
||||
assert f.rule_id == "GHSA-8q59-q68h-6hv4"
|
||||
assert f.severity == "high"
|
||||
assert f.tool == "pip-audit"
|
||||
assert "pyyaml" in f.message
|
||||
assert f.file == "requirements.txt"
|
||||
|
||||
|
||||
def test_no_finding_when_no_vulns():
|
||||
payload = '{"dependencies": [{"name": "x", "version": "1", "vulns": []}]}'
|
||||
assert parse_pip_audit_output(payload, manifest_path="requirements.txt") == []
|
||||
|
||||
|
||||
def test_handles_invalid():
|
||||
assert parse_pip_audit_output("nope", manifest_path="requirements.txt") == []
|
||||
@@ -0,0 +1,78 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.psscriptanalyzer import parse_psscriptanalyzer_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
_REPO = "/repo"
|
||||
|
||||
|
||||
def _pssa():
|
||||
payload = (_FIXTURES / "psscriptanalyzer_sample.json").read_text()
|
||||
return {f.rule_id: f for f in parse_psscriptanalyzer_output(payload, repo_root=_REPO)}
|
||||
|
||||
|
||||
def _injection():
|
||||
payload = (_FIXTURES / "injectionhunter_sample.json").read_text()
|
||||
return {
|
||||
f.rule_id: f
|
||||
for f in parse_psscriptanalyzer_output(payload, repo_root=_REPO, tool="injectionhunter")
|
||||
}
|
||||
|
||||
|
||||
def test_paths_are_relative_to_repo_root():
|
||||
f = _pssa()["PSAvoidUsingInvokeExpression"]
|
||||
assert f.file == "scripts/Deploy.ps1"
|
||||
assert f.tool == "psscriptanalyzer"
|
||||
|
||||
|
||||
def test_severity_mapping():
|
||||
by_rule = _pssa()
|
||||
assert by_rule["PSAvoidUsingConvertToSecureStringWithPlainText"].severity == "high"
|
||||
assert by_rule["PSAvoidUsingInvokeExpression"].severity == "medium"
|
||||
assert by_rule["PSUseDeclaredVarsMoreThanAssignments"].severity == "low"
|
||||
assert by_rule["TypeNotFound"].severity == "critical"
|
||||
|
||||
|
||||
def test_line_range_from_extent():
|
||||
f = _pssa()["PSAvoidUsingConvertToSecureStringWithPlainText"]
|
||||
assert f.line == 20
|
||||
assert f.end_line == 22
|
||||
|
||||
|
||||
def test_message_preserved():
|
||||
assert "Invoke-Expression" in _pssa()["PSAvoidUsingInvokeExpression"].message
|
||||
|
||||
|
||||
def test_injectionhunter_findings_floor_at_high():
|
||||
by_rule = _injection()
|
||||
assert by_rule["InjectionRisk.InvokeExpression"].severity == "high"
|
||||
assert by_rule["InjectionRisk.AddScriptBlock"].severity == "high"
|
||||
|
||||
|
||||
def test_injectionhunter_tool_name_and_location():
|
||||
f = _injection()["InjectionRisk.AddScriptBlock"]
|
||||
assert f.tool == "injectionhunter"
|
||||
assert f.file == "scripts/Deploy.ps1"
|
||||
assert f.line == 33
|
||||
assert f.end_line == 35
|
||||
|
||||
|
||||
def test_single_object_not_array():
|
||||
payload = '{"RuleName":"PSAvoidUsingWriteHost","Severity":"Warning","ScriptPath":"/repo/a.ps1","Line":3,"EndLine":3,"Message":"m"}'
|
||||
findings = parse_psscriptanalyzer_output(payload, repo_root=_REPO)
|
||||
assert len(findings) == 1
|
||||
assert findings[0].file == "a.ps1"
|
||||
|
||||
|
||||
def test_handles_empty_array():
|
||||
assert parse_psscriptanalyzer_output("[]", repo_root=_REPO) == []
|
||||
|
||||
|
||||
def test_handles_invalid_json():
|
||||
assert parse_psscriptanalyzer_output("not json", repo_root=_REPO) == []
|
||||
|
||||
|
||||
def test_unknown_severity_defaults_to_medium():
|
||||
payload = '[{"RuleName":"X","Severity":"Bogus","ScriptPath":"/repo/a.ps1","Line":1,"EndLine":1,"Message":"m"}]'
|
||||
assert parse_psscriptanalyzer_output(payload, repo_root=_REPO)[0].severity == "medium"
|
||||
@@ -0,0 +1,58 @@
|
||||
import json
|
||||
|
||||
from scripts.adapters.radon import parse_radon_output
|
||||
|
||||
|
||||
def test_parses_high_complexity_as_medium():
|
||||
payload = {
|
||||
"src/foo.py": [
|
||||
{"type": "function", "name": "bar", "lineno": 10, "endline": 25, "complexity": 12, "rank": "C"}
|
||||
]
|
||||
}
|
||||
findings = parse_radon_output(json.dumps(payload))
|
||||
assert len(findings) == 1
|
||||
f = findings[0]
|
||||
assert f.severity == "medium"
|
||||
assert f.rule_id == "radon:cc=12"
|
||||
assert f.file == "src/foo.py"
|
||||
assert f.line == 10
|
||||
assert f.end_line == 25
|
||||
|
||||
|
||||
def test_very_high_complexity_is_high():
|
||||
payload = {
|
||||
"src/foo.py": [
|
||||
{"type": "function", "name": "baz", "lineno": 30, "endline": 80, "complexity": 22, "rank": "E"}
|
||||
]
|
||||
}
|
||||
findings = parse_radon_output(json.dumps(payload))
|
||||
assert findings[0].severity == "high"
|
||||
|
||||
|
||||
def test_low_complexity_is_skipped():
|
||||
payload = {
|
||||
"src/foo.py": [
|
||||
{"type": "function", "name": "simple", "lineno": 1, "endline": 5, "complexity": 5, "rank": "A"}
|
||||
]
|
||||
}
|
||||
assert parse_radon_output(json.dumps(payload)) == []
|
||||
|
||||
|
||||
def test_multiple_files():
|
||||
payload = {
|
||||
"src/a.py": [
|
||||
{"type": "function", "name": "fn_a", "lineno": 1, "endline": 20, "complexity": 15, "rank": "C"}
|
||||
],
|
||||
"src/b.py": [
|
||||
{"type": "function", "name": "fn_b", "lineno": 5, "endline": 50, "complexity": 25, "rank": "E"}
|
||||
],
|
||||
}
|
||||
findings = parse_radon_output(json.dumps(payload))
|
||||
assert len(findings) == 2
|
||||
files = {f.file for f in findings}
|
||||
assert files == {"src/a.py", "src/b.py"}
|
||||
|
||||
|
||||
def test_empty_input():
|
||||
assert parse_radon_output("") == []
|
||||
assert parse_radon_output("not json") == []
|
||||
@@ -0,0 +1,33 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.ruff import parse_ruff_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_findings():
|
||||
payload = (_FIXTURES / "ruff_sample.json").read_text()
|
||||
findings = parse_ruff_output(payload)
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
assert "S608" in by_rule
|
||||
assert by_rule["S608"].severity == "high"
|
||||
assert by_rule["S608"].file == "src/api.py"
|
||||
assert by_rule["S608"].line == 45
|
||||
|
||||
|
||||
def test_severity_inferred_from_prefix():
|
||||
payload = (_FIXTURES / "ruff_sample.json").read_text()
|
||||
findings = parse_ruff_output(payload)
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
assert by_rule["S608"].severity == "high"
|
||||
assert by_rule["E501"].severity == "low"
|
||||
assert by_rule["F401"].severity == "low"
|
||||
|
||||
|
||||
def test_handles_empty():
|
||||
assert parse_ruff_output("[]") == []
|
||||
|
||||
|
||||
def test_handles_invalid_json():
|
||||
assert parse_ruff_output("not json") == []
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Parser-level tests for the idiom ruff adapter."""
|
||||
import json
|
||||
from scripts.adapters.ruff_idiom import parse_ruff_idiom_output
|
||||
|
||||
|
||||
def _ruff_item(code: str, file: str = "foo.py", row: int = 10) -> dict:
|
||||
return {
|
||||
"code": code,
|
||||
"filename": file,
|
||||
"location": {"row": row, "column": 1},
|
||||
"end_location": {"row": row, "column": 1},
|
||||
"message": f"{code} suggestion",
|
||||
}
|
||||
|
||||
|
||||
def test_idiom_severity_mapping():
|
||||
payload = json.dumps([
|
||||
_ruff_item("SIM117"),
|
||||
_ruff_item("UP008"),
|
||||
_ruff_item("PLR0913"),
|
||||
_ruff_item("C901"),
|
||||
_ruff_item("B008"),
|
||||
])
|
||||
findings = parse_ruff_idiom_output(payload)
|
||||
sev = {f.rule_id: f.severity for f in findings}
|
||||
assert sev["SIM117"] == "low"
|
||||
assert sev["UP008"] == "low"
|
||||
assert sev["PLR0913"] == "medium"
|
||||
assert sev["C901"] == "medium"
|
||||
assert sev["B008"] == "medium"
|
||||
|
||||
|
||||
def test_idiom_handles_empty_stdout():
|
||||
assert parse_ruff_idiom_output("") == []
|
||||
assert parse_ruff_idiom_output("not json") == []
|
||||
@@ -0,0 +1,51 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.selene import parse_selene_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_error_as_high():
|
||||
payload = (_FIXTURES / "selene_sample.json").read_text()
|
||||
findings = parse_selene_output(payload)
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
assert "undefined_variable" in by_rule
|
||||
f = by_rule["undefined_variable"]
|
||||
assert f.severity == "high"
|
||||
assert f.file == "src/game.lua"
|
||||
assert f.line == 5
|
||||
assert f.end_line == 5
|
||||
assert "undefined_var" in f.message
|
||||
assert f.tool == "selene"
|
||||
|
||||
|
||||
def test_parses_warning_as_medium():
|
||||
payload = (_FIXTURES / "selene_sample.json").read_text()
|
||||
findings = parse_selene_output(payload)
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
assert by_rule["undefined_field"].severity == "medium"
|
||||
assert by_rule["undefined_field"].line == 12
|
||||
assert by_rule["undefined_field"].end_line == 12
|
||||
|
||||
|
||||
def test_parses_note_as_low():
|
||||
payload = (_FIXTURES / "selene_sample.json").read_text()
|
||||
findings = parse_selene_output(payload)
|
||||
by_rule = {f.rule_id: f for f in findings}
|
||||
assert by_rule["unused_variable"].severity == "low"
|
||||
|
||||
|
||||
def test_handles_empty_array():
|
||||
findings = parse_selene_output("[]")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_handles_invalid_json():
|
||||
findings = parse_selene_output("not json")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_handles_non_array_json():
|
||||
findings = parse_selene_output('{"error": "unexpected"}')
|
||||
assert findings == []
|
||||
@@ -0,0 +1,24 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.tsc import parse_tsc_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_error_lines():
|
||||
txt = (_FIXTURES / "tsc_sample.txt").read_text()
|
||||
findings = parse_tsc_output(txt)
|
||||
by_file = {f.file: f for f in findings}
|
||||
assert "src/auth.ts" in by_file
|
||||
assert by_file["src/auth.ts"].line == 22
|
||||
assert by_file["src/auth.ts"].rule_id == "TS2322"
|
||||
assert by_file["src/auth.ts"].severity == "medium"
|
||||
|
||||
|
||||
def test_handles_empty():
|
||||
assert parse_tsc_output("") == []
|
||||
|
||||
|
||||
def test_handles_malformed_lines():
|
||||
assert parse_tsc_output("not a tsc line\n") == []
|
||||
@@ -0,0 +1,33 @@
|
||||
from scripts.adapters.vulture import parse_vulture_output
|
||||
|
||||
|
||||
def test_parses_unused_function():
|
||||
out = "src/foo.py:42: unused function 'bar' (60% confidence)\n"
|
||||
findings = parse_vulture_output(out)
|
||||
assert len(findings) == 1
|
||||
f = findings[0]
|
||||
assert f.tool == "vulture"
|
||||
assert f.file == "src/foo.py"
|
||||
assert f.line == 42
|
||||
assert "unused function 'bar'" in f.message
|
||||
assert f.severity == "low"
|
||||
|
||||
|
||||
def test_high_confidence_unused_is_medium():
|
||||
out = "src/foo.py:7: unused import 'os' (90% confidence)\n"
|
||||
findings = parse_vulture_output(out)
|
||||
assert findings[0].severity == "medium"
|
||||
|
||||
|
||||
def test_empty_stdout():
|
||||
assert parse_vulture_output("") == []
|
||||
|
||||
|
||||
def test_rule_id_contains_confidence():
|
||||
out = "src/foo.py:7: unused import 'os' (90% confidence)\n"
|
||||
findings = parse_vulture_output(out)
|
||||
assert findings[0].rule_id == "vulture:90pct"
|
||||
|
||||
|
||||
def test_malformed_line_skipped():
|
||||
assert parse_vulture_output("not a valid line\n") == []
|
||||
@@ -0,0 +1,56 @@
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.adapters.zizmor import parse_zizmor_output
|
||||
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parses_two_results():
|
||||
payload = (_FIXTURES / "zizmor_sample.json").read_text()
|
||||
findings = parse_zizmor_output(payload)
|
||||
assert len(findings) == 2
|
||||
|
||||
|
||||
def test_warning_maps_to_medium():
|
||||
payload = (_FIXTURES / "zizmor_sample.json").read_text()
|
||||
findings = parse_zizmor_output(payload)
|
||||
unpinned = next(f for f in findings if f.rule_id == "unpinned-uses")
|
||||
assert unpinned.tool == "zizmor"
|
||||
assert unpinned.severity == "medium"
|
||||
assert unpinned.file == ".github/workflows/ci.yml"
|
||||
assert unpinned.line == 12
|
||||
assert unpinned.end_line == 12
|
||||
assert "actions/checkout" in unpinned.message
|
||||
|
||||
|
||||
def test_error_maps_to_high():
|
||||
payload = (_FIXTURES / "zizmor_sample.json").read_text()
|
||||
findings = parse_zizmor_output(payload)
|
||||
perms = next(f for f in findings if f.rule_id == "excessive-permissions")
|
||||
assert perms.severity == "high"
|
||||
assert perms.line == 5
|
||||
|
||||
|
||||
def test_cwe_extracted_from_rule_tags():
|
||||
payload = (_FIXTURES / "zizmor_sample.json").read_text()
|
||||
findings = parse_zizmor_output(payload)
|
||||
unpinned = next(f for f in findings if f.rule_id == "unpinned-uses")
|
||||
assert unpinned.cwe == "CWE-829"
|
||||
|
||||
|
||||
def test_no_cwe_when_tags_empty():
|
||||
payload = (_FIXTURES / "zizmor_sample.json").read_text()
|
||||
findings = parse_zizmor_output(payload)
|
||||
perms = next(f for f in findings if f.rule_id == "excessive-permissions")
|
||||
assert perms.cwe is None
|
||||
|
||||
|
||||
def test_handles_empty_runs():
|
||||
findings = parse_zizmor_output('{"version": "2.1.0", "runs": []}')
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_handles_invalid_json():
|
||||
findings = parse_zizmor_output("not json")
|
||||
assert findings == []
|
||||
@@ -0,0 +1,195 @@
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
_SKILL_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def _load_cli():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"collect_findings", _SKILL_ROOT / "scripts" / "collect-findings.py"
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def _fake_subprocess(cmd, **kwargs):
|
||||
if cmd[:2] == ["git", "-C"]:
|
||||
sub = cmd[2:]
|
||||
else:
|
||||
sub = cmd
|
||||
if "symbolic-ref" in sub:
|
||||
return MagicMock(returncode=0, stdout="refs/remotes/origin/main\n", stderr="")
|
||||
if "fetch" in sub:
|
||||
return MagicMock(returncode=0, stdout="", stderr="")
|
||||
if "rev-parse" in sub and "--verify" in sub:
|
||||
return MagicMock(returncode=0, stdout="abc\n", stderr="")
|
||||
if cmd[:1] == ["git"] and "diff" in cmd:
|
||||
diff = (
|
||||
"diff --git a/src/api.py b/src/api.py\n"
|
||||
"index 1..2 100644\n"
|
||||
"--- a/src/api.py\n"
|
||||
"+++ b/src/api.py\n"
|
||||
"@@ -12 +12,1 @@\n"
|
||||
"+x = parse(user_input)\n"
|
||||
)
|
||||
return MagicMock(returncode=0, stdout=diff, stderr="")
|
||||
if Path(cmd[0]).name == "bandit":
|
||||
out = json.dumps({"results": [{
|
||||
"filename": "src/api.py", "line_number": 12, "line_range": [12, 12],
|
||||
"issue_severity": "HIGH", "issue_text": "Use of dynamic parsing", "test_id": "B307",
|
||||
}]})
|
||||
return MagicMock(returncode=0, stdout=out, stderr="")
|
||||
return MagicMock(returncode=0, stdout="", stderr="")
|
||||
|
||||
|
||||
def test_cli_aborts_when_no_supported_files(tmp_path):
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
def _fake(cmd, **kw):
|
||||
if cmd[:1] == ["git"] and "diff" in cmd:
|
||||
return MagicMock(returncode=0,
|
||||
stdout="diff --git a/README.md b/README.md\nindex 1..2 100644\n--- a/README.md\n+++ b/README.md\n@@ -1 +1,1 @@\n+x\n",
|
||||
stderr="")
|
||||
return _fake_subprocess(cmd, **kw)
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
with patch("shutil.which", return_value=None):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
assert rc == 1
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert any("no supported source files" in e.lower() for e in manifest["errors"])
|
||||
|
||||
|
||||
def test_cli_happy_path_python_only(tmp_path):
|
||||
from scripts import runner
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "src").mkdir()
|
||||
(repo / "src" / "api.py").write_text("x = 1\n")
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
empty_venv = tmp_path / "empty-venv"
|
||||
empty_venv.mkdir()
|
||||
def _which(binary):
|
||||
return "/usr/bin/bandit" if binary == "bandit" else None
|
||||
with patch.object(runner, "_SKILL_VENV_BIN", empty_venv):
|
||||
with patch("subprocess.run", side_effect=_fake_subprocess):
|
||||
with patch("shutil.which", side_effect=_which):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
assert rc == 0
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert manifest["mode"] == "local"
|
||||
assert manifest["language_breakdown"]["python"] == 1
|
||||
rule_ids = {f["rule_id"] for f in manifest["findings"]}
|
||||
assert "B307" in rule_ids
|
||||
for agent in ("security-triage", "type-safety", "dependency", "consistency", "secrets"):
|
||||
assert (out_dir / f"manifest-{agent}.json").exists()
|
||||
|
||||
|
||||
def test_cli_records_tool_unavailable(tmp_path):
|
||||
from scripts import runner
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "src").mkdir()
|
||||
(repo / "src" / "api.py").write_text("x = 1\n")
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
empty_venv = tmp_path / "empty-venv"
|
||||
empty_venv.mkdir()
|
||||
with patch.object(runner, "_SKILL_VENV_BIN", empty_venv):
|
||||
with patch("subprocess.run", side_effect=_fake_subprocess):
|
||||
with patch("shutil.which", return_value=None):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
assert rc == 0
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert "bandit" in manifest["tools_unavailable"]
|
||||
|
||||
|
||||
def test_cli_alerts_on_unsupported_languages(tmp_path):
|
||||
"""Mixed PR: Python supported, Go file alerts — but review still proceeds."""
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "src").mkdir()
|
||||
(repo / "src" / "api.py").write_text("x = 1\n")
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
|
||||
def _fake(cmd, **kw):
|
||||
if cmd[:1] == ["git"] and "diff" in cmd:
|
||||
diff = (
|
||||
"diff --git a/src/api.py b/src/api.py\n"
|
||||
"index 1..2 100644\n--- a/src/api.py\n+++ b/src/api.py\n"
|
||||
"@@ -1 +1,1 @@\n+x = 1\n"
|
||||
"diff --git a/cmd/server.go b/cmd/server.go\n"
|
||||
"index 3..4 100644\n--- a/cmd/server.go\n+++ b/cmd/server.go\n"
|
||||
"@@ -1 +1,1 @@\n+package main\n"
|
||||
)
|
||||
return MagicMock(returncode=0, stdout=diff, stderr="")
|
||||
return _fake_subprocess(cmd, **kw)
|
||||
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
with patch("shutil.which", return_value=None):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
|
||||
assert rc == 0, "review should continue when at least one supported file present"
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
assert any("Go" in e and "server.go" in e for e in manifest["errors"]), manifest["errors"]
|
||||
assert manifest["language_breakdown"]["python"] == 1
|
||||
|
||||
|
||||
def test_cli_alerts_on_unsupported_only_and_aborts(tmp_path):
|
||||
"""Diff contains ONLY unsupported languages: alert + abort."""
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
out_dir = tmp_path / "out"
|
||||
mod = _load_cli()
|
||||
|
||||
def _fake(cmd, **kw):
|
||||
if cmd[:1] == ["git"] and "diff" in cmd:
|
||||
diff = (
|
||||
"diff --git a/cmd/server.go b/cmd/server.go\n"
|
||||
"index 3..4 100644\n--- a/cmd/server.go\n+++ b/cmd/server.go\n"
|
||||
"@@ -1 +1,1 @@\n+package main\n"
|
||||
)
|
||||
return MagicMock(returncode=0, stdout=diff, stderr="")
|
||||
return _fake_subprocess(cmd, **kw)
|
||||
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
with patch("shutil.which", return_value=None):
|
||||
rc = mod.main([
|
||||
"--repo", str(repo),
|
||||
"--head", "HEAD",
|
||||
"--output-dir", str(out_dir),
|
||||
"--mode", "local",
|
||||
])
|
||||
|
||||
assert rc == 1
|
||||
manifest = json.loads((out_dir / "manifest.json").read_text())
|
||||
errors_blob = " | ".join(manifest["errors"])
|
||||
assert "Go" in errors_blob, errors_blob
|
||||
assert "no supported source files" in errors_blob.lower()
|
||||
@@ -0,0 +1,70 @@
|
||||
from scripts.diff_filter import filter_findings_by_diff
|
||||
from scripts.manifest import Finding, ChangedFile
|
||||
|
||||
|
||||
def _f(file: str, line: int, end: int = None) -> Finding:
|
||||
return Finding(
|
||||
tool="t", rule_id="R1", severity="medium",
|
||||
file=file, line=line, end_line=end or line,
|
||||
message="x",
|
||||
)
|
||||
|
||||
|
||||
def test_keeps_finding_inside_added_range():
|
||||
changed = [ChangedFile(path="src/a.py", language="python",
|
||||
added_lines=[(10, 15)])]
|
||||
findings = [_f("src/a.py", 12)]
|
||||
out = filter_findings_by_diff(findings, changed)
|
||||
assert len(out) == 1
|
||||
|
||||
|
||||
def test_drops_finding_outside_added_range():
|
||||
changed = [ChangedFile(path="src/a.py", language="python",
|
||||
added_lines=[(10, 15)])]
|
||||
findings = [_f("src/a.py", 20)]
|
||||
out = filter_findings_by_diff(findings, changed)
|
||||
assert out == []
|
||||
|
||||
|
||||
def test_drops_finding_in_unchanged_file():
|
||||
changed = [ChangedFile(path="src/a.py", language="python",
|
||||
added_lines=[(10, 15)])]
|
||||
findings = [_f("src/b.py", 12)]
|
||||
out = filter_findings_by_diff(findings, changed)
|
||||
assert out == []
|
||||
|
||||
|
||||
def test_keeps_finding_spanning_added_range():
|
||||
changed = [ChangedFile(path="src/a.py", language="python",
|
||||
added_lines=[(10, 15)])]
|
||||
findings = [_f("src/a.py", 8, end=12)]
|
||||
out = filter_findings_by_diff(findings, changed)
|
||||
assert len(out) == 1
|
||||
|
||||
|
||||
def test_keeps_finding_with_multiple_ranges():
|
||||
changed = [ChangedFile(path="src/a.py", language="python",
|
||||
added_lines=[(10, 15), (20, 25)])]
|
||||
findings = [_f("src/a.py", 22)]
|
||||
out = filter_findings_by_diff(findings, changed)
|
||||
assert len(out) == 1
|
||||
|
||||
|
||||
def test_normalizes_leading_dot_slash_prefix():
|
||||
"""Bandit/ruff/opengrep often emit `./src/a.py` while git diff yields `src/a.py`."""
|
||||
changed = [ChangedFile(path="src/a.py", language="python",
|
||||
added_lines=[(10, 15)])]
|
||||
findings = [_f("./src/a.py", 12)]
|
||||
out = filter_findings_by_diff(findings, changed)
|
||||
assert len(out) == 1
|
||||
assert out[0].file == "src/a.py"
|
||||
|
||||
|
||||
def test_normalizes_absolute_path_with_repo_root():
|
||||
"""Adapters may emit absolute paths; strip the repo root prefix."""
|
||||
changed = [ChangedFile(path="src/a.py", language="python",
|
||||
added_lines=[(10, 15)])]
|
||||
findings = [_f("/repo/src/a.py", 12)]
|
||||
out = filter_findings_by_diff(findings, changed, repo_root="/repo")
|
||||
assert len(out) == 1
|
||||
assert out[0].file == "src/a.py"
|
||||
@@ -0,0 +1,85 @@
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from scripts.git_diff import (
|
||||
resolve_default_branch, resolve_base_ref, changed_files_with_ranges,
|
||||
)
|
||||
|
||||
|
||||
_DIFF = """\
|
||||
diff --git a/src/api.py b/src/api.py
|
||||
index 1..2 100644
|
||||
--- a/src/api.py
|
||||
+++ b/src/api.py
|
||||
@@ -12 +12,2 @@
|
||||
-old line
|
||||
+new line one
|
||||
+new line two
|
||||
diff --git a/src/utils/helpers.ts b/src/utils/helpers.ts
|
||||
index 3..4 100644
|
||||
--- a/src/utils/helpers.ts
|
||||
+++ b/src/utils/helpers.ts
|
||||
@@ -5,0 +6,1 @@
|
||||
+ added line
|
||||
"""
|
||||
|
||||
|
||||
def test_resolve_default_branch_uses_symbolic_ref():
|
||||
fake = MagicMock(returncode=0, stdout="refs/remotes/origin/main\n", stderr="")
|
||||
with patch("subprocess.run", return_value=fake):
|
||||
assert resolve_default_branch("/repo") == "main"
|
||||
|
||||
|
||||
def test_resolve_default_branch_falls_back_to_master():
|
||||
def _fake(cmd, **kw):
|
||||
if "symbolic-ref" in cmd:
|
||||
return MagicMock(returncode=1, stdout="", stderr="")
|
||||
if "rev-parse" in cmd and "origin/main" in cmd:
|
||||
return MagicMock(returncode=1, stdout="", stderr="")
|
||||
if "rev-parse" in cmd and "origin/master" in cmd:
|
||||
return MagicMock(returncode=0, stdout="abc\n", stderr="")
|
||||
return MagicMock(returncode=1, stdout="", stderr="")
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
assert resolve_default_branch("/repo") == "master"
|
||||
|
||||
|
||||
def test_resolve_base_ref_uses_origin_when_available():
|
||||
def _fake(cmd, **kw):
|
||||
if "fetch" in cmd:
|
||||
return MagicMock(returncode=0, stdout="", stderr="")
|
||||
if "rev-parse" in cmd and "--verify" in cmd:
|
||||
return MagicMock(returncode=0, stdout="def\n", stderr="")
|
||||
return MagicMock(returncode=0, stdout="", stderr="")
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
assert resolve_base_ref("/repo", "main") == "origin/main"
|
||||
|
||||
|
||||
def test_resolve_base_ref_falls_back_when_origin_missing():
|
||||
def _fake(cmd, **kw):
|
||||
if "fetch" in cmd:
|
||||
return MagicMock(returncode=1, stdout="", stderr="no remote\n")
|
||||
if "rev-parse" in cmd and "--verify" in cmd:
|
||||
return MagicMock(returncode=1, stdout="", stderr="")
|
||||
return MagicMock(returncode=0, stdout="", stderr="")
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
assert resolve_base_ref("/repo", "main") == "main"
|
||||
|
||||
|
||||
def test_changed_files_with_ranges_parses_unified0_diff():
|
||||
fake = MagicMock(returncode=0, stdout=_DIFF, stderr="")
|
||||
with patch("subprocess.run", return_value=fake):
|
||||
out = changed_files_with_ranges("/repo", "origin/main", "HEAD")
|
||||
paths = {p: ranges for p, ranges in out}
|
||||
assert "src/api.py" in paths
|
||||
assert paths["src/api.py"] == [(12, 13)]
|
||||
assert "src/utils/helpers.ts" in paths
|
||||
assert paths["src/utils/helpers.ts"] == [(6, 6)]
|
||||
|
||||
|
||||
def test_changed_files_passes_no_ext_diff_flag():
|
||||
fake = MagicMock(returncode=0, stdout="", stderr="")
|
||||
with patch("subprocess.run", return_value=fake) as p:
|
||||
changed_files_with_ranges("/repo", "origin/main", "HEAD")
|
||||
args = p.call_args[0][0]
|
||||
assert "--no-ext-diff" in args
|
||||
assert "--unified=0" in args
|
||||
assert "diff.noprefix=false" in " ".join(args)
|
||||
@@ -0,0 +1,135 @@
|
||||
from scripts.language_detect import (
|
||||
detect_language, is_dep_manifest, is_gha_file, is_supported_source,
|
||||
known_unsupported_language,
|
||||
)
|
||||
|
||||
|
||||
def test_known_unsupported_languages():
|
||||
assert known_unsupported_language("cmd/main.go") == "Go"
|
||||
assert known_unsupported_language("lib/foo.rb") == "Ruby"
|
||||
assert known_unsupported_language("src/Main.java") == "Java"
|
||||
assert known_unsupported_language("src/main.rs") == "Rust"
|
||||
assert known_unsupported_language("src/App.kt") == "Kotlin"
|
||||
|
||||
|
||||
def test_known_unsupported_returns_none_for_supported():
|
||||
assert known_unsupported_language("src/api.py") is None
|
||||
assert known_unsupported_language("src/index.ts") is None
|
||||
|
||||
|
||||
def test_known_unsupported_returns_none_for_non_source():
|
||||
assert known_unsupported_language("README.md") is None
|
||||
assert known_unsupported_language("Dockerfile") is None
|
||||
assert known_unsupported_language("config.yaml") is None
|
||||
|
||||
|
||||
def test_python_file_detected():
|
||||
assert detect_language("src/api.py") == "python"
|
||||
|
||||
|
||||
def test_typescript_extensions():
|
||||
assert detect_language("src/index.ts") == "typescript"
|
||||
assert detect_language("src/component.tsx") == "typescript"
|
||||
|
||||
|
||||
def test_javascript_extensions():
|
||||
assert detect_language("src/index.js") == "javascript"
|
||||
assert detect_language("src/component.jsx") == "javascript"
|
||||
assert detect_language("src/module.mjs") == "javascript"
|
||||
assert detect_language("src/legacy.cjs") == "javascript"
|
||||
|
||||
|
||||
def test_csharp_file_detected():
|
||||
assert detect_language("Program.cs") == "csharp"
|
||||
|
||||
|
||||
def test_unsupported_source_returns_none():
|
||||
assert detect_language("main.go") is None
|
||||
assert detect_language("script.rb") is None
|
||||
assert detect_language("Main.java") is None
|
||||
|
||||
|
||||
def test_non_source_files_return_none():
|
||||
assert detect_language("README.md") is None
|
||||
assert detect_language("Dockerfile") is None
|
||||
assert detect_language("config.yaml") is None
|
||||
|
||||
|
||||
def test_is_dep_manifest_python():
|
||||
assert is_dep_manifest("requirements.txt")
|
||||
assert is_dep_manifest("requirements-dev.txt")
|
||||
assert is_dep_manifest("pyproject.toml")
|
||||
assert is_dep_manifest("poetry.lock")
|
||||
assert is_dep_manifest("Pipfile.lock")
|
||||
|
||||
|
||||
def test_is_dep_manifest_javascript():
|
||||
assert is_dep_manifest("package.json")
|
||||
assert is_dep_manifest("package-lock.json")
|
||||
assert is_dep_manifest("yarn.lock")
|
||||
assert is_dep_manifest("pnpm-lock.yaml")
|
||||
|
||||
|
||||
def test_is_dep_manifest_csharp():
|
||||
assert is_dep_manifest("MyApp.csproj")
|
||||
assert is_dep_manifest("MyApp.sln")
|
||||
assert is_dep_manifest("packages.config")
|
||||
|
||||
|
||||
def test_is_supported_source():
|
||||
assert is_supported_source("src/api.py")
|
||||
assert is_supported_source("src/index.ts")
|
||||
assert not is_supported_source("main.go")
|
||||
assert not is_supported_source("README.md")
|
||||
|
||||
|
||||
def test_lua_file_detected():
|
||||
assert detect_language("src/game.lua") == "lua"
|
||||
|
||||
|
||||
def test_lua_is_supported_source():
|
||||
assert is_supported_source("src/game.lua")
|
||||
|
||||
|
||||
def test_lua_not_in_known_unsupported():
|
||||
assert known_unsupported_language("src/game.lua") is None
|
||||
|
||||
|
||||
def test_powershell_extensions_detected():
|
||||
assert detect_language("scripts/Deploy.ps1") == "powershell"
|
||||
assert detect_language("module/Helpers.psm1") == "powershell"
|
||||
assert detect_language("module/Helpers.psd1") == "powershell"
|
||||
|
||||
|
||||
def test_powershell_is_supported_source():
|
||||
assert is_supported_source("scripts/Deploy.ps1")
|
||||
|
||||
|
||||
def test_powershell_not_in_known_unsupported():
|
||||
assert known_unsupported_language("scripts/Deploy.ps1") is None
|
||||
|
||||
|
||||
def test_gha_workflow_yml_detected():
|
||||
assert is_gha_file(".github/workflows/ci.yml")
|
||||
|
||||
|
||||
def test_gha_workflow_yaml_detected():
|
||||
assert is_gha_file(".github/workflows/deploy.yaml")
|
||||
|
||||
|
||||
def test_gha_actions_action_yml_detected():
|
||||
assert is_gha_file(".github/actions/my-action/action.yml")
|
||||
|
||||
|
||||
def test_non_gha_yaml_not_detected():
|
||||
assert not is_gha_file("config/app.yml")
|
||||
assert not is_gha_file("docker-compose.yaml")
|
||||
assert not is_gha_file("README.md")
|
||||
|
||||
|
||||
def test_gha_file_wrong_extension():
|
||||
assert not is_gha_file(".github/workflows/ci.json")
|
||||
|
||||
|
||||
def test_gha_file_windows_style_path():
|
||||
assert is_gha_file(".github\\workflows\\ci.yml")
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Tests for scripts/log-run.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
_SPEC = importlib.util.spec_from_file_location(
|
||||
"log_run",
|
||||
Path(__file__).parent.parent / "scripts" / "log-run.py",
|
||||
)
|
||||
log_run = importlib.util.module_from_spec(_SPEC)
|
||||
_SPEC.loader.exec_module(log_run)
|
||||
|
||||
|
||||
def _write_usage(tmp_path: Path, usage: dict) -> Path:
|
||||
p = tmp_path / "usage.json"
|
||||
p.write_text(json.dumps(usage), encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def _read_log(log: Path) -> list[dict]:
|
||||
return [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def test_writes_one_row_per_agent_with_finding_counts(tmp_path: Path) -> None:
|
||||
output = tmp_path / "out"
|
||||
output.mkdir()
|
||||
(output / "findings-security-triage-reviewer.json").write_text(
|
||||
json.dumps({"findings": [{"file": "a.py", "line": 1, "rule_id": "B1"},
|
||||
{"file": "b.py", "line": 2, "rule_id": "B2"}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(output / "findings-walkthrough-reviewer.json").write_text(
|
||||
json.dumps({"overview": "summary", "files": []}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
log = tmp_path / "runs.jsonl"
|
||||
usage = _write_usage(tmp_path, {
|
||||
"security-triage-reviewer": {"model": "sonnet", "input_tokens": 100,
|
||||
"output_tokens": 50, "duration_ms": 1234},
|
||||
"walkthrough-reviewer": {"model": "sonnet", "input_tokens": 200,
|
||||
"output_tokens": 80, "duration_ms": 4321},
|
||||
})
|
||||
|
||||
rc = log_run.main([
|
||||
"--output-dir", str(output), "--run-id", "abc123",
|
||||
"--repo", "/tmp/repo", "--mode", "local",
|
||||
"--log-path", str(log), "--usage-json", str(usage),
|
||||
])
|
||||
assert rc == 0
|
||||
|
||||
rows = _read_log(log)
|
||||
assert len(rows) == 2
|
||||
|
||||
by_agent = {r["agent"]: r for r in rows}
|
||||
assert by_agent["security-triage-reviewer"]["finding_count"] == 2
|
||||
assert by_agent["security-triage-reviewer"]["input_tokens"] == 100
|
||||
assert by_agent["security-triage-reviewer"]["model"] == "sonnet"
|
||||
assert by_agent["walkthrough-reviewer"]["finding_count"] == 0
|
||||
assert by_agent["walkthrough-reviewer"]["duration_ms"] == 4321
|
||||
|
||||
|
||||
def test_missing_findings_file_counts_zero(tmp_path: Path) -> None:
|
||||
output = tmp_path / "out"
|
||||
output.mkdir()
|
||||
log = tmp_path / "runs.jsonl"
|
||||
usage = _write_usage(tmp_path, {
|
||||
"phantom-reviewer": {"model": "sonnet", "input_tokens": 0,
|
||||
"output_tokens": 0, "duration_ms": 0},
|
||||
})
|
||||
|
||||
rc = log_run.main([
|
||||
"--output-dir", str(output), "--run-id", "x", "--repo", "/r",
|
||||
"--mode", "ref", "--log-path", str(log), "--usage-json", str(usage),
|
||||
])
|
||||
assert rc == 0
|
||||
rows = _read_log(log)
|
||||
assert rows[0]["finding_count"] == 0
|
||||
|
||||
|
||||
def test_malformed_findings_file_counts_zero(tmp_path: Path) -> None:
|
||||
output = tmp_path / "out"
|
||||
output.mkdir()
|
||||
(output / "findings-broken-reviewer.json").write_text("{not json", encoding="utf-8")
|
||||
log = tmp_path / "runs.jsonl"
|
||||
usage = _write_usage(tmp_path, {
|
||||
"broken-reviewer": {"model": "haiku", "input_tokens": 10,
|
||||
"output_tokens": 5, "duration_ms": 100},
|
||||
})
|
||||
|
||||
rc = log_run.main([
|
||||
"--output-dir", str(output), "--run-id", "x", "--repo", "/r",
|
||||
"--mode", "local", "--log-path", str(log), "--usage-json", str(usage),
|
||||
])
|
||||
assert rc == 0
|
||||
rows = _read_log(log)
|
||||
assert rows[0]["finding_count"] == 0
|
||||
|
||||
|
||||
def test_findings_as_bare_list_is_counted(tmp_path: Path) -> None:
|
||||
output = tmp_path / "out"
|
||||
output.mkdir()
|
||||
(output / "findings-x-reviewer.json").write_text(
|
||||
json.dumps([{"rule_id": "A"}, {"rule_id": "B"}, {"rule_id": "C"}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
log = tmp_path / "runs.jsonl"
|
||||
usage = _write_usage(tmp_path, {
|
||||
"x-reviewer": {"model": "sonnet", "input_tokens": 1,
|
||||
"output_tokens": 1, "duration_ms": 1},
|
||||
})
|
||||
|
||||
rc = log_run.main([
|
||||
"--output-dir", str(output), "--run-id", "x", "--repo", "/r",
|
||||
"--mode", "local", "--log-path", str(log), "--usage-json", str(usage),
|
||||
])
|
||||
assert rc == 0
|
||||
rows = _read_log(log)
|
||||
assert rows[0]["finding_count"] == 3
|
||||
|
||||
|
||||
def test_log_path_parent_is_created(tmp_path: Path) -> None:
|
||||
output = tmp_path / "out"
|
||||
output.mkdir()
|
||||
log = tmp_path / "nested" / "deeper" / "runs.jsonl"
|
||||
usage = _write_usage(tmp_path, {
|
||||
"x-reviewer": {"model": "sonnet", "input_tokens": 0,
|
||||
"output_tokens": 0, "duration_ms": 0},
|
||||
})
|
||||
|
||||
rc = log_run.main([
|
||||
"--output-dir", str(output), "--run-id", "x", "--repo", "/r",
|
||||
"--mode", "local", "--log-path", str(log), "--usage-json", str(usage),
|
||||
])
|
||||
assert rc == 0
|
||||
assert log.exists()
|
||||
@@ -0,0 +1,80 @@
|
||||
import json
|
||||
|
||||
from scripts.manifest import (
|
||||
Manifest, Finding, ChangedFile, LanguageBreakdown,
|
||||
PackageDiff, PackageEntry, PackageUpgrade, ToolStat,
|
||||
)
|
||||
|
||||
|
||||
def test_finding_to_dict_includes_all_fields():
|
||||
f = Finding(
|
||||
tool="bandit", rule_id="B608", severity="high",
|
||||
file="src/api.py", line=45, end_line=45,
|
||||
message="Possible SQL injection.",
|
||||
cwe="CWE-89", fix_suggestion=None,
|
||||
)
|
||||
d = f.to_dict()
|
||||
assert d["tool"] == "bandit"
|
||||
assert d["rule_id"] == "B608"
|
||||
assert d["cwe"] == "CWE-89"
|
||||
assert "fix_suggestion" in d
|
||||
|
||||
|
||||
def test_changed_file_serializes_ranges():
|
||||
cf = ChangedFile(
|
||||
path="src/api.py", language="python",
|
||||
added_lines=[(12, 14), (45, 45)],
|
||||
)
|
||||
d = cf.to_dict()
|
||||
assert d["language"] == "python"
|
||||
assert d["added_lines"] == [[12, 14], [45, 45]]
|
||||
|
||||
|
||||
def test_manifest_roundtrip():
|
||||
m = Manifest(
|
||||
mode="local", base_ref="origin/main", head_ref="HEAD",
|
||||
default_branch="main",
|
||||
language_breakdown=LanguageBreakdown(
|
||||
python=2, javascript=0, typescript=0, csharp=0,
|
||||
skipped_files=[],
|
||||
),
|
||||
changed_files=[],
|
||||
findings=[],
|
||||
package_diffs={
|
||||
"python": PackageDiff(),
|
||||
"javascript": PackageDiff(),
|
||||
"csharp": PackageDiff(),
|
||||
},
|
||||
tool_stats={},
|
||||
tools_unavailable=[],
|
||||
errors=[],
|
||||
)
|
||||
payload = json.loads(m.to_json())
|
||||
assert payload["mode"] == "local"
|
||||
assert payload["language_breakdown"]["python"] == 2
|
||||
assert payload["findings"] == []
|
||||
|
||||
|
||||
def test_package_diff_with_entries():
|
||||
pd = PackageDiff(
|
||||
added=[PackageEntry(name="pyyaml", version="6.0")],
|
||||
removed=[],
|
||||
upgraded=[PackageUpgrade(name="requests", from_version="2.28.0", to_version="2.31.0")],
|
||||
)
|
||||
d = pd.to_dict()
|
||||
assert d["added"][0]["name"] == "pyyaml"
|
||||
assert d["upgraded"][0]["from"] == "2.28.0"
|
||||
assert d["upgraded"][0]["to"] == "2.31.0"
|
||||
|
||||
|
||||
def test_tool_stat_records_ran_and_counts():
|
||||
ts = ToolStat(ran=True, pre_filter=47, post_filter=3)
|
||||
d = ts.to_dict()
|
||||
assert d == {"ran": True, "pre_filter": 47, "post_filter": 3}
|
||||
|
||||
|
||||
def test_tool_stat_records_skip_reason():
|
||||
ts = ToolStat(ran=False, reason="not on PATH")
|
||||
d = ts.to_dict()
|
||||
assert d["ran"] is False
|
||||
assert d["reason"] == "not on PATH"
|
||||
@@ -0,0 +1,75 @@
|
||||
from scripts.slicing import slice_for_agent
|
||||
|
||||
|
||||
def _manifest():
|
||||
return {
|
||||
"mode": "local",
|
||||
"base_ref": "origin/main",
|
||||
"head_ref": "HEAD",
|
||||
"default_branch": "main",
|
||||
"language_breakdown": {"python": 1, "javascript": 0, "typescript": 0, "csharp": 0, "skipped_files": []},
|
||||
"changed_files": [{"path": "foo.py", "language": "python", "added_lines": [[10, 50]]}],
|
||||
"findings": [
|
||||
{"tool": "vulture", "rule_id": "vulture:90pct", "severity": "medium",
|
||||
"file": "foo.py", "line": 10, "end_line": 10, "message": "unused function"},
|
||||
{"tool": "radon", "rule_id": "radon:cc=12", "severity": "medium",
|
||||
"file": "foo.py", "line": 20, "end_line": 40, "message": "high CCN"},
|
||||
{"tool": "interrogate", "rule_id": "interrogate:missing-docstring", "severity": "low",
|
||||
"file": "foo.py", "line": 5, "end_line": 5, "message": "missing docstring"},
|
||||
{"tool": "lizard", "rule_id": "lizard:ccn=22", "severity": "high",
|
||||
"file": "foo.py", "line": 30, "end_line": 60, "message": "high CCN"},
|
||||
{"tool": "knip", "rule_id": "knip:dead-export", "severity": "medium",
|
||||
"file": "src/a.ts", "line": 12, "end_line": 12, "message": "unused export"},
|
||||
{"tool": "jscpd", "rule_id": "jscpd:clone-40lines", "severity": "medium",
|
||||
"file": "foo.py", "line": 100, "end_line": 140, "message": "duplicate"},
|
||||
{"tool": "bandit", "rule_id": "B608", "severity": "high",
|
||||
"file": "foo.py", "line": 5, "end_line": 5, "message": "SQL injection"},
|
||||
{"tool": "ruff-idiom", "rule_id": "C901", "severity": "medium",
|
||||
"file": "foo.py", "line": 50, "end_line": 50, "message": "too complex"},
|
||||
{"tool": "ruff-idiom", "rule_id": "PLR0915", "severity": "medium",
|
||||
"file": "foo.py", "line": 60, "end_line": 60, "message": "too many statements"},
|
||||
{"tool": "ruff-idiom", "rule_id": "SIM117", "severity": "low",
|
||||
"file": "foo.py", "line": 70, "end_line": 70, "message": "combine with"},
|
||||
{"tool": "ruff-idiom", "rule_id": "PLR0913", "severity": "medium",
|
||||
"file": "foo.py", "line": 80, "end_line": 80, "message": "too many args"},
|
||||
],
|
||||
"package_diffs": {"python": {"added": [], "removed": [], "upgraded": []}},
|
||||
"tool_stats": {},
|
||||
"tools_unavailable": [],
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
|
||||
def test_maintainability_slice_includes_dedicated_tools():
|
||||
sliced = slice_for_agent(_manifest(), "maintainability")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert tools >= {"vulture", "radon", "interrogate", "lizard", "knip", "jscpd"}
|
||||
|
||||
|
||||
def test_maintainability_slice_includes_complexity_idioms():
|
||||
sliced = slice_for_agent(_manifest(), "maintainability")
|
||||
rules = {f["rule_id"] for f in sliced["findings"]}
|
||||
assert "C901" in rules
|
||||
assert "PLR0915" in rules
|
||||
|
||||
|
||||
def test_maintainability_slice_excludes_security_and_dependency():
|
||||
sliced = slice_for_agent(_manifest(), "maintainability")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert "bandit" not in tools
|
||||
|
||||
|
||||
def test_maintainability_slice_excludes_non_complexity_idioms():
|
||||
sliced = slice_for_agent(_manifest(), "maintainability")
|
||||
rules = {f["rule_id"] for f in sliced["findings"]}
|
||||
assert "SIM117" not in rules
|
||||
assert "PLR0913" not in rules
|
||||
|
||||
|
||||
def test_consistency_slice_excludes_complexity_idioms():
|
||||
sliced = slice_for_agent(_manifest(), "consistency")
|
||||
rules = {f["rule_id"] for f in sliced["findings"]}
|
||||
assert "C901" not in rules
|
||||
assert "PLR0915" not in rules
|
||||
assert "SIM117" in rules
|
||||
assert "PLR0913" in rules
|
||||
@@ -0,0 +1,48 @@
|
||||
from scripts.package_diff import diff_requirements_txt, diff_package_json
|
||||
|
||||
|
||||
def test_requirements_txt_diff():
|
||||
before = "requests==2.28.0\npyyaml==5.4.1\n"
|
||||
after = "requests==2.31.0\npyyaml==5.4.1\nclick==8.0.0\n"
|
||||
pd = diff_requirements_txt(before, after)
|
||||
assert {p.name for p in pd.added} == {"click"}
|
||||
assert {p.name for p in pd.removed} == set()
|
||||
assert len(pd.upgraded) == 1
|
||||
assert pd.upgraded[0].name == "requests"
|
||||
assert pd.upgraded[0].from_version == "2.28.0"
|
||||
assert pd.upgraded[0].to_version == "2.31.0"
|
||||
|
||||
|
||||
def test_requirements_txt_removed():
|
||||
before = "a==1.0.0\nb==1.0.0\n"
|
||||
after = "a==1.0.0\n"
|
||||
pd = diff_requirements_txt(before, after)
|
||||
assert {p.name for p in pd.removed} == {"b"}
|
||||
assert pd.added == [] and pd.upgraded == []
|
||||
|
||||
|
||||
def test_package_json_diff():
|
||||
before = '{"dependencies": {"axios": "0.21.0", "lodash": "4.17.20"}}'
|
||||
after = '{"dependencies": {"axios": "1.6.0", "react": "18.0.0"}}'
|
||||
pd = diff_package_json(before, after)
|
||||
assert {p.name for p in pd.added} == {"react"}
|
||||
assert {p.name for p in pd.removed} == {"lodash"}
|
||||
assert pd.upgraded[0].name == "axios"
|
||||
|
||||
|
||||
def test_package_json_handles_dev_dependencies():
|
||||
before = '{"dependencies": {}, "devDependencies": {"jest": "27.0.0"}}'
|
||||
after = '{"dependencies": {}, "devDependencies": {"jest": "29.0.0"}}'
|
||||
pd = diff_package_json(before, after)
|
||||
assert pd.upgraded[0].name == "jest"
|
||||
assert pd.upgraded[0].to_version == "29.0.0"
|
||||
|
||||
|
||||
def test_empty_before_treats_all_as_added():
|
||||
pd = diff_requirements_txt("", "click==8.0.0\n")
|
||||
assert {p.name for p in pd.added} == {"click"}
|
||||
|
||||
|
||||
def test_invalid_json_returns_empty_diff():
|
||||
pd = diff_package_json("not json", "not json")
|
||||
assert pd.added == [] and pd.removed == [] and pd.upgraded == []
|
||||
@@ -0,0 +1,52 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.review_stats import compute_stats
|
||||
|
||||
|
||||
def _write_log(path: Path, records: list[dict]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w") as f:
|
||||
for r in records:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
|
||||
|
||||
def test_precision_per_agent(tmp_path):
|
||||
log = tmp_path / "runs.jsonl"
|
||||
_write_log(log, [
|
||||
{"kind": "subagent_run", "run_id": "r1", "agent": "sec",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"input_tokens": 1000, "output_tokens": 200,
|
||||
"duration_ms": 100, "finding_count": 4, "ts": "x", "repo": "/r", "mode": "local"},
|
||||
{"kind": "verdict", "run_id": "r1", "agent": "sec", "rule_id": "B1", "file": "a", "line": 1, "verdict": "kept"},
|
||||
{"kind": "verdict", "run_id": "r1", "agent": "sec", "rule_id": "B2", "file": "a", "line": 2, "verdict": "kept"},
|
||||
{"kind": "verdict", "run_id": "r1", "agent": "sec", "rule_id": "B3", "file": "a", "line": 3, "verdict": "dismissed"},
|
||||
{"kind": "verdict", "run_id": "r1", "agent": "sec", "rule_id": "B4", "file": "a", "line": 4, "verdict": "false_positive"},
|
||||
])
|
||||
stats = compute_stats(log)
|
||||
sec = stats["by_agent"]["sec"]
|
||||
assert sec["kept"] == 2
|
||||
assert sec["total"] == 4
|
||||
assert abs(sec["precision"] - 0.5) < 1e-9
|
||||
assert sec["tokens"] == 1200
|
||||
assert sec["tokens_per_kept"] == 600.0
|
||||
|
||||
|
||||
def test_per_rule_precision(tmp_path):
|
||||
log = tmp_path / "runs.jsonl"
|
||||
_write_log(log, [
|
||||
{"kind": "verdict", "run_id": "r1", "agent": "sec", "rule_id": "B101", "file": "a", "line": 1, "verdict": "dismissed"},
|
||||
{"kind": "verdict", "run_id": "r2", "agent": "sec", "rule_id": "B101", "file": "b", "line": 1, "verdict": "dismissed"},
|
||||
{"kind": "verdict", "run_id": "r3", "agent": "sec", "rule_id": "B101", "file": "c", "line": 1, "verdict": "false_positive"},
|
||||
])
|
||||
stats = compute_stats(log)
|
||||
rule = stats["by_rule"]["sec/B101"]
|
||||
assert rule["kept"] == 0
|
||||
assert rule["total"] == 3
|
||||
assert rule["precision"] == 0.0
|
||||
|
||||
|
||||
def test_empty_log_returns_zero_stats(tmp_path):
|
||||
log = tmp_path / "runs.jsonl"
|
||||
stats = compute_stats(log)
|
||||
assert stats == {"by_agent": {}, "by_rule": {}, "runs": 0}
|
||||
@@ -0,0 +1,107 @@
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from scripts.runner import (
|
||||
run_tool, tool_available, run_tools_parallel,
|
||||
)
|
||||
|
||||
|
||||
def test_tool_available_returns_true_when_on_path():
|
||||
with patch("shutil.which", return_value="/usr/bin/bandit"):
|
||||
assert tool_available("bandit") is True
|
||||
|
||||
|
||||
def test_tool_available_returns_false_when_missing():
|
||||
with patch("shutil.which", return_value=None):
|
||||
assert tool_available("nope") is False
|
||||
|
||||
|
||||
def test_run_tool_captures_stdout():
|
||||
fake = MagicMock(returncode=0, stdout='{"results": []}', stderr="")
|
||||
with patch("subprocess.run", return_value=fake):
|
||||
with patch("shutil.which", return_value="/usr/bin/bandit"):
|
||||
r = run_tool(["bandit", "-r", "."], cwd="/repo")
|
||||
assert r.ran is True
|
||||
assert r.exit_code == 0
|
||||
assert r.stdout == '{"results": []}'
|
||||
|
||||
|
||||
def test_run_tool_reports_missing_binary():
|
||||
with patch("shutil.which", return_value=None):
|
||||
r = run_tool(["nope"], cwd="/repo")
|
||||
assert r.ran is False
|
||||
assert r.reason == "not on PATH"
|
||||
|
||||
|
||||
def test_run_tool_reports_timeout():
|
||||
import subprocess
|
||||
def _raise(*a, **kw):
|
||||
raise subprocess.TimeoutExpired(cmd="x", timeout=1)
|
||||
with patch("subprocess.run", side_effect=_raise):
|
||||
with patch("shutil.which", return_value="/usr/bin/x"):
|
||||
r = run_tool(["x"], cwd="/repo", timeout=1)
|
||||
assert r.ran is False
|
||||
assert "timeout" in r.reason.lower()
|
||||
|
||||
|
||||
def test_run_tools_parallel_respects_cap():
|
||||
def _fake(args, **kw):
|
||||
return MagicMock(returncode=0, stdout="", stderr="")
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
with patch("shutil.which", return_value="/usr/bin/x"):
|
||||
jobs = [(["a"], "/r"), (["b"], "/r"), (["c"], "/r")]
|
||||
results = run_tools_parallel(jobs, max_workers=2)
|
||||
assert len(results) == 3
|
||||
assert {r.binary for r in results} == {"a", "b", "c"}
|
||||
|
||||
|
||||
def test_resolve_tool_prefers_skill_venv(tmp_path):
|
||||
"""If a tool exists in the skill's .venv/bin/, prefer it over PATH."""
|
||||
from scripts import runner
|
||||
|
||||
venv_bin = tmp_path / "venv" / "bin"
|
||||
venv_bin.mkdir(parents=True)
|
||||
bandit_local = venv_bin / "bandit"
|
||||
bandit_local.write_text("#!/bin/sh\necho hi\n")
|
||||
bandit_local.chmod(0o755)
|
||||
|
||||
with patch.object(runner, "_SKILL_VENV_BIN", venv_bin):
|
||||
with patch("shutil.which", return_value="/usr/bin/bandit"):
|
||||
resolved = runner.resolve_tool("bandit")
|
||||
assert resolved == str(bandit_local)
|
||||
|
||||
|
||||
def test_resolve_tool_falls_back_to_path(tmp_path):
|
||||
"""Without a local venv copy, fall back to PATH."""
|
||||
from scripts import runner
|
||||
|
||||
venv_bin = tmp_path / "venv" / "bin"
|
||||
venv_bin.mkdir(parents=True)
|
||||
|
||||
with patch.object(runner, "_SKILL_VENV_BIN", venv_bin):
|
||||
with patch("shutil.which", return_value="/usr/bin/bandit"):
|
||||
resolved = runner.resolve_tool("bandit")
|
||||
assert resolved == "/usr/bin/bandit"
|
||||
|
||||
|
||||
def test_run_tool_uses_resolved_path(tmp_path):
|
||||
"""run_tool should invoke the resolved binary, not the bare name."""
|
||||
from scripts import runner
|
||||
|
||||
venv_bin = tmp_path / "venv" / "bin"
|
||||
venv_bin.mkdir(parents=True)
|
||||
bandit_local = venv_bin / "bandit"
|
||||
bandit_local.write_text("#!/bin/sh\necho hi\n")
|
||||
bandit_local.chmod(0o755)
|
||||
|
||||
captured: list[list[str]] = []
|
||||
|
||||
def _fake(args, **kw):
|
||||
captured.append(list(args))
|
||||
return MagicMock(returncode=0, stdout="", stderr="")
|
||||
|
||||
with patch.object(runner, "_SKILL_VENV_BIN", venv_bin):
|
||||
with patch("subprocess.run", side_effect=_fake):
|
||||
runner.run_tool(["bandit", "-r", "."], cwd="/repo")
|
||||
|
||||
assert captured[0][0] == str(bandit_local)
|
||||
assert captured[0][1:] == ["-r", "."]
|
||||
@@ -0,0 +1,189 @@
|
||||
from scripts.slicing import slice_for_agent
|
||||
|
||||
|
||||
def _manifest_dict():
|
||||
return {
|
||||
"mode": "local",
|
||||
"base_ref": "origin/main",
|
||||
"head_ref": "HEAD",
|
||||
"default_branch": "main",
|
||||
"language_breakdown": {"python": 1, "javascript": 0, "typescript": 0, "csharp": 0, "skipped_files": []},
|
||||
"changed_files": [{"path": "src/api.py", "language": "python", "added_lines": [[10, 15]]}],
|
||||
"findings": [
|
||||
{"tool": "bandit", "rule_id": "B608", "severity": "high", "file": "src/api.py", "line": 12, "end_line": 12, "message": "sqlinj"},
|
||||
{"tool": "mypy", "rule_id": "arg-type", "severity": "medium", "file": "src/api.py", "line": 14, "end_line": 14, "message": "type"},
|
||||
{"tool": "gitleaks", "rule_id": "aws-token", "severity": "critical", "file": "src/api.py", "line": 11, "end_line": 11, "message": "leak"},
|
||||
{"tool": "pip-audit", "rule_id": "GHSA-x", "severity": "high", "file": "requirements.txt", "line": 1, "end_line": 1, "message": "vuln"},
|
||||
{"tool": "opengrep", "rule_id": "py.dangerous","severity": "high", "file": "src/api.py", "line": 13, "end_line": 13, "message": "dangerous"},
|
||||
{"tool": "ruff", "rule_id": "S102", "severity": "high", "file": "src/api.py", "line": 10, "end_line": 10, "message": "issue"},
|
||||
{"tool": "ruff-idiom", "rule_id": "SIM102", "severity": "low", "file": "src/api.py", "line": 20, "end_line": 20, "message": "collapsible-if"},
|
||||
],
|
||||
"package_diffs": {"python": {"added": [], "removed": [], "upgraded": []}},
|
||||
"tool_stats": {},
|
||||
"tools_unavailable": [],
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
|
||||
def test_security_slice_only_security_tools():
|
||||
sliced = slice_for_agent(_manifest_dict(), "security-triage")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert tools == {"bandit", "opengrep", "ruff"}
|
||||
|
||||
|
||||
def test_type_safety_slice_only_type_tools():
|
||||
sliced = slice_for_agent(_manifest_dict(), "type-safety")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert tools == {"mypy"}
|
||||
|
||||
|
||||
def test_dependency_slice_includes_dep_tools_and_diffs():
|
||||
sliced = slice_for_agent(_manifest_dict(), "dependency")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert tools == {"pip-audit"}
|
||||
assert "package_diffs" in sliced
|
||||
|
||||
|
||||
def test_secrets_slice_only_gitleaks():
|
||||
sliced = slice_for_agent(_manifest_dict(), "secrets")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert tools == {"gitleaks"}
|
||||
|
||||
|
||||
def test_consistency_slice_excludes_non_idiom_tools():
|
||||
sliced = slice_for_agent(_manifest_dict(), "consistency")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
non_idiom = {"bandit", "mypy", "gitleaks", "pip-audit", "opengrep", "ruff"}
|
||||
assert tools & non_idiom == set()
|
||||
assert "changed_files" in sliced
|
||||
|
||||
|
||||
def test_consistency_slice_includes_ruff_idiom():
|
||||
sliced = slice_for_agent(_manifest_dict(), "consistency")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert tools == {"ruff-idiom"}
|
||||
rule_ids = {f["rule_id"] for f in sliced["findings"]}
|
||||
assert "SIM102" in rule_ids
|
||||
|
||||
|
||||
def test_security_slice_excludes_ruff_idiom():
|
||||
sliced = slice_for_agent(_manifest_dict(), "security-triage")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert "ruff-idiom" not in tools
|
||||
assert tools == {"bandit", "opengrep", "ruff"}
|
||||
|
||||
|
||||
def _manifest_with_lua_gha():
|
||||
"""Manifest with selene, luac, actionlint, and zizmor findings."""
|
||||
return {
|
||||
"mode": "local",
|
||||
"base_ref": "origin/main",
|
||||
"head_ref": "HEAD",
|
||||
"default_branch": "main",
|
||||
"language_breakdown": {
|
||||
"python": 0, "javascript": 0, "typescript": 0, "csharp": 0,
|
||||
"lua": 1, "github_actions": 1, "skipped_files": [],
|
||||
},
|
||||
"changed_files": [
|
||||
{"path": "src/game.lua", "language": "lua", "added_lines": [[5, 10]]},
|
||||
{"path": ".github/workflows/ci.yml", "language": "github-actions", "added_lines": [[12, 12]]},
|
||||
],
|
||||
"findings": [
|
||||
{"tool": "selene", "rule_id": "undefined_variable", "severity": "high", "file": "src/game.lua", "line": 5, "end_line": 5, "message": "undefined var"},
|
||||
{"tool": "luac", "rule_id": "syntax-error", "severity": "critical","file": "src/game.lua", "line": 8, "end_line": 8, "message": "syntax error"},
|
||||
{"tool": "actionlint", "rule_id": "expression", "severity": "high", "file": ".github/workflows/ci.yml", "line": 12, "end_line": 12, "message": "bad expr"},
|
||||
{"tool": "zizmor", "rule_id": "unpinned-uses", "severity": "medium", "file": ".github/workflows/ci.yml", "line": 15, "end_line": 15, "message": "unpin"},
|
||||
],
|
||||
"package_diffs": {},
|
||||
"tool_stats": {},
|
||||
"tools_unavailable": [],
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
|
||||
def test_gha_reviewer_slice_only_gha_tools():
|
||||
sliced = slice_for_agent(_manifest_with_lua_gha(), "gha-reviewer")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert tools == {"actionlint", "zizmor"}
|
||||
|
||||
|
||||
def test_gha_reviewer_slice_excludes_lua_tools():
|
||||
sliced = slice_for_agent(_manifest_with_lua_gha(), "gha-reviewer")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert "selene" not in tools
|
||||
assert "luac" not in tools
|
||||
|
||||
|
||||
def test_selene_routes_to_maintainability():
|
||||
sliced = slice_for_agent(_manifest_with_lua_gha(), "maintainability")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert "selene" in tools
|
||||
|
||||
|
||||
def test_luac_routes_to_security_triage():
|
||||
sliced = slice_for_agent(_manifest_with_lua_gha(), "security-triage")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert "luac" in tools
|
||||
|
||||
|
||||
def test_luac_routes_to_maintainability():
|
||||
sliced = slice_for_agent(_manifest_with_lua_gha(), "maintainability")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert "luac" in tools
|
||||
|
||||
|
||||
def test_security_triage_excludes_gha_tools():
|
||||
sliced = slice_for_agent(_manifest_with_lua_gha(), "security-triage")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert "actionlint" not in tools
|
||||
assert "zizmor" not in tools
|
||||
|
||||
|
||||
def _manifest_with_powershell():
|
||||
"""Manifest with psscriptanalyzer (security + style rules) and injectionhunter."""
|
||||
return {
|
||||
"mode": "local",
|
||||
"base_ref": "origin/main",
|
||||
"head_ref": "HEAD",
|
||||
"default_branch": "main",
|
||||
"language_breakdown": {
|
||||
"python": 0, "javascript": 0, "typescript": 0, "csharp": 0,
|
||||
"lua": 0, "powershell": 1, "github_actions": 0, "skipped_files": [],
|
||||
},
|
||||
"changed_files": [
|
||||
{"path": "scripts/Deploy.ps1", "language": "powershell", "added_lines": [[10, 40]]},
|
||||
],
|
||||
"findings": [
|
||||
{"tool": "psscriptanalyzer", "rule_id": "PSAvoidUsingInvokeExpression", "severity": "medium", "file": "scripts/Deploy.ps1", "line": 12, "end_line": 12, "message": "iex"},
|
||||
{"tool": "psscriptanalyzer", "rule_id": "PSAvoidUsingWriteHost", "severity": "medium", "file": "scripts/Deploy.ps1", "line": 5, "end_line": 5, "message": "write-host"},
|
||||
{"tool": "injectionhunter", "rule_id": "InjectionRisk.InvokeExpression","severity": "high", "file": "scripts/Deploy.ps1", "line": 12, "end_line": 12, "message": "injection"},
|
||||
],
|
||||
"package_diffs": {},
|
||||
"tool_stats": {},
|
||||
"tools_unavailable": [],
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
|
||||
def test_injectionhunter_routes_to_security_triage():
|
||||
sliced = slice_for_agent(_manifest_with_powershell(), "security-triage")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert "injectionhunter" in tools
|
||||
|
||||
|
||||
def test_psscriptanalyzer_security_rules_route_to_security_triage():
|
||||
sliced = slice_for_agent(_manifest_with_powershell(), "security-triage")
|
||||
rules = {f["rule_id"] for f in sliced["findings"] if f["tool"] == "psscriptanalyzer"}
|
||||
assert rules == {"PSAvoidUsingInvokeExpression"}
|
||||
|
||||
|
||||
def test_psscriptanalyzer_style_rules_route_to_maintainability():
|
||||
sliced = slice_for_agent(_manifest_with_powershell(), "maintainability")
|
||||
rules = {f["rule_id"] for f in sliced["findings"] if f["tool"] == "psscriptanalyzer"}
|
||||
assert rules == {"PSAvoidUsingWriteHost"}
|
||||
|
||||
|
||||
def test_maintainability_excludes_injectionhunter():
|
||||
sliced = slice_for_agent(_manifest_with_powershell(), "maintainability")
|
||||
tools = {f["tool"] for f in sliced["findings"]}
|
||||
assert "injectionhunter" not in tools
|
||||
@@ -0,0 +1,2 @@
|
||||
def test_imports():
|
||||
from scripts import manifest # noqa: F401
|
||||
@@ -0,0 +1,43 @@
|
||||
import pytest
|
||||
|
||||
from scripts.telemetry import append_subagent_run, append_verdict, read_runs
|
||||
|
||||
|
||||
def test_round_trip(tmp_path):
|
||||
log = tmp_path / "runs.jsonl"
|
||||
append_subagent_run(
|
||||
log, run_id="r1", repo="/r", mode="local",
|
||||
agent="security-triage-reviewer", model="claude-sonnet-4-6",
|
||||
input_tokens=100, output_tokens=20, duration_ms=1000, finding_count=3,
|
||||
)
|
||||
append_verdict(
|
||||
log, run_id="r1", agent="security-triage-reviewer",
|
||||
rule_id="bandit:B608", file="x.py", line=1,
|
||||
verdict="kept", notes="real",
|
||||
)
|
||||
records = read_runs(log)
|
||||
assert len(records) == 2
|
||||
assert records[0]["kind"] == "subagent_run"
|
||||
assert records[0]["input_tokens"] == 100
|
||||
assert records[1]["kind"] == "verdict"
|
||||
assert records[1]["verdict"] == "kept"
|
||||
|
||||
|
||||
def test_append_is_safe_with_no_parent_dir(tmp_path):
|
||||
log = tmp_path / "deep" / "nested" / "runs.jsonl"
|
||||
append_subagent_run(
|
||||
log, run_id="r2", repo="/r", mode="local",
|
||||
agent="x", model="y", input_tokens=0, output_tokens=0,
|
||||
duration_ms=0, finding_count=0,
|
||||
)
|
||||
assert log.exists()
|
||||
|
||||
|
||||
def test_invalid_verdict_raises(tmp_path):
|
||||
log = tmp_path / "runs.jsonl"
|
||||
with pytest.raises(ValueError, match="invalid verdict"):
|
||||
append_verdict(
|
||||
log, run_id="r3", agent="security-triage-reviewer",
|
||||
rule_id="bandit:B101", file="a.py", line=5,
|
||||
verdict="maybe", notes="",
|
||||
)
|
||||
Reference in New Issue
Block a user