diff --git a/docs/SECURITY_AUDIT.md b/docs/SECURITY_AUDIT.md index fd1bfda..65e1a38 100644 --- a/docs/SECURITY_AUDIT.md +++ b/docs/SECURITY_AUDIT.md @@ -34,16 +34,28 @@ tools/checks/security-audit/run.sh --mode full --scope govoplan ``` Reports are written to `audit-reports/`, which is intentionally ignored by git. +Each run records tool versions, report checksums, and start/end repository +revision plus worktree fingerprints. A repository change during scanning makes +the run fail so a mixed code snapshot cannot be reported as a valid audit. +Step exit codes are recorded separately from findings: report-only mode may +accept findings, but scanner execution errors and malformed JSON/SARIF reports +always fail the run. The manifest lists the expected, present, and missing +reports for that invocation. Validation and checksums use that explicit set, so +reusing a report directory cannot make stale output look like part of a new run. -The wrapper tags the toolbox image by a fingerprint of the Dockerfile and -`requirements-audit.txt`. If those inputs have not changed, subsequent runs reuse -the existing local image instead of reinstalling all tools. The stable alias is -`govoplan/security-audit:local` unless `SECURITY_AUDIT_IMAGE` is set. +The wrapper tags the toolbox image by a fingerprint of the Dockerfile, +`requirements-audit.txt`, and the Semgrep smoke-test inputs. If those inputs have +not changed, subsequent runs reuse the existing local image instead of +reinstalling all tools. The stable alias is `govoplan/security-audit:local` +unless `SECURITY_AUDIT_IMAGE` is set. Semgrep is installed separately in the toolbox image because current Semgrep -packages pin Click to an affected `8.1.x` line. The image upgrades Click after -Semgrep installation and fails during build if Semgrep no longer starts with the -patched Click version. +packages pin affected Click and MCP versions. The image upgrades both after +Semgrep installation, runs an actual local-rule scan, and audits the final +toolbox Python environment during the image build. The compatibility releases +are pinned exactly to keep the tested override reproducible. Remove either +compatibility override only after Semgrep's own dependency bounds include a +fixed version. Force a cached rebuild: @@ -70,6 +82,10 @@ tools/checks/security-audit/run.sh --mode quick --scope current --update --build - `ci`: quick plus Semgrep public registry rulesets, Trivy, pip-audit, npm audit. - `full`: ci plus OSV-Scanner, jscpd, Radon, and Xenon. +Semgrep and Trivy are invoked with finding-sensitive exit codes. Their exit 1 +is therefore a finding under the wrapper contract; higher exit codes, missing +output, invalid JSON/SARIF, and scanner error payloads are execution failures. + Bandit and Ruff security reports are split by source kind. Production code under `src/` is written to `bandit.json` and `ruff-security.json` and controls strict mode. Test code under `tests/` is still scanned for visibility, but its findings @@ -153,7 +169,10 @@ cd /mnt/DATA/git/govoplan python -m venv .venv ./.venv/bin/python -m pip install -r requirements-audit.txt ./.venv/bin/python -m pip install 'semgrep>=1.140,<2' -./.venv/bin/python -m pip install --upgrade --no-deps 'click>=8.3.3' +./.venv/bin/python -m pip install --upgrade --no-deps 'click==8.3.3' +./.venv/bin/python -m pip install --upgrade --no-deps 'mcp==1.28.1' +./.venv/bin/semgrep scan --metrics=off --config tools/checks/security-audit/semgrep-govoplan.yml tools/checks/check-version-alignment.py +./.venv/bin/pip-audit --progress-spinner off ``` Then install the non-Python tools (`gitleaks`, `trivy`, `osv-scanner`, `jscpd`) diff --git a/tests/test_security_audit_wrapper.py b/tests/test_security_audit_wrapper.py new file mode 100644 index 0000000..ee81fd9 --- /dev/null +++ b/tests/test_security_audit_wrapper.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import textwrap +import unittest + + +META_ROOT = Path(__file__).resolve().parents[1] +AUDIT_SCRIPT = META_ROOT / "tools" / "checks" / "check-security-audit.sh" + + +class SecurityAuditWrapperTests(unittest.TestCase): + def setUp(self) -> None: + self._temporary_directory = tempfile.TemporaryDirectory() + self.temporary_root = Path(self._temporary_directory.name) + self.checkout = self.temporary_root / "checkout" + checks = self.checkout / "tools" / "checks" + checks.mkdir(parents=True) + shutil.copy2(AUDIT_SCRIPT, checks / AUDIT_SCRIPT.name) + (self.checkout / "README.md").write_text("audit fixture\n", encoding="utf-8") + + subprocess.run(["git", "init", "-q", str(self.checkout)], check=True) + subprocess.run( + [ + "git", + "-C", + str(self.checkout), + "config", + "user.email", + "audit@example.invalid", + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(self.checkout), "config", "user.name", "Audit Test"], + check=True, + ) + subprocess.run(["git", "-C", str(self.checkout), "add", "."], check=True) + subprocess.run( + ["git", "-C", str(self.checkout), "commit", "-qm", "audit fixture"], + check=True, + ) + + self.stub_bin = self.temporary_root / "bin" + self.stub_bin.mkdir() + self._write_stub( + "semgrep", + r""" + if [[ " $* " == *" --version "* ]]; then + echo 'semgrep stub 1.0' + exit 0 + fi + if [[ " $* " != *" --error "* ]]; then + echo 'missing --error' >&2 + exit 3 + fi + output='' + while [[ $# -gt 0 ]]; do + if [[ "$1" == '--output' ]]; then + output="$2" + shift 2 + else + shift + fi + done + case "${STUB_SEMGREP_REPORT:-valid}" in + valid) + printf '%s\n' '{"version":"2.1.0","runs":[]}' > "$output" + ;; + malformed) + printf '%s\n' '{' > "$output" + ;; + invalid-sarif) + printf '%s\n' '{}' > "$output" + ;; + missing) + ;; + esac + if [[ -n "${STUB_MUTATE_PATH:-}" ]]; then + printf '%s\n' 'changed during audit' >> "$STUB_MUTATE_PATH" + fi + exit "${STUB_SEMGREP_EXIT:-0}" + """, + ) + self._write_stub( + "gitleaks", + r""" + if [[ "${1:-}" == '--version' || "${1:-}" == 'version' ]]; then + echo 'gitleaks stub 1.0' + exit 0 + fi + if [[ "${1:-}" == 'git' && "${2:-}" == '--help' ]]; then + exit 0 + fi + output='' + while [[ $# -gt 0 ]]; do + if [[ "$1" == '--report-path' ]]; then + output="$2" + shift 2 + else + shift + fi + done + mkdir -p "$(dirname "$output")" + printf '%s\n' '[]' > "$output" + exit "${STUB_GITLEAKS_EXIT:-0}" + """, + ) + for tool in ("bandit", "ruff"): + self._write_stub( + tool, + f""" + if [[ " $* " == *" --version "* ]]; then + echo '{tool} stub 1.0' + fi + exit 0 + """, + ) + + def tearDown(self) -> None: + self._temporary_directory.cleanup() + + def _write_stub(self, name: str, body: str) -> None: + path = self.stub_bin / name + path.write_text( + "#!/usr/bin/env bash\nset -euo pipefail\n" + textwrap.dedent(body).lstrip(), + encoding="utf-8", + ) + path.chmod(0o755) + + def _run( + self, *arguments: str, **environment: str + ) -> tuple[subprocess.CompletedProcess[str], Path]: + reports = self.temporary_root / "reports" + env = os.environ.copy() + env.update( + { + "PATH": f"{self.stub_bin}:{env['PATH']}", + "SECURITY_AUDIT_REQUIRE_TOOLS": "1", + "SECURITY_AUDIT_TOOLBOX_FINGERPRINT": 'stub-"safe', + **environment, + } + ) + result = subprocess.run( + [ + "bash", + str(self.checkout / "tools" / "checks" / AUDIT_SCRIPT.name), + "--mode", + "quick", + "--scope", + "current", + "--reports-dir", + str(reports), + *arguments, + ], + cwd=self.checkout, + env=env, + check=False, + capture_output=True, + text=True, + ) + return result, reports + + @staticmethod + def _manifest(reports: Path) -> dict[str, object]: + return json.loads((reports / "manifest.json").read_text(encoding="utf-8")) + + def test_report_only_findings_are_nonfatal_and_strict_findings_fail(self) -> None: + result, reports = self._run("--report-only", STUB_SEMGREP_EXIT="1") + + self.assertEqual(0, result.returncode, result.stderr) + manifest = self._manifest(reports) + self.assertEqual(0, manifest["overall_status"]) + self.assertEqual(1, manifest["finding_status"]) + self.assertEqual(0, manifest["execution_error_status"]) + self.assertIn( + "Semgrep SAST\t1\tfindings", (reports / "step-status.tsv").read_text() + ) + + strict_result, strict_reports = self._run("--strict", STUB_SEMGREP_EXIT="1") + self.assertEqual(1, strict_result.returncode) + self.assertEqual(1, self._manifest(strict_reports)["overall_status"]) + + def test_scanner_execution_error_is_fatal_in_report_only_mode(self) -> None: + result, reports = self._run("--report-only", STUB_SEMGREP_EXIT="2") + + self.assertEqual(1, result.returncode) + manifest = self._manifest(reports) + self.assertEqual(1, manifest["overall_status"]) + self.assertEqual(1, manifest["execution_error_status"]) + self.assertIn( + "Semgrep SAST\t2\texecution-error", + (reports / "step-status.tsv").read_text(), + ) + + def test_malformed_or_missing_machine_report_is_fatal(self) -> None: + for report_kind in ("malformed", "invalid-sarif", "missing"): + with self.subTest(report_kind=report_kind): + result, reports = self._run( + "--report-only", + STUB_SEMGREP_REPORT=report_kind, + ) + self.assertEqual(1, result.returncode) + manifest = self._manifest(reports) + self.assertEqual(1, manifest["execution_error_status"]) + if report_kind == "missing": + self.assertIn("semgrep.sarif", manifest["missing_reports"]) + self.assertIn( + "Validate machine-readable audit reports\t2\texecution-error", + (reports / "step-status.tsv").read_text(), + ) + + def test_workspace_mutation_invalidates_the_audit(self) -> None: + result, reports = self._run( + "--report-only", + STUB_MUTATE_PATH=str(self.checkout / "README.md"), + ) + + self.assertEqual(1, result.returncode) + manifest = self._manifest(reports) + self.assertFalse(manifest["workspace_unchanged"]) + self.assertEqual(1, manifest["overall_status"]) + self.assertNotEqual( + (reports / "workspace-state-start.tsv").read_text(), + (reports / "workspace-state-end.tsv").read_text(), + ) + + def test_manifest_and_checksums_cover_only_current_run_reports(self) -> None: + reports = self.temporary_root / "reports" + reports.mkdir() + (reports / "stale-malformed.json").write_text("{", encoding="utf-8") + + result, reports = self._run("--report-only") + + self.assertEqual(0, result.returncode, result.stderr) + manifest = self._manifest(reports) + self.assertEqual('stub-"safe', manifest["toolbox_fingerprint"]) + self.assertNotIn("stale-malformed.json", manifest["reports"]) + self.assertEqual([], manifest["missing_reports"]) + checksums = (reports / "report-sha256.txt").read_text(encoding="utf-8") + self.assertNotIn("stale-malformed.json", checksums) + verification = subprocess.run( + ["sha256sum", "--check", "report-sha256.txt"], + cwd=reports, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(0, verification.returncode, verification.stderr) + checksummed_paths = { + line.split(" ", 1)[1] for line in checksums.splitlines() if " " in line + } + self.assertEqual(set(manifest["reports"]), checksummed_paths) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/checks/check-security-audit.sh b/tools/checks/check-security-audit.sh index 5f3bf79..df35171 100644 --- a/tools/checks/check-security-audit.sh +++ b/tools/checks/check-security-audit.sh @@ -77,6 +77,15 @@ if [[ "$REPORTS_DIR" != /* ]]; then REPORTS_DIR="$ROOT/$REPORTS_DIR" fi mkdir -p "$REPORTS_DIR" +REPORTS_DIR="$(cd "$REPORTS_DIR" && pwd -P)" + +for flag_name in FAIL_ON_FINDINGS REQUIRE_TOOLS; do + flag_value="${!flag_name}" + if [[ "$flag_value" != "0" && "$flag_value" != "1" ]]; then + echo "$flag_name must be 0 or 1, got: $flag_value" >&2 + exit 2 + fi +done declare -a REPOS=() if [[ "$SCOPE" == "govoplan" ]]; then @@ -118,7 +127,7 @@ for repo in "${REPOS[@]}"; do done safe_name() { - basename "$1" | tr -c 'A-Za-z0-9_.-' '_' + printf '%s' "$(basename "$1")" | tr -c 'A-Za-z0-9_.-' '_' } has_tool() { @@ -128,58 +137,340 @@ has_tool() { overall_status=0 finding_status=0 missing_status=0 +execution_status=0 +audit_started_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" +audit_completed_at="" +workspace_unchanged="unknown" +audit_toolbox_fingerprint="${SECURITY_AUDIT_TOOLBOX_FINGERPRINT:-direct-host}" +declare -A REPORT_FILES=() +declare -A MACHINE_REPORTS=() + +register_report() { + local path="$1" + local report_kind="${2:-text}" + case "$path" in + "$REPORTS_DIR"/*) ;; + *) + echo "Refusing to register an audit report outside $REPORTS_DIR: $path" >&2 + return 2 + ;; + esac + REPORT_FILES["$path"]=1 + if [[ "$report_kind" == "machine" ]]; then + MACHINE_REPORTS["$path"]=1 + fi +} + +prepare_machine_report() { + local path="$1" + register_report "$path" machine || return 2 + if [[ -e "$path" || -L "$path" ]]; then + rm -f -- "$path" || return 2 + fi +} run_step() { local name="$1" - shift + local exit_contract="$2" + shift 2 echo echo "==> $name" "$@" local status=$? - if [[ "$status" -ne 0 ]]; then - echo "Audit step reported findings or failed: $name (exit $status)" >&2 + local outcome="passed" + if [[ "$status" -eq 0 ]]; then + : + elif [[ "$exit_contract" == "findings-exit-one" && "$status" -eq 1 ]]; then + outcome="findings" + echo "Audit step reported findings: $name (exit $status)" >&2 finding_status=1 if [[ "$FAIL_ON_FINDINGS" == "1" ]]; then overall_status=1 fi + else + outcome="execution-error" + echo "Audit step failed to execute successfully: $name (exit $status)" >&2 + execution_status=1 + overall_status=1 fi + printf '%s\t%s\t%s\n' "$name" "$status" "$outcome" >> "$REPORTS_DIR/step-status.tsv" } skip_or_fail_missing() { local tool="$1" echo "Skipping $tool: command not found. Use tools/checks/security-audit/run.sh for the containerized toolbox." >&2 + missing_status=1 if [[ "$REQUIRE_TOOLS" == "1" ]]; then - missing_status=1 overall_status=1 fi + printf '%s\t127\tmissing\n' "$tool" >> "$REPORTS_DIR/step-status.tsv" } write_manifest() { - { - printf '{\n' - printf ' "mode": "%s",\n' "$MODE" - printf ' "scope": "%s",\n' "$SCOPE" - printf ' "fail_on_findings": "%s",\n' "$FAIL_ON_FINDINGS" - printf ' "repositories": [\n' - local index=0 - for repo in "${REPOS[@]}"; do - [[ "$index" -gt 0 ]] && printf ',\n' - printf ' "%s"' "$repo" - index=$((index + 1)) - done - printf '\n ]\n' - printf '}\n' - } > "$REPORTS_DIR/manifest.json" + local -a present_reports=() + local -a expected_reports=() + local path + while IFS= read -r path; do + expected_reports+=("${path#"$REPORTS_DIR"/}") + if [[ -f "$path" ]]; then + present_reports+=("${path#"$REPORTS_DIR"/}") + fi + done < <(printf '%s\n' "${!REPORT_FILES[@]}" | sort) + + python3 - \ + "$REPORTS_DIR/manifest.json" \ + "$MODE" \ + "$SCOPE" \ + "$FAIL_ON_FINDINGS" \ + "$audit_started_at" \ + "$audit_completed_at" \ + "$workspace_unchanged" \ + "$overall_status" \ + "$finding_status" \ + "$execution_status" \ + "$missing_status" \ + "$audit_toolbox_fingerprint" \ + "${#REPOS[@]}" \ + "${REPOS[@]}" \ + "${#present_reports[@]}" \ + "${present_reports[@]}" \ + "${expected_reports[@]}" <<'PY' +from __future__ import annotations + +import json +from pathlib import Path +import sys + + +( + manifest_path, + mode, + scope, + fail_on_findings, + started_at, + completed_at, + workspace_unchanged, + overall_status, + finding_status, + execution_status, + missing_status, + toolbox_fingerprint, + repository_count, + *remaining, +) = sys.argv[1:] +repo_count = int(repository_count) +repositories = remaining[:repo_count] +remaining = remaining[repo_count:] +report_count = int(remaining[0]) +reports = remaining[1 : report_count + 1] +expected_reports = remaining[report_count + 1 :] +payload = { + "mode": mode, + "scope": scope, + "fail_on_findings": fail_on_findings == "1", + "started_at_utc": started_at, + "completed_at_utc": completed_at, + "toolbox_fingerprint": toolbox_fingerprint, + "workspace_unchanged": workspace_unchanged == "true", + "overall_status": int(overall_status), + "finding_status": int(finding_status), + "execution_error_status": int(execution_status), + "missing_tool_status": int(missing_status), + "tool_versions": "tool-versions.txt", + "workspace_state_start": "workspace-state-start.tsv", + "workspace_state_end": "workspace-state-end.tsv", + "report_checksums": "report-sha256.txt", + "step_status": "step-status.tsv", + "repositories": repositories, + "reports": reports, + "expected_reports": expected_reports, + "missing_reports": sorted(set(expected_reports) - set(reports)), +} +Path(manifest_path).write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", +) +PY +} + +accumulate_exit_status() { + local -n aggregate="$1" + local child_status="$2" + if [[ "$child_status" -gt 1 ]]; then + aggregate=2 + elif [[ "$child_status" -eq 1 && "$aggregate" -eq 0 ]]; then + aggregate=1 + fi +} + +repository_fingerprint() { + local repo="$1" + ( + set -e + cd "$repo" || exit 1 + git rev-parse HEAD + git status --porcelain=v1 -z + git diff --binary --no-ext-diff HEAD -- + while IFS= read -r -d '' path; do + printf '%s\0' "$path" + if [[ -L "$path" ]]; then + printf 'symlink\0' + readlink -- "$path" + else + sha256sum -- "$path" + fi + done < <(git ls-files --others --exclude-standard -z) + ) | sha256sum | cut -d' ' -f1 +} + +write_workspace_state() { + local destination="$1" + local head + local fingerprint + : > "$destination" + for repo in "${REPOS[@]}"; do + if ! head="$(git -C "$repo" rev-parse HEAD)"; then + return 2 + fi + if ! fingerprint="$(repository_fingerprint "$repo")"; then + return 2 + fi + printf '%s\t%s\t%s\n' \ + "$repo" \ + "$head" \ + "$fingerprint" \ + >> "$destination" + done +} + +write_tool_versions() { + local destination="$REPORTS_DIR/tool-versions.txt" + local status=0 + local version + : > "$destination" + local tool + for tool in semgrep bandit ruff gitleaks trivy pip-audit npm osv-scanner jscpd radon xenon; do + has_tool "$tool" || continue + if [[ "$tool" == "gitleaks" ]]; then + version="$("$tool" version 2>&1)" + else + version="$("$tool" --version 2>&1)" + fi + local version_status=$? + version="${version%%$'\n'*}" + if [[ "$version_status" -ne 0 || -z "$version" ]]; then + printf '%s\t%s\n' "$tool" "version-command-failed:$version_status" >> "$destination" + status=2 + else + printf '%s\t%s\n' "$tool" "$version" >> "$destination" + fi + done + return "$status" +} + +write_report_checksums() { + local path + local relative_path + : > "$REPORTS_DIR/report-sha256.txt" + while IFS= read -r path; do + [[ -f "$path" ]] || continue + relative_path="${path#"$REPORTS_DIR"/}" + (cd "$REPORTS_DIR" && sha256sum -- "$relative_path") \ + >> "$REPORTS_DIR/report-sha256.txt" || return 2 + done < <(printf '%s\n' "${!REPORT_FILES[@]}" | sort) +} + +validate_machine_reports() { + python3 - "$REPORTS_DIR" "${!MACHINE_REPORTS[@]}" <<'PY' +from __future__ import annotations + +import json +from pathlib import Path +import sys + + +root = Path(sys.argv[1]).resolve() +errors: list[str] = [] +validated = 0 +for raw_path in sorted(set(sys.argv[2:])): + path = Path(raw_path) + try: + relative_path = path.resolve().relative_to(root) + except (OSError, ValueError) as exc: + errors.append(f"{path}: invalid report path: {exc}") + continue + if not path.is_file(): + errors.append(f"{relative_path}: expected report was not produced") + continue + if path.stat().st_size == 0: + errors.append(f"{relative_path}: report is empty") + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + errors.append(f"{relative_path}: invalid JSON: {exc}") + continue + validated += 1 + if path.suffix == ".sarif" and ( + not isinstance(payload, dict) + or not isinstance(payload.get("version"), str) + or not isinstance(payload.get("runs"), list) + ): + errors.append(f"{relative_path}: invalid SARIF document") + continue + report_name = path.name + if report_name.startswith("bandit") and ( + not isinstance(payload, dict) + or not isinstance(payload.get("errors"), list) + or not isinstance(payload.get("metrics"), dict) + or not isinstance(payload.get("results"), list) + ): + errors.append(f"{relative_path}: invalid Bandit report") + elif report_name.startswith("ruff-security") and not isinstance(payload, list): + errors.append(f"{relative_path}: invalid Ruff report") + elif report_name.startswith("gitleaks-") and not isinstance(payload, list): + errors.append(f"{relative_path}: invalid Gitleaks report") + elif report_name.startswith("pip-audit-") and ( + not isinstance(payload, dict) + or not isinstance(payload.get("dependencies"), list) + ): + errors.append(f"{relative_path}: invalid pip-audit report") + elif report_name.startswith("npm-audit-") and ( + not isinstance(payload, dict) + or payload.get("error") + or not isinstance(payload.get("metadata"), dict) + or not isinstance(payload.get("vulnerabilities"), dict) + ): + errors.append(f"{relative_path}: npm audit returned an invalid or error report") + elif report_name.startswith("osv-scanner-") and ( + not isinstance(payload, dict) + or not isinstance(payload.get("results"), list) + ): + errors.append(f"{relative_path}: invalid OSV-Scanner report") + elif report_name == "jscpd-report.json" and ( + not isinstance(payload, dict) + or not isinstance(payload.get("duplicates"), list) + or not isinstance(payload.get("statistics"), dict) + ): + errors.append(f"{relative_path}: invalid jscpd report") + +if errors: + print("\n".join(errors), file=sys.stderr) + raise SystemExit(2) +print(f"Validated {validated} JSON/SARIF reports.") +PY } run_semgrep() { local report="$REPORTS_DIR/semgrep.sarif" + prepare_machine_report "$report" || return 2 local config_args=(--config "$ROOT/tools/checks/security-audit/semgrep-govoplan.yml") if [[ "$MODE" != "quick" ]]; then config_args+=(--config p/default --config p/owasp-top-ten) fi semgrep scan \ --metrics=off \ + --error \ --no-git-ignore \ --sarif \ --output "$report" \ @@ -198,10 +489,15 @@ run_semgrep() { run_bandit() { local status=0 if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then - bandit -r "${PY_PROD_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json" || status=1 + prepare_machine_report "$REPORTS_DIR/bandit.json" || return 2 + bandit -r "${PY_PROD_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json" + accumulate_exit_status status "$?" fi if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then - bandit -r "${PY_TEST_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit-tests.json" || true + prepare_machine_report "$REPORTS_DIR/bandit-tests.json" || return 2 + bandit -r "${PY_TEST_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit-tests.json" + local test_status=$? + [[ "$test_status" -le 1 ]] || status=2 fi return "$status" } @@ -209,10 +505,15 @@ run_bandit() { run_ruff_security() { local status=0 if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then - ruff check --select S --output-format json "${PY_PROD_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json" || status=1 + prepare_machine_report "$REPORTS_DIR/ruff-security.json" || return 2 + ruff check --select S --output-format json "${PY_PROD_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json" + accumulate_exit_status status "$?" fi if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then - ruff check --select S --output-format json "${PY_TEST_ROOTS[@]}" > "$REPORTS_DIR/ruff-security-tests.json" || true + prepare_machine_report "$REPORTS_DIR/ruff-security-tests.json" || return 2 + ruff check --select S --output-format json "${PY_TEST_ROOTS[@]}" > "$REPORTS_DIR/ruff-security-tests.json" + local test_status=$? + [[ "$test_status" -le 1 ]] || status=2 fi return "$status" } @@ -223,12 +524,15 @@ run_gitleaks() { local name name="$(safe_name "$repo")" if gitleaks git --help >/dev/null 2>&1; then + prepare_machine_report "$REPORTS_DIR/gitleaks-history-$name.json" || return 2 + prepare_machine_report "$REPORTS_DIR/gitleaks-worktree-$name.json" || return 2 gitleaks git \ --config "$ROOT/.gitleaks.toml" \ --report-format json \ --report-path "$REPORTS_DIR/gitleaks-history-$name.json" \ --no-banner \ - "$repo" || status=1 + "$repo" + accumulate_exit_status status "$?" # History scanning does not include new or modified working-tree files. # Scan the directory as well so pre-commit audits cover the exact code # under review, while retaining the history scan above. @@ -237,14 +541,17 @@ run_gitleaks() { --report-format json \ --report-path "$REPORTS_DIR/gitleaks-worktree-$name.json" \ --no-banner \ - "$repo" || status=1 + "$repo" + accumulate_exit_status status "$?" else + prepare_machine_report "$REPORTS_DIR/gitleaks-$name.json" || return 2 gitleaks detect \ --source "$repo" \ --config "$ROOT/.gitleaks.toml" \ --report-format json \ --report-path "$REPORTS_DIR/gitleaks-$name.json" \ - --no-banner || status=1 + --no-banner + accumulate_exit_status status "$?" fi done return "$status" @@ -255,6 +562,7 @@ run_trivy() { for repo in "${REPOS[@]}"; do local name name="$(safe_name "$repo")" + prepare_machine_report "$REPORTS_DIR/trivy-$name.sarif" || return 2 trivy fs \ --scanners vuln,secret,misconfig \ --skip-dirs node_modules \ @@ -262,8 +570,10 @@ run_trivy() { --skip-dirs dist \ --skip-dirs build \ --format sarif \ + --exit-code 1 \ --output "$REPORTS_DIR/trivy-$name.sarif" \ - "$repo" || status=1 + "$repo" + accumulate_exit_status status "$?" done return "$status" } @@ -274,9 +584,11 @@ run_pip_audit_manifests() { [[ -f "$repo/requirements.txt" ]] || continue local name name="$(safe_name "$repo")" + prepare_machine_report "$REPORTS_DIR/pip-audit-$name.json" || return 2 # Requirements may contain project-relative entries such as `.[server]`. # Resolve them from the owning repository instead of the meta-repository. - (cd "$repo" && pip-audit -r requirements.txt --progress-spinner off --format json --output "$REPORTS_DIR/pip-audit-$name.json") || status=1 + (cd "$repo" && pip-audit -r requirements.txt --progress-spinner off --format json --output "$REPORTS_DIR/pip-audit-$name.json") + accumulate_exit_status status "$?" done return "$status" } @@ -287,7 +599,9 @@ run_npm_audit_manifests() { [[ -f "$repo/webui/package-lock.json" ]] || continue local name name="$(safe_name "$repo")" - (cd "$repo/webui" && npm audit --omit=dev --json > "$REPORTS_DIR/npm-audit-$name.json") || status=1 + prepare_machine_report "$REPORTS_DIR/npm-audit-$name.json" || return 2 + (cd "$repo/webui" && npm audit --omit=dev --json > "$REPORTS_DIR/npm-audit-$name.json") + accumulate_exit_status status "$?" done return "$status" } @@ -298,19 +612,24 @@ run_osv_scanner() { local name name="$(safe_name "$repo")" if osv-scanner scan --help >/dev/null 2>&1; then + prepare_machine_report "$REPORTS_DIR/osv-scanner-$name.json" || return 2 osv-scanner scan -r \ --allow-no-lockfiles \ --format json \ --output-file "$REPORTS_DIR/osv-scanner-$name.json" \ - "$repo" || status=1 + "$repo" + accumulate_exit_status status "$?" else - osv-scanner -r --format json --output "$REPORTS_DIR/osv-scanner-$name.json" "$repo" || status=1 + prepare_machine_report "$REPORTS_DIR/osv-scanner-$name.json" || return 2 + osv-scanner -r --format json --output "$REPORTS_DIR/osv-scanner-$name.json" "$repo" + accumulate_exit_status status "$?" fi done return "$status" } run_jscpd() { + prepare_machine_report "$REPORTS_DIR/jscpd/jscpd-report.json" || return 2 local ignored_paths ignored_paths=( "**/.git/**" @@ -349,55 +668,66 @@ run_jscpd() { run_radon() { [[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0 + register_report "$REPORTS_DIR/radon-cc.txt" || return 2 radon cc -s -a "${PY_ROOTS[@]}" > "$REPORTS_DIR/radon-cc.txt" } run_xenon() { [[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0 + register_report "$REPORTS_DIR/xenon.txt" || return 2 xenon --max-absolute C --max-modules B --max-average A "${PY_ROOTS[@]}" > "$REPORTS_DIR/xenon.txt" 2>&1 } -write_manifest +register_report "$REPORTS_DIR/step-status.tsv" +register_report "$REPORTS_DIR/tool-versions.txt" +register_report "$REPORTS_DIR/workspace-state-start.tsv" +register_report "$REPORTS_DIR/workspace-state-end.tsv" +: > "$REPORTS_DIR/step-status.tsv" +if ! write_workspace_state "$REPORTS_DIR/workspace-state-start.tsv"; then + echo "Could not capture the repository state before the audit." >&2 + exit 1 +fi +run_step "Record audit tool versions" execution-only write_tool_versions if has_tool semgrep; then - run_step "Semgrep SAST" run_semgrep + run_step "Semgrep SAST" findings-exit-one run_semgrep else skip_or_fail_missing semgrep fi if has_tool bandit; then - run_step "Bandit Python security scan" run_bandit + run_step "Bandit Python security scan" findings-exit-one run_bandit else skip_or_fail_missing bandit fi if has_tool ruff; then - run_step "Ruff flake8-bandit security rules" run_ruff_security + run_step "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security else skip_or_fail_missing ruff fi if has_tool gitleaks; then - run_step "Gitleaks secret scan" run_gitleaks + run_step "Gitleaks secret scan" findings-exit-one run_gitleaks else skip_or_fail_missing gitleaks fi if [[ "$MODE" != "quick" ]]; then if has_tool trivy; then - run_step "Trivy filesystem vulnerability/secret/misconfig scan" run_trivy + run_step "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy else skip_or_fail_missing trivy fi if has_tool pip-audit; then - run_step "pip-audit requirements scan" run_pip_audit_manifests + run_step "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests else skip_or_fail_missing pip-audit fi if has_tool npm; then - run_step "npm audit lockfile scan" run_npm_audit_manifests + run_step "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests else skip_or_fail_missing npm fi @@ -405,37 +735,62 @@ fi if [[ "$MODE" == "full" ]]; then if has_tool osv-scanner; then - run_step "OSV-Scanner recursive dependency scan" run_osv_scanner + run_step "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner else skip_or_fail_missing osv-scanner fi if has_tool jscpd; then - run_step "jscpd duplicate code scan" run_jscpd + run_step "jscpd duplicate code scan" execution-only run_jscpd else skip_or_fail_missing jscpd fi if has_tool radon; then - run_step "Radon complexity report" run_radon + run_step "Radon complexity report" execution-only run_radon else skip_or_fail_missing radon fi if has_tool xenon; then - run_step "Xenon complexity threshold scan" run_xenon + run_step "Xenon complexity threshold scan" findings-exit-one run_xenon else skip_or_fail_missing xenon fi fi +run_step "Validate machine-readable audit reports" execution-only validate_machine_reports + echo echo "Security audit reports written to: $REPORTS_DIR" if [[ "$finding_status" -ne 0 && "$FAIL_ON_FINDINGS" != "1" ]]; then echo "Findings were reported, but this run is report-only. Re-run with --strict to fail on findings." fi if [[ "$missing_status" -ne 0 ]]; then - echo "One or more required tools were missing." >&2 + echo "One or more audit tools were missing." >&2 +fi + +if ! write_workspace_state "$REPORTS_DIR/workspace-state-end.tsv"; then + workspace_unchanged=false + execution_status=1 + overall_status=1 + echo "Could not capture the repository state after the audit." >&2 +elif cmp -s "$REPORTS_DIR/workspace-state-start.tsv" "$REPORTS_DIR/workspace-state-end.tsv"; then + workspace_unchanged=true +else + workspace_unchanged=false + overall_status=1 + echo "Repository state changed during the audit; discard this mixed snapshot and rerun." >&2 +fi +audit_completed_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" +if ! write_report_checksums; then + execution_status=1 + overall_status=1 + echo "Could not generate report checksums." >&2 +fi +if ! write_manifest; then + echo "Could not write the audit manifest." >&2 + exit 1 fi exit "$overall_status" diff --git a/tools/checks/security-audit/Dockerfile b/tools/checks/security-audit/Dockerfile index d066a2f..8de5970 100644 --- a/tools/checks/security-audit/Dockerfile +++ b/tools/checks/security-audit/Dockerfile @@ -31,19 +31,24 @@ RUN apt-get update \ COPY --from=go-tools /go/bin/gitleaks /usr/local/bin/gitleaks COPY --from=go-tools /go/bin/osv-scanner /usr/local/bin/osv-scanner COPY requirements-audit.txt /tmp/requirements-audit.txt +COPY tools/checks/security-audit/semgrep-govoplan.yml /tmp/semgrep-govoplan.yml +COPY tools/checks/check-version-alignment.py /tmp/semgrep-smoke.py RUN python -m pip install --upgrade pip \ && python -m pip install --no-cache-dir -r /tmp/requirements-audit.txt \ && python -m pip install --no-cache-dir "semgrep>=1.140,<2" \ - && python -m pip install --no-cache-dir --upgrade --no-deps "click>=8.3.3" \ + && python -m pip install --no-cache-dir --upgrade --no-deps "click==8.3.3" \ + && python -m pip install --no-cache-dir --upgrade --no-deps "mcp==1.28.1" \ && npm install -g jscpd \ && semgrep --version \ + && semgrep scan --metrics=off --config /tmp/semgrep-govoplan.yml /tmp/semgrep-smoke.py \ && bandit --version \ && ruff --version \ && gitleaks version \ && osv-scanner --version \ && trivy --version \ - && jscpd --version + && jscpd --version \ + && pip-audit --progress-spinner off RUN mkdir -p /workspace \ && chmod 0777 /workspace diff --git a/tools/checks/security-audit/run.sh b/tools/checks/security-audit/run.sh index 3c6f4ca..85b7be8 100644 --- a/tools/checks/security-audit/run.sh +++ b/tools/checks/security-audit/run.sh @@ -84,6 +84,8 @@ IMAGE_FINGERPRINT="$( { sha256sum "$ROOT/tools/checks/security-audit/Dockerfile" sha256sum "$ROOT/requirements-audit.txt" + sha256sum "$ROOT/tools/checks/security-audit/semgrep-govoplan.yml" + sha256sum "$ROOT/tools/checks/check-version-alignment.py" } | sha256sum | cut -c1-16 )" FINGERPRINT_IMAGE="${SECURITY_AUDIT_FINGERPRINT_IMAGE:-$IMAGE_REPOSITORY:$IMAGE_FINGERPRINT}" @@ -129,6 +131,7 @@ docker run --rm \ -e SECURITY_AUDIT_REPORTS_DIR="$REPORTS_DIR" \ -e SECURITY_AUDIT_FAIL_ON_FINDINGS="${SECURITY_AUDIT_FAIL_ON_FINDINGS:-0}" \ -e SECURITY_AUDIT_REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}" \ + -e SECURITY_AUDIT_TOOLBOX_FINGERPRINT="$IMAGE_FINGERPRINT" \ "${EXTRA_ENV[@]}" \ -v "$MOUNT_ROOT:/workspace" \ -w "$WORKDIR" \