diff --git a/.gitea/workflows/security-audit.yml b/.gitea/workflows/security-audit.yml index 82bf94e..912024f 100644 --- a/.gitea/workflows/security-audit.yml +++ b/.gitea/workflows/security-audit.yml @@ -12,7 +12,7 @@ jobs: security-audit: runs-on: ubuntu-latest env: - SECURITY_AUDIT_MODE: ci + SECURITY_AUDIT_MODE: full SECURITY_AUDIT_SCOPE: govoplan SECURITY_AUDIT_FAIL_ON_FINDINGS: "0" SECURITY_AUDIT_REQUIRE_TOOLS: "1" diff --git a/docs/SECURITY_AUDIT.md b/docs/SECURITY_AUDIT.md index 34e86d5..ea882bc 100644 --- a/docs/SECURITY_AUDIT.md +++ b/docs/SECURITY_AUDIT.md @@ -50,6 +50,10 @@ accept findings, but scanner execution errors and malformed JSON/SARIF reports always fail the run. The manifest lists the expected, present, and missing reports for that invocation. Validation and checksums use that explicit set, so reusing a report directory cannot make stale output look like part of a new run. +It also contains `coverage_status` and structured `scanner_coverage` entries. +Every required scanner is recorded as `no-findings`, `findings`, +`scanner-failure`, or `skipped`; this makes an incomplete local run visible +without treating it as a clean audit. The wrapper tags the toolbox image by a fingerprint of the Dockerfile, `requirements-audit.txt`, and the Semgrep smoke-test inputs. If those inputs have @@ -90,6 +94,12 @@ tools/checks/security-audit/run.sh --mode quick --scope current --update --build - `ci`: quick plus Semgrep public registry rulesets, Trivy, pip-audit, npm audit. - `full`: ci plus OSV-Scanner, jscpd, Radon, and Xenon. +The Gitea workflow uses `full` mode so its coverage contract includes every +scanner above. Missing scanners fail strict runs and all Actions runs, even +while actual findings remain report-only. Local report-only runs may finish +with missing tools for diagnostics, but their manifest is marked +`coverage_status: incomplete`. + Semgrep and Trivy are invoked with finding-sensitive exit codes. Their exit 1 is therefore a finding under the wrapper contract; higher exit codes, missing output, invalid JSON/SARIF, and scanner error payloads are execution failures. @@ -103,7 +113,7 @@ baseline. ## Gating -The initial Gitea workflow runs in report-only mode: +The Gitea workflow currently runs findings in report-only mode: ```bash SECURITY_AUDIT_FAIL_ON_FINDINGS=0 @@ -119,7 +129,7 @@ SECURITY_AUDIT_FAIL_ON_FINDINGS=1 or run locally with: ```bash -tools/checks/security-audit/run.sh --mode ci --scope current --strict +tools/checks/security-audit/run.sh --mode full --scope govoplan --strict ``` ## Audit Burndown Workflow diff --git a/tests/test_security_audit_wrapper.py b/tests/test_security_audit_wrapper.py index e77cf02..9c7b443 100644 --- a/tests/test_security_audit_wrapper.py +++ b/tests/test_security_audit_wrapper.py @@ -253,6 +253,12 @@ class SecurityAuditWrapperTests(unittest.TestCase): self.assertEqual(0, manifest["overall_status"]) self.assertEqual(1, manifest["finding_status"]) self.assertEqual(0, manifest["execution_error_status"]) + self.assertEqual("complete", manifest["coverage_status"]) + scanner_results = { + result["id"]: result["status"] + for result in manifest["scanner_coverage"]["results"] + } + self.assertEqual("findings", scanner_results["semgrep"]) self.assertIn( "Semgrep SAST\t1\tfindings", (reports / "step-status.tsv").read_text() ) @@ -312,6 +318,86 @@ class SecurityAuditWrapperTests(unittest.TestCase): ) self.assertEqual(1, self._manifest(invalid_reports)["execution_error_status"]) + def test_missing_tool_is_machine_readable_and_strictly_enforced(self) -> None: + gitleaks_stub = self.stub_bin / "gitleaks" + disabled_stub = self.stub_bin / "gitleaks.disabled" + gitleaks_stub.rename(disabled_stub) + try: + result, reports = self._run( + "--report-only", + SECURITY_AUDIT_REQUIRE_TOOLS="0", + CI="false", + GITEA_ACTIONS="false", + ) + self.assertEqual(0, result.returncode, result.stderr) + manifest = self._manifest(reports) + self.assertEqual("incomplete", manifest["coverage_status"]) + self.assertEqual(["gitleaks"], manifest["scanner_coverage"]["incomplete"]) + scanner_results = { + item["id"]: item for item in manifest["scanner_coverage"]["results"] + } + self.assertEqual("skipped", scanner_results["gitleaks"]["status"]) + self.assertEqual(127, scanner_results["gitleaks"]["exit_code"]) + self.assertIn( + "gitleaks\tmissing", + (reports / "tool-versions.txt").read_text(encoding="utf-8"), + ) + + strict_result, strict_reports = self._run( + "--strict", + SECURITY_AUDIT_REQUIRE_TOOLS="0", + CI="false", + GITEA_ACTIONS="false", + ) + self.assertEqual(1, strict_result.returncode) + self.assertEqual( + "incomplete", + self._manifest(strict_reports)["coverage_status"], + ) + + ci_result, ci_reports = self._run( + "--report-only", + SECURITY_AUDIT_REQUIRE_TOOLS="0", + CI="true", + GITEA_ACTIONS="false", + ) + self.assertEqual(1, ci_result.returncode) + self.assertEqual( + "incomplete", + self._manifest(ci_reports)["coverage_status"], + ) + finally: + disabled_stub.rename(gitleaks_stub) + + def test_full_mode_records_every_required_scanner(self) -> None: + self._install_full_mode_stubs() + + result, reports = self._run("--report-only", mode="full") + + self.assertEqual(0, result.returncode, result.stderr) + manifest = self._manifest(reports) + self.assertEqual("complete", manifest["coverage_status"]) + required = manifest["scanner_coverage"]["required"] + results = manifest["scanner_coverage"]["results"] + self.assertEqual( + { + "semgrep", + "bandit", + "ruff-security", + "gitleaks", + "trivy", + "pip-audit", + "npm-audit", + "osv-scanner", + "jscpd", + "radon", + "xenon", + }, + set(required), + ) + self.assertEqual(set(required), {item["id"] for item in results}) + self.assertTrue(all(item["status"] == "no-findings" for item in results)) + def test_workspace_mutation_invalidates_the_audit(self) -> None: result, reports = self._run( "--report-only", @@ -498,6 +584,7 @@ class SecurityAuditContainerRunnerTests(unittest.TestCase): else [] ) self.assertEqual(expected_roots, repository_roots) + self.assertIn("SECURITY_AUDIT_REQUIRE_TOOLS=1", run_command) def test_non_actions_runner_keeps_the_scoped_bind_mount(self) -> None: result, commands = self._run(scope="govoplan") diff --git a/tools/checks/check-security-audit.sh b/tools/checks/check-security-audit.sh index facdedb..60ae309 100644 --- a/tools/checks/check-security-audit.sh +++ b/tools/checks/check-security-audit.sh @@ -87,6 +87,37 @@ for flag_name in FAIL_ON_FINDINGS REQUIRE_TOOLS; do fi done +# A strict audit is only meaningful when every scanner in the selected mode is +# available. CI must enforce the same coverage even while findings are +# temporarily report-only. +if [[ "$FAIL_ON_FINDINGS" == "1" \ + || "${CI:-false}" == "true" \ + || "${GITEA_ACTIONS:-false}" == "true" ]]; then + REQUIRE_TOOLS=1 +fi + +declare -a REQUIRED_SCANNERS=( + semgrep + bandit + ruff-security + gitleaks +) +if [[ "$MODE" != "quick" ]]; then + REQUIRED_SCANNERS+=( + trivy + pip-audit + npm-audit + ) +fi +if [[ "$MODE" == "full" ]]; then + REQUIRED_SCANNERS+=( + osv-scanner + jscpd + radon + xenon + ) +fi + declare -a REPOS=() if [[ "$SCOPE" == "govoplan" ]]; then REPOS_ROOT="${GOVOPLAN_REPOS_ROOT:-$(dirname "$ROOT")}" @@ -138,12 +169,14 @@ overall_status=0 finding_status=0 missing_status=0 execution_status=0 +coverage_complete=true audit_started_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" audit_completed_at="" workspace_unchanged="unknown" audit_toolbox_fingerprint="${SECURITY_AUDIT_TOOLBOX_FINGERPRINT:-direct-host}" declare -A REPORT_FILES=() declare -A MACHINE_REPORTS=() +declare -A SCANNER_TERMINAL_STATUSES=() register_report() { local path="$1" @@ -169,7 +202,9 @@ prepare_machine_report() { fi } -run_step() { +run_step_internal() { + local scanner_id="$1" + shift local name="$1" local exit_contract="$2" shift 2 @@ -178,10 +213,12 @@ run_step() { "$@" local status=$? local outcome="passed" + local scanner_outcome="no-findings" if [[ "$status" -eq 0 ]]; then : elif [[ "$exit_contract" == "findings-exit-one" && "$status" -eq 1 ]]; then outcome="findings" + scanner_outcome="findings" echo "Audit step reported findings: $name (exit $status)" >&2 finding_status=1 if [[ "$FAIL_ON_FINDINGS" == "1" ]]; then @@ -189,21 +226,77 @@ run_step() { fi else outcome="execution-error" + scanner_outcome="scanner-failure" echo "Audit step failed to execute successfully: $name (exit $status)" >&2 execution_status=1 overall_status=1 fi printf '%s\t%s\t%s\n' "$name" "$status" "$outcome" >> "$REPORTS_DIR/step-status.tsv" + if [[ -n "$scanner_id" ]]; then + SCANNER_TERMINAL_STATUSES["$scanner_id"]="$scanner_outcome" + if [[ "$scanner_outcome" == "scanner-failure" ]]; then + coverage_complete=false + fi + printf '%s\t%s\t%s\t%s\n' \ + "$scanner_id" \ + "$name" \ + "$status" \ + "$scanner_outcome" \ + >> "$REPORTS_DIR/scanner-status.tsv" + fi +} + +run_step() { + run_step_internal "" "$@" +} + +run_scanner_step() { + local scanner_id="$1" + shift + run_step_internal "$scanner_id" "$@" } skip_or_fail_missing() { - local tool="$1" + local scanner_id="$1" + local tool="$2" echo "Skipping $tool: command not found. Use tools/checks/security-audit/run.sh for the containerized toolbox." >&2 missing_status=1 + coverage_complete=false + SCANNER_TERMINAL_STATUSES["$scanner_id"]="skipped" if [[ "$REQUIRE_TOOLS" == "1" ]]; then overall_status=1 fi printf '%s\t127\tmissing\n' "$tool" >> "$REPORTS_DIR/step-status.tsv" + printf '%s\t%s\t127\tskipped\n' \ + "$scanner_id" \ + "$tool" \ + >> "$REPORTS_DIR/scanner-status.tsv" +} + +validate_scanner_status_contract() { + local scanner_id + local scanner_status + local invalid_status=0 + for scanner_id in "${REQUIRED_SCANNERS[@]}"; do + scanner_status="${SCANNER_TERMINAL_STATUSES[$scanner_id]:-}" + case "$scanner_status" in + no-findings|findings) ;; + scanner-failure|skipped) + coverage_complete=false + ;; + "") + echo "Required scanner did not record a terminal status: $scanner_id" >&2 + coverage_complete=false + invalid_status=2 + ;; + *) + echo "Required scanner recorded an invalid status: $scanner_id ($scanner_status)" >&2 + coverage_complete=false + invalid_status=2 + ;; + esac + done + return "$invalid_status" } write_manifest() { @@ -229,7 +322,10 @@ write_manifest() { "$finding_status" \ "$execution_status" \ "$missing_status" \ + "$coverage_complete" \ "$audit_toolbox_fingerprint" \ + "${#REQUIRED_SCANNERS[@]}" \ + "${REQUIRED_SCANNERS[@]}" \ "${#REPOS[@]}" \ "${REPOS[@]}" \ "${#present_reports[@]}" \ @@ -254,16 +350,51 @@ import sys finding_status, execution_status, missing_status, + coverage_complete, toolbox_fingerprint, - repository_count, *remaining, ) = sys.argv[1:] +required_scanner_count = int(remaining[0]) +required_scanners = remaining[1 : required_scanner_count + 1] +remaining = remaining[required_scanner_count + 1 :] +repository_count = remaining[0] +remaining = remaining[1:] repo_count = int(repository_count) repositories = remaining[:repo_count] remaining = remaining[repo_count:] report_count = int(remaining[0]) reports = remaining[1 : report_count + 1] expected_reports = remaining[report_count + 1 :] +scanner_results = [] +scanner_status_path = Path(manifest_path).with_name("scanner-status.tsv") +if scanner_status_path.is_file(): + for line_number, raw_line in enumerate( + scanner_status_path.read_text(encoding="utf-8").splitlines(), + start=1, + ): + fields = raw_line.split("\t") + if len(fields) != 4: + raise SystemExit( + f"{scanner_status_path}:{line_number}: expected four fields" + ) + scanner_id, name, exit_code, status = fields + scanner_results.append( + { + "id": scanner_id, + "name": name, + "exit_code": int(exit_code), + "status": status, + } + ) +result_ids = {result["id"] for result in scanner_results} +incomplete_scanners = sorted( + { + result["id"] + for result in scanner_results + if result["status"] in {"scanner-failure", "skipped"} + } + | (set(required_scanners) - result_ids) +) payload = { "mode": mode, "scope": scope, @@ -276,6 +407,12 @@ payload = { "finding_status": int(finding_status), "execution_error_status": int(execution_status), "missing_tool_status": int(missing_status), + "coverage_status": "complete" if coverage_complete == "true" else "incomplete", + "scanner_coverage": { + "required": required_scanners, + "results": scanner_results, + "incomplete": incomplete_scanners, + }, "tool_versions": "tool-versions.txt", "workspace_state_start": "workspace-state-start.tsv", "workspace_state_end": "workspace-state-end.tsv", @@ -350,7 +487,10 @@ write_tool_versions() { : > "$destination" local tool for tool in semgrep bandit ruff gitleaks trivy pip-audit npm osv-scanner jscpd radon xenon; do - has_tool "$tool" || continue + if ! has_tool "$tool"; then + printf '%s\t%s\n' "$tool" "missing" >> "$destination" + continue + fi if [[ "$tool" == "gitleaks" ]]; then version="$("$tool" version 2>&1)" else @@ -599,9 +739,9 @@ run_pip_audit_manifests() { report_path="$REPORTS_DIR/pip-audit-$repo_name-$manifest_name.json" prepare_machine_report "$report_path" || return 2 # Requirements may contain project-relative entries such as `.[server]`. - # Resolve them from the owning repository instead of the meta-repository. + # Resolve them from the directory containing the manifest. ( - cd "$repo" && + cd "$(dirname "$requirements_file")" && pip-audit \ -r "$(basename "$requirements_file")" \ --progress-spinner off \ @@ -611,10 +751,17 @@ run_pip_audit_manifests() { accumulate_exit_status status "$?" done < <( find "$repo" \ - -maxdepth 1 \ - -type f \ - -name 'requirements*.txt' \ - -print0 | + -type d \( \ + -name .git \ + -o -name .venv \ + -o -name node_modules \ + -o -name dist \ + -o -name build \ + -o -name runtime \ + -o -name audit-reports \ + -o -name '.*-test-build' \ + \) -prune \ + -o -type f -name 'requirements*.txt' -print0 | sort -z ) done @@ -649,11 +796,17 @@ run_npm_audit_manifests() { accumulate_exit_status status "$?" done < <( find "$repo" \ - -maxdepth 3 \ - -type f \ - -name package-lock.json \ - -not -path '*/node_modules/*' \ - -print0 | + -type d \( \ + -name .git \ + -o -name .venv \ + -o -name node_modules \ + -o -name dist \ + -o -name build \ + -o -name runtime \ + -o -name audit-reports \ + -o -name '.*-test-build' \ + \) -prune \ + -o -type f -name package-lock.json -print0 | sort -z ) done @@ -718,6 +871,19 @@ run_jscpd() { --output "$REPORTS_DIR/jscpd" \ --ignore "$ignored_path_pattern" \ "${REPOS[@]}" + local jscpd_status=$? + [[ "$jscpd_status" -eq 0 ]] || return 2 + python3 - "$REPORTS_DIR/jscpd/jscpd-report.json" <<'PY' +from __future__ import annotations + +import json +from pathlib import Path +import sys + + +report = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +raise SystemExit(1 if report.get("duplicates") else 0) +PY } run_radon() { @@ -733,10 +899,12 @@ run_xenon() { } register_report "$REPORTS_DIR/step-status.tsv" +register_report "$REPORTS_DIR/scanner-status.tsv" register_report "$REPORTS_DIR/tool-versions.txt" register_report "$REPORTS_DIR/workspace-state-start.tsv" register_report "$REPORTS_DIR/workspace-state-end.tsv" : > "$REPORTS_DIR/step-status.tsv" +: > "$REPORTS_DIR/scanner-status.tsv" if ! write_workspace_state "$REPORTS_DIR/workspace-state-start.tsv"; then echo "Could not capture the repository state before the audit." >&2 exit 1 @@ -744,75 +912,76 @@ fi run_step "Record audit tool versions" execution-only write_tool_versions if has_tool semgrep; then - run_step "Semgrep SAST" findings-exit-one run_semgrep + run_scanner_step semgrep "Semgrep SAST" findings-exit-one run_semgrep else - skip_or_fail_missing semgrep + skip_or_fail_missing semgrep semgrep fi if has_tool bandit; then - run_step "Bandit Python security scan" findings-exit-one run_bandit + run_scanner_step bandit "Bandit Python security scan" findings-exit-one run_bandit else - skip_or_fail_missing bandit + skip_or_fail_missing bandit bandit fi if has_tool ruff; then - run_step "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security + run_scanner_step ruff-security "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security else - skip_or_fail_missing ruff + skip_or_fail_missing ruff-security ruff fi if has_tool gitleaks; then - run_step "Gitleaks secret scan" findings-exit-one run_gitleaks + run_scanner_step gitleaks "Gitleaks secret scan" findings-exit-one run_gitleaks else - skip_or_fail_missing gitleaks + skip_or_fail_missing gitleaks gitleaks fi if [[ "$MODE" != "quick" ]]; then if has_tool trivy; then - run_step "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy + run_scanner_step trivy "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy else - skip_or_fail_missing trivy + skip_or_fail_missing trivy trivy fi if has_tool pip-audit; then - run_step "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests + run_scanner_step pip-audit "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests else - skip_or_fail_missing pip-audit + skip_or_fail_missing pip-audit pip-audit fi if has_tool npm; then - run_step "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests + run_scanner_step npm-audit "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests else - skip_or_fail_missing npm + skip_or_fail_missing npm-audit npm fi fi if [[ "$MODE" == "full" ]]; then if has_tool osv-scanner; then - run_step "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner + run_scanner_step osv-scanner "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner else - skip_or_fail_missing osv-scanner + skip_or_fail_missing osv-scanner osv-scanner fi if has_tool jscpd; then - run_step "jscpd duplicate code scan" execution-only run_jscpd + run_scanner_step jscpd "jscpd duplicate code scan" findings-exit-one run_jscpd else - skip_or_fail_missing jscpd + skip_or_fail_missing jscpd jscpd fi if has_tool radon; then - run_step "Radon complexity report" execution-only run_radon + run_scanner_step radon "Radon complexity report" execution-only run_radon else - skip_or_fail_missing radon + skip_or_fail_missing radon radon fi if has_tool xenon; then - run_step "Xenon complexity threshold scan" findings-exit-one run_xenon + run_scanner_step xenon "Xenon complexity threshold scan" findings-exit-one run_xenon else - skip_or_fail_missing xenon + skip_or_fail_missing xenon xenon fi fi +run_step "Validate required scanner coverage records" execution-only validate_scanner_status_contract run_step "Validate machine-readable audit reports" execution-only validate_machine_reports echo diff --git a/tools/checks/security-audit/Dockerfile b/tools/checks/security-audit/Dockerfile index 8de5970..ae3d4d7 100644 --- a/tools/checks/security-audit/Dockerfile +++ b/tools/checks/security-audit/Dockerfile @@ -48,7 +48,10 @@ RUN python -m pip install --upgrade pip \ && osv-scanner --version \ && trivy --version \ && jscpd --version \ - && pip-audit --progress-spinner off + && pip-audit --progress-spinner off \ + && npm --version \ + && radon --version \ + && xenon --version RUN mkdir -p /workspace \ && chmod 0777 /workspace diff --git a/tools/checks/security-audit/run.sh b/tools/checks/security-audit/run.sh index 47850cd..41ff0c5 100755 --- a/tools/checks/security-audit/run.sh +++ b/tools/checks/security-audit/run.sh @@ -70,6 +70,10 @@ done REBUILD="${REBUILD:-0}" UPDATE="${UPDATE:-0}" BUILD_ONLY="${BUILD_ONLY:-0}" +REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}" +if [[ "${CI:-false}" == "true" || "${GITEA_ACTIONS:-false}" == "true" ]]; then + REQUIRE_TOOLS=1 +fi declare -a DOCKER_CLI=() DOCKER_LOCATION="" @@ -196,7 +200,7 @@ fi -e SECURITY_AUDIT_MODE="$MODE" \ -e SECURITY_AUDIT_REPORTS_DIR="$REPORTS_DIR" \ -e SECURITY_AUDIT_FAIL_ON_FINDINGS="${SECURITY_AUDIT_FAIL_ON_FINDINGS:-0}" \ - -e SECURITY_AUDIT_REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}" \ + -e SECURITY_AUDIT_REQUIRE_TOOLS="$REQUIRE_TOOLS" \ -e SECURITY_AUDIT_TOOLBOX_FINGERPRINT="$IMAGE_FINGERPRINT" \ "${EXTRA_ENV[@]}" \ "${WORKSPACE_MOUNT[@]}" \