ci: enforce complete security audit coverage

.gitea/workflows/security-audit.yml

@@ -12,7 +12,7 @@ jobs:
   security-audit:
     runs-on: ubuntu-latest
     env:
-      SECURITY_AUDIT_MODE: ci
+      SECURITY_AUDIT_MODE: full
       SECURITY_AUDIT_SCOPE: govoplan
       SECURITY_AUDIT_FAIL_ON_FINDINGS: "0"
       SECURITY_AUDIT_REQUIRE_TOOLS: "1"

docs/SECURITY_AUDIT.md

@@ -50,6 +50,10 @@ accept findings, but scanner execution errors and malformed JSON/SARIF reports
 always fail the run. The manifest lists the expected, present, and missing
 reports for that invocation. Validation and checksums use that explicit set, so
 reusing a report directory cannot make stale output look like part of a new run.
+It also contains `coverage_status` and structured `scanner_coverage` entries.
+Every required scanner is recorded as `no-findings`, `findings`,
+`scanner-failure`, or `skipped`; this makes an incomplete local run visible
+without treating it as a clean audit.
 
 The wrapper tags the toolbox image by a fingerprint of the Dockerfile,
 `requirements-audit.txt`, and the Semgrep smoke-test inputs. If those inputs have
@@ -90,6 +94,12 @@ tools/checks/security-audit/run.sh --mode quick --scope current --update --build
 - `ci`: quick plus Semgrep public registry rulesets, Trivy, pip-audit, npm audit.
 - `full`: ci plus OSV-Scanner, jscpd, Radon, and Xenon.
 
+The Gitea workflow uses `full` mode so its coverage contract includes every
+scanner above. Missing scanners fail strict runs and all Actions runs, even
+while actual findings remain report-only. Local report-only runs may finish
+with missing tools for diagnostics, but their manifest is marked
+`coverage_status: incomplete`.
+
 Semgrep and Trivy are invoked with finding-sensitive exit codes. Their exit 1
 is therefore a finding under the wrapper contract; higher exit codes, missing
 output, invalid JSON/SARIF, and scanner error payloads are execution failures.
@@ -103,7 +113,7 @@ baseline.
 
 ## Gating
 
-The initial Gitea workflow runs in report-only mode:
+The Gitea workflow currently runs findings in report-only mode:
 
 ```bash
 SECURITY_AUDIT_FAIL_ON_FINDINGS=0
@@ -119,7 +129,7 @@ SECURITY_AUDIT_FAIL_ON_FINDINGS=1
 or run locally with:
 
 ```bash
-tools/checks/security-audit/run.sh --mode ci --scope current --strict
+tools/checks/security-audit/run.sh --mode full --scope govoplan --strict
 ```
 
 ## Audit Burndown Workflow

tests/test_security_audit_wrapper.py

@@ -253,6 +253,12 @@ class SecurityAuditWrapperTests(unittest.TestCase):
         self.assertEqual(0, manifest["overall_status"])
         self.assertEqual(1, manifest["finding_status"])
         self.assertEqual(0, manifest["execution_error_status"])
+        self.assertEqual("complete", manifest["coverage_status"])
+        scanner_results = {
+            result["id"]: result["status"]
+            for result in manifest["scanner_coverage"]["results"]
+        }
+        self.assertEqual("findings", scanner_results["semgrep"])
         self.assertIn(
             "Semgrep SAST\t1\tfindings", (reports / "step-status.tsv").read_text()
         )
@@ -312,6 +318,86 @@ class SecurityAuditWrapperTests(unittest.TestCase):
         )
         self.assertEqual(1, self._manifest(invalid_reports)["execution_error_status"])
 
+    def test_missing_tool_is_machine_readable_and_strictly_enforced(self) -> None:
+        gitleaks_stub = self.stub_bin / "gitleaks"
+        disabled_stub = self.stub_bin / "gitleaks.disabled"
+        gitleaks_stub.rename(disabled_stub)
+        try:
+            result, reports = self._run(
+                "--report-only",
+                SECURITY_AUDIT_REQUIRE_TOOLS="0",
+                CI="false",
+                GITEA_ACTIONS="false",
+            )
+            self.assertEqual(0, result.returncode, result.stderr)
+            manifest = self._manifest(reports)
+            self.assertEqual("incomplete", manifest["coverage_status"])
+            self.assertEqual(["gitleaks"], manifest["scanner_coverage"]["incomplete"])
+            scanner_results = {
+                item["id"]: item for item in manifest["scanner_coverage"]["results"]
+            }
+            self.assertEqual("skipped", scanner_results["gitleaks"]["status"])
+            self.assertEqual(127, scanner_results["gitleaks"]["exit_code"])
+            self.assertIn(
+                "gitleaks\tmissing",
+                (reports / "tool-versions.txt").read_text(encoding="utf-8"),
+            )
+
+            strict_result, strict_reports = self._run(
+                "--strict",
+                SECURITY_AUDIT_REQUIRE_TOOLS="0",
+                CI="false",
+                GITEA_ACTIONS="false",
+            )
+            self.assertEqual(1, strict_result.returncode)
+            self.assertEqual(
+                "incomplete",
+                self._manifest(strict_reports)["coverage_status"],
+            )
+
+            ci_result, ci_reports = self._run(
+                "--report-only",
+                SECURITY_AUDIT_REQUIRE_TOOLS="0",
+                CI="true",
+                GITEA_ACTIONS="false",
+            )
+            self.assertEqual(1, ci_result.returncode)
+            self.assertEqual(
+                "incomplete",
+                self._manifest(ci_reports)["coverage_status"],
+            )
+        finally:
+            disabled_stub.rename(gitleaks_stub)
+
+    def test_full_mode_records_every_required_scanner(self) -> None:
+        self._install_full_mode_stubs()
+
+        result, reports = self._run("--report-only", mode="full")
+
+        self.assertEqual(0, result.returncode, result.stderr)
+        manifest = self._manifest(reports)
+        self.assertEqual("complete", manifest["coverage_status"])
+        required = manifest["scanner_coverage"]["required"]
+        results = manifest["scanner_coverage"]["results"]
+        self.assertEqual(
+            {
+                "semgrep",
+                "bandit",
+                "ruff-security",
+                "gitleaks",
+                "trivy",
+                "pip-audit",
+                "npm-audit",
+                "osv-scanner",
+                "jscpd",
+                "radon",
+                "xenon",
+            },
+            set(required),
+        )
+        self.assertEqual(set(required), {item["id"] for item in results})
+        self.assertTrue(all(item["status"] == "no-findings" for item in results))
+
     def test_workspace_mutation_invalidates_the_audit(self) -> None:
         result, reports = self._run(
             "--report-only",
@@ -498,6 +584,7 @@ class SecurityAuditContainerRunnerTests(unittest.TestCase):
                     else []
                 )
                 self.assertEqual(expected_roots, repository_roots)
+                self.assertIn("SECURITY_AUDIT_REQUIRE_TOOLS=1", run_command)
 
     def test_non_actions_runner_keeps_the_scoped_bind_mount(self) -> None:
         result, commands = self._run(scope="govoplan")

tools/checks/check-security-audit.sh

@@ -87,6 +87,37 @@ for flag_name in FAIL_ON_FINDINGS REQUIRE_TOOLS; do
   fi
 done
 
+# A strict audit is only meaningful when every scanner in the selected mode is
+# available. CI must enforce the same coverage even while findings are
+# temporarily report-only.
+if [[ "$FAIL_ON_FINDINGS" == "1" \
+  || "${CI:-false}" == "true" \
+  || "${GITEA_ACTIONS:-false}" == "true" ]]; then
+  REQUIRE_TOOLS=1
+fi
+
+declare -a REQUIRED_SCANNERS=(
+  semgrep
+  bandit
+  ruff-security
+  gitleaks
+)
+if [[ "$MODE" != "quick" ]]; then
+  REQUIRED_SCANNERS+=(
+    trivy
+    pip-audit
+    npm-audit
+  )
+fi
+if [[ "$MODE" == "full" ]]; then
+  REQUIRED_SCANNERS+=(
+    osv-scanner
+    jscpd
+    radon
+    xenon
+  )
+fi
+
 declare -a REPOS=()
 if [[ "$SCOPE" == "govoplan" ]]; then
   REPOS_ROOT="${GOVOPLAN_REPOS_ROOT:-$(dirname "$ROOT")}"
@@ -138,12 +169,14 @@ overall_status=0
 finding_status=0
 missing_status=0
 execution_status=0
+coverage_complete=true
 audit_started_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
 audit_completed_at=""
 workspace_unchanged="unknown"
 audit_toolbox_fingerprint="${SECURITY_AUDIT_TOOLBOX_FINGERPRINT:-direct-host}"
 declare -A REPORT_FILES=()
 declare -A MACHINE_REPORTS=()
+declare -A SCANNER_TERMINAL_STATUSES=()
 
 register_report() {
   local path="$1"
@@ -169,7 +202,9 @@ prepare_machine_report() {
   fi
 }
 
-run_step() {
+run_step_internal() {
+  local scanner_id="$1"
+  shift
   local name="$1"
   local exit_contract="$2"
   shift 2
@@ -178,10 +213,12 @@ run_step() {
   "$@"
   local status=$?
   local outcome="passed"
+  local scanner_outcome="no-findings"
   if [[ "$status" -eq 0 ]]; then
     :
   elif [[ "$exit_contract" == "findings-exit-one" && "$status" -eq 1 ]]; then
     outcome="findings"
+    scanner_outcome="findings"
     echo "Audit step reported findings: $name (exit $status)" >&2
     finding_status=1
     if [[ "$FAIL_ON_FINDINGS" == "1" ]]; then
@@ -189,21 +226,77 @@ run_step() {
     fi
   else
     outcome="execution-error"
+    scanner_outcome="scanner-failure"
     echo "Audit step failed to execute successfully: $name (exit $status)" >&2
     execution_status=1
     overall_status=1
   fi
   printf '%s\t%s\t%s\n' "$name" "$status" "$outcome" >> "$REPORTS_DIR/step-status.tsv"
+  if [[ -n "$scanner_id" ]]; then
+    SCANNER_TERMINAL_STATUSES["$scanner_id"]="$scanner_outcome"
+    if [[ "$scanner_outcome" == "scanner-failure" ]]; then
+      coverage_complete=false
+    fi
+    printf '%s\t%s\t%s\t%s\n' \
+      "$scanner_id" \
+      "$name" \
+      "$status" \
+      "$scanner_outcome" \
+      >> "$REPORTS_DIR/scanner-status.tsv"
+  fi
+}
+
+run_step() {
+  run_step_internal "" "$@"
+}
+
+run_scanner_step() {
+  local scanner_id="$1"
+  shift
+  run_step_internal "$scanner_id" "$@"
 }
 
 skip_or_fail_missing() {
-  local tool="$1"
+  local scanner_id="$1"
+  local tool="$2"
   echo "Skipping $tool: command not found. Use tools/checks/security-audit/run.sh for the containerized toolbox." >&2
   missing_status=1
+  coverage_complete=false
+  SCANNER_TERMINAL_STATUSES["$scanner_id"]="skipped"
   if [[ "$REQUIRE_TOOLS" == "1" ]]; then
     overall_status=1
   fi
   printf '%s\t127\tmissing\n' "$tool" >> "$REPORTS_DIR/step-status.tsv"
+  printf '%s\t%s\t127\tskipped\n' \
+    "$scanner_id" \
+    "$tool" \
+    >> "$REPORTS_DIR/scanner-status.tsv"
+}
+
+validate_scanner_status_contract() {
+  local scanner_id
+  local scanner_status
+  local invalid_status=0
+  for scanner_id in "${REQUIRED_SCANNERS[@]}"; do
+    scanner_status="${SCANNER_TERMINAL_STATUSES[$scanner_id]:-}"
+    case "$scanner_status" in
+      no-findings|findings) ;;
+      scanner-failure|skipped)
+        coverage_complete=false
+        ;;
+      "")
+        echo "Required scanner did not record a terminal status: $scanner_id" >&2
+        coverage_complete=false
+        invalid_status=2
+        ;;
+      *)
+        echo "Required scanner recorded an invalid status: $scanner_id ($scanner_status)" >&2
+        coverage_complete=false
+        invalid_status=2
+        ;;
+    esac
+  done
+  return "$invalid_status"
 }
 
 write_manifest() {
@@ -229,7 +322,10 @@ write_manifest() {
     "$finding_status" \
     "$execution_status" \
     "$missing_status" \
+    "$coverage_complete" \
     "$audit_toolbox_fingerprint" \
+    "${#REQUIRED_SCANNERS[@]}" \
+    "${REQUIRED_SCANNERS[@]}" \
     "${#REPOS[@]}" \
     "${REPOS[@]}" \
     "${#present_reports[@]}" \
@@ -254,16 +350,51 @@ import sys
     finding_status,
     execution_status,
     missing_status,
+    coverage_complete,
     toolbox_fingerprint,
-    repository_count,
     *remaining,
 ) = sys.argv[1:]
+required_scanner_count = int(remaining[0])
+required_scanners = remaining[1 : required_scanner_count + 1]
+remaining = remaining[required_scanner_count + 1 :]
+repository_count = remaining[0]
+remaining = remaining[1:]
 repo_count = int(repository_count)
 repositories = remaining[:repo_count]
 remaining = remaining[repo_count:]
 report_count = int(remaining[0])
 reports = remaining[1 : report_count + 1]
 expected_reports = remaining[report_count + 1 :]
+scanner_results = []
+scanner_status_path = Path(manifest_path).with_name("scanner-status.tsv")
+if scanner_status_path.is_file():
+    for line_number, raw_line in enumerate(
+        scanner_status_path.read_text(encoding="utf-8").splitlines(),
+        start=1,
+    ):
+        fields = raw_line.split("\t")
+        if len(fields) != 4:
+            raise SystemExit(
+                f"{scanner_status_path}:{line_number}: expected four fields"
+            )
+        scanner_id, name, exit_code, status = fields
+        scanner_results.append(
+            {
+                "id": scanner_id,
+                "name": name,
+                "exit_code": int(exit_code),
+                "status": status,
+            }
+        )
+result_ids = {result["id"] for result in scanner_results}
+incomplete_scanners = sorted(
+    {
+        result["id"]
+        for result in scanner_results
+        if result["status"] in {"scanner-failure", "skipped"}
+    }
+    | (set(required_scanners) - result_ids)
+)
 payload = {
     "mode": mode,
     "scope": scope,
@@ -276,6 +407,12 @@ payload = {
     "finding_status": int(finding_status),
     "execution_error_status": int(execution_status),
     "missing_tool_status": int(missing_status),
+    "coverage_status": "complete" if coverage_complete == "true" else "incomplete",
+    "scanner_coverage": {
+        "required": required_scanners,
+        "results": scanner_results,
+        "incomplete": incomplete_scanners,
+    },
     "tool_versions": "tool-versions.txt",
     "workspace_state_start": "workspace-state-start.tsv",
     "workspace_state_end": "workspace-state-end.tsv",
@@ -350,7 +487,10 @@ write_tool_versions() {
   : > "$destination"
   local tool
   for tool in semgrep bandit ruff gitleaks trivy pip-audit npm osv-scanner jscpd radon xenon; do
-    has_tool "$tool" || continue
+    if ! has_tool "$tool"; then
+      printf '%s\t%s\n' "$tool" "missing" >> "$destination"
+      continue
+    fi
     if [[ "$tool" == "gitleaks" ]]; then
       version="$("$tool" version 2>&1)"
     else
@@ -599,9 +739,9 @@ run_pip_audit_manifests() {
       report_path="$REPORTS_DIR/pip-audit-$repo_name-$manifest_name.json"
       prepare_machine_report "$report_path" || return 2
       # Requirements may contain project-relative entries such as `.[server]`.
-      # Resolve them from the owning repository instead of the meta-repository.
+      # Resolve them from the directory containing the manifest.
       (
-        cd "$repo" &&
+        cd "$(dirname "$requirements_file")" &&
           pip-audit \
             -r "$(basename "$requirements_file")" \
             --progress-spinner off \
@@ -611,10 +751,17 @@ run_pip_audit_manifests() {
       accumulate_exit_status status "$?"
     done < <(
       find "$repo" \
-        -maxdepth 1 \
+        -type d \( \
-        -type f \
+          -name .git \
-        -name 'requirements*.txt' \
+          -o -name .venv \
-        -print0 |
+          -o -name node_modules \
+          -o -name dist \
+          -o -name build \
+          -o -name runtime \
+          -o -name audit-reports \
+          -o -name '.*-test-build' \
+        \) -prune \
+        -o -type f -name 'requirements*.txt' -print0 |
         sort -z
     )
   done
@@ -649,11 +796,17 @@ run_npm_audit_manifests() {
       accumulate_exit_status status "$?"
     done < <(
       find "$repo" \
-        -maxdepth 3 \
+        -type d \( \
-        -type f \
+          -name .git \
-        -name package-lock.json \
+          -o -name .venv \
-        -not -path '*/node_modules/*' \
+          -o -name node_modules \
-        -print0 |
+          -o -name dist \
+          -o -name build \
+          -o -name runtime \
+          -o -name audit-reports \
+          -o -name '.*-test-build' \
+        \) -prune \
+        -o -type f -name package-lock.json -print0 |
         sort -z
     )
   done
@@ -718,6 +871,19 @@ run_jscpd() {
     --output "$REPORTS_DIR/jscpd" \
     --ignore "$ignored_path_pattern" \
     "${REPOS[@]}"
+  local jscpd_status=$?
+  [[ "$jscpd_status" -eq 0 ]] || return 2
+  python3 - "$REPORTS_DIR/jscpd/jscpd-report.json" <<'PY'
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import sys
+
+
+report = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
+raise SystemExit(1 if report.get("duplicates") else 0)
+PY
 }
 
 run_radon() {
@@ -733,10 +899,12 @@ run_xenon() {
 }
 
 register_report "$REPORTS_DIR/step-status.tsv"
+register_report "$REPORTS_DIR/scanner-status.tsv"
 register_report "$REPORTS_DIR/tool-versions.txt"
 register_report "$REPORTS_DIR/workspace-state-start.tsv"
 register_report "$REPORTS_DIR/workspace-state-end.tsv"
 : > "$REPORTS_DIR/step-status.tsv"
+: > "$REPORTS_DIR/scanner-status.tsv"
 if ! write_workspace_state "$REPORTS_DIR/workspace-state-start.tsv"; then
   echo "Could not capture the repository state before the audit." >&2
   exit 1
@@ -744,75 +912,76 @@ fi
 run_step "Record audit tool versions" execution-only write_tool_versions
 
 if has_tool semgrep; then
-  run_step "Semgrep SAST" findings-exit-one run_semgrep
+  run_scanner_step semgrep "Semgrep SAST" findings-exit-one run_semgrep
 else
-  skip_or_fail_missing semgrep
+  skip_or_fail_missing semgrep semgrep
 fi
 
 if has_tool bandit; then
-  run_step "Bandit Python security scan" findings-exit-one run_bandit
+  run_scanner_step bandit "Bandit Python security scan" findings-exit-one run_bandit
 else
-  skip_or_fail_missing bandit
+  skip_or_fail_missing bandit bandit
 fi
 
 if has_tool ruff; then
-  run_step "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security
+  run_scanner_step ruff-security "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security
 else
-  skip_or_fail_missing ruff
+  skip_or_fail_missing ruff-security ruff
 fi
 
 if has_tool gitleaks; then
-  run_step "Gitleaks secret scan" findings-exit-one run_gitleaks
+  run_scanner_step gitleaks "Gitleaks secret scan" findings-exit-one run_gitleaks
 else
-  skip_or_fail_missing gitleaks
+  skip_or_fail_missing gitleaks gitleaks
 fi
 
 if [[ "$MODE" != "quick" ]]; then
   if has_tool trivy; then
-    run_step "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy
+    run_scanner_step trivy "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy
   else
-    skip_or_fail_missing trivy
+    skip_or_fail_missing trivy trivy
   fi
 
   if has_tool pip-audit; then
-    run_step "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests
+    run_scanner_step pip-audit "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests
   else
-    skip_or_fail_missing pip-audit
+    skip_or_fail_missing pip-audit pip-audit
   fi
 
   if has_tool npm; then
-    run_step "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests
+    run_scanner_step npm-audit "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests
   else
-    skip_or_fail_missing npm
+    skip_or_fail_missing npm-audit npm
   fi
 fi
 
 if [[ "$MODE" == "full" ]]; then
   if has_tool osv-scanner; then
-    run_step "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner
+    run_scanner_step osv-scanner "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner
   else
-    skip_or_fail_missing osv-scanner
+    skip_or_fail_missing osv-scanner osv-scanner
   fi
 
   if has_tool jscpd; then
-    run_step "jscpd duplicate code scan" execution-only run_jscpd
+    run_scanner_step jscpd "jscpd duplicate code scan" findings-exit-one run_jscpd
   else
-    skip_or_fail_missing jscpd
+    skip_or_fail_missing jscpd jscpd
   fi
 
   if has_tool radon; then
-    run_step "Radon complexity report" execution-only run_radon
+    run_scanner_step radon "Radon complexity report" execution-only run_radon
   else
-    skip_or_fail_missing radon
+    skip_or_fail_missing radon radon
   fi
 
   if has_tool xenon; then
-    run_step "Xenon complexity threshold scan" findings-exit-one run_xenon
+    run_scanner_step xenon "Xenon complexity threshold scan" findings-exit-one run_xenon
   else
-    skip_or_fail_missing xenon
+    skip_or_fail_missing xenon xenon
   fi
 fi
 
+run_step "Validate required scanner coverage records" execution-only validate_scanner_status_contract
 run_step "Validate machine-readable audit reports" execution-only validate_machine_reports
 
 echo

tools/checks/security-audit/Dockerfile

@@ -48,7 +48,10 @@ RUN python -m pip install --upgrade pip \
   && osv-scanner --version \
   && trivy --version \
   && jscpd --version \
-  && pip-audit --progress-spinner off
+  && pip-audit --progress-spinner off \
+  && npm --version \
+  && radon --version \
+  && xenon --version
 
 RUN mkdir -p /workspace \
   && chmod 0777 /workspace

tools/checks/security-audit/run.sh

@@ -70,6 +70,10 @@ done
 REBUILD="${REBUILD:-0}"
 UPDATE="${UPDATE:-0}"
 BUILD_ONLY="${BUILD_ONLY:-0}"
+REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}"
+if [[ "${CI:-false}" == "true" || "${GITEA_ACTIONS:-false}" == "true" ]]; then
+  REQUIRE_TOOLS=1
+fi
 
 declare -a DOCKER_CLI=()
 DOCKER_LOCATION=""
@@ -196,7 +200,7 @@ fi
   -e SECURITY_AUDIT_MODE="$MODE" \
   -e SECURITY_AUDIT_REPORTS_DIR="$REPORTS_DIR" \
   -e SECURITY_AUDIT_FAIL_ON_FINDINGS="${SECURITY_AUDIT_FAIL_ON_FINDINGS:-0}" \
-  -e SECURITY_AUDIT_REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}" \
+  -e SECURITY_AUDIT_REQUIRE_TOOLS="$REQUIRE_TOOLS" \
   -e SECURITY_AUDIT_TOOLBOX_FINGERPRINT="$IMAGE_FINGERPRINT" \
   "${EXTRA_ENV[@]}" \
   "${WORKSPACE_MOUNT[@]}" \