ci: enforce complete security audit coverage

tools/checks/check-security-audit.sh

@@ -87,6 +87,37 @@ for flag_name in FAIL_ON_FINDINGS REQUIRE_TOOLS; do
   fi
 done
 
+# A strict audit is only meaningful when every scanner in the selected mode is
+# available. CI must enforce the same coverage even while findings are
+# temporarily report-only.
+if [[ "$FAIL_ON_FINDINGS" == "1" \
+  || "${CI:-false}" == "true" \
+  || "${GITEA_ACTIONS:-false}" == "true" ]]; then
+  REQUIRE_TOOLS=1
+fi
+
+declare -a REQUIRED_SCANNERS=(
+  semgrep
+  bandit
+  ruff-security
+  gitleaks
+)
+if [[ "$MODE" != "quick" ]]; then
+  REQUIRED_SCANNERS+=(
+    trivy
+    pip-audit
+    npm-audit
+  )
+fi
+if [[ "$MODE" == "full" ]]; then
+  REQUIRED_SCANNERS+=(
+    osv-scanner
+    jscpd
+    radon
+    xenon
+  )
+fi
+
 declare -a REPOS=()
 if [[ "$SCOPE" == "govoplan" ]]; then
   REPOS_ROOT="${GOVOPLAN_REPOS_ROOT:-$(dirname "$ROOT")}"
@@ -138,12 +169,14 @@ overall_status=0
 finding_status=0
 missing_status=0
 execution_status=0
+coverage_complete=true
 audit_started_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
 audit_completed_at=""
 workspace_unchanged="unknown"
 audit_toolbox_fingerprint="${SECURITY_AUDIT_TOOLBOX_FINGERPRINT:-direct-host}"
 declare -A REPORT_FILES=()
 declare -A MACHINE_REPORTS=()
+declare -A SCANNER_TERMINAL_STATUSES=()
 
 register_report() {
   local path="$1"
@@ -169,7 +202,9 @@ prepare_machine_report() {
   fi
 }
 
-run_step() {
+run_step_internal() {
+  local scanner_id="$1"
+  shift
   local name="$1"
   local exit_contract="$2"
   shift 2
@@ -178,10 +213,12 @@ run_step() {
   "$@"
   local status=$?
   local outcome="passed"
+  local scanner_outcome="no-findings"
   if [[ "$status" -eq 0 ]]; then
     :
   elif [[ "$exit_contract" == "findings-exit-one" && "$status" -eq 1 ]]; then
     outcome="findings"
+    scanner_outcome="findings"
     echo "Audit step reported findings: $name (exit $status)" >&2
     finding_status=1
     if [[ "$FAIL_ON_FINDINGS" == "1" ]]; then
@@ -189,21 +226,77 @@ run_step() {
     fi
   else
     outcome="execution-error"
+    scanner_outcome="scanner-failure"
     echo "Audit step failed to execute successfully: $name (exit $status)" >&2
     execution_status=1
     overall_status=1
   fi
   printf '%s\t%s\t%s\n' "$name" "$status" "$outcome" >> "$REPORTS_DIR/step-status.tsv"
+  if [[ -n "$scanner_id" ]]; then
+    SCANNER_TERMINAL_STATUSES["$scanner_id"]="$scanner_outcome"
+    if [[ "$scanner_outcome" == "scanner-failure" ]]; then
+      coverage_complete=false
+    fi
+    printf '%s\t%s\t%s\t%s\n' \
+      "$scanner_id" \
+      "$name" \
+      "$status" \
+      "$scanner_outcome" \
+      >> "$REPORTS_DIR/scanner-status.tsv"
+  fi
+}
+
+run_step() {
+  run_step_internal "" "$@"
+}
+
+run_scanner_step() {
+  local scanner_id="$1"
+  shift
+  run_step_internal "$scanner_id" "$@"
 }
 
 skip_or_fail_missing() {
-  local tool="$1"
+  local scanner_id="$1"
+  local tool="$2"
   echo "Skipping $tool: command not found. Use tools/checks/security-audit/run.sh for the containerized toolbox." >&2
   missing_status=1
+  coverage_complete=false
+  SCANNER_TERMINAL_STATUSES["$scanner_id"]="skipped"
   if [[ "$REQUIRE_TOOLS" == "1" ]]; then
     overall_status=1
   fi
   printf '%s\t127\tmissing\n' "$tool" >> "$REPORTS_DIR/step-status.tsv"
+  printf '%s\t%s\t127\tskipped\n' \
+    "$scanner_id" \
+    "$tool" \
+    >> "$REPORTS_DIR/scanner-status.tsv"
+}
+
+validate_scanner_status_contract() {
+  local scanner_id
+  local scanner_status
+  local invalid_status=0
+  for scanner_id in "${REQUIRED_SCANNERS[@]}"; do
+    scanner_status="${SCANNER_TERMINAL_STATUSES[$scanner_id]:-}"
+    case "$scanner_status" in
+      no-findings|findings) ;;
+      scanner-failure|skipped)
+        coverage_complete=false
+        ;;
+      "")
+        echo "Required scanner did not record a terminal status: $scanner_id" >&2
+        coverage_complete=false
+        invalid_status=2
+        ;;
+      *)
+        echo "Required scanner recorded an invalid status: $scanner_id ($scanner_status)" >&2
+        coverage_complete=false
+        invalid_status=2
+        ;;
+    esac
+  done
+  return "$invalid_status"
 }
 
 write_manifest() {
@@ -229,7 +322,10 @@ write_manifest() {
     "$finding_status" \
     "$execution_status" \
     "$missing_status" \
+    "$coverage_complete" \
     "$audit_toolbox_fingerprint" \
+    "${#REQUIRED_SCANNERS[@]}" \
+    "${REQUIRED_SCANNERS[@]}" \
     "${#REPOS[@]}" \
     "${REPOS[@]}" \
     "${#present_reports[@]}" \
@@ -254,16 +350,51 @@ import sys
     finding_status,
     execution_status,
     missing_status,
+    coverage_complete,
     toolbox_fingerprint,
-    repository_count,
     *remaining,
 ) = sys.argv[1:]
+required_scanner_count = int(remaining[0])
+required_scanners = remaining[1 : required_scanner_count + 1]
+remaining = remaining[required_scanner_count + 1 :]
+repository_count = remaining[0]
+remaining = remaining[1:]
 repo_count = int(repository_count)
 repositories = remaining[:repo_count]
 remaining = remaining[repo_count:]
 report_count = int(remaining[0])
 reports = remaining[1 : report_count + 1]
 expected_reports = remaining[report_count + 1 :]
+scanner_results = []
+scanner_status_path = Path(manifest_path).with_name("scanner-status.tsv")
+if scanner_status_path.is_file():
+    for line_number, raw_line in enumerate(
+        scanner_status_path.read_text(encoding="utf-8").splitlines(),
+        start=1,
+    ):
+        fields = raw_line.split("\t")
+        if len(fields) != 4:
+            raise SystemExit(
+                f"{scanner_status_path}:{line_number}: expected four fields"
+            )
+        scanner_id, name, exit_code, status = fields
+        scanner_results.append(
+            {
+                "id": scanner_id,
+                "name": name,
+                "exit_code": int(exit_code),
+                "status": status,
+            }
+        )
+result_ids = {result["id"] for result in scanner_results}
+incomplete_scanners = sorted(
+    {
+        result["id"]
+        for result in scanner_results
+        if result["status"] in {"scanner-failure", "skipped"}
+    }
+    | (set(required_scanners) - result_ids)
+)
 payload = {
     "mode": mode,
     "scope": scope,
@@ -276,6 +407,12 @@ payload = {
     "finding_status": int(finding_status),
     "execution_error_status": int(execution_status),
     "missing_tool_status": int(missing_status),
+    "coverage_status": "complete" if coverage_complete == "true" else "incomplete",
+    "scanner_coverage": {
+        "required": required_scanners,
+        "results": scanner_results,
+        "incomplete": incomplete_scanners,
+    },
     "tool_versions": "tool-versions.txt",
     "workspace_state_start": "workspace-state-start.tsv",
     "workspace_state_end": "workspace-state-end.tsv",
@@ -350,7 +487,10 @@ write_tool_versions() {
   : > "$destination"
   local tool
   for tool in semgrep bandit ruff gitleaks trivy pip-audit npm osv-scanner jscpd radon xenon; do
-    has_tool "$tool" || continue
+    if ! has_tool "$tool"; then
+      printf '%s\t%s\n' "$tool" "missing" >> "$destination"
+      continue
+    fi
     if [[ "$tool" == "gitleaks" ]]; then
       version="$("$tool" version 2>&1)"
     else
@@ -599,9 +739,9 @@ run_pip_audit_manifests() {
       report_path="$REPORTS_DIR/pip-audit-$repo_name-$manifest_name.json"
       prepare_machine_report "$report_path" || return 2
       # Requirements may contain project-relative entries such as `.[server]`.
-      # Resolve them from the owning repository instead of the meta-repository.
+      # Resolve them from the directory containing the manifest.
       (
-        cd "$repo" &&
+        cd "$(dirname "$requirements_file")" &&
           pip-audit \
             -r "$(basename "$requirements_file")" \
             --progress-spinner off \
@@ -611,10 +751,17 @@ run_pip_audit_manifests() {
       accumulate_exit_status status "$?"
     done < <(
       find "$repo" \
-        -maxdepth 1 \
-        -type f \
-        -name 'requirements*.txt' \
-        -print0 |
+        -type d \( \
+          -name .git \
+          -o -name .venv \
+          -o -name node_modules \
+          -o -name dist \
+          -o -name build \
+          -o -name runtime \
+          -o -name audit-reports \
+          -o -name '.*-test-build' \
+        \) -prune \
+        -o -type f -name 'requirements*.txt' -print0 |
         sort -z
     )
   done
@@ -649,11 +796,17 @@ run_npm_audit_manifests() {
       accumulate_exit_status status "$?"
     done < <(
       find "$repo" \
-        -maxdepth 3 \
-        -type f \
-        -name package-lock.json \
-        -not -path '*/node_modules/*' \
-        -print0 |
+        -type d \( \
+          -name .git \
+          -o -name .venv \
+          -o -name node_modules \
+          -o -name dist \
+          -o -name build \
+          -o -name runtime \
+          -o -name audit-reports \
+          -o -name '.*-test-build' \
+        \) -prune \
+        -o -type f -name package-lock.json -print0 |
         sort -z
     )
   done
@@ -718,6 +871,19 @@ run_jscpd() {
     --output "$REPORTS_DIR/jscpd" \
     --ignore "$ignored_path_pattern" \
     "${REPOS[@]}"
+  local jscpd_status=$?
+  [[ "$jscpd_status" -eq 0 ]] || return 2
+  python3 - "$REPORTS_DIR/jscpd/jscpd-report.json" <<'PY'
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import sys
+
+
+report = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
+raise SystemExit(1 if report.get("duplicates") else 0)
+PY
 }
 
 run_radon() {
@@ -733,10 +899,12 @@ run_xenon() {
 }
 
 register_report "$REPORTS_DIR/step-status.tsv"
+register_report "$REPORTS_DIR/scanner-status.tsv"
 register_report "$REPORTS_DIR/tool-versions.txt"
 register_report "$REPORTS_DIR/workspace-state-start.tsv"
 register_report "$REPORTS_DIR/workspace-state-end.tsv"
 : > "$REPORTS_DIR/step-status.tsv"
+: > "$REPORTS_DIR/scanner-status.tsv"
 if ! write_workspace_state "$REPORTS_DIR/workspace-state-start.tsv"; then
   echo "Could not capture the repository state before the audit." >&2
   exit 1
@@ -744,75 +912,76 @@ fi
 run_step "Record audit tool versions" execution-only write_tool_versions
 
 if has_tool semgrep; then
-  run_step "Semgrep SAST" findings-exit-one run_semgrep
+  run_scanner_step semgrep "Semgrep SAST" findings-exit-one run_semgrep
 else
-  skip_or_fail_missing semgrep
+  skip_or_fail_missing semgrep semgrep
 fi
 
 if has_tool bandit; then
-  run_step "Bandit Python security scan" findings-exit-one run_bandit
+  run_scanner_step bandit "Bandit Python security scan" findings-exit-one run_bandit
 else
-  skip_or_fail_missing bandit
+  skip_or_fail_missing bandit bandit
 fi
 
 if has_tool ruff; then
-  run_step "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security
+  run_scanner_step ruff-security "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security
 else
-  skip_or_fail_missing ruff
+  skip_or_fail_missing ruff-security ruff
 fi
 
 if has_tool gitleaks; then
-  run_step "Gitleaks secret scan" findings-exit-one run_gitleaks
+  run_scanner_step gitleaks "Gitleaks secret scan" findings-exit-one run_gitleaks
 else
-  skip_or_fail_missing gitleaks
+  skip_or_fail_missing gitleaks gitleaks
 fi
 
 if [[ "$MODE" != "quick" ]]; then
   if has_tool trivy; then
-    run_step "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy
+    run_scanner_step trivy "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy
   else
-    skip_or_fail_missing trivy
+    skip_or_fail_missing trivy trivy
   fi
 
   if has_tool pip-audit; then
-    run_step "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests
+    run_scanner_step pip-audit "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests
   else
-    skip_or_fail_missing pip-audit
+    skip_or_fail_missing pip-audit pip-audit
   fi
 
   if has_tool npm; then
-    run_step "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests
+    run_scanner_step npm-audit "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests
   else
-    skip_or_fail_missing npm
+    skip_or_fail_missing npm-audit npm
   fi
 fi
 
 if [[ "$MODE" == "full" ]]; then
   if has_tool osv-scanner; then
-    run_step "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner
+    run_scanner_step osv-scanner "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner
   else
-    skip_or_fail_missing osv-scanner
+    skip_or_fail_missing osv-scanner osv-scanner
   fi
 
   if has_tool jscpd; then
-    run_step "jscpd duplicate code scan" execution-only run_jscpd
+    run_scanner_step jscpd "jscpd duplicate code scan" findings-exit-one run_jscpd
   else
-    skip_or_fail_missing jscpd
+    skip_or_fail_missing jscpd jscpd
   fi
 
   if has_tool radon; then
-    run_step "Radon complexity report" execution-only run_radon
+    run_scanner_step radon "Radon complexity report" execution-only run_radon
   else
-    skip_or_fail_missing radon
+    skip_or_fail_missing radon radon
   fi
 
   if has_tool xenon; then
-    run_step "Xenon complexity threshold scan" findings-exit-one run_xenon
+    run_scanner_step xenon "Xenon complexity threshold scan" findings-exit-one run_xenon
   else
-    skip_or_fail_missing xenon
+    skip_or_fail_missing xenon xenon
   fi
 fi
 
+run_step "Validate required scanner coverage records" execution-only validate_scanner_status_contract
 run_step "Validate machine-readable audit reports" execution-only validate_machine_reports
 
 echo