#!/usr/bin/env bash set -uo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" MODE="${SECURITY_AUDIT_MODE:-ci}" SCOPE="${SECURITY_AUDIT_SCOPE:-current}" REPORTS_DIR="${SECURITY_AUDIT_REPORTS_DIR:-$ROOT/audit-reports}" FAIL_ON_FINDINGS="${SECURITY_AUDIT_FAIL_ON_FINDINGS:-0}" REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}" while [[ $# -gt 0 ]]; do case "$1" in --mode) MODE="${2:?missing mode}" shift 2 ;; --scope) SCOPE="${2:?missing scope}" shift 2 ;; --reports-dir) REPORTS_DIR="${2:?missing reports dir}" shift 2 ;; --strict) FAIL_ON_FINDINGS=1 shift ;; --report-only) FAIL_ON_FINDINGS=0 shift ;; -h|--help) cat <<'EOF' Usage: tools/checks/check-security-audit.sh [--mode ci|quick|full] [--scope current|govoplan] [--reports-dir DIR] [--strict|--report-only] Runs GovOPlaN static/security audit tools and writes machine-readable reports. Modes: quick Semgrep local rules, Bandit, Ruff security rules, Gitleaks ci quick + Semgrep registry auto config, Trivy, dependency manifest scans full ci + OSV-Scanner, jscpd duplicate scan, Radon/Xenon complexity reports Scopes: current scan this repository checkout govoplan scan sibling govoplan-* repositories below GOVOPLAN_REPOS_ROOT or the parent of this repo By default findings are reported but do not fail the process. Use --strict or SECURITY_AUDIT_FAIL_ON_FINDINGS=1 once the baseline is clean. EOF exit 0 ;; *) echo "Unknown argument: $1" >&2 exit 2 ;; esac done case "$MODE" in quick|ci|full) ;; *) echo "Unknown SECURITY_AUDIT_MODE=$MODE; expected quick, ci, or full." >&2 exit 2 ;; esac case "$SCOPE" in current|govoplan) ;; *) echo "Unknown SECURITY_AUDIT_SCOPE=$SCOPE; expected current or govoplan." >&2 exit 2 ;; esac if [[ "$REPORTS_DIR" != /* ]]; then REPORTS_DIR="$ROOT/$REPORTS_DIR" fi mkdir -p "$REPORTS_DIR" REPORTS_DIR="$(cd "$REPORTS_DIR" && pwd -P)" for flag_name in FAIL_ON_FINDINGS REQUIRE_TOOLS; do flag_value="${!flag_name}" if [[ "$flag_value" != "0" && "$flag_value" != "1" ]]; then echo "$flag_name must be 0 or 1, got: $flag_value" >&2 exit 2 fi done declare -a REPOS=() if [[ "$SCOPE" == "govoplan" ]]; then REPOS_ROOT="${GOVOPLAN_REPOS_ROOT:-$(dirname "$ROOT")}" [[ -d "$ROOT/.git" ]] && REPOS+=("$ROOT") while IFS= read -r repo; do [[ -d "$repo/.git" ]] || continue REPOS+=("$repo") done < <(find "$REPOS_ROOT" -maxdepth 1 -type d \( -name 'govoplan-*' -o -name 'addideas-govoplan-*' \) | sort) else REPOS=("$ROOT") fi if [[ "${#REPOS[@]}" -eq 0 ]]; then echo "No repositories found for scope $SCOPE." >&2 exit 1 fi echo "Security audit mode: $MODE" echo "Security audit scope: $SCOPE" echo "Security audit reports: $REPORTS_DIR" echo "Security audit repositories: ${#REPOS[@]}" if [[ "${#REPOS[@]}" -le 12 ]]; then printf ' %s\n' "${REPOS[@]}" fi declare -a PY_ROOTS=() declare -a PY_PROD_ROOTS=() declare -a PY_TEST_ROOTS=() for repo in "${REPOS[@]}"; do if [[ -d "$repo/src" ]]; then PY_ROOTS+=("$repo/src") PY_PROD_ROOTS+=("$repo/src") fi if [[ -d "$repo/tests" ]]; then PY_ROOTS+=("$repo/tests") PY_TEST_ROOTS+=("$repo/tests") fi done safe_name() { printf '%s' "$(basename "$1")" | tr -c 'A-Za-z0-9_.-' '_' } has_tool() { command -v "$1" >/dev/null 2>&1 } overall_status=0 finding_status=0 missing_status=0 execution_status=0 audit_started_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" audit_completed_at="" workspace_unchanged="unknown" audit_toolbox_fingerprint="${SECURITY_AUDIT_TOOLBOX_FINGERPRINT:-direct-host}" declare -A REPORT_FILES=() declare -A MACHINE_REPORTS=() register_report() { local path="$1" local report_kind="${2:-text}" case "$path" in "$REPORTS_DIR"/*) ;; *) echo "Refusing to register an audit report outside $REPORTS_DIR: $path" >&2 return 2 ;; esac REPORT_FILES["$path"]=1 if [[ "$report_kind" == "machine" ]]; then MACHINE_REPORTS["$path"]=1 fi } prepare_machine_report() { local path="$1" register_report "$path" machine || return 2 if [[ -e "$path" || -L "$path" ]]; then rm -f -- "$path" || return 2 fi } run_step() { local name="$1" local exit_contract="$2" shift 2 echo echo "==> $name" "$@" local status=$? local outcome="passed" if [[ "$status" -eq 0 ]]; then : elif [[ "$exit_contract" == "findings-exit-one" && "$status" -eq 1 ]]; then outcome="findings" echo "Audit step reported findings: $name (exit $status)" >&2 finding_status=1 if [[ "$FAIL_ON_FINDINGS" == "1" ]]; then overall_status=1 fi else outcome="execution-error" echo "Audit step failed to execute successfully: $name (exit $status)" >&2 execution_status=1 overall_status=1 fi printf '%s\t%s\t%s\n' "$name" "$status" "$outcome" >> "$REPORTS_DIR/step-status.tsv" } skip_or_fail_missing() { local tool="$1" echo "Skipping $tool: command not found. Use tools/checks/security-audit/run.sh for the containerized toolbox." >&2 missing_status=1 if [[ "$REQUIRE_TOOLS" == "1" ]]; then overall_status=1 fi printf '%s\t127\tmissing\n' "$tool" >> "$REPORTS_DIR/step-status.tsv" } write_manifest() { local -a present_reports=() local -a expected_reports=() local path while IFS= read -r path; do expected_reports+=("${path#"$REPORTS_DIR"/}") if [[ -f "$path" ]]; then present_reports+=("${path#"$REPORTS_DIR"/}") fi done < <(printf '%s\n' "${!REPORT_FILES[@]}" | sort) python3 - \ "$REPORTS_DIR/manifest.json" \ "$MODE" \ "$SCOPE" \ "$FAIL_ON_FINDINGS" \ "$audit_started_at" \ "$audit_completed_at" \ "$workspace_unchanged" \ "$overall_status" \ "$finding_status" \ "$execution_status" \ "$missing_status" \ "$audit_toolbox_fingerprint" \ "${#REPOS[@]}" \ "${REPOS[@]}" \ "${#present_reports[@]}" \ "${present_reports[@]}" \ "${expected_reports[@]}" <<'PY' from __future__ import annotations import json from pathlib import Path import sys ( manifest_path, mode, scope, fail_on_findings, started_at, completed_at, workspace_unchanged, overall_status, finding_status, execution_status, missing_status, toolbox_fingerprint, repository_count, *remaining, ) = sys.argv[1:] repo_count = int(repository_count) repositories = remaining[:repo_count] remaining = remaining[repo_count:] report_count = int(remaining[0]) reports = remaining[1 : report_count + 1] expected_reports = remaining[report_count + 1 :] payload = { "mode": mode, "scope": scope, "fail_on_findings": fail_on_findings == "1", "started_at_utc": started_at, "completed_at_utc": completed_at, "toolbox_fingerprint": toolbox_fingerprint, "workspace_unchanged": workspace_unchanged == "true", "overall_status": int(overall_status), "finding_status": int(finding_status), "execution_error_status": int(execution_status), "missing_tool_status": int(missing_status), "tool_versions": "tool-versions.txt", "workspace_state_start": "workspace-state-start.tsv", "workspace_state_end": "workspace-state-end.tsv", "report_checksums": "report-sha256.txt", "step_status": "step-status.tsv", "repositories": repositories, "reports": reports, "expected_reports": expected_reports, "missing_reports": sorted(set(expected_reports) - set(reports)), } Path(manifest_path).write_text( json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) PY } accumulate_exit_status() { local -n aggregate="$1" local child_status="$2" if [[ "$child_status" -gt 1 ]]; then aggregate=2 elif [[ "$child_status" -eq 1 && "$aggregate" -eq 0 ]]; then aggregate=1 fi } repository_fingerprint() { local repo="$1" ( set -e cd "$repo" || exit 1 git rev-parse HEAD git status --porcelain=v1 -z git diff --binary --no-ext-diff HEAD -- while IFS= read -r -d '' path; do printf '%s\0' "$path" if [[ -L "$path" ]]; then printf 'symlink\0' readlink -- "$path" else sha256sum -- "$path" fi done < <(git ls-files --others --exclude-standard -z) ) | sha256sum | cut -d' ' -f1 } write_workspace_state() { local destination="$1" local head local fingerprint : > "$destination" for repo in "${REPOS[@]}"; do if ! head="$(git -C "$repo" rev-parse HEAD)"; then return 2 fi if ! fingerprint="$(repository_fingerprint "$repo")"; then return 2 fi printf '%s\t%s\t%s\n' \ "$repo" \ "$head" \ "$fingerprint" \ >> "$destination" done } write_tool_versions() { local destination="$REPORTS_DIR/tool-versions.txt" local status=0 local version : > "$destination" local tool for tool in semgrep bandit ruff gitleaks trivy pip-audit npm osv-scanner jscpd radon xenon; do has_tool "$tool" || continue if [[ "$tool" == "gitleaks" ]]; then version="$("$tool" version 2>&1)" else version="$("$tool" --version 2>&1)" fi local version_status=$? version="${version%%$'\n'*}" if [[ "$version_status" -ne 0 || -z "$version" ]]; then printf '%s\t%s\n' "$tool" "version-command-failed:$version_status" >> "$destination" status=2 else printf '%s\t%s\n' "$tool" "$version" >> "$destination" fi done return "$status" } write_report_checksums() { local path local relative_path : > "$REPORTS_DIR/report-sha256.txt" while IFS= read -r path; do [[ -f "$path" ]] || continue relative_path="${path#"$REPORTS_DIR"/}" (cd "$REPORTS_DIR" && sha256sum -- "$relative_path") \ >> "$REPORTS_DIR/report-sha256.txt" || return 2 done < <(printf '%s\n' "${!REPORT_FILES[@]}" | sort) } validate_machine_reports() { python3 - "$REPORTS_DIR" "${!MACHINE_REPORTS[@]}" <<'PY' from __future__ import annotations import json from pathlib import Path import sys root = Path(sys.argv[1]).resolve() errors: list[str] = [] validated = 0 for raw_path in sorted(set(sys.argv[2:])): path = Path(raw_path) try: relative_path = path.resolve().relative_to(root) except (OSError, ValueError) as exc: errors.append(f"{path}: invalid report path: {exc}") continue if not path.is_file(): errors.append(f"{relative_path}: expected report was not produced") continue if path.stat().st_size == 0: errors.append(f"{relative_path}: report is empty") continue try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError) as exc: errors.append(f"{relative_path}: invalid JSON: {exc}") continue validated += 1 if path.suffix == ".sarif" and ( not isinstance(payload, dict) or not isinstance(payload.get("version"), str) or not isinstance(payload.get("runs"), list) ): errors.append(f"{relative_path}: invalid SARIF document") continue report_name = path.name if report_name.startswith("bandit") and ( not isinstance(payload, dict) or not isinstance(payload.get("errors"), list) or not isinstance(payload.get("metrics"), dict) or not isinstance(payload.get("results"), list) ): errors.append(f"{relative_path}: invalid Bandit report") elif report_name.startswith("ruff-security") and not isinstance(payload, list): errors.append(f"{relative_path}: invalid Ruff report") elif report_name.startswith("gitleaks-") and not isinstance(payload, list): errors.append(f"{relative_path}: invalid Gitleaks report") elif report_name.startswith("pip-audit-") and ( not isinstance(payload, dict) or not isinstance(payload.get("dependencies"), list) ): errors.append(f"{relative_path}: invalid pip-audit report") elif report_name.startswith("npm-audit-") and ( not isinstance(payload, dict) or payload.get("error") or not isinstance(payload.get("metadata"), dict) or not isinstance(payload.get("vulnerabilities"), dict) ): errors.append(f"{relative_path}: npm audit returned an invalid or error report") elif report_name.startswith("osv-scanner-") and ( not isinstance(payload, dict) or not isinstance(payload.get("results"), list) ): errors.append(f"{relative_path}: invalid OSV-Scanner report") elif report_name == "jscpd-report.json" and ( not isinstance(payload, dict) or not isinstance(payload.get("duplicates"), list) or not isinstance(payload.get("statistics"), dict) ): errors.append(f"{relative_path}: invalid jscpd report") if errors: print("\n".join(errors), file=sys.stderr) raise SystemExit(2) print(f"Validated {validated} JSON/SARIF reports.") PY } run_semgrep() { local report="$REPORTS_DIR/semgrep.sarif" prepare_machine_report "$report" || return 2 local config_args=(--config "$ROOT/tools/checks/security-audit/semgrep-govoplan.yml") if [[ "$MODE" != "quick" ]]; then config_args+=(--config p/default --config p/owasp-top-ten) fi semgrep scan \ --metrics=off \ --error \ --no-git-ignore \ --sarif \ --output "$report" \ --exclude .git \ --exclude node_modules \ --exclude .venv \ --exclude dist \ --exclude build \ --exclude runtime \ --exclude audit-reports \ --exclude '**/.*-test-build/**' \ "${config_args[@]}" \ "${REPOS[@]}" } run_bandit() { local status=0 if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then prepare_machine_report "$REPORTS_DIR/bandit.json" || return 2 bandit -r "${PY_PROD_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json" accumulate_exit_status status "$?" fi if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then prepare_machine_report "$REPORTS_DIR/bandit-tests.json" || return 2 bandit -r "${PY_TEST_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit-tests.json" local test_status=$? [[ "$test_status" -le 1 ]] || status=2 fi return "$status" } run_ruff_security() { local status=0 if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then prepare_machine_report "$REPORTS_DIR/ruff-security.json" || return 2 ruff check --select S --output-format json "${PY_PROD_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json" accumulate_exit_status status "$?" fi if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then prepare_machine_report "$REPORTS_DIR/ruff-security-tests.json" || return 2 ruff check --select S --output-format json "${PY_TEST_ROOTS[@]}" > "$REPORTS_DIR/ruff-security-tests.json" local test_status=$? [[ "$test_status" -le 1 ]] || status=2 fi return "$status" } run_gitleaks() { local status=0 for repo in "${REPOS[@]}"; do local name name="$(safe_name "$repo")" if gitleaks git --help >/dev/null 2>&1; then prepare_machine_report "$REPORTS_DIR/gitleaks-history-$name.json" || return 2 prepare_machine_report "$REPORTS_DIR/gitleaks-worktree-$name.json" || return 2 gitleaks git \ --config "$ROOT/.gitleaks.toml" \ --report-format json \ --report-path "$REPORTS_DIR/gitleaks-history-$name.json" \ --no-banner \ "$repo" accumulate_exit_status status "$?" # History scanning does not include new or modified working-tree files. # Scan the directory as well so pre-commit audits cover the exact code # under review, while retaining the history scan above. gitleaks dir \ --config "$ROOT/.gitleaks.toml" \ --report-format json \ --report-path "$REPORTS_DIR/gitleaks-worktree-$name.json" \ --no-banner \ "$repo" accumulate_exit_status status "$?" else prepare_machine_report "$REPORTS_DIR/gitleaks-$name.json" || return 2 gitleaks detect \ --source "$repo" \ --config "$ROOT/.gitleaks.toml" \ --report-format json \ --report-path "$REPORTS_DIR/gitleaks-$name.json" \ --no-banner accumulate_exit_status status "$?" fi done return "$status" } run_trivy() { local status=0 for repo in "${REPOS[@]}"; do local name name="$(safe_name "$repo")" prepare_machine_report "$REPORTS_DIR/trivy-$name.sarif" || return 2 trivy fs \ --scanners vuln,secret,misconfig \ --skip-dirs node_modules \ --skip-dirs .venv \ --skip-dirs dist \ --skip-dirs build \ --format sarif \ --exit-code 1 \ --output "$REPORTS_DIR/trivy-$name.sarif" \ "$repo" accumulate_exit_status status "$?" done return "$status" } run_pip_audit_manifests() { local status=0 for repo in "${REPOS[@]}"; do [[ -f "$repo/requirements.txt" ]] || continue local name name="$(safe_name "$repo")" prepare_machine_report "$REPORTS_DIR/pip-audit-$name.json" || return 2 # Requirements may contain project-relative entries such as `.[server]`. # Resolve them from the owning repository instead of the meta-repository. (cd "$repo" && pip-audit -r requirements.txt --progress-spinner off --format json --output "$REPORTS_DIR/pip-audit-$name.json") accumulate_exit_status status "$?" done return "$status" } run_npm_audit_manifests() { local status=0 for repo in "${REPOS[@]}"; do [[ -f "$repo/webui/package-lock.json" ]] || continue local name name="$(safe_name "$repo")" prepare_machine_report "$REPORTS_DIR/npm-audit-$name.json" || return 2 (cd "$repo/webui" && npm audit --omit=dev --json > "$REPORTS_DIR/npm-audit-$name.json") accumulate_exit_status status "$?" done return "$status" } run_osv_scanner() { local status=0 for repo in "${REPOS[@]}"; do local name name="$(safe_name "$repo")" if osv-scanner scan --help >/dev/null 2>&1; then prepare_machine_report "$REPORTS_DIR/osv-scanner-$name.json" || return 2 osv-scanner scan -r \ --allow-no-lockfiles \ --format json \ --output-file "$REPORTS_DIR/osv-scanner-$name.json" \ "$repo" accumulate_exit_status status "$?" else prepare_machine_report "$REPORTS_DIR/osv-scanner-$name.json" || return 2 osv-scanner -r --format json --output "$REPORTS_DIR/osv-scanner-$name.json" "$repo" accumulate_exit_status status "$?" fi done return "$status" } run_jscpd() { prepare_machine_report "$REPORTS_DIR/jscpd/jscpd-report.json" || return 2 local ignored_paths ignored_paths=( "**/.git/**" "**/node_modules/**" "**/.venv/**" "**/dist/**" "**/build/**" "**/audit-reports/**" "**/package-lock.json" "**/package-lock.release.json" "**/package.json" "package.json" "**/generatedTranslations.ts" "**/docs/gitea-labels.json" "**/*.md" "**/*.md:*" "*.md" "*.md:*" "**/*.svg" "*.svg" ".gitea/workflows/*.yml" "**/.gitea/workflows/*.yml" "**/backend/schema/*.schema.json" ) local ignored_path_pattern ignored_path_pattern="$(printf '%s,' "${ignored_paths[@]}")" ignored_path_pattern="${ignored_path_pattern%,}" jscpd \ --silent \ --min-tokens 80 \ --reporters json \ --output "$REPORTS_DIR/jscpd" \ --ignore "$ignored_path_pattern" \ "${REPOS[@]}" } run_radon() { [[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0 register_report "$REPORTS_DIR/radon-cc.txt" || return 2 radon cc -s -a "${PY_ROOTS[@]}" > "$REPORTS_DIR/radon-cc.txt" } run_xenon() { [[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0 register_report "$REPORTS_DIR/xenon.txt" || return 2 xenon --max-absolute C --max-modules B --max-average A "${PY_ROOTS[@]}" > "$REPORTS_DIR/xenon.txt" 2>&1 } register_report "$REPORTS_DIR/step-status.tsv" register_report "$REPORTS_DIR/tool-versions.txt" register_report "$REPORTS_DIR/workspace-state-start.tsv" register_report "$REPORTS_DIR/workspace-state-end.tsv" : > "$REPORTS_DIR/step-status.tsv" if ! write_workspace_state "$REPORTS_DIR/workspace-state-start.tsv"; then echo "Could not capture the repository state before the audit." >&2 exit 1 fi run_step "Record audit tool versions" execution-only write_tool_versions if has_tool semgrep; then run_step "Semgrep SAST" findings-exit-one run_semgrep else skip_or_fail_missing semgrep fi if has_tool bandit; then run_step "Bandit Python security scan" findings-exit-one run_bandit else skip_or_fail_missing bandit fi if has_tool ruff; then run_step "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security else skip_or_fail_missing ruff fi if has_tool gitleaks; then run_step "Gitleaks secret scan" findings-exit-one run_gitleaks else skip_or_fail_missing gitleaks fi if [[ "$MODE" != "quick" ]]; then if has_tool trivy; then run_step "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy else skip_or_fail_missing trivy fi if has_tool pip-audit; then run_step "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests else skip_or_fail_missing pip-audit fi if has_tool npm; then run_step "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests else skip_or_fail_missing npm fi fi if [[ "$MODE" == "full" ]]; then if has_tool osv-scanner; then run_step "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner else skip_or_fail_missing osv-scanner fi if has_tool jscpd; then run_step "jscpd duplicate code scan" execution-only run_jscpd else skip_or_fail_missing jscpd fi if has_tool radon; then run_step "Radon complexity report" execution-only run_radon else skip_or_fail_missing radon fi if has_tool xenon; then run_step "Xenon complexity threshold scan" findings-exit-one run_xenon else skip_or_fail_missing xenon fi fi run_step "Validate machine-readable audit reports" execution-only validate_machine_reports echo echo "Security audit reports written to: $REPORTS_DIR" if [[ "$finding_status" -ne 0 && "$FAIL_ON_FINDINGS" != "1" ]]; then echo "Findings were reported, but this run is report-only. Re-run with --strict to fail on findings." fi if [[ "$missing_status" -ne 0 ]]; then echo "One or more audit tools were missing." >&2 fi if ! write_workspace_state "$REPORTS_DIR/workspace-state-end.tsv"; then workspace_unchanged=false execution_status=1 overall_status=1 echo "Could not capture the repository state after the audit." >&2 elif cmp -s "$REPORTS_DIR/workspace-state-start.tsv" "$REPORTS_DIR/workspace-state-end.tsv"; then workspace_unchanged=true else workspace_unchanged=false overall_status=1 echo "Repository state changed during the audit; discard this mixed snapshot and rerun." >&2 fi audit_completed_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" if ! write_report_checksums; then execution_status=1 overall_status=1 echo "Could not generate report checksums." >&2 fi if ! write_manifest; then echo "Could not write the audit manifest." >&2 exit 1 fi exit "$overall_status"