851 lines
24 KiB
Bash
851 lines
24 KiB
Bash
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
MODE="${SECURITY_AUDIT_MODE:-ci}"
|
|
SCOPE="${SECURITY_AUDIT_SCOPE:-current}"
|
|
REPORTS_DIR="${SECURITY_AUDIT_REPORTS_DIR:-$ROOT/audit-reports}"
|
|
FAIL_ON_FINDINGS="${SECURITY_AUDIT_FAIL_ON_FINDINGS:-0}"
|
|
REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}"
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--mode)
|
|
MODE="${2:?missing mode}"
|
|
shift 2
|
|
;;
|
|
--scope)
|
|
SCOPE="${2:?missing scope}"
|
|
shift 2
|
|
;;
|
|
--reports-dir)
|
|
REPORTS_DIR="${2:?missing reports dir}"
|
|
shift 2
|
|
;;
|
|
--strict)
|
|
FAIL_ON_FINDINGS=1
|
|
shift
|
|
;;
|
|
--report-only)
|
|
FAIL_ON_FINDINGS=0
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
cat <<'EOF'
|
|
Usage: tools/checks/check-security-audit.sh [--mode ci|quick|full] [--scope current|govoplan] [--reports-dir DIR] [--strict|--report-only]
|
|
|
|
Runs GovOPlaN static/security audit tools and writes machine-readable reports.
|
|
|
|
Modes:
|
|
quick Semgrep local rules, Bandit, Ruff security rules, Gitleaks
|
|
ci quick + Semgrep registry auto config, Trivy, dependency manifest scans
|
|
full ci + OSV-Scanner, jscpd duplicate scan, Radon/Xenon complexity reports
|
|
|
|
Scopes:
|
|
current scan this repository checkout
|
|
govoplan scan sibling govoplan-* repositories below GOVOPLAN_REPOS_ROOT or the parent of this repo
|
|
|
|
By default findings are reported but do not fail the process. Use --strict or
|
|
SECURITY_AUDIT_FAIL_ON_FINDINGS=1 once the baseline is clean.
|
|
EOF
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unknown argument: $1" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
case "$MODE" in
|
|
quick|ci|full) ;;
|
|
*)
|
|
echo "Unknown SECURITY_AUDIT_MODE=$MODE; expected quick, ci, or full." >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
case "$SCOPE" in
|
|
current|govoplan) ;;
|
|
*)
|
|
echo "Unknown SECURITY_AUDIT_SCOPE=$SCOPE; expected current or govoplan." >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
if [[ "$REPORTS_DIR" != /* ]]; then
|
|
REPORTS_DIR="$ROOT/$REPORTS_DIR"
|
|
fi
|
|
mkdir -p "$REPORTS_DIR"
|
|
REPORTS_DIR="$(cd "$REPORTS_DIR" && pwd -P)"
|
|
|
|
for flag_name in FAIL_ON_FINDINGS REQUIRE_TOOLS; do
|
|
flag_value="${!flag_name}"
|
|
if [[ "$flag_value" != "0" && "$flag_value" != "1" ]]; then
|
|
echo "$flag_name must be 0 or 1, got: $flag_value" >&2
|
|
exit 2
|
|
fi
|
|
done
|
|
|
|
declare -a REPOS=()
|
|
if [[ "$SCOPE" == "govoplan" ]]; then
|
|
REPOS_ROOT="${GOVOPLAN_REPOS_ROOT:-$(dirname "$ROOT")}"
|
|
[[ -d "$ROOT/.git" ]] && REPOS+=("$ROOT")
|
|
while IFS= read -r repo; do
|
|
[[ -d "$repo/.git" ]] || continue
|
|
REPOS+=("$repo")
|
|
done < <(find "$REPOS_ROOT" -maxdepth 1 -type d \( -name 'govoplan-*' -o -name 'addideas-govoplan-*' \) | sort)
|
|
else
|
|
REPOS=("$ROOT")
|
|
fi
|
|
|
|
if [[ "${#REPOS[@]}" -eq 0 ]]; then
|
|
echo "No repositories found for scope $SCOPE." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Security audit mode: $MODE"
|
|
echo "Security audit scope: $SCOPE"
|
|
echo "Security audit reports: $REPORTS_DIR"
|
|
echo "Security audit repositories: ${#REPOS[@]}"
|
|
if [[ "${#REPOS[@]}" -le 12 ]]; then
|
|
printf ' %s\n' "${REPOS[@]}"
|
|
fi
|
|
|
|
declare -a PY_ROOTS=()
|
|
declare -a PY_PROD_ROOTS=()
|
|
declare -a PY_TEST_ROOTS=()
|
|
for repo in "${REPOS[@]}"; do
|
|
if [[ -d "$repo/src" ]]; then
|
|
PY_ROOTS+=("$repo/src")
|
|
PY_PROD_ROOTS+=("$repo/src")
|
|
fi
|
|
if [[ -d "$repo/tests" ]]; then
|
|
PY_ROOTS+=("$repo/tests")
|
|
PY_TEST_ROOTS+=("$repo/tests")
|
|
fi
|
|
done
|
|
|
|
safe_name() {
|
|
printf '%s' "$(basename "$1")" | tr -c 'A-Za-z0-9_.-' '_'
|
|
}
|
|
|
|
has_tool() {
|
|
command -v "$1" >/dev/null 2>&1
|
|
}
|
|
|
|
overall_status=0
|
|
finding_status=0
|
|
missing_status=0
|
|
execution_status=0
|
|
audit_started_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
|
audit_completed_at=""
|
|
workspace_unchanged="unknown"
|
|
audit_toolbox_fingerprint="${SECURITY_AUDIT_TOOLBOX_FINGERPRINT:-direct-host}"
|
|
declare -A REPORT_FILES=()
|
|
declare -A MACHINE_REPORTS=()
|
|
|
|
register_report() {
|
|
local path="$1"
|
|
local report_kind="${2:-text}"
|
|
case "$path" in
|
|
"$REPORTS_DIR"/*) ;;
|
|
*)
|
|
echo "Refusing to register an audit report outside $REPORTS_DIR: $path" >&2
|
|
return 2
|
|
;;
|
|
esac
|
|
REPORT_FILES["$path"]=1
|
|
if [[ "$report_kind" == "machine" ]]; then
|
|
MACHINE_REPORTS["$path"]=1
|
|
fi
|
|
}
|
|
|
|
prepare_machine_report() {
|
|
local path="$1"
|
|
register_report "$path" machine || return 2
|
|
if [[ -e "$path" || -L "$path" ]]; then
|
|
rm -f -- "$path" || return 2
|
|
fi
|
|
}
|
|
|
|
run_step() {
|
|
local name="$1"
|
|
local exit_contract="$2"
|
|
shift 2
|
|
echo
|
|
echo "==> $name"
|
|
"$@"
|
|
local status=$?
|
|
local outcome="passed"
|
|
if [[ "$status" -eq 0 ]]; then
|
|
:
|
|
elif [[ "$exit_contract" == "findings-exit-one" && "$status" -eq 1 ]]; then
|
|
outcome="findings"
|
|
echo "Audit step reported findings: $name (exit $status)" >&2
|
|
finding_status=1
|
|
if [[ "$FAIL_ON_FINDINGS" == "1" ]]; then
|
|
overall_status=1
|
|
fi
|
|
else
|
|
outcome="execution-error"
|
|
echo "Audit step failed to execute successfully: $name (exit $status)" >&2
|
|
execution_status=1
|
|
overall_status=1
|
|
fi
|
|
printf '%s\t%s\t%s\n' "$name" "$status" "$outcome" >> "$REPORTS_DIR/step-status.tsv"
|
|
}
|
|
|
|
skip_or_fail_missing() {
|
|
local tool="$1"
|
|
echo "Skipping $tool: command not found. Use tools/checks/security-audit/run.sh for the containerized toolbox." >&2
|
|
missing_status=1
|
|
if [[ "$REQUIRE_TOOLS" == "1" ]]; then
|
|
overall_status=1
|
|
fi
|
|
printf '%s\t127\tmissing\n' "$tool" >> "$REPORTS_DIR/step-status.tsv"
|
|
}
|
|
|
|
write_manifest() {
|
|
local -a present_reports=()
|
|
local -a expected_reports=()
|
|
local path
|
|
while IFS= read -r path; do
|
|
expected_reports+=("${path#"$REPORTS_DIR"/}")
|
|
if [[ -f "$path" ]]; then
|
|
present_reports+=("${path#"$REPORTS_DIR"/}")
|
|
fi
|
|
done < <(printf '%s\n' "${!REPORT_FILES[@]}" | sort)
|
|
|
|
python3 - \
|
|
"$REPORTS_DIR/manifest.json" \
|
|
"$MODE" \
|
|
"$SCOPE" \
|
|
"$FAIL_ON_FINDINGS" \
|
|
"$audit_started_at" \
|
|
"$audit_completed_at" \
|
|
"$workspace_unchanged" \
|
|
"$overall_status" \
|
|
"$finding_status" \
|
|
"$execution_status" \
|
|
"$missing_status" \
|
|
"$audit_toolbox_fingerprint" \
|
|
"${#REPOS[@]}" \
|
|
"${REPOS[@]}" \
|
|
"${#present_reports[@]}" \
|
|
"${present_reports[@]}" \
|
|
"${expected_reports[@]}" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
|
|
(
|
|
manifest_path,
|
|
mode,
|
|
scope,
|
|
fail_on_findings,
|
|
started_at,
|
|
completed_at,
|
|
workspace_unchanged,
|
|
overall_status,
|
|
finding_status,
|
|
execution_status,
|
|
missing_status,
|
|
toolbox_fingerprint,
|
|
repository_count,
|
|
*remaining,
|
|
) = sys.argv[1:]
|
|
repo_count = int(repository_count)
|
|
repositories = remaining[:repo_count]
|
|
remaining = remaining[repo_count:]
|
|
report_count = int(remaining[0])
|
|
reports = remaining[1 : report_count + 1]
|
|
expected_reports = remaining[report_count + 1 :]
|
|
payload = {
|
|
"mode": mode,
|
|
"scope": scope,
|
|
"fail_on_findings": fail_on_findings == "1",
|
|
"started_at_utc": started_at,
|
|
"completed_at_utc": completed_at,
|
|
"toolbox_fingerprint": toolbox_fingerprint,
|
|
"workspace_unchanged": workspace_unchanged == "true",
|
|
"overall_status": int(overall_status),
|
|
"finding_status": int(finding_status),
|
|
"execution_error_status": int(execution_status),
|
|
"missing_tool_status": int(missing_status),
|
|
"tool_versions": "tool-versions.txt",
|
|
"workspace_state_start": "workspace-state-start.tsv",
|
|
"workspace_state_end": "workspace-state-end.tsv",
|
|
"report_checksums": "report-sha256.txt",
|
|
"step_status": "step-status.tsv",
|
|
"repositories": repositories,
|
|
"reports": reports,
|
|
"expected_reports": expected_reports,
|
|
"missing_reports": sorted(set(expected_reports) - set(reports)),
|
|
}
|
|
Path(manifest_path).write_text(
|
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
PY
|
|
}
|
|
|
|
accumulate_exit_status() {
|
|
local -n aggregate="$1"
|
|
local child_status="$2"
|
|
if [[ "$child_status" -gt 1 ]]; then
|
|
aggregate=2
|
|
elif [[ "$child_status" -eq 1 && "$aggregate" -eq 0 ]]; then
|
|
aggregate=1
|
|
fi
|
|
}
|
|
|
|
repository_fingerprint() {
|
|
local repo="$1"
|
|
(
|
|
set -e
|
|
cd "$repo" || exit 1
|
|
git rev-parse HEAD
|
|
git status --porcelain=v1 -z
|
|
git diff --binary --no-ext-diff HEAD --
|
|
while IFS= read -r -d '' path; do
|
|
printf '%s\0' "$path"
|
|
if [[ -L "$path" ]]; then
|
|
printf 'symlink\0'
|
|
readlink -- "$path"
|
|
else
|
|
sha256sum -- "$path"
|
|
fi
|
|
done < <(git ls-files --others --exclude-standard -z)
|
|
) | sha256sum | cut -d' ' -f1
|
|
}
|
|
|
|
write_workspace_state() {
|
|
local destination="$1"
|
|
local head
|
|
local fingerprint
|
|
: > "$destination"
|
|
for repo in "${REPOS[@]}"; do
|
|
if ! head="$(git -C "$repo" rev-parse HEAD)"; then
|
|
return 2
|
|
fi
|
|
if ! fingerprint="$(repository_fingerprint "$repo")"; then
|
|
return 2
|
|
fi
|
|
printf '%s\t%s\t%s\n' \
|
|
"$repo" \
|
|
"$head" \
|
|
"$fingerprint" \
|
|
>> "$destination"
|
|
done
|
|
}
|
|
|
|
write_tool_versions() {
|
|
local destination="$REPORTS_DIR/tool-versions.txt"
|
|
local status=0
|
|
local version
|
|
: > "$destination"
|
|
local tool
|
|
for tool in semgrep bandit ruff gitleaks trivy pip-audit npm osv-scanner jscpd radon xenon; do
|
|
has_tool "$tool" || continue
|
|
if [[ "$tool" == "gitleaks" ]]; then
|
|
version="$("$tool" version 2>&1)"
|
|
else
|
|
version="$("$tool" --version 2>&1)"
|
|
fi
|
|
local version_status=$?
|
|
version="${version%%$'\n'*}"
|
|
if [[ "$version_status" -ne 0 || -z "$version" ]]; then
|
|
printf '%s\t%s\n' "$tool" "version-command-failed:$version_status" >> "$destination"
|
|
status=2
|
|
else
|
|
printf '%s\t%s\n' "$tool" "$version" >> "$destination"
|
|
fi
|
|
done
|
|
return "$status"
|
|
}
|
|
|
|
write_report_checksums() {
|
|
local path
|
|
local relative_path
|
|
: > "$REPORTS_DIR/report-sha256.txt"
|
|
while IFS= read -r path; do
|
|
[[ -f "$path" ]] || continue
|
|
relative_path="${path#"$REPORTS_DIR"/}"
|
|
(cd "$REPORTS_DIR" && sha256sum -- "$relative_path") \
|
|
>> "$REPORTS_DIR/report-sha256.txt" || return 2
|
|
done < <(printf '%s\n' "${!REPORT_FILES[@]}" | sort)
|
|
}
|
|
|
|
validate_machine_reports() {
|
|
python3 - "$REPORTS_DIR" "${!MACHINE_REPORTS[@]}" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
|
|
root = Path(sys.argv[1]).resolve()
|
|
errors: list[str] = []
|
|
validated = 0
|
|
for raw_path in sorted(set(sys.argv[2:])):
|
|
path = Path(raw_path)
|
|
try:
|
|
relative_path = path.resolve().relative_to(root)
|
|
except (OSError, ValueError) as exc:
|
|
errors.append(f"{path}: invalid report path: {exc}")
|
|
continue
|
|
if not path.is_file():
|
|
errors.append(f"{relative_path}: expected report was not produced")
|
|
continue
|
|
if path.stat().st_size == 0:
|
|
errors.append(f"{relative_path}: report is empty")
|
|
continue
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
errors.append(f"{relative_path}: invalid JSON: {exc}")
|
|
continue
|
|
validated += 1
|
|
if path.suffix == ".sarif" and (
|
|
not isinstance(payload, dict)
|
|
or not isinstance(payload.get("version"), str)
|
|
or not isinstance(payload.get("runs"), list)
|
|
):
|
|
errors.append(f"{relative_path}: invalid SARIF document")
|
|
continue
|
|
report_name = path.name
|
|
if report_name.startswith("bandit") and (
|
|
not isinstance(payload, dict)
|
|
or not isinstance(payload.get("errors"), list)
|
|
or not isinstance(payload.get("metrics"), dict)
|
|
or not isinstance(payload.get("results"), list)
|
|
):
|
|
errors.append(f"{relative_path}: invalid Bandit report")
|
|
elif report_name.startswith("ruff-security") and not isinstance(payload, list):
|
|
errors.append(f"{relative_path}: invalid Ruff report")
|
|
elif report_name.startswith("gitleaks-") and not isinstance(payload, list):
|
|
errors.append(f"{relative_path}: invalid Gitleaks report")
|
|
elif report_name.startswith("pip-audit-") and (
|
|
not isinstance(payload, dict)
|
|
or not isinstance(payload.get("dependencies"), list)
|
|
):
|
|
errors.append(f"{relative_path}: invalid pip-audit report")
|
|
elif report_name.startswith("npm-audit-") and (
|
|
not isinstance(payload, dict)
|
|
or payload.get("error")
|
|
or not isinstance(payload.get("metadata"), dict)
|
|
or not isinstance(payload.get("vulnerabilities"), dict)
|
|
):
|
|
errors.append(f"{relative_path}: npm audit returned an invalid or error report")
|
|
elif report_name.startswith("osv-scanner-") and (
|
|
not isinstance(payload, dict)
|
|
or "results" not in payload
|
|
or (payload["results"] is not None and not isinstance(payload["results"], list))
|
|
):
|
|
errors.append(f"{relative_path}: invalid OSV-Scanner report")
|
|
elif report_name == "jscpd-report.json" and (
|
|
not isinstance(payload, dict)
|
|
or not isinstance(payload.get("duplicates"), list)
|
|
or not isinstance(payload.get("statistics"), dict)
|
|
):
|
|
errors.append(f"{relative_path}: invalid jscpd report")
|
|
|
|
if errors:
|
|
print("\n".join(errors), file=sys.stderr)
|
|
raise SystemExit(2)
|
|
print(f"Validated {validated} JSON/SARIF reports.")
|
|
PY
|
|
}
|
|
|
|
run_semgrep() {
|
|
local report="$REPORTS_DIR/semgrep.sarif"
|
|
prepare_machine_report "$report" || return 2
|
|
local config_args=(--config "$ROOT/tools/checks/security-audit/semgrep-govoplan.yml")
|
|
if [[ "$MODE" != "quick" ]]; then
|
|
config_args+=(--config p/default --config p/owasp-top-ten)
|
|
fi
|
|
semgrep scan \
|
|
--metrics=off \
|
|
--error \
|
|
--no-git-ignore \
|
|
--sarif \
|
|
--output "$report" \
|
|
--exclude .git \
|
|
--exclude node_modules \
|
|
--exclude .venv \
|
|
--exclude dist \
|
|
--exclude build \
|
|
--exclude runtime \
|
|
--exclude audit-reports \
|
|
--exclude '**/.*-test-build/**' \
|
|
"${config_args[@]}" \
|
|
"${REPOS[@]}"
|
|
}
|
|
|
|
run_bandit() {
|
|
local status=0
|
|
if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then
|
|
prepare_machine_report "$REPORTS_DIR/bandit.json" || return 2
|
|
bandit -r "${PY_PROD_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json"
|
|
accumulate_exit_status status "$?"
|
|
fi
|
|
if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then
|
|
prepare_machine_report "$REPORTS_DIR/bandit-tests.json" || return 2
|
|
# Tests intentionally use assertions and synthetic credentials. Keep the
|
|
# separate test scan useful by excluding only those test-specific signals.
|
|
bandit -r "${PY_TEST_ROOTS[@]}" \
|
|
--skip B101,B105,B106,B107 \
|
|
-f json \
|
|
-o "$REPORTS_DIR/bandit-tests.json"
|
|
local test_status=$?
|
|
[[ "$test_status" -le 1 ]] || status=2
|
|
fi
|
|
return "$status"
|
|
}
|
|
|
|
run_ruff_security() {
|
|
local status=0
|
|
if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then
|
|
prepare_machine_report "$REPORTS_DIR/ruff-security.json" || return 2
|
|
ruff check --select S --output-format json "${PY_PROD_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json"
|
|
accumulate_exit_status status "$?"
|
|
fi
|
|
if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then
|
|
prepare_machine_report "$REPORTS_DIR/ruff-security-tests.json" || return 2
|
|
ruff check \
|
|
--select S \
|
|
--ignore S101,S105,S106,S107 \
|
|
--output-format json \
|
|
"${PY_TEST_ROOTS[@]}" > "$REPORTS_DIR/ruff-security-tests.json"
|
|
local test_status=$?
|
|
[[ "$test_status" -le 1 ]] || status=2
|
|
fi
|
|
return "$status"
|
|
}
|
|
|
|
run_gitleaks() {
|
|
local status=0
|
|
for repo in "${REPOS[@]}"; do
|
|
local name
|
|
name="$(safe_name "$repo")"
|
|
if gitleaks git --help >/dev/null 2>&1; then
|
|
prepare_machine_report "$REPORTS_DIR/gitleaks-history-$name.json" || return 2
|
|
prepare_machine_report "$REPORTS_DIR/gitleaks-worktree-$name.json" || return 2
|
|
gitleaks git \
|
|
--config "$ROOT/.gitleaks.toml" \
|
|
--report-format json \
|
|
--report-path "$REPORTS_DIR/gitleaks-history-$name.json" \
|
|
--no-banner \
|
|
"$repo"
|
|
accumulate_exit_status status "$?"
|
|
# History scanning does not include new or modified working-tree files.
|
|
# Scan the directory as well so pre-commit audits cover the exact code
|
|
# under review, while retaining the history scan above.
|
|
gitleaks dir \
|
|
--config "$ROOT/.gitleaks.toml" \
|
|
--report-format json \
|
|
--report-path "$REPORTS_DIR/gitleaks-worktree-$name.json" \
|
|
--no-banner \
|
|
"$repo"
|
|
accumulate_exit_status status "$?"
|
|
else
|
|
prepare_machine_report "$REPORTS_DIR/gitleaks-$name.json" || return 2
|
|
gitleaks detect \
|
|
--source "$repo" \
|
|
--config "$ROOT/.gitleaks.toml" \
|
|
--report-format json \
|
|
--report-path "$REPORTS_DIR/gitleaks-$name.json" \
|
|
--no-banner
|
|
accumulate_exit_status status "$?"
|
|
fi
|
|
done
|
|
return "$status"
|
|
}
|
|
|
|
run_trivy() {
|
|
local status=0
|
|
for repo in "${REPOS[@]}"; do
|
|
local name
|
|
name="$(safe_name "$repo")"
|
|
prepare_machine_report "$REPORTS_DIR/trivy-$name.sarif" || return 2
|
|
trivy fs \
|
|
--scanners vuln,secret,misconfig \
|
|
--skip-dirs node_modules \
|
|
--skip-dirs .venv \
|
|
--skip-dirs dist \
|
|
--skip-dirs build \
|
|
--format sarif \
|
|
--exit-code 1 \
|
|
--output "$REPORTS_DIR/trivy-$name.sarif" \
|
|
"$repo"
|
|
accumulate_exit_status status "$?"
|
|
done
|
|
return "$status"
|
|
}
|
|
|
|
run_pip_audit_manifests() {
|
|
local status=0
|
|
for repo in "${REPOS[@]}"; do
|
|
local repo_name
|
|
repo_name="$(safe_name "$repo")"
|
|
while IFS= read -r -d '' requirements_file; do
|
|
local manifest_name report_path
|
|
manifest_name="$(safe_name "$requirements_file")"
|
|
report_path="$REPORTS_DIR/pip-audit-$repo_name-$manifest_name.json"
|
|
prepare_machine_report "$report_path" || return 2
|
|
# Requirements may contain project-relative entries such as `.[server]`.
|
|
# Resolve them from the owning repository instead of the meta-repository.
|
|
(
|
|
cd "$repo" &&
|
|
pip-audit \
|
|
-r "$(basename "$requirements_file")" \
|
|
--progress-spinner off \
|
|
--format json \
|
|
--output "$report_path"
|
|
)
|
|
accumulate_exit_status status "$?"
|
|
done < <(
|
|
find "$repo" \
|
|
-maxdepth 1 \
|
|
-type f \
|
|
-name 'requirements*.txt' \
|
|
-print0 |
|
|
sort -z
|
|
)
|
|
done
|
|
return "$status"
|
|
}
|
|
|
|
run_npm_audit_manifests() {
|
|
local status=0
|
|
for repo in "${REPOS[@]}"; do
|
|
local repo_name
|
|
repo_name="$(safe_name "$repo")"
|
|
while IFS= read -r -d '' lock_file; do
|
|
local lock_dir lock_name runtime_report all_report
|
|
lock_dir="$(dirname "$lock_file")"
|
|
lock_name="$(
|
|
printf '%s' "${lock_file#"$repo"/}" |
|
|
tr -c 'A-Za-z0-9_.-' '_'
|
|
)"
|
|
runtime_report="$REPORTS_DIR/npm-audit-runtime-$repo_name-$lock_name.json"
|
|
all_report="$REPORTS_DIR/npm-audit-all-$repo_name-$lock_name.json"
|
|
prepare_machine_report "$runtime_report" || return 2
|
|
(
|
|
cd "$lock_dir" &&
|
|
npm audit --omit=dev --json > "$runtime_report"
|
|
)
|
|
accumulate_exit_status status "$?"
|
|
prepare_machine_report "$all_report" || return 2
|
|
(
|
|
cd "$lock_dir" &&
|
|
npm audit --json > "$all_report"
|
|
)
|
|
accumulate_exit_status status "$?"
|
|
done < <(
|
|
find "$repo" \
|
|
-maxdepth 3 \
|
|
-type f \
|
|
-name package-lock.json \
|
|
-not -path '*/node_modules/*' \
|
|
-print0 |
|
|
sort -z
|
|
)
|
|
done
|
|
return "$status"
|
|
}
|
|
|
|
run_osv_scanner() {
|
|
local status=0
|
|
for repo in "${REPOS[@]}"; do
|
|
local name
|
|
name="$(safe_name "$repo")"
|
|
if osv-scanner scan --help >/dev/null 2>&1; then
|
|
prepare_machine_report "$REPORTS_DIR/osv-scanner-$name.json" || return 2
|
|
osv-scanner scan -r \
|
|
--allow-no-lockfiles \
|
|
--format json \
|
|
--output-file "$REPORTS_DIR/osv-scanner-$name.json" \
|
|
"$repo"
|
|
accumulate_exit_status status "$?"
|
|
else
|
|
prepare_machine_report "$REPORTS_DIR/osv-scanner-$name.json" || return 2
|
|
osv-scanner -r --format json --output "$REPORTS_DIR/osv-scanner-$name.json" "$repo"
|
|
accumulate_exit_status status "$?"
|
|
fi
|
|
done
|
|
return "$status"
|
|
}
|
|
|
|
run_jscpd() {
|
|
prepare_machine_report "$REPORTS_DIR/jscpd/jscpd-report.json" || return 2
|
|
local ignored_paths
|
|
ignored_paths=(
|
|
"**/.git/**"
|
|
"**/node_modules/**"
|
|
"**/.venv/**"
|
|
"**/dist/**"
|
|
"**/build/**"
|
|
"**/audit-reports/**"
|
|
"**/package-lock.json"
|
|
"**/package-lock.release.json"
|
|
"**/package.json"
|
|
"package.json"
|
|
"**/generatedTranslations.ts"
|
|
"**/docs/gitea-labels.json"
|
|
"**/*.md"
|
|
"**/*.md:*"
|
|
"*.md"
|
|
"*.md:*"
|
|
"**/*.svg"
|
|
"*.svg"
|
|
".gitea/workflows/*.yml"
|
|
"**/.gitea/workflows/*.yml"
|
|
"**/backend/schema/*.schema.json"
|
|
)
|
|
local ignored_path_pattern
|
|
ignored_path_pattern="$(printf '%s,' "${ignored_paths[@]}")"
|
|
ignored_path_pattern="${ignored_path_pattern%,}"
|
|
jscpd \
|
|
--silent \
|
|
--min-tokens 80 \
|
|
--reporters json \
|
|
--output "$REPORTS_DIR/jscpd" \
|
|
--ignore "$ignored_path_pattern" \
|
|
"${REPOS[@]}"
|
|
}
|
|
|
|
run_radon() {
|
|
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
|
|
register_report "$REPORTS_DIR/radon-cc.txt" || return 2
|
|
radon cc -s -a "${PY_ROOTS[@]}" > "$REPORTS_DIR/radon-cc.txt"
|
|
}
|
|
|
|
run_xenon() {
|
|
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
|
|
register_report "$REPORTS_DIR/xenon.txt" || return 2
|
|
xenon --max-absolute C --max-modules B --max-average A "${PY_ROOTS[@]}" > "$REPORTS_DIR/xenon.txt" 2>&1
|
|
}
|
|
|
|
register_report "$REPORTS_DIR/step-status.tsv"
|
|
register_report "$REPORTS_DIR/tool-versions.txt"
|
|
register_report "$REPORTS_DIR/workspace-state-start.tsv"
|
|
register_report "$REPORTS_DIR/workspace-state-end.tsv"
|
|
: > "$REPORTS_DIR/step-status.tsv"
|
|
if ! write_workspace_state "$REPORTS_DIR/workspace-state-start.tsv"; then
|
|
echo "Could not capture the repository state before the audit." >&2
|
|
exit 1
|
|
fi
|
|
run_step "Record audit tool versions" execution-only write_tool_versions
|
|
|
|
if has_tool semgrep; then
|
|
run_step "Semgrep SAST" findings-exit-one run_semgrep
|
|
else
|
|
skip_or_fail_missing semgrep
|
|
fi
|
|
|
|
if has_tool bandit; then
|
|
run_step "Bandit Python security scan" findings-exit-one run_bandit
|
|
else
|
|
skip_or_fail_missing bandit
|
|
fi
|
|
|
|
if has_tool ruff; then
|
|
run_step "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security
|
|
else
|
|
skip_or_fail_missing ruff
|
|
fi
|
|
|
|
if has_tool gitleaks; then
|
|
run_step "Gitleaks secret scan" findings-exit-one run_gitleaks
|
|
else
|
|
skip_or_fail_missing gitleaks
|
|
fi
|
|
|
|
if [[ "$MODE" != "quick" ]]; then
|
|
if has_tool trivy; then
|
|
run_step "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy
|
|
else
|
|
skip_or_fail_missing trivy
|
|
fi
|
|
|
|
if has_tool pip-audit; then
|
|
run_step "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests
|
|
else
|
|
skip_or_fail_missing pip-audit
|
|
fi
|
|
|
|
if has_tool npm; then
|
|
run_step "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests
|
|
else
|
|
skip_or_fail_missing npm
|
|
fi
|
|
fi
|
|
|
|
if [[ "$MODE" == "full" ]]; then
|
|
if has_tool osv-scanner; then
|
|
run_step "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner
|
|
else
|
|
skip_or_fail_missing osv-scanner
|
|
fi
|
|
|
|
if has_tool jscpd; then
|
|
run_step "jscpd duplicate code scan" execution-only run_jscpd
|
|
else
|
|
skip_or_fail_missing jscpd
|
|
fi
|
|
|
|
if has_tool radon; then
|
|
run_step "Radon complexity report" execution-only run_radon
|
|
else
|
|
skip_or_fail_missing radon
|
|
fi
|
|
|
|
if has_tool xenon; then
|
|
run_step "Xenon complexity threshold scan" findings-exit-one run_xenon
|
|
else
|
|
skip_or_fail_missing xenon
|
|
fi
|
|
fi
|
|
|
|
run_step "Validate machine-readable audit reports" execution-only validate_machine_reports
|
|
|
|
echo
|
|
echo "Security audit reports written to: $REPORTS_DIR"
|
|
if [[ "$finding_status" -ne 0 && "$FAIL_ON_FINDINGS" != "1" ]]; then
|
|
echo "Findings were reported, but this run is report-only. Re-run with --strict to fail on findings."
|
|
fi
|
|
if [[ "$missing_status" -ne 0 ]]; then
|
|
echo "One or more audit tools were missing." >&2
|
|
fi
|
|
|
|
if ! write_workspace_state "$REPORTS_DIR/workspace-state-end.tsv"; then
|
|
workspace_unchanged=false
|
|
execution_status=1
|
|
overall_status=1
|
|
echo "Could not capture the repository state after the audit." >&2
|
|
elif cmp -s "$REPORTS_DIR/workspace-state-start.tsv" "$REPORTS_DIR/workspace-state-end.tsv"; then
|
|
workspace_unchanged=true
|
|
else
|
|
workspace_unchanged=false
|
|
overall_status=1
|
|
echo "Repository state changed during the audit; discard this mixed snapshot and rerun." >&2
|
|
fi
|
|
audit_completed_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
|
if ! write_report_checksums; then
|
|
execution_status=1
|
|
overall_status=1
|
|
echo "Could not generate report checksums." >&2
|
|
fi
|
|
if ! write_manifest; then
|
|
echo "Could not write the audit manifest." >&2
|
|
exit 1
|
|
fi
|
|
|
|
exit "$overall_status"
|