feat: reconcile sanctions screening freshness

2026-07-30 01:55:03 +02:00
parent 11404ac42f
commit da1baede36
10 changed files with 926 additions and 36 deletions
@@ -16,6 +16,7 @@ from govoplan_risk_compliance.backend.db.models import (
    RiskSanctionsEntry,
    RiskSanctionsListSnapshot,
    RiskScreeningCandidate,
+    RiskScreeningDisposition,
    RiskScreeningException,
    RiskScreeningRun,
    RiskScreeningSubjectSnapshot,
@@ -44,6 +45,10 @@ MAX_MATCH_ENTRIES = 5_000
 MAX_CANDIDATES = 100
 DEFAULT_FUZZY_THRESHOLD = 0.88
 DEFAULT_MAX_SNAPSHOT_AGE_DAYS = 7
+SCREENING_EVIDENCE_PREFIX = "risk-screening:"
+SCREENING_FAILURE_POLICIES = frozenset(
+    {"block", "review", "degraded"}
+)


@dataclass(frozen=True, slots=True)
@@ -99,6 +104,21 @@ class ScreeningPolicy:
        }


+@dataclass(frozen=True, slots=True)
+class ScreeningFreshnessAssessment:
+    run: RiskScreeningRun
+    fresh: bool
+    reasons: tuple[str, ...]
+    checked_at: datetime
+    current_list_snapshot_id: str | None
+    gate_decision: str
+    gate_reasons: tuple[str, ...]
+
+    @property
+    def evidence_ref(self) -> str:
+        return screening_evidence_ref(self.run.id)
+
+
 def run_screening(
    session: Session,
    principal: ApiPrincipal,
@@ -284,6 +304,183 @@ def run_screening(
    ), True


+def screening_evidence_ref(run_id: str) -> str:
+    clean_run_id = str(run_id or "").strip()
+    if not clean_run_id:
+        raise RiskSanctionsConflictError(
+            "A screening run is required for evidence."
+        )
+    return f"{SCREENING_EVIDENCE_PREFIX}{clean_run_id}"
+
+
+def assess_screening_freshness(
+    session: Session,
+    principal: ApiPrincipal,
+    *,
+    evidence_ref: str,
+    current_subject: ScreeningSubject | None = None,
+    expected_list_snapshot_id: str | None = None,
+    policy: ScreeningPolicy | None = None,
+    failure_policy: str = "block",
+    now: datetime | None = None,
+) -> ScreeningFreshnessAssessment:
+    if failure_policy not in SCREENING_FAILURE_POLICIES:
+        raise RiskSanctionsConflictError(
+            "Failure policy must be block, review, or degraded."
+        )
+    run = get_screening_run(
+        session,
+        principal,
+        run_id=_run_id_from_evidence_ref(evidence_ref),
+    )
+    checked_at = now or utcnow()
+    effective_policy = policy or _policy_from_run(run)
+    reasons: list[str] = []
+
+    if expected_list_snapshot_id:
+        current_snapshot = _visible_list_snapshot(
+            session,
+            principal,
+            snapshot_id=expected_list_snapshot_id,
+        )
+    else:
+        current_snapshot = _latest_visible_list_snapshot(
+            session,
+            principal,
+            baseline=run.list_snapshot,
+        )
+    reasons.extend(
+        _snapshot_freshness_reasons(
+            run,
+            current_snapshot=current_snapshot,
+            checked_at=checked_at,
+            policy=effective_policy,
+        )
+    )
+
+    if current_subject is not None:
+        current_fingerprint = subject_fingerprint(
+            _canonical_subject(current_subject)
+        )
+        if current_fingerprint != run.subject_snapshot.fingerprint:
+            reasons.append("subject_changed")
+    reasons.extend(
+        _record_freshness_reasons(
+            session,
+            run=run,
+            checked_at=checked_at,
+        )
+    )
+    if policy is not None and not _run_policy_matches(run, policy):
+        reasons.append("policy_changed")
+
+    unique_reasons = tuple(dict.fromkeys(reasons))
+    gate_decision, gate_reasons = _screening_gate(
+        run,
+        freshness_reasons=unique_reasons,
+        failure_policy=failure_policy,
+    )
+    return ScreeningFreshnessAssessment(
+        run=run,
+        fresh=not unique_reasons,
+        reasons=unique_reasons,
+        checked_at=checked_at,
+        current_list_snapshot_id=(
+            current_snapshot.id
+            if current_snapshot is not None
+            else None
+        ),
+        gate_decision=gate_decision,
+        gate_reasons=gate_reasons,
+    )
+
+
+def list_rescreening_requirements(
+    session: Session,
+    principal: ApiPrincipal,
+    *,
+    limit: int = 100,
+    now: datetime | None = None,
+) -> tuple[ScreeningFreshnessAssessment, ...]:
+    _require_scope(principal, SANCTIONS_READ_SCOPE)
+    bounded_limit = max(1, min(limit, 500))
+    items = tuple(
+        session.scalars(
+            select(RiskScreeningRun)
+            .where(RiskScreeningRun.tenant_id == principal.tenant_id)
+            .options(
+                selectinload(RiskScreeningRun.subject_snapshot),
+                selectinload(RiskScreeningRun.list_snapshot),
+                selectinload(RiskScreeningRun.candidates),
+            )
+            .order_by(
+                RiskScreeningRun.created_at.desc(),
+                RiskScreeningRun.id.desc(),
+            )
+            .limit(bounded_limit * 5)
+        ).unique()
+    )
+    latest_runs: list[RiskScreeningRun] = []
+    seen_subjects: set[str] = set()
+    for item in items:
+        subject_key = (
+            item.subject_snapshot.subject_ref
+            or item.subject_snapshot.fingerprint
+        )
+        if subject_key in seen_subjects:
+            continue
+        seen_subjects.add(subject_key)
+        latest_runs.append(item)
+
+    disposition_ids = {
+        candidate.current_disposition_id
+        for item in latest_runs
+        for candidate in item.candidates
+        if candidate.current_disposition_id
+    }
+    dispositions = {
+        item.id: item
+        for item in session.scalars(
+            select(RiskScreeningDisposition).where(
+                RiskScreeningDisposition.tenant_id
+                == principal.tenant_id,
+                RiskScreeningDisposition.id.in_(disposition_ids),
+            )
+        )
+    } if disposition_ids else {}
+    latest_snapshots: dict[
+        tuple[str, str],
+        RiskSanctionsListSnapshot | None,
+    ] = {}
+    requirements: list[ScreeningFreshnessAssessment] = []
+    checked_at = now or utcnow()
+    for item in latest_runs:
+        source_key = (
+            item.list_snapshot.provider_id,
+            item.list_snapshot.source_id,
+        )
+        if source_key not in latest_snapshots:
+            latest_snapshots[source_key] = (
+                _latest_visible_list_snapshot(
+                    session,
+                    principal,
+                    baseline=item.list_snapshot,
+                )
+            )
+        assessment = _assess_loaded_run(
+            session,
+            run=item,
+            checked_at=checked_at,
+            current_snapshot=latest_snapshots[source_key],
+            dispositions=dispositions,
+        )
+        if not assessment.fresh:
+            requirements.append(assessment)
+        if len(requirements) >= bounded_limit:
+            break
+    return tuple(requirements)
+
+
 def get_screening_run(
    session: Session,
    principal: ApiPrincipal,
@@ -364,6 +561,234 @@ def _visible_list_snapshot(
    return item


+def _latest_visible_list_snapshot(
+    session: Session,
+    principal: ApiPrincipal,
+    *,
+    baseline: RiskSanctionsListSnapshot,
+) -> RiskSanctionsListSnapshot | None:
+    return session.scalar(
+        select(RiskSanctionsListSnapshot)
+        .where(
+            RiskSanctionsListSnapshot.provider_id
+            == baseline.provider_id,
+            RiskSanctionsListSnapshot.source_id == baseline.source_id,
+            or_(
+                RiskSanctionsListSnapshot.tenant_id
+                == principal.tenant_id,
+                RiskSanctionsListSnapshot.visibility == "global",
+            ),
+        )
+        .order_by(
+            RiskSanctionsListSnapshot.acquired_at.desc(),
+            RiskSanctionsListSnapshot.imported_at.desc(),
+            RiskSanctionsListSnapshot.id.desc(),
+        )
+        .limit(1)
+    )
+
+
+def _assess_loaded_run(
+    session: Session,
+    *,
+    run: RiskScreeningRun,
+    checked_at: datetime,
+    current_snapshot: RiskSanctionsListSnapshot | None,
+    dispositions: dict[str, RiskScreeningDisposition],
+) -> ScreeningFreshnessAssessment:
+    policy = _policy_from_run(run)
+    reasons = list(
+        _snapshot_freshness_reasons(
+            run,
+            current_snapshot=current_snapshot,
+            checked_at=checked_at,
+            policy=policy,
+        )
+    )
+    reasons.extend(
+        _record_freshness_reasons(
+            session,
+            run=run,
+            checked_at=checked_at,
+            dispositions=dispositions,
+        )
+    )
+    unique_reasons = tuple(dict.fromkeys(reasons))
+    gate_decision, gate_reasons = _screening_gate(
+        run,
+        freshness_reasons=unique_reasons,
+        failure_policy="block",
+    )
+    return ScreeningFreshnessAssessment(
+        run=run,
+        fresh=not unique_reasons,
+        reasons=unique_reasons,
+        checked_at=checked_at,
+        current_list_snapshot_id=(
+            current_snapshot.id
+            if current_snapshot is not None
+            else None
+        ),
+        gate_decision=gate_decision,
+        gate_reasons=gate_reasons,
+    )
+
+
+def _snapshot_freshness_reasons(
+    run: RiskScreeningRun,
+    *,
+    current_snapshot: RiskSanctionsListSnapshot | None,
+    checked_at: datetime,
+    policy: ScreeningPolicy,
+) -> tuple[str, ...]:
+    if current_snapshot is None:
+        return ("source_unavailable",)
+    reasons: list[str] = []
+    if current_snapshot.id != run.list_snapshot_id:
+        reasons.append("source_snapshot_changed")
+    state = _list_state(
+        current_snapshot,
+        now=checked_at,
+        max_age_days=policy.max_snapshot_age_days,
+    )
+    if state == "stale":
+        reasons.append("source_snapshot_stale")
+    elif state == "unavailable":
+        reasons.append("source_unavailable")
+    return tuple(reasons)
+
+
+def _record_freshness_reasons(
+    session: Session,
+    *,
+    run: RiskScreeningRun,
+    checked_at: datetime,
+    dispositions: dict[str, RiskScreeningDisposition] | None = None,
+) -> tuple[str, ...]:
+    reasons: list[str] = []
+    if run.matcher_version != MATCHER_VERSION:
+        reasons.append("matcher_version_changed")
+    if run.normalization_version != NORMALIZATION_VERSION:
+        reasons.append("normalization_version_changed")
+    if run.policy_version != POLICY_VERSION:
+        reasons.append("policy_version_changed")
+    if run.status != "completed":
+        reasons.append("screening_incomplete")
+    if run.outcome in {"stale", "unavailable", "insufficient"}:
+        reasons.append(f"screening_outcome_{run.outcome}")
+
+    disposition_ids = {
+        candidate.current_disposition_id
+        for candidate in run.candidates
+        if candidate.current_disposition_id
+    }
+    if not disposition_ids:
+        return tuple(reasons)
+    current_dispositions = dispositions
+    if current_dispositions is None:
+        current_dispositions = {
+            item.id: item
+            for item in session.scalars(
+                select(RiskScreeningDisposition).where(
+                    RiskScreeningDisposition.tenant_id == run.tenant_id,
+                    RiskScreeningDisposition.id.in_(disposition_ids),
+                )
+            )
+        }
+    for disposition_id in sorted(disposition_ids):
+        item = current_dispositions.get(disposition_id)
+        if item is None:
+            reasons.append("disposition_evidence_unavailable")
+            continue
+        if (
+            item.expires_at is not None
+            and _aware_datetime(item.expires_at)
+            <= _aware_datetime(checked_at)
+        ):
+            reasons.append("disposition_expired")
+        if (
+            item.review_at is not None
+            and _aware_datetime(item.review_at)
+            <= _aware_datetime(checked_at)
+        ):
+            reasons.append("disposition_review_due")
+    return tuple(dict.fromkeys(reasons))
+
+
+def _policy_from_run(run: RiskScreeningRun) -> ScreeningPolicy:
+    values = dict(run.policy_snapshot)
+    return ScreeningPolicy(
+        fuzzy_threshold=float(
+            values.get("fuzzy_threshold", DEFAULT_FUZZY_THRESHOLD)
+        ),
+        max_snapshot_age_days=int(
+            values.get(
+                "max_snapshot_age_days",
+                DEFAULT_MAX_SNAPSHOT_AGE_DAYS,
+            )
+        ),
+        max_candidates=int(
+            values.get("max_candidates", MAX_CANDIDATES)
+        ),
+    )
+
+
+def _run_policy_matches(
+    run: RiskScreeningRun,
+    policy: ScreeningPolicy,
+) -> bool:
+    recorded = dict(run.policy_snapshot)
+    expected = policy.to_dict()
+    return all(
+        recorded.get(key) == value
+        for key, value in expected.items()
+    )
+
+
+def _screening_gate(
+    run: RiskScreeningRun,
+    *,
+    freshness_reasons: tuple[str, ...],
+    failure_policy: str,
+) -> tuple[str, tuple[str, ...]]:
+    if freshness_reasons:
+        return (
+            failure_policy,
+            ("rescreening_required", *freshness_reasons),
+        )
+    if run.outcome == "clear":
+        return "allow", ("screening_clear",)
+    if run.outcome == "potential":
+        statuses = {
+            candidate.review_status
+            for candidate in run.candidates
+        }
+        if "confirmed" in statuses:
+            return "block", ("confirmed_match",)
+        unresolved = statuses - {"false_positive"}
+        if unresolved or not statuses:
+            return "review", ("candidate_review_required",)
+        return "allow", ("all_candidates_cleared",)
+    return (
+        failure_policy,
+        (f"screening_outcome_{run.outcome}",),
+    )
+
+
+def _run_id_from_evidence_ref(evidence_ref: str) -> str:
+    clean_ref = str(evidence_ref or "").strip()
+    if not clean_ref.startswith(SCREENING_EVIDENCE_PREFIX):
+        raise RiskSanctionsConflictError(
+            "Screening evidence reference is invalid."
+        )
+    run_id = clean_ref.removeprefix(SCREENING_EVIDENCE_PREFIX)
+    if not run_id or ":" in run_id:
+        raise RiskSanctionsConflictError(
+            "Screening evidence reference is invalid."
+        )
+    return run_id
+
+
 def _canonical_subject(subject: ScreeningSubject) -> dict[str, Any]:
    aliases = sorted(
        {
@@ -606,6 +1031,12 @@ def _list_state(
    return "active"


+def _aware_datetime(value: datetime) -> datetime:
+    if value.tzinfo is None:
+        return value.replace(tzinfo=timezone.utc)
+    return value
+
+
 def _bounded(value: object, length: int) -> str | None:
    clean = " ".join(str(value or "").split())
    return clean[:length] or None
@@ -640,9 +1071,15 @@ __all__ = [
    "MATCHER_VERSION",
    "MAX_CANDIDATES",
    "POLICY_VERSION",
+    "SCREENING_EVIDENCE_PREFIX",
+    "SCREENING_FAILURE_POLICIES",
+    "ScreeningFreshnessAssessment",
    "ScreeningPolicy",
    "ScreeningSubject",
+    "assess_screening_freshness",
    "get_screening_run",
+    "list_rescreening_requirements",
    "list_screening_runs",
    "run_screening",
+    "screening_evidence_ref",
 ]