1115 lines
33 KiB
Python
1115 lines
33 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
from difflib import SequenceMatcher
|
|
import hashlib
|
|
import json
|
|
from typing import Any
|
|
|
|
from sqlalchemy import or_, select
|
|
from sqlalchemy.orm import Session, selectinload
|
|
|
|
from govoplan_core.auth import ApiPrincipal, has_scope
|
|
from govoplan_core.db.base import utcnow
|
|
from govoplan_risk_compliance.backend.db.models import (
|
|
RiskSanctionsEntry,
|
|
RiskSanctionsListSnapshot,
|
|
RiskScreeningCandidate,
|
|
RiskScreeningDisposition,
|
|
RiskScreeningException,
|
|
RiskScreeningRun,
|
|
RiskScreeningSubjectSnapshot,
|
|
)
|
|
from govoplan_risk_compliance.backend.normalization import (
|
|
NORMALIZATION_VERSION,
|
|
normalize_identifier,
|
|
normalize_name,
|
|
subject_fingerprint,
|
|
)
|
|
from govoplan_risk_compliance.backend.permissions import (
|
|
SANCTIONS_ADMIN_SCOPE,
|
|
SANCTIONS_READ_SCOPE,
|
|
SANCTIONS_SCREEN_SCOPE,
|
|
)
|
|
from govoplan_risk_compliance.backend.sanctions_catalog import (
|
|
RiskSanctionsAccessError,
|
|
RiskSanctionsConflictError,
|
|
RiskSanctionsNotFoundError,
|
|
)
|
|
|
|
|
|
MATCHER_VERSION = "sanctions-matcher-v1"
|
|
POLICY_VERSION = "sanctions-policy-v1"
|
|
MAX_MATCH_ENTRIES = 5_000
|
|
MAX_CANDIDATES = 100
|
|
DEFAULT_FUZZY_THRESHOLD = 0.88
|
|
DEFAULT_MAX_SNAPSHOT_AGE_DAYS = 7
|
|
SCREENING_EVIDENCE_PREFIX = "risk-screening:"
|
|
SCREENING_FAILURE_POLICIES = frozenset(
|
|
{"block", "review", "degraded"}
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ScreeningSubject:
|
|
subject_type: str
|
|
primary_name: str | None = None
|
|
subject_ref: str | None = None
|
|
aliases: tuple[str, ...] = ()
|
|
identifiers: tuple[dict[str, str], ...] = ()
|
|
dates: tuple[str, ...] = ()
|
|
addresses: tuple[dict[str, str], ...] = ()
|
|
|
|
def __post_init__(self) -> None:
|
|
if self.subject_type not in {"person", "entity"}:
|
|
raise RiskSanctionsConflictError(
|
|
"Screening subject type must be person or entity."
|
|
)
|
|
if not normalize_name(self.primary_name) and not any(
|
|
normalize_identifier(item.get("value"))
|
|
for item in self.identifiers
|
|
):
|
|
raise RiskSanctionsConflictError(
|
|
"A screening subject needs a name or identifier."
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ScreeningPolicy:
|
|
fuzzy_threshold: float = DEFAULT_FUZZY_THRESHOLD
|
|
max_snapshot_age_days: int = DEFAULT_MAX_SNAPSHOT_AGE_DAYS
|
|
max_candidates: int = MAX_CANDIDATES
|
|
|
|
def __post_init__(self) -> None:
|
|
if not 0.8 <= self.fuzzy_threshold <= 1:
|
|
raise RiskSanctionsConflictError(
|
|
"Fuzzy threshold must be between 0.8 and 1."
|
|
)
|
|
if not 1 <= self.max_snapshot_age_days <= 365:
|
|
raise RiskSanctionsConflictError(
|
|
"Snapshot age must be between 1 and 365 days."
|
|
)
|
|
if not 1 <= self.max_candidates <= MAX_CANDIDATES:
|
|
raise RiskSanctionsConflictError(
|
|
f"Candidate limit must be between 1 and {MAX_CANDIDATES}."
|
|
)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"fuzzy_threshold": self.fuzzy_threshold,
|
|
"max_snapshot_age_days": self.max_snapshot_age_days,
|
|
"max_candidates": self.max_candidates,
|
|
"fuzzy_auto_confirms": False,
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ScreeningFreshnessAssessment:
|
|
run: RiskScreeningRun
|
|
fresh: bool
|
|
reasons: tuple[str, ...]
|
|
checked_at: datetime
|
|
current_list_snapshot_id: str | None
|
|
gate_decision: str
|
|
gate_reasons: tuple[str, ...]
|
|
|
|
@property
|
|
def evidence_ref(self) -> str:
|
|
return screening_evidence_ref(self.run.id)
|
|
|
|
|
|
def run_screening(
|
|
session: Session,
|
|
principal: ApiPrincipal,
|
|
*,
|
|
list_snapshot_id: str,
|
|
idempotency_key: str,
|
|
subject: ScreeningSubject,
|
|
policy: ScreeningPolicy | None = None,
|
|
) -> tuple[RiskScreeningRun, bool]:
|
|
_require_scope(principal, SANCTIONS_SCREEN_SCOPE)
|
|
clean_key = idempotency_key.strip()
|
|
if not clean_key or len(clean_key) > 255:
|
|
raise RiskSanctionsConflictError(
|
|
"Idempotency key must contain 1 to 255 characters."
|
|
)
|
|
effective_policy = policy or ScreeningPolicy()
|
|
canonical_subject = _canonical_subject(subject)
|
|
request_hash = _request_hash(
|
|
list_snapshot_id=list_snapshot_id,
|
|
subject=canonical_subject,
|
|
policy=effective_policy,
|
|
)
|
|
existing = session.scalar(
|
|
select(RiskScreeningRun).where(
|
|
RiskScreeningRun.tenant_id == principal.tenant_id,
|
|
RiskScreeningRun.idempotency_key == clean_key,
|
|
)
|
|
)
|
|
if existing is not None:
|
|
if existing.request_hash != request_hash:
|
|
raise RiskSanctionsConflictError(
|
|
"Idempotency key was already used for another screening request."
|
|
)
|
|
return _screening_result(
|
|
session,
|
|
principal,
|
|
run_id=existing.id,
|
|
created=False,
|
|
)
|
|
|
|
list_snapshot = _visible_list_snapshot(
|
|
session,
|
|
principal,
|
|
snapshot_id=list_snapshot_id,
|
|
)
|
|
actor_id = _actor_id(principal)
|
|
subject_snapshot = RiskScreeningSubjectSnapshot(
|
|
tenant_id=principal.tenant_id,
|
|
subject_ref=_bounded(subject.subject_ref, 500),
|
|
subject_type=subject.subject_type,
|
|
primary_name=_bounded(subject.primary_name, 1000),
|
|
normalized_name=canonical_subject["primary_name"],
|
|
aliases=list(canonical_subject["aliases"]),
|
|
identifiers=list(canonical_subject["identifiers"]),
|
|
dates=list(canonical_subject["dates"]),
|
|
addresses=list(canonical_subject["addresses"]),
|
|
fingerprint=subject_fingerprint(canonical_subject),
|
|
submitted_by=actor_id,
|
|
)
|
|
now = utcnow()
|
|
run = RiskScreeningRun(
|
|
tenant_id=principal.tenant_id,
|
|
subject_snapshot=subject_snapshot,
|
|
list_snapshot_id=list_snapshot.id,
|
|
idempotency_key=clean_key,
|
|
request_hash=request_hash,
|
|
matcher_version=MATCHER_VERSION,
|
|
normalization_version=NORMALIZATION_VERSION,
|
|
policy_version=POLICY_VERSION,
|
|
policy_snapshot=effective_policy.to_dict(),
|
|
status="running",
|
|
outcome="insufficient",
|
|
candidate_count=0,
|
|
started_at=now,
|
|
created_by=actor_id,
|
|
)
|
|
session.add(run)
|
|
session.flush()
|
|
|
|
list_state = _list_state(
|
|
list_snapshot,
|
|
now=now,
|
|
max_age_days=effective_policy.max_snapshot_age_days,
|
|
)
|
|
if list_state in {"unavailable", "stale"}:
|
|
run.status = "completed"
|
|
run.outcome = list_state
|
|
run.completed_at = utcnow()
|
|
session.flush()
|
|
return _screening_result(
|
|
session,
|
|
principal,
|
|
run_id=run.id,
|
|
created=True,
|
|
)
|
|
|
|
entries = tuple(
|
|
session.scalars(
|
|
select(RiskSanctionsEntry)
|
|
.where(RiskSanctionsEntry.snapshot_id == list_snapshot.id)
|
|
.options(
|
|
selectinload(RiskSanctionsEntry.aliases),
|
|
selectinload(RiskSanctionsEntry.identifiers),
|
|
selectinload(RiskSanctionsEntry.dates),
|
|
selectinload(RiskSanctionsEntry.addresses),
|
|
)
|
|
.limit(MAX_MATCH_ENTRIES + 1)
|
|
)
|
|
)
|
|
if len(entries) > MAX_MATCH_ENTRIES:
|
|
run.status = "completed"
|
|
run.outcome = "unavailable"
|
|
run.policy_snapshot = {
|
|
**run.policy_snapshot,
|
|
"failure": "entry_limit_exceeded",
|
|
}
|
|
run.completed_at = utcnow()
|
|
session.flush()
|
|
return _screening_result(
|
|
session,
|
|
principal,
|
|
run_id=run.id,
|
|
created=True,
|
|
)
|
|
|
|
candidates = sorted(
|
|
(
|
|
candidate
|
|
for entry in entries
|
|
if (
|
|
candidate := _match_entry(
|
|
subject_snapshot,
|
|
entry,
|
|
fuzzy_threshold=effective_policy.fuzzy_threshold,
|
|
)
|
|
)
|
|
is not None
|
|
),
|
|
key=lambda item: (-item["score"], item["entry"].source_entry_id),
|
|
)[: effective_policy.max_candidates]
|
|
active_exceptions = _active_exceptions(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
subject_fingerprint_value=subject_snapshot.fingerprint,
|
|
entry_refs={
|
|
item["entry"].source_entry_id
|
|
for item in candidates
|
|
},
|
|
now=now,
|
|
)
|
|
for item in candidates:
|
|
entry = item["entry"]
|
|
evidence = list(item["evidence"])
|
|
exception = active_exceptions.get(entry.source_entry_id)
|
|
review_status = "pending"
|
|
if exception is not None:
|
|
review_status = "exception_review"
|
|
evidence.append(
|
|
{
|
|
"kind": "prior_exception",
|
|
"exception_id": exception.id,
|
|
"expires_at": exception.expires_at.isoformat(),
|
|
"effect": "review_required",
|
|
}
|
|
)
|
|
session.add(
|
|
RiskScreeningCandidate(
|
|
tenant_id=principal.tenant_id,
|
|
run_id=run.id,
|
|
entry_id=entry.id,
|
|
score=item["score"],
|
|
match_kind=item["match_kind"],
|
|
evidence=evidence,
|
|
review_status=review_status,
|
|
)
|
|
)
|
|
run.candidate_count = len(candidates)
|
|
run.status = "completed"
|
|
run.outcome = "potential" if candidates else "clear"
|
|
run.completed_at = utcnow()
|
|
session.flush()
|
|
return _screening_result(
|
|
session,
|
|
principal,
|
|
run_id=run.id,
|
|
created=True,
|
|
)
|
|
|
|
|
|
def screening_evidence_ref(run_id: str) -> str:
|
|
clean_run_id = str(run_id or "").strip()
|
|
if not clean_run_id:
|
|
raise RiskSanctionsConflictError(
|
|
"A screening run is required for evidence."
|
|
)
|
|
return f"{SCREENING_EVIDENCE_PREFIX}{clean_run_id}"
|
|
|
|
|
|
def _screening_result(
|
|
session: Session,
|
|
principal: ApiPrincipal,
|
|
*,
|
|
run_id: str,
|
|
created: bool,
|
|
) -> tuple[RiskScreeningRun, bool]:
|
|
run = get_screening_run(
|
|
session,
|
|
principal,
|
|
run_id=run_id,
|
|
)
|
|
if run.status == "completed":
|
|
from govoplan_risk_compliance.backend.assurance import (
|
|
record_sanctions_assurance,
|
|
)
|
|
|
|
record_sanctions_assurance(
|
|
session,
|
|
principal,
|
|
run=run,
|
|
)
|
|
return run, created
|
|
|
|
|
|
def assess_screening_freshness(
|
|
session: Session,
|
|
principal: ApiPrincipal,
|
|
*,
|
|
evidence_ref: str,
|
|
current_subject: ScreeningSubject | None = None,
|
|
expected_list_snapshot_id: str | None = None,
|
|
policy: ScreeningPolicy | None = None,
|
|
failure_policy: str = "block",
|
|
now: datetime | None = None,
|
|
) -> ScreeningFreshnessAssessment:
|
|
if failure_policy not in SCREENING_FAILURE_POLICIES:
|
|
raise RiskSanctionsConflictError(
|
|
"Failure policy must be block, review, or degraded."
|
|
)
|
|
run = get_screening_run(
|
|
session,
|
|
principal,
|
|
run_id=_run_id_from_evidence_ref(evidence_ref),
|
|
)
|
|
checked_at = now or utcnow()
|
|
effective_policy = policy or _policy_from_run(run)
|
|
reasons: list[str] = []
|
|
|
|
if expected_list_snapshot_id:
|
|
current_snapshot = _visible_list_snapshot(
|
|
session,
|
|
principal,
|
|
snapshot_id=expected_list_snapshot_id,
|
|
)
|
|
else:
|
|
current_snapshot = _latest_visible_list_snapshot(
|
|
session,
|
|
principal,
|
|
baseline=run.list_snapshot,
|
|
)
|
|
reasons.extend(
|
|
_snapshot_freshness_reasons(
|
|
run,
|
|
current_snapshot=current_snapshot,
|
|
checked_at=checked_at,
|
|
policy=effective_policy,
|
|
)
|
|
)
|
|
|
|
if current_subject is not None:
|
|
current_fingerprint = subject_fingerprint(
|
|
_canonical_subject(current_subject)
|
|
)
|
|
if current_fingerprint != run.subject_snapshot.fingerprint:
|
|
reasons.append("subject_changed")
|
|
reasons.extend(
|
|
_record_freshness_reasons(
|
|
session,
|
|
run=run,
|
|
checked_at=checked_at,
|
|
)
|
|
)
|
|
if policy is not None and not _run_policy_matches(run, policy):
|
|
reasons.append("policy_changed")
|
|
|
|
unique_reasons = tuple(dict.fromkeys(reasons))
|
|
gate_decision, gate_reasons = _screening_gate(
|
|
run,
|
|
freshness_reasons=unique_reasons,
|
|
failure_policy=failure_policy,
|
|
)
|
|
return ScreeningFreshnessAssessment(
|
|
run=run,
|
|
fresh=not unique_reasons,
|
|
reasons=unique_reasons,
|
|
checked_at=checked_at,
|
|
current_list_snapshot_id=(
|
|
current_snapshot.id
|
|
if current_snapshot is not None
|
|
else None
|
|
),
|
|
gate_decision=gate_decision,
|
|
gate_reasons=gate_reasons,
|
|
)
|
|
|
|
|
|
def list_rescreening_requirements(
|
|
session: Session,
|
|
principal: ApiPrincipal,
|
|
*,
|
|
limit: int = 100,
|
|
now: datetime | None = None,
|
|
) -> tuple[ScreeningFreshnessAssessment, ...]:
|
|
_require_scope(principal, SANCTIONS_READ_SCOPE)
|
|
bounded_limit = max(1, min(limit, 500))
|
|
items = tuple(
|
|
session.scalars(
|
|
select(RiskScreeningRun)
|
|
.where(RiskScreeningRun.tenant_id == principal.tenant_id)
|
|
.options(
|
|
selectinload(RiskScreeningRun.subject_snapshot),
|
|
selectinload(RiskScreeningRun.list_snapshot),
|
|
selectinload(RiskScreeningRun.candidates),
|
|
)
|
|
.order_by(
|
|
RiskScreeningRun.created_at.desc(),
|
|
RiskScreeningRun.id.desc(),
|
|
)
|
|
.limit(bounded_limit * 5)
|
|
).unique()
|
|
)
|
|
latest_runs: list[RiskScreeningRun] = []
|
|
seen_subjects: set[str] = set()
|
|
for item in items:
|
|
subject_key = (
|
|
item.subject_snapshot.subject_ref
|
|
or item.subject_snapshot.fingerprint
|
|
)
|
|
if subject_key in seen_subjects:
|
|
continue
|
|
seen_subjects.add(subject_key)
|
|
latest_runs.append(item)
|
|
|
|
disposition_ids = {
|
|
candidate.current_disposition_id
|
|
for item in latest_runs
|
|
for candidate in item.candidates
|
|
if candidate.current_disposition_id
|
|
}
|
|
dispositions = {
|
|
item.id: item
|
|
for item in session.scalars(
|
|
select(RiskScreeningDisposition).where(
|
|
RiskScreeningDisposition.tenant_id
|
|
== principal.tenant_id,
|
|
RiskScreeningDisposition.id.in_(disposition_ids),
|
|
)
|
|
)
|
|
} if disposition_ids else {}
|
|
latest_snapshots: dict[
|
|
tuple[str, str],
|
|
RiskSanctionsListSnapshot | None,
|
|
] = {}
|
|
requirements: list[ScreeningFreshnessAssessment] = []
|
|
checked_at = now or utcnow()
|
|
for item in latest_runs:
|
|
source_key = (
|
|
item.list_snapshot.provider_id,
|
|
item.list_snapshot.source_id,
|
|
)
|
|
if source_key not in latest_snapshots:
|
|
latest_snapshots[source_key] = (
|
|
_latest_visible_list_snapshot(
|
|
session,
|
|
principal,
|
|
baseline=item.list_snapshot,
|
|
)
|
|
)
|
|
assessment = _assess_loaded_run(
|
|
session,
|
|
run=item,
|
|
checked_at=checked_at,
|
|
current_snapshot=latest_snapshots[source_key],
|
|
dispositions=dispositions,
|
|
)
|
|
if not assessment.fresh:
|
|
requirements.append(assessment)
|
|
if len(requirements) >= bounded_limit:
|
|
break
|
|
return tuple(requirements)
|
|
|
|
|
|
def get_screening_run(
|
|
session: Session,
|
|
principal: ApiPrincipal,
|
|
*,
|
|
run_id: str,
|
|
) -> RiskScreeningRun:
|
|
_require_scope(principal, SANCTIONS_READ_SCOPE)
|
|
item = session.scalar(
|
|
select(RiskScreeningRun)
|
|
.where(
|
|
RiskScreeningRun.id == run_id,
|
|
RiskScreeningRun.tenant_id == principal.tenant_id,
|
|
)
|
|
.options(
|
|
selectinload(RiskScreeningRun.subject_snapshot),
|
|
selectinload(RiskScreeningRun.list_snapshot),
|
|
selectinload(RiskScreeningRun.candidates).options(
|
|
selectinload(RiskScreeningCandidate.entry).options(
|
|
selectinload(RiskSanctionsEntry.aliases),
|
|
selectinload(RiskSanctionsEntry.identifiers),
|
|
selectinload(RiskSanctionsEntry.dates),
|
|
selectinload(RiskSanctionsEntry.addresses),
|
|
)
|
|
),
|
|
)
|
|
)
|
|
if item is None:
|
|
raise RiskSanctionsNotFoundError(
|
|
"Screening run was not found."
|
|
)
|
|
return item
|
|
|
|
|
|
def list_screening_runs(
|
|
session: Session,
|
|
principal: ApiPrincipal,
|
|
*,
|
|
limit: int = 100,
|
|
) -> tuple[RiskScreeningRun, ...]:
|
|
_require_scope(principal, SANCTIONS_READ_SCOPE)
|
|
return tuple(
|
|
session.scalars(
|
|
select(RiskScreeningRun)
|
|
.where(RiskScreeningRun.tenant_id == principal.tenant_id)
|
|
.options(
|
|
selectinload(RiskScreeningRun.subject_snapshot),
|
|
selectinload(RiskScreeningRun.list_snapshot),
|
|
)
|
|
.order_by(
|
|
RiskScreeningRun.created_at.desc(),
|
|
RiskScreeningRun.id.desc(),
|
|
)
|
|
.limit(max(1, min(limit, 500)))
|
|
)
|
|
)
|
|
|
|
|
|
def _visible_list_snapshot(
|
|
session: Session,
|
|
principal: ApiPrincipal,
|
|
*,
|
|
snapshot_id: str,
|
|
) -> RiskSanctionsListSnapshot:
|
|
item = session.scalar(
|
|
select(RiskSanctionsListSnapshot).where(
|
|
RiskSanctionsListSnapshot.id == snapshot_id,
|
|
or_(
|
|
RiskSanctionsListSnapshot.tenant_id
|
|
== principal.tenant_id,
|
|
RiskSanctionsListSnapshot.visibility == "global",
|
|
),
|
|
)
|
|
)
|
|
if item is None:
|
|
raise RiskSanctionsNotFoundError(
|
|
"Sanctions list snapshot was not found."
|
|
)
|
|
return item
|
|
|
|
|
|
def _latest_visible_list_snapshot(
|
|
session: Session,
|
|
principal: ApiPrincipal,
|
|
*,
|
|
baseline: RiskSanctionsListSnapshot,
|
|
) -> RiskSanctionsListSnapshot | None:
|
|
return session.scalar(
|
|
select(RiskSanctionsListSnapshot)
|
|
.where(
|
|
RiskSanctionsListSnapshot.provider_id
|
|
== baseline.provider_id,
|
|
RiskSanctionsListSnapshot.source_id == baseline.source_id,
|
|
or_(
|
|
RiskSanctionsListSnapshot.tenant_id
|
|
== principal.tenant_id,
|
|
RiskSanctionsListSnapshot.visibility == "global",
|
|
),
|
|
)
|
|
.order_by(
|
|
RiskSanctionsListSnapshot.acquired_at.desc(),
|
|
RiskSanctionsListSnapshot.imported_at.desc(),
|
|
RiskSanctionsListSnapshot.id.desc(),
|
|
)
|
|
.limit(1)
|
|
)
|
|
|
|
|
|
def _assess_loaded_run(
|
|
session: Session,
|
|
*,
|
|
run: RiskScreeningRun,
|
|
checked_at: datetime,
|
|
current_snapshot: RiskSanctionsListSnapshot | None,
|
|
dispositions: dict[str, RiskScreeningDisposition],
|
|
) -> ScreeningFreshnessAssessment:
|
|
policy = _policy_from_run(run)
|
|
reasons = list(
|
|
_snapshot_freshness_reasons(
|
|
run,
|
|
current_snapshot=current_snapshot,
|
|
checked_at=checked_at,
|
|
policy=policy,
|
|
)
|
|
)
|
|
reasons.extend(
|
|
_record_freshness_reasons(
|
|
session,
|
|
run=run,
|
|
checked_at=checked_at,
|
|
dispositions=dispositions,
|
|
)
|
|
)
|
|
unique_reasons = tuple(dict.fromkeys(reasons))
|
|
gate_decision, gate_reasons = _screening_gate(
|
|
run,
|
|
freshness_reasons=unique_reasons,
|
|
failure_policy="block",
|
|
)
|
|
return ScreeningFreshnessAssessment(
|
|
run=run,
|
|
fresh=not unique_reasons,
|
|
reasons=unique_reasons,
|
|
checked_at=checked_at,
|
|
current_list_snapshot_id=(
|
|
current_snapshot.id
|
|
if current_snapshot is not None
|
|
else None
|
|
),
|
|
gate_decision=gate_decision,
|
|
gate_reasons=gate_reasons,
|
|
)
|
|
|
|
|
|
def _snapshot_freshness_reasons(
|
|
run: RiskScreeningRun,
|
|
*,
|
|
current_snapshot: RiskSanctionsListSnapshot | None,
|
|
checked_at: datetime,
|
|
policy: ScreeningPolicy,
|
|
) -> tuple[str, ...]:
|
|
if current_snapshot is None:
|
|
return ("source_unavailable",)
|
|
reasons: list[str] = []
|
|
if current_snapshot.id != run.list_snapshot_id:
|
|
reasons.append("source_snapshot_changed")
|
|
state = _list_state(
|
|
current_snapshot,
|
|
now=checked_at,
|
|
max_age_days=policy.max_snapshot_age_days,
|
|
)
|
|
if state == "stale":
|
|
reasons.append("source_snapshot_stale")
|
|
elif state == "unavailable":
|
|
reasons.append("source_unavailable")
|
|
return tuple(reasons)
|
|
|
|
|
|
def _record_freshness_reasons(
|
|
session: Session,
|
|
*,
|
|
run: RiskScreeningRun,
|
|
checked_at: datetime,
|
|
dispositions: dict[str, RiskScreeningDisposition] | None = None,
|
|
) -> tuple[str, ...]:
|
|
reasons: list[str] = []
|
|
if run.matcher_version != MATCHER_VERSION:
|
|
reasons.append("matcher_version_changed")
|
|
if run.normalization_version != NORMALIZATION_VERSION:
|
|
reasons.append("normalization_version_changed")
|
|
if run.policy_version != POLICY_VERSION:
|
|
reasons.append("policy_version_changed")
|
|
if run.status != "completed":
|
|
reasons.append("screening_incomplete")
|
|
if run.outcome in {"stale", "unavailable", "insufficient"}:
|
|
reasons.append(f"screening_outcome_{run.outcome}")
|
|
|
|
disposition_ids = {
|
|
candidate.current_disposition_id
|
|
for candidate in run.candidates
|
|
if candidate.current_disposition_id
|
|
}
|
|
if not disposition_ids:
|
|
return tuple(reasons)
|
|
current_dispositions = dispositions
|
|
if current_dispositions is None:
|
|
current_dispositions = {
|
|
item.id: item
|
|
for item in session.scalars(
|
|
select(RiskScreeningDisposition).where(
|
|
RiskScreeningDisposition.tenant_id == run.tenant_id,
|
|
RiskScreeningDisposition.id.in_(disposition_ids),
|
|
)
|
|
)
|
|
}
|
|
for disposition_id in sorted(disposition_ids):
|
|
item = current_dispositions.get(disposition_id)
|
|
if item is None:
|
|
reasons.append("disposition_evidence_unavailable")
|
|
continue
|
|
if (
|
|
item.expires_at is not None
|
|
and _aware_datetime(item.expires_at)
|
|
<= _aware_datetime(checked_at)
|
|
):
|
|
reasons.append("disposition_expired")
|
|
if (
|
|
item.review_at is not None
|
|
and _aware_datetime(item.review_at)
|
|
<= _aware_datetime(checked_at)
|
|
):
|
|
reasons.append("disposition_review_due")
|
|
return tuple(dict.fromkeys(reasons))
|
|
|
|
|
|
def _policy_from_run(run: RiskScreeningRun) -> ScreeningPolicy:
|
|
values = dict(run.policy_snapshot)
|
|
return ScreeningPolicy(
|
|
fuzzy_threshold=float(
|
|
values.get("fuzzy_threshold", DEFAULT_FUZZY_THRESHOLD)
|
|
),
|
|
max_snapshot_age_days=int(
|
|
values.get(
|
|
"max_snapshot_age_days",
|
|
DEFAULT_MAX_SNAPSHOT_AGE_DAYS,
|
|
)
|
|
),
|
|
max_candidates=int(
|
|
values.get("max_candidates", MAX_CANDIDATES)
|
|
),
|
|
)
|
|
|
|
|
|
def _run_policy_matches(
|
|
run: RiskScreeningRun,
|
|
policy: ScreeningPolicy,
|
|
) -> bool:
|
|
recorded = dict(run.policy_snapshot)
|
|
expected = policy.to_dict()
|
|
return all(
|
|
recorded.get(key) == value
|
|
for key, value in expected.items()
|
|
)
|
|
|
|
|
|
def _screening_gate(
|
|
run: RiskScreeningRun,
|
|
*,
|
|
freshness_reasons: tuple[str, ...],
|
|
failure_policy: str,
|
|
) -> tuple[str, tuple[str, ...]]:
|
|
if freshness_reasons:
|
|
return (
|
|
failure_policy,
|
|
("rescreening_required", *freshness_reasons),
|
|
)
|
|
if run.outcome == "clear":
|
|
return "allow", ("screening_clear",)
|
|
if run.outcome == "potential":
|
|
statuses = {
|
|
candidate.review_status
|
|
for candidate in run.candidates
|
|
}
|
|
if "confirmed" in statuses:
|
|
return "block", ("confirmed_match",)
|
|
unresolved = statuses - {"false_positive"}
|
|
if unresolved or not statuses:
|
|
return "review", ("candidate_review_required",)
|
|
return "allow", ("all_candidates_cleared",)
|
|
return (
|
|
failure_policy,
|
|
(f"screening_outcome_{run.outcome}",),
|
|
)
|
|
|
|
|
|
def _run_id_from_evidence_ref(evidence_ref: str) -> str:
|
|
clean_ref = str(evidence_ref or "").strip()
|
|
if not clean_ref.startswith(SCREENING_EVIDENCE_PREFIX):
|
|
raise RiskSanctionsConflictError(
|
|
"Screening evidence reference is invalid."
|
|
)
|
|
run_id = clean_ref.removeprefix(SCREENING_EVIDENCE_PREFIX)
|
|
if not run_id or ":" in run_id:
|
|
raise RiskSanctionsConflictError(
|
|
"Screening evidence reference is invalid."
|
|
)
|
|
return run_id
|
|
|
|
|
|
def _canonical_subject(subject: ScreeningSubject) -> dict[str, Any]:
|
|
aliases = sorted(
|
|
{
|
|
normalized
|
|
for value in subject.aliases
|
|
if (normalized := normalize_name(value))
|
|
}
|
|
)
|
|
identifiers = sorted(
|
|
(
|
|
{
|
|
"type": _bounded(
|
|
item.get("type") or item.get("identifier_type"),
|
|
100,
|
|
)
|
|
or "document",
|
|
"value": normalized,
|
|
}
|
|
for item in subject.identifiers
|
|
if (
|
|
normalized := normalize_identifier(item.get("value"))
|
|
)
|
|
),
|
|
key=lambda item: (item["type"], item["value"]),
|
|
)
|
|
dates = sorted(
|
|
{
|
|
str(value).strip()[:100]
|
|
for value in subject.dates
|
|
if str(value).strip()
|
|
}
|
|
)
|
|
addresses = sorted(
|
|
(
|
|
{
|
|
key: clean
|
|
for key in (
|
|
"street",
|
|
"city",
|
|
"region",
|
|
"postal_code",
|
|
"country",
|
|
)
|
|
if (clean := normalize_name(item.get(key)))
|
|
}
|
|
for item in subject.addresses
|
|
),
|
|
key=lambda item: json.dumps(item, sort_keys=True),
|
|
)
|
|
return {
|
|
"subject_type": subject.subject_type,
|
|
"primary_name": normalize_name(subject.primary_name),
|
|
"aliases": aliases,
|
|
"identifiers": identifiers,
|
|
"dates": dates,
|
|
"addresses": addresses,
|
|
}
|
|
|
|
|
|
def _request_hash(
|
|
*,
|
|
list_snapshot_id: str,
|
|
subject: dict[str, Any],
|
|
policy: ScreeningPolicy,
|
|
) -> str:
|
|
encoded = json.dumps(
|
|
{
|
|
"list_snapshot_id": list_snapshot_id,
|
|
"subject": subject,
|
|
"policy": policy.to_dict(),
|
|
"matcher_version": MATCHER_VERSION,
|
|
"normalization_version": NORMALIZATION_VERSION,
|
|
"policy_version": POLICY_VERSION,
|
|
},
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
ensure_ascii=False,
|
|
).encode("utf-8")
|
|
return hashlib.sha256(encoded).hexdigest()
|
|
|
|
|
|
def _match_entry(
|
|
subject: RiskScreeningSubjectSnapshot,
|
|
entry: RiskSanctionsEntry,
|
|
*,
|
|
fuzzy_threshold: float,
|
|
) -> dict[str, Any] | None:
|
|
if subject.subject_type != entry.subject_type:
|
|
return None
|
|
subject_names = {
|
|
name
|
|
for name in (
|
|
subject.normalized_name,
|
|
*(normalize_name(value) for value in subject.aliases),
|
|
)
|
|
if name
|
|
}
|
|
entry_names = {
|
|
("primary_name", entry.normalized_name),
|
|
*(
|
|
("alias", alias.normalized_name)
|
|
for alias in entry.aliases
|
|
if alias.normalized_name
|
|
),
|
|
}
|
|
subject_identifiers = {
|
|
item.get("value", "")
|
|
for item in subject.identifiers
|
|
if item.get("value")
|
|
}
|
|
entry_identifiers = {
|
|
item.normalized_value
|
|
for item in entry.identifiers
|
|
if item.normalized_value
|
|
}
|
|
identifier_matches = sorted(
|
|
subject_identifiers & entry_identifiers
|
|
)
|
|
if identifier_matches:
|
|
return {
|
|
"entry": entry,
|
|
"score": 100,
|
|
"match_kind": "identifier_exact",
|
|
"evidence": [
|
|
{
|
|
"kind": "identifier_exact",
|
|
"subject_field": "identifiers",
|
|
"list_field": "identifiers",
|
|
"source_entry_ref": entry.source_entry_id,
|
|
"raw_evidence_locator": entry.raw_evidence_locator,
|
|
}
|
|
],
|
|
}
|
|
|
|
exact = next(
|
|
(
|
|
field
|
|
for field, entry_name in entry_names
|
|
if entry_name in subject_names
|
|
),
|
|
None,
|
|
)
|
|
if exact is not None:
|
|
score = 98 if exact == "alias" else 100
|
|
return {
|
|
"entry": entry,
|
|
"score": score,
|
|
"match_kind": (
|
|
"alias_normalized"
|
|
if exact == "alias"
|
|
else "name_normalized"
|
|
),
|
|
"evidence": [
|
|
{
|
|
"kind": "normalized_name",
|
|
"subject_field": "name",
|
|
"list_field": exact,
|
|
"source_entry_ref": entry.source_entry_id,
|
|
"raw_evidence_locator": entry.raw_evidence_locator,
|
|
}
|
|
],
|
|
}
|
|
|
|
best: tuple[float, str] | None = None
|
|
for subject_name in subject_names:
|
|
if len(subject_name) < 5:
|
|
continue
|
|
for field, entry_name in entry_names:
|
|
if len(entry_name) < 5:
|
|
continue
|
|
ratio = SequenceMatcher(
|
|
None,
|
|
subject_name,
|
|
entry_name,
|
|
autojunk=False,
|
|
).ratio()
|
|
if best is None or ratio > best[0]:
|
|
best = (ratio, field)
|
|
if best is None or best[0] < fuzzy_threshold:
|
|
return None
|
|
return {
|
|
"entry": entry,
|
|
"score": int(round(best[0] * 100)),
|
|
"match_kind": "name_fuzzy",
|
|
"evidence": [
|
|
{
|
|
"kind": "fuzzy_name",
|
|
"subject_field": "name",
|
|
"list_field": best[1],
|
|
"similarity": round(best[0], 4),
|
|
"threshold": fuzzy_threshold,
|
|
"auto_confirmed": False,
|
|
"source_entry_ref": entry.source_entry_id,
|
|
"raw_evidence_locator": entry.raw_evidence_locator,
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
def _active_exceptions(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
subject_fingerprint_value: str,
|
|
entry_refs: set[str],
|
|
now: datetime,
|
|
) -> dict[str, RiskScreeningException]:
|
|
if not entry_refs:
|
|
return {}
|
|
items = session.scalars(
|
|
select(RiskScreeningException).where(
|
|
RiskScreeningException.tenant_id == tenant_id,
|
|
RiskScreeningException.subject_fingerprint
|
|
== subject_fingerprint_value,
|
|
RiskScreeningException.source_entry_ref.in_(entry_refs),
|
|
RiskScreeningException.status == "active",
|
|
RiskScreeningException.starts_at <= now,
|
|
RiskScreeningException.expires_at > now,
|
|
)
|
|
)
|
|
return {item.source_entry_ref: item for item in items}
|
|
|
|
|
|
def _list_state(
|
|
snapshot: RiskSanctionsListSnapshot,
|
|
*,
|
|
now: datetime,
|
|
max_age_days: int,
|
|
) -> str:
|
|
if snapshot.status != "active":
|
|
return "unavailable"
|
|
acquired_at = snapshot.acquired_at
|
|
if acquired_at.tzinfo is None:
|
|
acquired_at = acquired_at.replace(tzinfo=timezone.utc)
|
|
compare_now = now
|
|
if compare_now.tzinfo is None:
|
|
compare_now = compare_now.replace(tzinfo=timezone.utc)
|
|
if compare_now - acquired_at > timedelta(days=max_age_days):
|
|
return "stale"
|
|
return "active"
|
|
|
|
|
|
def _aware_datetime(value: datetime) -> datetime:
|
|
if value.tzinfo is None:
|
|
return value.replace(tzinfo=timezone.utc)
|
|
return value
|
|
|
|
|
|
def _bounded(value: object, length: int) -> str | None:
|
|
clean = " ".join(str(value or "").split())
|
|
return clean[:length] or None
|
|
|
|
|
|
def _require_scope(
|
|
principal: ApiPrincipal,
|
|
scope: str,
|
|
) -> None:
|
|
if not isinstance(principal, ApiPrincipal):
|
|
raise RiskSanctionsAccessError(
|
|
"A tenant API principal is required."
|
|
)
|
|
if not (
|
|
has_scope(principal, scope)
|
|
or has_scope(principal, SANCTIONS_ADMIN_SCOPE)
|
|
):
|
|
raise RiskSanctionsAccessError(f"Missing scope: {scope}")
|
|
|
|
|
|
def _actor_id(principal: ApiPrincipal) -> str:
|
|
return (
|
|
principal.account_id
|
|
or principal.membership_id
|
|
or principal.identity_id
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"DEFAULT_FUZZY_THRESHOLD",
|
|
"DEFAULT_MAX_SNAPSHOT_AGE_DAYS",
|
|
"MATCHER_VERSION",
|
|
"MAX_CANDIDATES",
|
|
"POLICY_VERSION",
|
|
"SCREENING_EVIDENCE_PREFIX",
|
|
"SCREENING_FAILURE_POLICIES",
|
|
"ScreeningFreshnessAssessment",
|
|
"ScreeningPolicy",
|
|
"ScreeningSubject",
|
|
"assess_screening_freshness",
|
|
"get_screening_run",
|
|
"list_rescreening_requirements",
|
|
"list_screening_runs",
|
|
"run_screening",
|
|
"screening_evidence_ref",
|
|
]
|