feat: implement sanctions screening vertical
This commit is contained in:
@@ -0,0 +1,648 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from difflib import SequenceMatcher
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session, selectinload
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, has_scope
|
||||
from govoplan_core.db.base import utcnow
|
||||
from govoplan_risk_compliance.backend.db.models import (
|
||||
RiskSanctionsEntry,
|
||||
RiskSanctionsListSnapshot,
|
||||
RiskScreeningCandidate,
|
||||
RiskScreeningException,
|
||||
RiskScreeningRun,
|
||||
RiskScreeningSubjectSnapshot,
|
||||
)
|
||||
from govoplan_risk_compliance.backend.normalization import (
|
||||
NORMALIZATION_VERSION,
|
||||
normalize_identifier,
|
||||
normalize_name,
|
||||
subject_fingerprint,
|
||||
)
|
||||
from govoplan_risk_compliance.backend.permissions import (
|
||||
SANCTIONS_ADMIN_SCOPE,
|
||||
SANCTIONS_READ_SCOPE,
|
||||
SANCTIONS_SCREEN_SCOPE,
|
||||
)
|
||||
from govoplan_risk_compliance.backend.sanctions_catalog import (
|
||||
RiskSanctionsAccessError,
|
||||
RiskSanctionsConflictError,
|
||||
RiskSanctionsNotFoundError,
|
||||
)
|
||||
|
||||
|
||||
MATCHER_VERSION = "sanctions-matcher-v1"
|
||||
POLICY_VERSION = "sanctions-policy-v1"
|
||||
MAX_MATCH_ENTRIES = 5_000
|
||||
MAX_CANDIDATES = 100
|
||||
DEFAULT_FUZZY_THRESHOLD = 0.88
|
||||
DEFAULT_MAX_SNAPSHOT_AGE_DAYS = 7
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ScreeningSubject:
|
||||
subject_type: str
|
||||
primary_name: str | None = None
|
||||
subject_ref: str | None = None
|
||||
aliases: tuple[str, ...] = ()
|
||||
identifiers: tuple[dict[str, str], ...] = ()
|
||||
dates: tuple[str, ...] = ()
|
||||
addresses: tuple[dict[str, str], ...] = ()
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.subject_type not in {"person", "entity"}:
|
||||
raise RiskSanctionsConflictError(
|
||||
"Screening subject type must be person or entity."
|
||||
)
|
||||
if not normalize_name(self.primary_name) and not any(
|
||||
normalize_identifier(item.get("value"))
|
||||
for item in self.identifiers
|
||||
):
|
||||
raise RiskSanctionsConflictError(
|
||||
"A screening subject needs a name or identifier."
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ScreeningPolicy:
|
||||
fuzzy_threshold: float = DEFAULT_FUZZY_THRESHOLD
|
||||
max_snapshot_age_days: int = DEFAULT_MAX_SNAPSHOT_AGE_DAYS
|
||||
max_candidates: int = MAX_CANDIDATES
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not 0.8 <= self.fuzzy_threshold <= 1:
|
||||
raise RiskSanctionsConflictError(
|
||||
"Fuzzy threshold must be between 0.8 and 1."
|
||||
)
|
||||
if not 1 <= self.max_snapshot_age_days <= 365:
|
||||
raise RiskSanctionsConflictError(
|
||||
"Snapshot age must be between 1 and 365 days."
|
||||
)
|
||||
if not 1 <= self.max_candidates <= MAX_CANDIDATES:
|
||||
raise RiskSanctionsConflictError(
|
||||
f"Candidate limit must be between 1 and {MAX_CANDIDATES}."
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"fuzzy_threshold": self.fuzzy_threshold,
|
||||
"max_snapshot_age_days": self.max_snapshot_age_days,
|
||||
"max_candidates": self.max_candidates,
|
||||
"fuzzy_auto_confirms": False,
|
||||
}
|
||||
|
||||
|
||||
def run_screening(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
list_snapshot_id: str,
|
||||
idempotency_key: str,
|
||||
subject: ScreeningSubject,
|
||||
policy: ScreeningPolicy | None = None,
|
||||
) -> tuple[RiskScreeningRun, bool]:
|
||||
_require_scope(principal, SANCTIONS_SCREEN_SCOPE)
|
||||
clean_key = idempotency_key.strip()
|
||||
if not clean_key or len(clean_key) > 255:
|
||||
raise RiskSanctionsConflictError(
|
||||
"Idempotency key must contain 1 to 255 characters."
|
||||
)
|
||||
effective_policy = policy or ScreeningPolicy()
|
||||
canonical_subject = _canonical_subject(subject)
|
||||
request_hash = _request_hash(
|
||||
list_snapshot_id=list_snapshot_id,
|
||||
subject=canonical_subject,
|
||||
policy=effective_policy,
|
||||
)
|
||||
existing = session.scalar(
|
||||
select(RiskScreeningRun).where(
|
||||
RiskScreeningRun.tenant_id == principal.tenant_id,
|
||||
RiskScreeningRun.idempotency_key == clean_key,
|
||||
)
|
||||
)
|
||||
if existing is not None:
|
||||
if existing.request_hash != request_hash:
|
||||
raise RiskSanctionsConflictError(
|
||||
"Idempotency key was already used for another screening request."
|
||||
)
|
||||
return get_screening_run(
|
||||
session,
|
||||
principal,
|
||||
run_id=existing.id,
|
||||
), False
|
||||
|
||||
list_snapshot = _visible_list_snapshot(
|
||||
session,
|
||||
principal,
|
||||
snapshot_id=list_snapshot_id,
|
||||
)
|
||||
actor_id = _actor_id(principal)
|
||||
subject_snapshot = RiskScreeningSubjectSnapshot(
|
||||
tenant_id=principal.tenant_id,
|
||||
subject_ref=_bounded(subject.subject_ref, 500),
|
||||
subject_type=subject.subject_type,
|
||||
primary_name=_bounded(subject.primary_name, 1000),
|
||||
normalized_name=canonical_subject["primary_name"],
|
||||
aliases=list(canonical_subject["aliases"]),
|
||||
identifiers=list(canonical_subject["identifiers"]),
|
||||
dates=list(canonical_subject["dates"]),
|
||||
addresses=list(canonical_subject["addresses"]),
|
||||
fingerprint=subject_fingerprint(canonical_subject),
|
||||
submitted_by=actor_id,
|
||||
)
|
||||
now = utcnow()
|
||||
run = RiskScreeningRun(
|
||||
tenant_id=principal.tenant_id,
|
||||
subject_snapshot=subject_snapshot,
|
||||
list_snapshot_id=list_snapshot.id,
|
||||
idempotency_key=clean_key,
|
||||
request_hash=request_hash,
|
||||
matcher_version=MATCHER_VERSION,
|
||||
normalization_version=NORMALIZATION_VERSION,
|
||||
policy_version=POLICY_VERSION,
|
||||
policy_snapshot=effective_policy.to_dict(),
|
||||
status="running",
|
||||
outcome="insufficient",
|
||||
candidate_count=0,
|
||||
started_at=now,
|
||||
created_by=actor_id,
|
||||
)
|
||||
session.add(run)
|
||||
session.flush()
|
||||
|
||||
list_state = _list_state(
|
||||
list_snapshot,
|
||||
now=now,
|
||||
max_age_days=effective_policy.max_snapshot_age_days,
|
||||
)
|
||||
if list_state in {"unavailable", "stale"}:
|
||||
run.status = "completed"
|
||||
run.outcome = list_state
|
||||
run.completed_at = utcnow()
|
||||
session.flush()
|
||||
return get_screening_run(
|
||||
session,
|
||||
principal,
|
||||
run_id=run.id,
|
||||
), True
|
||||
|
||||
entries = tuple(
|
||||
session.scalars(
|
||||
select(RiskSanctionsEntry)
|
||||
.where(RiskSanctionsEntry.snapshot_id == list_snapshot.id)
|
||||
.options(
|
||||
selectinload(RiskSanctionsEntry.aliases),
|
||||
selectinload(RiskSanctionsEntry.identifiers),
|
||||
selectinload(RiskSanctionsEntry.dates),
|
||||
selectinload(RiskSanctionsEntry.addresses),
|
||||
)
|
||||
.limit(MAX_MATCH_ENTRIES + 1)
|
||||
)
|
||||
)
|
||||
if len(entries) > MAX_MATCH_ENTRIES:
|
||||
run.status = "completed"
|
||||
run.outcome = "unavailable"
|
||||
run.policy_snapshot = {
|
||||
**run.policy_snapshot,
|
||||
"failure": "entry_limit_exceeded",
|
||||
}
|
||||
run.completed_at = utcnow()
|
||||
session.flush()
|
||||
return get_screening_run(
|
||||
session,
|
||||
principal,
|
||||
run_id=run.id,
|
||||
), True
|
||||
|
||||
candidates = sorted(
|
||||
(
|
||||
candidate
|
||||
for entry in entries
|
||||
if (
|
||||
candidate := _match_entry(
|
||||
subject_snapshot,
|
||||
entry,
|
||||
fuzzy_threshold=effective_policy.fuzzy_threshold,
|
||||
)
|
||||
)
|
||||
is not None
|
||||
),
|
||||
key=lambda item: (-item["score"], item["entry"].source_entry_id),
|
||||
)[: effective_policy.max_candidates]
|
||||
active_exceptions = _active_exceptions(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
subject_fingerprint_value=subject_snapshot.fingerprint,
|
||||
entry_refs={
|
||||
item["entry"].source_entry_id
|
||||
for item in candidates
|
||||
},
|
||||
now=now,
|
||||
)
|
||||
for item in candidates:
|
||||
entry = item["entry"]
|
||||
evidence = list(item["evidence"])
|
||||
exception = active_exceptions.get(entry.source_entry_id)
|
||||
review_status = "pending"
|
||||
if exception is not None:
|
||||
review_status = "exception_review"
|
||||
evidence.append(
|
||||
{
|
||||
"kind": "prior_exception",
|
||||
"exception_id": exception.id,
|
||||
"expires_at": exception.expires_at.isoformat(),
|
||||
"effect": "review_required",
|
||||
}
|
||||
)
|
||||
session.add(
|
||||
RiskScreeningCandidate(
|
||||
tenant_id=principal.tenant_id,
|
||||
run_id=run.id,
|
||||
entry_id=entry.id,
|
||||
score=item["score"],
|
||||
match_kind=item["match_kind"],
|
||||
evidence=evidence,
|
||||
review_status=review_status,
|
||||
)
|
||||
)
|
||||
run.candidate_count = len(candidates)
|
||||
run.status = "completed"
|
||||
run.outcome = "potential" if candidates else "clear"
|
||||
run.completed_at = utcnow()
|
||||
session.flush()
|
||||
return get_screening_run(
|
||||
session,
|
||||
principal,
|
||||
run_id=run.id,
|
||||
), True
|
||||
|
||||
|
||||
def get_screening_run(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
run_id: str,
|
||||
) -> RiskScreeningRun:
|
||||
_require_scope(principal, SANCTIONS_READ_SCOPE)
|
||||
item = session.scalar(
|
||||
select(RiskScreeningRun)
|
||||
.where(
|
||||
RiskScreeningRun.id == run_id,
|
||||
RiskScreeningRun.tenant_id == principal.tenant_id,
|
||||
)
|
||||
.options(
|
||||
selectinload(RiskScreeningRun.subject_snapshot),
|
||||
selectinload(RiskScreeningRun.list_snapshot),
|
||||
selectinload(RiskScreeningRun.candidates).options(
|
||||
selectinload(RiskScreeningCandidate.entry).options(
|
||||
selectinload(RiskSanctionsEntry.aliases),
|
||||
selectinload(RiskSanctionsEntry.identifiers),
|
||||
selectinload(RiskSanctionsEntry.dates),
|
||||
selectinload(RiskSanctionsEntry.addresses),
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
if item is None:
|
||||
raise RiskSanctionsNotFoundError(
|
||||
"Screening run was not found."
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
def list_screening_runs(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
limit: int = 100,
|
||||
) -> tuple[RiskScreeningRun, ...]:
|
||||
_require_scope(principal, SANCTIONS_READ_SCOPE)
|
||||
return tuple(
|
||||
session.scalars(
|
||||
select(RiskScreeningRun)
|
||||
.where(RiskScreeningRun.tenant_id == principal.tenant_id)
|
||||
.options(
|
||||
selectinload(RiskScreeningRun.subject_snapshot),
|
||||
selectinload(RiskScreeningRun.list_snapshot),
|
||||
)
|
||||
.order_by(
|
||||
RiskScreeningRun.created_at.desc(),
|
||||
RiskScreeningRun.id.desc(),
|
||||
)
|
||||
.limit(max(1, min(limit, 500)))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _visible_list_snapshot(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
snapshot_id: str,
|
||||
) -> RiskSanctionsListSnapshot:
|
||||
item = session.scalar(
|
||||
select(RiskSanctionsListSnapshot).where(
|
||||
RiskSanctionsListSnapshot.id == snapshot_id,
|
||||
or_(
|
||||
RiskSanctionsListSnapshot.tenant_id
|
||||
== principal.tenant_id,
|
||||
RiskSanctionsListSnapshot.visibility == "global",
|
||||
),
|
||||
)
|
||||
)
|
||||
if item is None:
|
||||
raise RiskSanctionsNotFoundError(
|
||||
"Sanctions list snapshot was not found."
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
def _canonical_subject(subject: ScreeningSubject) -> dict[str, Any]:
|
||||
aliases = sorted(
|
||||
{
|
||||
normalized
|
||||
for value in subject.aliases
|
||||
if (normalized := normalize_name(value))
|
||||
}
|
||||
)
|
||||
identifiers = sorted(
|
||||
(
|
||||
{
|
||||
"type": _bounded(
|
||||
item.get("type") or item.get("identifier_type"),
|
||||
100,
|
||||
)
|
||||
or "document",
|
||||
"value": normalized,
|
||||
}
|
||||
for item in subject.identifiers
|
||||
if (
|
||||
normalized := normalize_identifier(item.get("value"))
|
||||
)
|
||||
),
|
||||
key=lambda item: (item["type"], item["value"]),
|
||||
)
|
||||
dates = sorted(
|
||||
{
|
||||
str(value).strip()[:100]
|
||||
for value in subject.dates
|
||||
if str(value).strip()
|
||||
}
|
||||
)
|
||||
addresses = sorted(
|
||||
(
|
||||
{
|
||||
key: clean
|
||||
for key in (
|
||||
"street",
|
||||
"city",
|
||||
"region",
|
||||
"postal_code",
|
||||
"country",
|
||||
)
|
||||
if (clean := normalize_name(item.get(key)))
|
||||
}
|
||||
for item in subject.addresses
|
||||
),
|
||||
key=lambda item: json.dumps(item, sort_keys=True),
|
||||
)
|
||||
return {
|
||||
"subject_type": subject.subject_type,
|
||||
"primary_name": normalize_name(subject.primary_name),
|
||||
"aliases": aliases,
|
||||
"identifiers": identifiers,
|
||||
"dates": dates,
|
||||
"addresses": addresses,
|
||||
}
|
||||
|
||||
|
||||
def _request_hash(
|
||||
*,
|
||||
list_snapshot_id: str,
|
||||
subject: dict[str, Any],
|
||||
policy: ScreeningPolicy,
|
||||
) -> str:
|
||||
encoded = json.dumps(
|
||||
{
|
||||
"list_snapshot_id": list_snapshot_id,
|
||||
"subject": subject,
|
||||
"policy": policy.to_dict(),
|
||||
"matcher_version": MATCHER_VERSION,
|
||||
"normalization_version": NORMALIZATION_VERSION,
|
||||
"policy_version": POLICY_VERSION,
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _match_entry(
|
||||
subject: RiskScreeningSubjectSnapshot,
|
||||
entry: RiskSanctionsEntry,
|
||||
*,
|
||||
fuzzy_threshold: float,
|
||||
) -> dict[str, Any] | None:
|
||||
if subject.subject_type != entry.subject_type:
|
||||
return None
|
||||
subject_names = {
|
||||
name
|
||||
for name in (
|
||||
subject.normalized_name,
|
||||
*(normalize_name(value) for value in subject.aliases),
|
||||
)
|
||||
if name
|
||||
}
|
||||
entry_names = {
|
||||
("primary_name", entry.normalized_name),
|
||||
*(
|
||||
("alias", alias.normalized_name)
|
||||
for alias in entry.aliases
|
||||
if alias.normalized_name
|
||||
),
|
||||
}
|
||||
subject_identifiers = {
|
||||
item.get("value", "")
|
||||
for item in subject.identifiers
|
||||
if item.get("value")
|
||||
}
|
||||
entry_identifiers = {
|
||||
item.normalized_value
|
||||
for item in entry.identifiers
|
||||
if item.normalized_value
|
||||
}
|
||||
identifier_matches = sorted(
|
||||
subject_identifiers & entry_identifiers
|
||||
)
|
||||
if identifier_matches:
|
||||
return {
|
||||
"entry": entry,
|
||||
"score": 100,
|
||||
"match_kind": "identifier_exact",
|
||||
"evidence": [
|
||||
{
|
||||
"kind": "identifier_exact",
|
||||
"subject_field": "identifiers",
|
||||
"list_field": "identifiers",
|
||||
"source_entry_ref": entry.source_entry_id,
|
||||
"raw_evidence_locator": entry.raw_evidence_locator,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
exact = next(
|
||||
(
|
||||
field
|
||||
for field, entry_name in entry_names
|
||||
if entry_name in subject_names
|
||||
),
|
||||
None,
|
||||
)
|
||||
if exact is not None:
|
||||
score = 98 if exact == "alias" else 100
|
||||
return {
|
||||
"entry": entry,
|
||||
"score": score,
|
||||
"match_kind": (
|
||||
"alias_normalized"
|
||||
if exact == "alias"
|
||||
else "name_normalized"
|
||||
),
|
||||
"evidence": [
|
||||
{
|
||||
"kind": "normalized_name",
|
||||
"subject_field": "name",
|
||||
"list_field": exact,
|
||||
"source_entry_ref": entry.source_entry_id,
|
||||
"raw_evidence_locator": entry.raw_evidence_locator,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
best: tuple[float, str] | None = None
|
||||
for subject_name in subject_names:
|
||||
if len(subject_name) < 5:
|
||||
continue
|
||||
for field, entry_name in entry_names:
|
||||
if len(entry_name) < 5:
|
||||
continue
|
||||
ratio = SequenceMatcher(
|
||||
None,
|
||||
subject_name,
|
||||
entry_name,
|
||||
autojunk=False,
|
||||
).ratio()
|
||||
if best is None or ratio > best[0]:
|
||||
best = (ratio, field)
|
||||
if best is None or best[0] < fuzzy_threshold:
|
||||
return None
|
||||
return {
|
||||
"entry": entry,
|
||||
"score": int(round(best[0] * 100)),
|
||||
"match_kind": "name_fuzzy",
|
||||
"evidence": [
|
||||
{
|
||||
"kind": "fuzzy_name",
|
||||
"subject_field": "name",
|
||||
"list_field": best[1],
|
||||
"similarity": round(best[0], 4),
|
||||
"threshold": fuzzy_threshold,
|
||||
"auto_confirmed": False,
|
||||
"source_entry_ref": entry.source_entry_id,
|
||||
"raw_evidence_locator": entry.raw_evidence_locator,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _active_exceptions(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
subject_fingerprint_value: str,
|
||||
entry_refs: set[str],
|
||||
now: datetime,
|
||||
) -> dict[str, RiskScreeningException]:
|
||||
if not entry_refs:
|
||||
return {}
|
||||
items = session.scalars(
|
||||
select(RiskScreeningException).where(
|
||||
RiskScreeningException.tenant_id == tenant_id,
|
||||
RiskScreeningException.subject_fingerprint
|
||||
== subject_fingerprint_value,
|
||||
RiskScreeningException.source_entry_ref.in_(entry_refs),
|
||||
RiskScreeningException.status == "active",
|
||||
RiskScreeningException.starts_at <= now,
|
||||
RiskScreeningException.expires_at > now,
|
||||
)
|
||||
)
|
||||
return {item.source_entry_ref: item for item in items}
|
||||
|
||||
|
||||
def _list_state(
|
||||
snapshot: RiskSanctionsListSnapshot,
|
||||
*,
|
||||
now: datetime,
|
||||
max_age_days: int,
|
||||
) -> str:
|
||||
if snapshot.status != "active":
|
||||
return "unavailable"
|
||||
acquired_at = snapshot.acquired_at
|
||||
if acquired_at.tzinfo is None:
|
||||
acquired_at = acquired_at.replace(tzinfo=timezone.utc)
|
||||
compare_now = now
|
||||
if compare_now.tzinfo is None:
|
||||
compare_now = compare_now.replace(tzinfo=timezone.utc)
|
||||
if compare_now - acquired_at > timedelta(days=max_age_days):
|
||||
return "stale"
|
||||
return "active"
|
||||
|
||||
|
||||
def _bounded(value: object, length: int) -> str | None:
|
||||
clean = " ".join(str(value or "").split())
|
||||
return clean[:length] or None
|
||||
|
||||
|
||||
def _require_scope(
|
||||
principal: ApiPrincipal,
|
||||
scope: str,
|
||||
) -> None:
|
||||
if not isinstance(principal, ApiPrincipal):
|
||||
raise RiskSanctionsAccessError(
|
||||
"A tenant API principal is required."
|
||||
)
|
||||
if not (
|
||||
has_scope(principal, scope)
|
||||
or has_scope(principal, SANCTIONS_ADMIN_SCOPE)
|
||||
):
|
||||
raise RiskSanctionsAccessError(f"Missing scope: {scope}")
|
||||
|
||||
|
||||
def _actor_id(principal: ApiPrincipal) -> str:
|
||||
return (
|
||||
principal.account_id
|
||||
or principal.membership_id
|
||||
or principal.identity_id
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_FUZZY_THRESHOLD",
|
||||
"DEFAULT_MAX_SNAPSHOT_AGE_DAYS",
|
||||
"MATCHER_VERSION",
|
||||
"MAX_CANDIDATES",
|
||||
"POLICY_VERSION",
|
||||
"ScreeningPolicy",
|
||||
"ScreeningSubject",
|
||||
"get_screening_run",
|
||||
"list_screening_runs",
|
||||
"run_screening",
|
||||
]
|
||||
Reference in New Issue
Block a user