533 lines
18 KiB
Python
533 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import date
|
|
from decimal import Decimal
|
|
|
|
from mousehold_domain import (
|
|
EventObservationRelation,
|
|
FinancialEventStatus,
|
|
RelationType,
|
|
SourceType,
|
|
)
|
|
from mousehold_matching import score_event_match
|
|
from mousehold_matching.fingerprints import normalize_counterparty
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session, selectinload
|
|
|
|
from .. import models
|
|
|
|
ANNOUNCEMENT_SOURCES = {
|
|
SourceType.PAYPAL_EMAIL,
|
|
SourceType.KLARNA_EMAIL,
|
|
SourceType.ORDER_CONFIRMATION_EMAIL,
|
|
SourceType.IMAP_EMAIL,
|
|
}
|
|
CONFIRMATION_SOURCES = {
|
|
SourceType.PAYPAL_API,
|
|
SourceType.PAYPAL_CSV,
|
|
SourceType.KLARNA_CSV,
|
|
SourceType.BANK_CSV,
|
|
SourceType.FINTS,
|
|
SourceType.PSD2_AGGREGATOR,
|
|
}
|
|
BANK_CONFIRMATION_SOURCES = {SourceType.BANK_CSV, SourceType.FINTS, SourceType.PSD2_AGGREGATOR}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EventSummary:
|
|
id: str
|
|
source_type: str | None
|
|
event_type: str
|
|
status: str
|
|
event_date: date
|
|
amount: Decimal
|
|
currency: str
|
|
merchant: str | None
|
|
counterparty: str | None
|
|
description: str | None
|
|
source_reference: str | None
|
|
provider_transaction_id: str | None
|
|
observation_id: str | None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReconciliationSuggestion:
|
|
id: str
|
|
kind: str
|
|
relation_type: RelationType
|
|
primary_event: EventSummary
|
|
confirming_event: EventSummary
|
|
confidence: Decimal
|
|
reasons: list[str]
|
|
action: str
|
|
|
|
|
|
def list_reconciliation_suggestions(
|
|
session: Session,
|
|
*,
|
|
household_id: str,
|
|
limit: int = 50,
|
|
include_ignored: bool = False,
|
|
) -> list[ReconciliationSuggestion]:
|
|
events = _events_with_observations(session, household_id=household_id)
|
|
suggestions: list[ReconciliationSuggestion] = []
|
|
existing_pairs = _existing_relation_pairs(session, household_id=household_id)
|
|
ignored_suggestion_ids = (
|
|
set() if include_ignored else _decision_suggestion_ids(session, household_id=household_id, decision="ignored")
|
|
)
|
|
|
|
for primary in events:
|
|
primary_source = _first_source_type(primary)
|
|
if primary_source not in ANNOUNCEMENT_SOURCES:
|
|
continue
|
|
if primary.status in {
|
|
FinancialEventStatus.CONFIRMED,
|
|
FinancialEventStatus.CANCELLED,
|
|
FinancialEventStatus.IGNORED,
|
|
}:
|
|
continue
|
|
for confirming in events:
|
|
if primary.id == confirming.id or (primary.id, confirming.id) in existing_pairs:
|
|
continue
|
|
confirming_source = _first_source_type(confirming)
|
|
if confirming_source not in CONFIRMATION_SOURCES:
|
|
continue
|
|
suggestion = _suggest_event_pair(primary, confirming)
|
|
if suggestion:
|
|
suggestions.append(suggestion)
|
|
|
|
for source in events:
|
|
source_type = _first_source_type(source)
|
|
if source_type not in {SourceType.PAYPAL_CSV, SourceType.PAYPAL_API, SourceType.KLARNA_CSV}:
|
|
continue
|
|
for bank_event in events:
|
|
if source.id == bank_event.id or (source.id, bank_event.id) in existing_pairs:
|
|
continue
|
|
if _first_source_type(bank_event) not in BANK_CONFIRMATION_SOURCES:
|
|
continue
|
|
suggestion = _suggest_event_pair(source, bank_event, kind="statement_bank")
|
|
if suggestion:
|
|
suggestions.append(suggestion)
|
|
|
|
unique: dict[str, ReconciliationSuggestion] = {}
|
|
for suggestion in sorted(suggestions, key=lambda item: item.confidence, reverse=True):
|
|
if suggestion.id not in ignored_suggestion_ids:
|
|
unique.setdefault(suggestion.id, suggestion)
|
|
return list(unique.values())[:limit]
|
|
|
|
|
|
def accept_reconciliation_suggestion(
|
|
session: Session,
|
|
*,
|
|
household_id: str,
|
|
suggestion_id: str,
|
|
) -> ReconciliationSuggestion:
|
|
suggestions = list_reconciliation_suggestions(session, household_id=household_id, limit=500)
|
|
suggestion = next((item for item in suggestions if item.id == suggestion_id), None)
|
|
if suggestion is None:
|
|
raise LookupError("Reconciliation suggestion not found")
|
|
|
|
primary = session.get(models.FinancialEvent, suggestion.primary_event.id)
|
|
confirming = session.get(models.FinancialEvent, suggestion.confirming_event.id)
|
|
if not primary or not confirming:
|
|
raise LookupError("Reconciliation event not found")
|
|
if primary.household_id != household_id or confirming.household_id != household_id:
|
|
raise LookupError("Reconciliation event not found")
|
|
|
|
_ensure_event_relation(
|
|
session,
|
|
from_event_id=primary.id,
|
|
to_event_id=confirming.id,
|
|
relation_type=suggestion.relation_type,
|
|
confidence=suggestion.confidence,
|
|
)
|
|
if suggestion.confirming_event.observation_id:
|
|
_ensure_observation_link(
|
|
session,
|
|
event_id=primary.id,
|
|
observation_id=suggestion.confirming_event.observation_id,
|
|
relation=EventObservationRelation.EVIDENCE_FOR,
|
|
confidence=suggestion.confidence,
|
|
)
|
|
if suggestion.primary_event.observation_id:
|
|
_ensure_observation_link(
|
|
session,
|
|
event_id=confirming.id,
|
|
observation_id=suggestion.primary_event.observation_id,
|
|
relation=EventObservationRelation.EVIDENCE_FOR,
|
|
confidence=suggestion.confidence,
|
|
)
|
|
|
|
if confirming.status == FinancialEventStatus.CONFIRMED:
|
|
primary.status = FinancialEventStatus.CONFIRMED
|
|
if not primary.payment_account_id and confirming.payment_account_id:
|
|
primary.payment_account_id = confirming.payment_account_id
|
|
if not primary.source_account_id and confirming.source_account_id:
|
|
primary.source_account_id = confirming.source_account_id
|
|
if not primary.payer_user_id and confirming.payer_user_id:
|
|
primary.payer_user_id = confirming.payer_user_id
|
|
session.add(primary)
|
|
_record_reconciliation_decision(
|
|
session,
|
|
household_id=household_id,
|
|
suggestion=suggestion,
|
|
decision="accepted",
|
|
)
|
|
return suggestion
|
|
|
|
|
|
def ignore_reconciliation_suggestion(
|
|
session: Session,
|
|
*,
|
|
household_id: str,
|
|
suggestion_id: str,
|
|
actor_user_id: str | None = None,
|
|
reason: str | None = None,
|
|
) -> tuple[ReconciliationSuggestion, models.ReconciliationDecision]:
|
|
suggestions = list_reconciliation_suggestions(
|
|
session,
|
|
household_id=household_id,
|
|
limit=500,
|
|
include_ignored=True,
|
|
)
|
|
suggestion = next((item for item in suggestions if item.id == suggestion_id), None)
|
|
if suggestion is None:
|
|
raise LookupError("Reconciliation suggestion not found")
|
|
decision = _record_reconciliation_decision(
|
|
session,
|
|
household_id=household_id,
|
|
suggestion=suggestion,
|
|
decision="ignored",
|
|
actor_user_id=actor_user_id,
|
|
reason=reason,
|
|
)
|
|
return suggestion, decision
|
|
|
|
|
|
def list_reconciliation_decisions(
|
|
session: Session,
|
|
*,
|
|
household_id: str,
|
|
) -> list[models.ReconciliationDecision]:
|
|
return list(
|
|
session.scalars(
|
|
select(models.ReconciliationDecision)
|
|
.where(models.ReconciliationDecision.household_id == household_id)
|
|
.order_by(models.ReconciliationDecision.updated_at.desc())
|
|
).all()
|
|
)
|
|
|
|
|
|
def _suggest_event_pair(
|
|
primary: models.FinancialEvent,
|
|
confirming: models.FinancialEvent,
|
|
*,
|
|
kind: str = "evidence_confirmation",
|
|
) -> ReconciliationSuggestion | None:
|
|
if primary.currency.upper() != confirming.currency.upper():
|
|
return None
|
|
if abs(primary.amount - confirming.amount) > Decimal("0.01"):
|
|
return None
|
|
days = abs((primary.event_date - confirming.event_date).days)
|
|
if days > _max_date_window(primary, confirming, kind):
|
|
return None
|
|
|
|
reasons: list[str] = ["amount_exact"]
|
|
if days == 0:
|
|
reasons.append("same_date")
|
|
else:
|
|
reasons.append(f"date_within_{days}_days")
|
|
|
|
reference_score = _reference_score(primary, confirming)
|
|
if reference_score:
|
|
reasons.append("shared_reference")
|
|
|
|
party_a = _party_text(primary)
|
|
party_b = _party_text(confirming)
|
|
score = Decimal(
|
|
str(
|
|
score_event_match(
|
|
amount_a=primary.amount,
|
|
amount_b=confirming.amount,
|
|
currency_a=primary.currency,
|
|
currency_b=confirming.currency,
|
|
date_a=primary.event_date,
|
|
date_b=confirming.event_date,
|
|
party_a=party_a,
|
|
party_b=party_b,
|
|
)
|
|
)
|
|
)
|
|
party_overlap = _party_overlap_score(party_a, party_b)
|
|
if party_overlap >= Decimal("0.60"):
|
|
reasons.append("party_overlap")
|
|
if _provider_bridge(primary, confirming):
|
|
reasons.append("provider_bridge")
|
|
score += Decimal("0.12")
|
|
if kind == "statement_bank" and str(confirming.event_type) == "liability_payment":
|
|
reasons.append("liability_payment")
|
|
score += Decimal("0.10")
|
|
score += reference_score
|
|
if kind == "statement_bank":
|
|
score += Decimal("0.05")
|
|
|
|
confidence = min(score, Decimal("0.99")).quantize(Decimal("0.01"))
|
|
if confidence < Decimal("0.72"):
|
|
return None
|
|
|
|
relation_type = (
|
|
RelationType.SAME_ECONOMIC_EVENT_AS
|
|
if kind == "evidence_confirmation"
|
|
else RelationType.SETTLES
|
|
)
|
|
primary_summary = _event_summary(primary)
|
|
confirming_summary = _event_summary(confirming)
|
|
return ReconciliationSuggestion(
|
|
id=_suggestion_id(kind, primary.id, confirming.id),
|
|
kind=kind,
|
|
relation_type=relation_type,
|
|
primary_event=primary_summary,
|
|
confirming_event=confirming_summary,
|
|
confidence=confidence,
|
|
reasons=reasons,
|
|
action=(
|
|
"Attach confirming evidence and mark primary event confirmed"
|
|
if kind == "evidence_confirmation"
|
|
else "Link statement event to cash movement"
|
|
),
|
|
)
|
|
|
|
|
|
def _events_with_observations(session: Session, *, household_id: str) -> list[models.FinancialEvent]:
|
|
return list(
|
|
session.scalars(
|
|
select(models.FinancialEvent)
|
|
.where(models.FinancialEvent.household_id == household_id)
|
|
.options(
|
|
selectinload(models.FinancialEvent.observation_links).selectinload(
|
|
models.EventObservationLink.observation
|
|
)
|
|
)
|
|
.order_by(models.FinancialEvent.event_date.desc(), models.FinancialEvent.created_at.desc())
|
|
).all()
|
|
)
|
|
|
|
|
|
def _existing_relation_pairs(session: Session, *, household_id: str) -> set[tuple[str, str]]:
|
|
event_ids = select(models.FinancialEvent.id).where(models.FinancialEvent.household_id == household_id)
|
|
rows = session.execute(
|
|
select(models.EventRelation.from_event_id, models.EventRelation.to_event_id)
|
|
.where(models.EventRelation.from_event_id.in_(event_ids))
|
|
).all()
|
|
return {(row[0], row[1]) for row in rows} | {(row[1], row[0]) for row in rows}
|
|
|
|
|
|
def _decision_suggestion_ids(session: Session, *, household_id: str, decision: str) -> set[str]:
|
|
return set(
|
|
session.scalars(
|
|
select(models.ReconciliationDecision.suggestion_id)
|
|
.where(models.ReconciliationDecision.household_id == household_id)
|
|
.where(models.ReconciliationDecision.decision == decision)
|
|
).all()
|
|
)
|
|
|
|
|
|
def _record_reconciliation_decision(
|
|
session: Session,
|
|
*,
|
|
household_id: str,
|
|
suggestion: ReconciliationSuggestion,
|
|
decision: str,
|
|
actor_user_id: str | None = None,
|
|
reason: str | None = None,
|
|
) -> models.ReconciliationDecision:
|
|
existing = session.scalars(
|
|
select(models.ReconciliationDecision)
|
|
.where(models.ReconciliationDecision.household_id == household_id)
|
|
.where(models.ReconciliationDecision.suggestion_id == suggestion.id)
|
|
.limit(1)
|
|
).first()
|
|
if existing:
|
|
existing.decision = decision
|
|
existing.kind = suggestion.kind
|
|
existing.relation_type = suggestion.relation_type
|
|
existing.primary_event_id = suggestion.primary_event.id
|
|
existing.confirming_event_id = suggestion.confirming_event.id
|
|
existing.confidence = suggestion.confidence
|
|
existing.actor_user_id = actor_user_id
|
|
existing.reason = reason
|
|
session.add(existing)
|
|
return existing
|
|
row = models.ReconciliationDecision(
|
|
household_id=household_id,
|
|
suggestion_id=suggestion.id,
|
|
decision=decision,
|
|
kind=suggestion.kind,
|
|
relation_type=suggestion.relation_type,
|
|
primary_event_id=suggestion.primary_event.id,
|
|
confirming_event_id=suggestion.confirming_event.id,
|
|
confidence=suggestion.confidence,
|
|
actor_user_id=actor_user_id,
|
|
reason=reason,
|
|
)
|
|
session.add(row)
|
|
session.flush()
|
|
return row
|
|
|
|
|
|
def _first_observation(event: models.FinancialEvent) -> models.RawObservation | None:
|
|
for link in event.observation_links:
|
|
if link.observation:
|
|
return link.observation
|
|
return None
|
|
|
|
|
|
def _first_source_type(event: models.FinancialEvent) -> SourceType | None:
|
|
observation = _first_observation(event)
|
|
return observation.source_type if observation else None
|
|
|
|
|
|
def _event_summary(event: models.FinancialEvent) -> EventSummary:
|
|
observation = _first_observation(event)
|
|
return EventSummary(
|
|
id=event.id,
|
|
source_type=str(observation.source_type) if observation else None,
|
|
event_type=str(event.event_type),
|
|
status=str(event.status),
|
|
event_date=event.event_date,
|
|
amount=event.amount,
|
|
currency=event.currency,
|
|
merchant=event.merchant,
|
|
counterparty=event.counterparty,
|
|
description=event.description,
|
|
source_reference=observation.source_reference if observation else None,
|
|
provider_transaction_id=observation.provider_transaction_id if observation else None,
|
|
observation_id=observation.id if observation else None,
|
|
)
|
|
|
|
|
|
def _suggestion_id(kind: str, primary_id: str, confirming_id: str) -> str:
|
|
return f"{kind}:{primary_id}:{confirming_id}"
|
|
|
|
|
|
def _party_text(event: models.FinancialEvent) -> str:
|
|
parts = [event.merchant, event.counterparty, event.description]
|
|
observation = _first_observation(event)
|
|
if observation and observation.raw_payload_json:
|
|
refs = observation.raw_payload_json.get("references")
|
|
if isinstance(refs, dict):
|
|
parts.extend(str(value) for value in refs.values() if value)
|
|
return " ".join(part for part in parts if part)
|
|
|
|
|
|
def _party_overlap_score(a: str, b: str) -> Decimal:
|
|
left = set(normalize_counterparty(a).split())
|
|
right = set(normalize_counterparty(b).split())
|
|
if not left or not right:
|
|
return Decimal("0")
|
|
return Decimal(len(left & right)) / Decimal(max(len(left), len(right)))
|
|
|
|
|
|
def _reference_score(a: models.FinancialEvent, b: models.FinancialEvent) -> Decimal:
|
|
left = _references(a)
|
|
right = _references(b)
|
|
if not left or not right:
|
|
return Decimal("0")
|
|
return Decimal("0.22") if left & right else Decimal("0")
|
|
|
|
|
|
def _references(event: models.FinancialEvent) -> set[str]:
|
|
refs: set[str] = set()
|
|
observation = _first_observation(event)
|
|
if not observation:
|
|
return refs
|
|
for value in (observation.source_reference, observation.provider_transaction_id, observation.message_id):
|
|
if value:
|
|
refs.add(normalize_counterparty(value))
|
|
payload = observation.raw_payload_json or {}
|
|
raw_refs = payload.get("references")
|
|
if isinstance(raw_refs, dict):
|
|
refs.update(normalize_counterparty(str(value)) for value in raw_refs.values() if value)
|
|
return {value for value in refs if value}
|
|
|
|
|
|
def _provider_bridge(primary: models.FinancialEvent, confirming: models.FinancialEvent) -> bool:
|
|
party = normalize_counterparty(_party_text(confirming))
|
|
primary_source = _first_source_type(primary)
|
|
confirming_source = _first_source_type(confirming)
|
|
if primary_source in {SourceType.PAYPAL_EMAIL, SourceType.PAYPAL_CSV, SourceType.PAYPAL_API}:
|
|
return "paypal" in party or confirming_source in {SourceType.PAYPAL_CSV, SourceType.PAYPAL_API}
|
|
if primary_source in {SourceType.KLARNA_EMAIL, SourceType.KLARNA_CSV}:
|
|
return "klarna" in party or confirming_source == SourceType.KLARNA_CSV
|
|
return False
|
|
|
|
|
|
def _max_date_window(primary: models.FinancialEvent, confirming: models.FinancialEvent, kind: str) -> int:
|
|
primary_source = _first_source_type(primary)
|
|
confirming_source = _first_source_type(confirming)
|
|
if kind == "statement_bank":
|
|
return 45
|
|
if primary_source in {SourceType.KLARNA_EMAIL, SourceType.PAYPAL_EMAIL} and confirming_source in {
|
|
SourceType.KLARNA_CSV,
|
|
SourceType.PAYPAL_CSV,
|
|
SourceType.PAYPAL_API,
|
|
}:
|
|
return 45
|
|
return 21
|
|
|
|
|
|
def _ensure_event_relation(
|
|
session: Session,
|
|
*,
|
|
from_event_id: str,
|
|
to_event_id: str,
|
|
relation_type: RelationType,
|
|
confidence: Decimal,
|
|
) -> None:
|
|
existing = session.scalars(
|
|
select(models.EventRelation)
|
|
.where(models.EventRelation.from_event_id == from_event_id)
|
|
.where(models.EventRelation.to_event_id == to_event_id)
|
|
.where(models.EventRelation.relation_type == relation_type)
|
|
.limit(1)
|
|
).first()
|
|
if existing:
|
|
return
|
|
session.add(
|
|
models.EventRelation(
|
|
from_event_id=from_event_id,
|
|
to_event_id=to_event_id,
|
|
relation_type=relation_type,
|
|
confidence=confidence,
|
|
created_by="reconciliation",
|
|
)
|
|
)
|
|
|
|
|
|
def _ensure_observation_link(
|
|
session: Session,
|
|
*,
|
|
event_id: str,
|
|
observation_id: str,
|
|
relation: EventObservationRelation,
|
|
confidence: Decimal,
|
|
) -> None:
|
|
existing = session.scalars(
|
|
select(models.EventObservationLink)
|
|
.where(models.EventObservationLink.event_id == event_id)
|
|
.where(models.EventObservationLink.observation_id == observation_id)
|
|
.where(models.EventObservationLink.relation == relation)
|
|
.limit(1)
|
|
).first()
|
|
if existing:
|
|
return
|
|
session.add(
|
|
models.EventObservationLink(
|
|
event_id=event_id,
|
|
observation_id=observation_id,
|
|
relation=relation,
|
|
confidence=confidence,
|
|
)
|
|
)
|