from __future__ import annotations from dataclasses import dataclass from datetime import date from decimal import Decimal from mousehold_domain import ( EventObservationRelation, FinancialEventStatus, RelationType, SourceType, ) from mousehold_matching import score_event_match from mousehold_matching.fingerprints import normalize_counterparty from sqlalchemy import select from sqlalchemy.orm import Session, selectinload from .. import models ANNOUNCEMENT_SOURCES = { SourceType.PAYPAL_EMAIL, SourceType.KLARNA_EMAIL, SourceType.ORDER_CONFIRMATION_EMAIL, SourceType.IMAP_EMAIL, } CONFIRMATION_SOURCES = { SourceType.PAYPAL_API, SourceType.PAYPAL_CSV, SourceType.KLARNA_CSV, SourceType.BANK_CSV, SourceType.FINTS, SourceType.PSD2_AGGREGATOR, } BANK_CONFIRMATION_SOURCES = {SourceType.BANK_CSV, SourceType.FINTS, SourceType.PSD2_AGGREGATOR} @dataclass(frozen=True) class EventSummary: id: str source_type: str | None event_type: str status: str event_date: date amount: Decimal currency: str merchant: str | None counterparty: str | None description: str | None source_reference: str | None provider_transaction_id: str | None observation_id: str | None @dataclass(frozen=True) class ReconciliationSuggestion: id: str kind: str relation_type: RelationType primary_event: EventSummary confirming_event: EventSummary confidence: Decimal reasons: list[str] action: str def list_reconciliation_suggestions( session: Session, *, household_id: str, limit: int = 50, include_ignored: bool = False, ) -> list[ReconciliationSuggestion]: events = _events_with_observations(session, household_id=household_id) suggestions: list[ReconciliationSuggestion] = [] existing_pairs = _existing_relation_pairs(session, household_id=household_id) ignored_suggestion_ids = ( set() if include_ignored else _decision_suggestion_ids(session, household_id=household_id, decision="ignored") ) for primary in events: primary_source = _first_source_type(primary) if primary_source not in ANNOUNCEMENT_SOURCES: continue if primary.status in { FinancialEventStatus.CONFIRMED, FinancialEventStatus.CANCELLED, FinancialEventStatus.IGNORED, }: continue for confirming in events: if primary.id == confirming.id or (primary.id, confirming.id) in existing_pairs: continue confirming_source = _first_source_type(confirming) if confirming_source not in CONFIRMATION_SOURCES: continue suggestion = _suggest_event_pair(primary, confirming) if suggestion: suggestions.append(suggestion) for source in events: source_type = _first_source_type(source) if source_type not in {SourceType.PAYPAL_CSV, SourceType.PAYPAL_API, SourceType.KLARNA_CSV}: continue for bank_event in events: if source.id == bank_event.id or (source.id, bank_event.id) in existing_pairs: continue if _first_source_type(bank_event) not in BANK_CONFIRMATION_SOURCES: continue suggestion = _suggest_event_pair(source, bank_event, kind="statement_bank") if suggestion: suggestions.append(suggestion) unique: dict[str, ReconciliationSuggestion] = {} for suggestion in sorted(suggestions, key=lambda item: item.confidence, reverse=True): if suggestion.id not in ignored_suggestion_ids: unique.setdefault(suggestion.id, suggestion) return list(unique.values())[:limit] def accept_reconciliation_suggestion( session: Session, *, household_id: str, suggestion_id: str, ) -> ReconciliationSuggestion: suggestions = list_reconciliation_suggestions(session, household_id=household_id, limit=500) suggestion = next((item for item in suggestions if item.id == suggestion_id), None) if suggestion is None: raise LookupError("Reconciliation suggestion not found") primary = session.get(models.FinancialEvent, suggestion.primary_event.id) confirming = session.get(models.FinancialEvent, suggestion.confirming_event.id) if not primary or not confirming: raise LookupError("Reconciliation event not found") if primary.household_id != household_id or confirming.household_id != household_id: raise LookupError("Reconciliation event not found") _ensure_event_relation( session, from_event_id=primary.id, to_event_id=confirming.id, relation_type=suggestion.relation_type, confidence=suggestion.confidence, ) if suggestion.confirming_event.observation_id: _ensure_observation_link( session, event_id=primary.id, observation_id=suggestion.confirming_event.observation_id, relation=EventObservationRelation.EVIDENCE_FOR, confidence=suggestion.confidence, ) if suggestion.primary_event.observation_id: _ensure_observation_link( session, event_id=confirming.id, observation_id=suggestion.primary_event.observation_id, relation=EventObservationRelation.EVIDENCE_FOR, confidence=suggestion.confidence, ) if confirming.status == FinancialEventStatus.CONFIRMED: primary.status = FinancialEventStatus.CONFIRMED if not primary.payment_account_id and confirming.payment_account_id: primary.payment_account_id = confirming.payment_account_id if not primary.source_account_id and confirming.source_account_id: primary.source_account_id = confirming.source_account_id if not primary.payer_user_id and confirming.payer_user_id: primary.payer_user_id = confirming.payer_user_id session.add(primary) _record_reconciliation_decision( session, household_id=household_id, suggestion=suggestion, decision="accepted", ) return suggestion def ignore_reconciliation_suggestion( session: Session, *, household_id: str, suggestion_id: str, actor_user_id: str | None = None, reason: str | None = None, ) -> tuple[ReconciliationSuggestion, models.ReconciliationDecision]: suggestions = list_reconciliation_suggestions( session, household_id=household_id, limit=500, include_ignored=True, ) suggestion = next((item for item in suggestions if item.id == suggestion_id), None) if suggestion is None: raise LookupError("Reconciliation suggestion not found") decision = _record_reconciliation_decision( session, household_id=household_id, suggestion=suggestion, decision="ignored", actor_user_id=actor_user_id, reason=reason, ) return suggestion, decision def list_reconciliation_decisions( session: Session, *, household_id: str, ) -> list[models.ReconciliationDecision]: return list( session.scalars( select(models.ReconciliationDecision) .where(models.ReconciliationDecision.household_id == household_id) .order_by(models.ReconciliationDecision.updated_at.desc()) ).all() ) def _suggest_event_pair( primary: models.FinancialEvent, confirming: models.FinancialEvent, *, kind: str = "evidence_confirmation", ) -> ReconciliationSuggestion | None: if primary.currency.upper() != confirming.currency.upper(): return None if abs(primary.amount - confirming.amount) > Decimal("0.01"): return None days = abs((primary.event_date - confirming.event_date).days) if days > _max_date_window(primary, confirming, kind): return None reasons: list[str] = ["amount_exact"] if days == 0: reasons.append("same_date") else: reasons.append(f"date_within_{days}_days") reference_score = _reference_score(primary, confirming) if reference_score: reasons.append("shared_reference") party_a = _party_text(primary) party_b = _party_text(confirming) score = Decimal( str( score_event_match( amount_a=primary.amount, amount_b=confirming.amount, currency_a=primary.currency, currency_b=confirming.currency, date_a=primary.event_date, date_b=confirming.event_date, party_a=party_a, party_b=party_b, ) ) ) party_overlap = _party_overlap_score(party_a, party_b) if party_overlap >= Decimal("0.60"): reasons.append("party_overlap") if _provider_bridge(primary, confirming): reasons.append("provider_bridge") score += Decimal("0.12") if kind == "statement_bank" and str(confirming.event_type) == "liability_payment": reasons.append("liability_payment") score += Decimal("0.10") score += reference_score if kind == "statement_bank": score += Decimal("0.05") confidence = min(score, Decimal("0.99")).quantize(Decimal("0.01")) if confidence < Decimal("0.72"): return None relation_type = ( RelationType.SAME_ECONOMIC_EVENT_AS if kind == "evidence_confirmation" else RelationType.SETTLES ) primary_summary = _event_summary(primary) confirming_summary = _event_summary(confirming) return ReconciliationSuggestion( id=_suggestion_id(kind, primary.id, confirming.id), kind=kind, relation_type=relation_type, primary_event=primary_summary, confirming_event=confirming_summary, confidence=confidence, reasons=reasons, action=( "Attach confirming evidence and mark primary event confirmed" if kind == "evidence_confirmation" else "Link statement event to cash movement" ), ) def _events_with_observations(session: Session, *, household_id: str) -> list[models.FinancialEvent]: return list( session.scalars( select(models.FinancialEvent) .where(models.FinancialEvent.household_id == household_id) .options( selectinload(models.FinancialEvent.observation_links).selectinload( models.EventObservationLink.observation ) ) .order_by(models.FinancialEvent.event_date.desc(), models.FinancialEvent.created_at.desc()) ).all() ) def _existing_relation_pairs(session: Session, *, household_id: str) -> set[tuple[str, str]]: event_ids = select(models.FinancialEvent.id).where(models.FinancialEvent.household_id == household_id) rows = session.execute( select(models.EventRelation.from_event_id, models.EventRelation.to_event_id) .where(models.EventRelation.from_event_id.in_(event_ids)) ).all() return {(row[0], row[1]) for row in rows} | {(row[1], row[0]) for row in rows} def _decision_suggestion_ids(session: Session, *, household_id: str, decision: str) -> set[str]: return set( session.scalars( select(models.ReconciliationDecision.suggestion_id) .where(models.ReconciliationDecision.household_id == household_id) .where(models.ReconciliationDecision.decision == decision) ).all() ) def _record_reconciliation_decision( session: Session, *, household_id: str, suggestion: ReconciliationSuggestion, decision: str, actor_user_id: str | None = None, reason: str | None = None, ) -> models.ReconciliationDecision: existing = session.scalars( select(models.ReconciliationDecision) .where(models.ReconciliationDecision.household_id == household_id) .where(models.ReconciliationDecision.suggestion_id == suggestion.id) .limit(1) ).first() if existing: existing.decision = decision existing.kind = suggestion.kind existing.relation_type = suggestion.relation_type existing.primary_event_id = suggestion.primary_event.id existing.confirming_event_id = suggestion.confirming_event.id existing.confidence = suggestion.confidence existing.actor_user_id = actor_user_id existing.reason = reason session.add(existing) return existing row = models.ReconciliationDecision( household_id=household_id, suggestion_id=suggestion.id, decision=decision, kind=suggestion.kind, relation_type=suggestion.relation_type, primary_event_id=suggestion.primary_event.id, confirming_event_id=suggestion.confirming_event.id, confidence=suggestion.confidence, actor_user_id=actor_user_id, reason=reason, ) session.add(row) session.flush() return row def _first_observation(event: models.FinancialEvent) -> models.RawObservation | None: for link in event.observation_links: if link.observation: return link.observation return None def _first_source_type(event: models.FinancialEvent) -> SourceType | None: observation = _first_observation(event) return observation.source_type if observation else None def _event_summary(event: models.FinancialEvent) -> EventSummary: observation = _first_observation(event) return EventSummary( id=event.id, source_type=str(observation.source_type) if observation else None, event_type=str(event.event_type), status=str(event.status), event_date=event.event_date, amount=event.amount, currency=event.currency, merchant=event.merchant, counterparty=event.counterparty, description=event.description, source_reference=observation.source_reference if observation else None, provider_transaction_id=observation.provider_transaction_id if observation else None, observation_id=observation.id if observation else None, ) def _suggestion_id(kind: str, primary_id: str, confirming_id: str) -> str: return f"{kind}:{primary_id}:{confirming_id}" def _party_text(event: models.FinancialEvent) -> str: parts = [event.merchant, event.counterparty, event.description] observation = _first_observation(event) if observation and observation.raw_payload_json: refs = observation.raw_payload_json.get("references") if isinstance(refs, dict): parts.extend(str(value) for value in refs.values() if value) return " ".join(part for part in parts if part) def _party_overlap_score(a: str, b: str) -> Decimal: left = set(normalize_counterparty(a).split()) right = set(normalize_counterparty(b).split()) if not left or not right: return Decimal("0") return Decimal(len(left & right)) / Decimal(max(len(left), len(right))) def _reference_score(a: models.FinancialEvent, b: models.FinancialEvent) -> Decimal: left = _references(a) right = _references(b) if not left or not right: return Decimal("0") return Decimal("0.22") if left & right else Decimal("0") def _references(event: models.FinancialEvent) -> set[str]: refs: set[str] = set() observation = _first_observation(event) if not observation: return refs for value in (observation.source_reference, observation.provider_transaction_id, observation.message_id): if value: refs.add(normalize_counterparty(value)) payload = observation.raw_payload_json or {} raw_refs = payload.get("references") if isinstance(raw_refs, dict): refs.update(normalize_counterparty(str(value)) for value in raw_refs.values() if value) return {value for value in refs if value} def _provider_bridge(primary: models.FinancialEvent, confirming: models.FinancialEvent) -> bool: party = normalize_counterparty(_party_text(confirming)) primary_source = _first_source_type(primary) confirming_source = _first_source_type(confirming) if primary_source in {SourceType.PAYPAL_EMAIL, SourceType.PAYPAL_CSV, SourceType.PAYPAL_API}: return "paypal" in party or confirming_source in {SourceType.PAYPAL_CSV, SourceType.PAYPAL_API} if primary_source in {SourceType.KLARNA_EMAIL, SourceType.KLARNA_CSV}: return "klarna" in party or confirming_source == SourceType.KLARNA_CSV return False def _max_date_window(primary: models.FinancialEvent, confirming: models.FinancialEvent, kind: str) -> int: primary_source = _first_source_type(primary) confirming_source = _first_source_type(confirming) if kind == "statement_bank": return 45 if primary_source in {SourceType.KLARNA_EMAIL, SourceType.PAYPAL_EMAIL} and confirming_source in { SourceType.KLARNA_CSV, SourceType.PAYPAL_CSV, SourceType.PAYPAL_API, }: return 45 return 21 def _ensure_event_relation( session: Session, *, from_event_id: str, to_event_id: str, relation_type: RelationType, confidence: Decimal, ) -> None: existing = session.scalars( select(models.EventRelation) .where(models.EventRelation.from_event_id == from_event_id) .where(models.EventRelation.to_event_id == to_event_id) .where(models.EventRelation.relation_type == relation_type) .limit(1) ).first() if existing: return session.add( models.EventRelation( from_event_id=from_event_id, to_event_id=to_event_id, relation_type=relation_type, confidence=confidence, created_by="reconciliation", ) ) def _ensure_observation_link( session: Session, *, event_id: str, observation_id: str, relation: EventObservationRelation, confidence: Decimal, ) -> None: existing = session.scalars( select(models.EventObservationLink) .where(models.EventObservationLink.event_id == event_id) .where(models.EventObservationLink.observation_id == observation_id) .where(models.EventObservationLink.relation == relation) .limit(1) ).first() if existing: return session.add( models.EventObservationLink( event_id=event_id, observation_id=observation_id, relation=relation, confidence=confidence, ) )