first ruggedy draft
This commit is contained in:
1
apps/api/mousehold_api/services/__init__.py
Normal file
1
apps/api/mousehold_api/services/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Application services for ledger ingestion, reconciliation, and summaries."""
|
||||
192
apps/api/mousehold_api/services/ingestion.py
Normal file
192
apps/api/mousehold_api/services/ingestion.py
Normal file
@@ -0,0 +1,192 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
from mousehold_domain import (
|
||||
EventObservationRelation,
|
||||
FinancialEventStatus,
|
||||
FinancialEventType,
|
||||
ObservationStatus,
|
||||
SourceType,
|
||||
)
|
||||
from mousehold_matching import (
|
||||
compute_normalized_transaction_fingerprint,
|
||||
compute_raw_observation_hash,
|
||||
)
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .. import models
|
||||
from ..schemas import ManualObservationCreate, ObservationImportCreate
|
||||
|
||||
|
||||
def infer_event_status(payload: ManualObservationCreate) -> FinancialEventStatus:
|
||||
if payload.status:
|
||||
return payload.status
|
||||
if payload.source_type in {
|
||||
SourceType.BANK_CSV,
|
||||
SourceType.FINTS,
|
||||
SourceType.PSD2_AGGREGATOR,
|
||||
SourceType.PAYPAL_API,
|
||||
SourceType.PAYPAL_CSV,
|
||||
SourceType.KLARNA_CSV,
|
||||
}:
|
||||
return FinancialEventStatus.CONFIRMED
|
||||
if (
|
||||
payload.source_type in {SourceType.KLARNA_EMAIL, SourceType.PAYPAL_EMAIL}
|
||||
and payload.due_date
|
||||
):
|
||||
return FinancialEventStatus.OUTSTANDING
|
||||
if payload.source_type in {
|
||||
SourceType.ORDER_CONFIRMATION_EMAIL,
|
||||
SourceType.IMAP_EMAIL,
|
||||
SourceType.PDF_INVOICE,
|
||||
}:
|
||||
return FinancialEventStatus.ANNOUNCED
|
||||
return FinancialEventStatus.CANDIDATE
|
||||
|
||||
|
||||
def infer_event_type(payload: ManualObservationCreate) -> FinancialEventType:
|
||||
if payload.event_type != FinancialEventType.EXPENSE:
|
||||
return payload.event_type
|
||||
if payload.source_type not in {SourceType.BANK_CSV, SourceType.FINTS, SourceType.PSD2_AGGREGATOR}:
|
||||
return payload.event_type
|
||||
text = f"{payload.merchant or ''} {payload.counterparty or ''} {payload.description or ''}".lower()
|
||||
liability_markers = (
|
||||
"paypal",
|
||||
"klarna",
|
||||
"visa",
|
||||
"mastercard",
|
||||
"amex",
|
||||
"american express",
|
||||
"credit card",
|
||||
"kreditkarte",
|
||||
"kartenabrechnung",
|
||||
"card repayment",
|
||||
)
|
||||
if any(marker in text for marker in liability_markers):
|
||||
return FinancialEventType.LIABILITY_PAYMENT
|
||||
return payload.event_type
|
||||
|
||||
|
||||
def create_observation_with_candidate(
|
||||
session: Session,
|
||||
payload: ManualObservationCreate | ObservationImportCreate,
|
||||
) -> tuple[models.RawObservation, models.FinancialEvent | None]:
|
||||
raw_payload = payload.raw_payload_json or payload.model_dump(
|
||||
mode="json", exclude={"split_lines"}
|
||||
)
|
||||
event_type = infer_event_type(payload)
|
||||
event_status = infer_event_status(payload)
|
||||
raw_hash = compute_raw_observation_hash(
|
||||
source_type=str(payload.source_type),
|
||||
source_reference=payload.source_reference
|
||||
or payload.provider_transaction_id
|
||||
or payload.message_id,
|
||||
raw_text=payload.raw_text,
|
||||
raw_payload=raw_payload,
|
||||
)
|
||||
|
||||
duplicate = session.scalars(
|
||||
select(models.RawObservation)
|
||||
.where(models.RawObservation.household_id == payload.household_id)
|
||||
.where(models.RawObservation.source_type == payload.source_type)
|
||||
.where(models.RawObservation.raw_hash == raw_hash)
|
||||
.limit(1)
|
||||
).first()
|
||||
|
||||
observation = models.RawObservation(
|
||||
household_id=payload.household_id,
|
||||
source_type=payload.source_type,
|
||||
source_account_id=payload.source_account_id,
|
||||
owner_user_id=payload.owner_user_id,
|
||||
observed_at=payload.observed_at,
|
||||
raw_hash=raw_hash,
|
||||
source_reference=payload.source_reference,
|
||||
provider_transaction_id=payload.provider_transaction_id,
|
||||
message_id=payload.message_id,
|
||||
raw_text=payload.raw_text,
|
||||
raw_payload_json=raw_payload,
|
||||
parsed_json={
|
||||
"amount": str(payload.amount),
|
||||
"currency": payload.currency,
|
||||
"event_date": payload.event_date.isoformat(),
|
||||
"booking_date": payload.booking_date.isoformat() if payload.booking_date else None,
|
||||
"due_date": payload.due_date.isoformat() if payload.due_date else None,
|
||||
"merchant": payload.merchant,
|
||||
"counterparty": payload.counterparty,
|
||||
"description": payload.description,
|
||||
"event_type": str(event_type),
|
||||
"status": str(event_status),
|
||||
},
|
||||
parser_name="manual-or-normalized-import",
|
||||
parser_version="0.1.0",
|
||||
confidence=Decimal("1.00"),
|
||||
status=ObservationStatus.DUPLICATE_SOURCE
|
||||
if duplicate
|
||||
else ObservationStatus.CANDIDATE_CREATED,
|
||||
)
|
||||
session.add(observation)
|
||||
session.flush()
|
||||
|
||||
if duplicate:
|
||||
return observation, None
|
||||
|
||||
fingerprint = compute_normalized_transaction_fingerprint(
|
||||
amount=payload.amount,
|
||||
currency=payload.currency,
|
||||
event_date=payload.event_date,
|
||||
merchant=payload.merchant,
|
||||
counterparty=payload.counterparty,
|
||||
external_id=payload.provider_transaction_id,
|
||||
)
|
||||
event = models.FinancialEvent(
|
||||
household_id=payload.household_id,
|
||||
source_account_id=payload.source_account_id,
|
||||
payment_account_id=payload.payment_account_id or payload.source_account_id,
|
||||
payer_user_id=payload.payer_user_id or payload.owner_user_id,
|
||||
event_type=event_type,
|
||||
status=event_status,
|
||||
event_date=payload.event_date,
|
||||
booking_date=payload.booking_date,
|
||||
due_date=payload.due_date,
|
||||
amount=payload.amount,
|
||||
currency=payload.currency,
|
||||
merchant=payload.merchant,
|
||||
counterparty=payload.counterparty,
|
||||
description=payload.description,
|
||||
category_id=payload.category_id,
|
||||
project_id=payload.project_id,
|
||||
created_by_user_id=payload.owner_user_id,
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
session.add(event)
|
||||
session.flush()
|
||||
|
||||
session.add(
|
||||
models.EventObservationLink(
|
||||
event_id=event.id,
|
||||
observation_id=observation.id,
|
||||
relation=EventObservationRelation.IMPORTED_FROM,
|
||||
confidence=Decimal("1.00"),
|
||||
)
|
||||
)
|
||||
for split in payload.split_lines:
|
||||
session.add(
|
||||
models.SplitLine(
|
||||
event_id=event.id,
|
||||
user_id=split.user_id,
|
||||
share_amount=split.share_amount,
|
||||
share_percent=split.share_percent,
|
||||
reason=split.reason,
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
return observation, event
|
||||
|
||||
|
||||
def confirm_event(session: Session, event: models.FinancialEvent) -> models.FinancialEvent:
|
||||
event.status = FinancialEventStatus.CONFIRMED
|
||||
session.add(event)
|
||||
session.flush()
|
||||
return event
|
||||
136
apps/api/mousehold_api/services/projects.py
Normal file
136
apps/api/mousehold_api/services/projects.py
Normal file
@@ -0,0 +1,136 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
from mousehold_domain import (
|
||||
FinancialEventStatus,
|
||||
PaymentLineStatus,
|
||||
PaymentStatus,
|
||||
PlanningStatus,
|
||||
ReconciliationStatus,
|
||||
SettlementStatus,
|
||||
)
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .. import models
|
||||
from ..schemas import ProjectSummary
|
||||
|
||||
|
||||
def _money(value: Decimal | int | str = "0.00") -> Decimal:
|
||||
return Decimal(value).quantize(Decimal("0.01"))
|
||||
|
||||
|
||||
def calculate_project_summary(session: Session, project_id: str) -> ProjectSummary:
|
||||
project = session.get(models.Project, project_id)
|
||||
if not project:
|
||||
raise ValueError("Project not found")
|
||||
|
||||
budget_lines = list(
|
||||
session.scalars(
|
||||
select(models.ProjectBudgetLine).where(
|
||||
models.ProjectBudgetLine.project_id == project_id
|
||||
)
|
||||
).all()
|
||||
)
|
||||
planned_items = list(
|
||||
session.scalars(
|
||||
select(models.ProjectPlannedItem).where(
|
||||
models.ProjectPlannedItem.project_id == project_id
|
||||
)
|
||||
).all()
|
||||
)
|
||||
payment_lines = list(
|
||||
session.scalars(
|
||||
select(models.PaymentLine)
|
||||
.join(
|
||||
models.ProjectPlannedItem,
|
||||
models.PaymentLine.planned_item_id == models.ProjectPlannedItem.id,
|
||||
)
|
||||
.where(models.ProjectPlannedItem.project_id == project_id)
|
||||
).all()
|
||||
)
|
||||
linked_events = list(
|
||||
session.scalars(
|
||||
select(models.FinancialEvent).where(models.FinancialEvent.project_id == project_id)
|
||||
).all()
|
||||
)
|
||||
|
||||
budgeted_total = _money(sum((line.budgeted_amount for line in budget_lines), Decimal("0.00")))
|
||||
committed_total = _money(
|
||||
sum(
|
||||
(
|
||||
item.committed_amount
|
||||
if item.committed_amount is not None
|
||||
else item.estimated_amount
|
||||
if item.planning_status == PlanningStatus.COMMITTED
|
||||
else Decimal("0.00")
|
||||
)
|
||||
for item in planned_items
|
||||
)
|
||||
)
|
||||
paid_total = _money(
|
||||
sum(
|
||||
(
|
||||
line.amount
|
||||
for line in payment_lines
|
||||
if line.status in {PaymentLineStatus.PAID, PaymentLineStatus.RECONCILED}
|
||||
),
|
||||
Decimal("0.00"),
|
||||
)
|
||||
+ sum(
|
||||
(
|
||||
event.amount
|
||||
for event in linked_events
|
||||
if event.status
|
||||
in {
|
||||
FinancialEventStatus.CONFIRMED,
|
||||
FinancialEventStatus.BOOKED,
|
||||
FinancialEventStatus.SETTLED,
|
||||
}
|
||||
),
|
||||
Decimal("0.00"),
|
||||
)
|
||||
)
|
||||
reconciled_total = _money(
|
||||
sum(
|
||||
(line.amount for line in payment_lines if line.status == PaymentLineStatus.RECONCILED),
|
||||
Decimal("0.00"),
|
||||
)
|
||||
)
|
||||
settled_total = _money(
|
||||
sum(
|
||||
(
|
||||
item.committed_amount or item.estimated_amount
|
||||
for item in planned_items
|
||||
if item.settlement_status == SettlementStatus.SETTLED
|
||||
),
|
||||
Decimal("0.00"),
|
||||
)
|
||||
)
|
||||
due_later_total = _money(max(Decimal("0.00"), committed_total - paid_total))
|
||||
remaining_uncommitted_budget = _money(budgeted_total - committed_total)
|
||||
|
||||
open_actions: list[str] = []
|
||||
for item in planned_items:
|
||||
expected = item.committed_amount or item.estimated_amount
|
||||
if item.planning_status == PlanningStatus.COMMITTED and item.payment_status in {
|
||||
PaymentStatus.UNPAID,
|
||||
PaymentStatus.PARTIALLY_PAID,
|
||||
}:
|
||||
open_actions.append(f"{item.description} still has payment due.")
|
||||
if item.reconciliation_status == ReconciliationStatus.NO_EVIDENCE and expected > 0:
|
||||
open_actions.append(f"{item.description} needs evidence.")
|
||||
|
||||
return ProjectSummary(
|
||||
project_id=project.id,
|
||||
currency=project.currency,
|
||||
budgeted_total=budgeted_total,
|
||||
committed_total=committed_total,
|
||||
paid_total=paid_total,
|
||||
reconciled_total=reconciled_total,
|
||||
settled_total=settled_total,
|
||||
due_later_total=due_later_total,
|
||||
remaining_uncommitted_budget=remaining_uncommitted_budget,
|
||||
open_actions=open_actions,
|
||||
)
|
||||
532
apps/api/mousehold_api/services/reconciliation.py
Normal file
532
apps/api/mousehold_api/services/reconciliation.py
Normal file
@@ -0,0 +1,532 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
from mousehold_domain import (
|
||||
EventObservationRelation,
|
||||
FinancialEventStatus,
|
||||
RelationType,
|
||||
SourceType,
|
||||
)
|
||||
from mousehold_matching import score_event_match
|
||||
from mousehold_matching.fingerprints import normalize_counterparty
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, selectinload
|
||||
|
||||
from .. import models
|
||||
|
||||
ANNOUNCEMENT_SOURCES = {
|
||||
SourceType.PAYPAL_EMAIL,
|
||||
SourceType.KLARNA_EMAIL,
|
||||
SourceType.ORDER_CONFIRMATION_EMAIL,
|
||||
SourceType.IMAP_EMAIL,
|
||||
}
|
||||
CONFIRMATION_SOURCES = {
|
||||
SourceType.PAYPAL_API,
|
||||
SourceType.PAYPAL_CSV,
|
||||
SourceType.KLARNA_CSV,
|
||||
SourceType.BANK_CSV,
|
||||
SourceType.FINTS,
|
||||
SourceType.PSD2_AGGREGATOR,
|
||||
}
|
||||
BANK_CONFIRMATION_SOURCES = {SourceType.BANK_CSV, SourceType.FINTS, SourceType.PSD2_AGGREGATOR}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EventSummary:
|
||||
id: str
|
||||
source_type: str | None
|
||||
event_type: str
|
||||
status: str
|
||||
event_date: date
|
||||
amount: Decimal
|
||||
currency: str
|
||||
merchant: str | None
|
||||
counterparty: str | None
|
||||
description: str | None
|
||||
source_reference: str | None
|
||||
provider_transaction_id: str | None
|
||||
observation_id: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReconciliationSuggestion:
|
||||
id: str
|
||||
kind: str
|
||||
relation_type: RelationType
|
||||
primary_event: EventSummary
|
||||
confirming_event: EventSummary
|
||||
confidence: Decimal
|
||||
reasons: list[str]
|
||||
action: str
|
||||
|
||||
|
||||
def list_reconciliation_suggestions(
|
||||
session: Session,
|
||||
*,
|
||||
household_id: str,
|
||||
limit: int = 50,
|
||||
include_ignored: bool = False,
|
||||
) -> list[ReconciliationSuggestion]:
|
||||
events = _events_with_observations(session, household_id=household_id)
|
||||
suggestions: list[ReconciliationSuggestion] = []
|
||||
existing_pairs = _existing_relation_pairs(session, household_id=household_id)
|
||||
ignored_suggestion_ids = (
|
||||
set() if include_ignored else _decision_suggestion_ids(session, household_id=household_id, decision="ignored")
|
||||
)
|
||||
|
||||
for primary in events:
|
||||
primary_source = _first_source_type(primary)
|
||||
if primary_source not in ANNOUNCEMENT_SOURCES:
|
||||
continue
|
||||
if primary.status in {
|
||||
FinancialEventStatus.CONFIRMED,
|
||||
FinancialEventStatus.CANCELLED,
|
||||
FinancialEventStatus.IGNORED,
|
||||
}:
|
||||
continue
|
||||
for confirming in events:
|
||||
if primary.id == confirming.id or (primary.id, confirming.id) in existing_pairs:
|
||||
continue
|
||||
confirming_source = _first_source_type(confirming)
|
||||
if confirming_source not in CONFIRMATION_SOURCES:
|
||||
continue
|
||||
suggestion = _suggest_event_pair(primary, confirming)
|
||||
if suggestion:
|
||||
suggestions.append(suggestion)
|
||||
|
||||
for source in events:
|
||||
source_type = _first_source_type(source)
|
||||
if source_type not in {SourceType.PAYPAL_CSV, SourceType.PAYPAL_API, SourceType.KLARNA_CSV}:
|
||||
continue
|
||||
for bank_event in events:
|
||||
if source.id == bank_event.id or (source.id, bank_event.id) in existing_pairs:
|
||||
continue
|
||||
if _first_source_type(bank_event) not in BANK_CONFIRMATION_SOURCES:
|
||||
continue
|
||||
suggestion = _suggest_event_pair(source, bank_event, kind="statement_bank")
|
||||
if suggestion:
|
||||
suggestions.append(suggestion)
|
||||
|
||||
unique: dict[str, ReconciliationSuggestion] = {}
|
||||
for suggestion in sorted(suggestions, key=lambda item: item.confidence, reverse=True):
|
||||
if suggestion.id not in ignored_suggestion_ids:
|
||||
unique.setdefault(suggestion.id, suggestion)
|
||||
return list(unique.values())[:limit]
|
||||
|
||||
|
||||
def accept_reconciliation_suggestion(
|
||||
session: Session,
|
||||
*,
|
||||
household_id: str,
|
||||
suggestion_id: str,
|
||||
) -> ReconciliationSuggestion:
|
||||
suggestions = list_reconciliation_suggestions(session, household_id=household_id, limit=500)
|
||||
suggestion = next((item for item in suggestions if item.id == suggestion_id), None)
|
||||
if suggestion is None:
|
||||
raise LookupError("Reconciliation suggestion not found")
|
||||
|
||||
primary = session.get(models.FinancialEvent, suggestion.primary_event.id)
|
||||
confirming = session.get(models.FinancialEvent, suggestion.confirming_event.id)
|
||||
if not primary or not confirming:
|
||||
raise LookupError("Reconciliation event not found")
|
||||
if primary.household_id != household_id or confirming.household_id != household_id:
|
||||
raise LookupError("Reconciliation event not found")
|
||||
|
||||
_ensure_event_relation(
|
||||
session,
|
||||
from_event_id=primary.id,
|
||||
to_event_id=confirming.id,
|
||||
relation_type=suggestion.relation_type,
|
||||
confidence=suggestion.confidence,
|
||||
)
|
||||
if suggestion.confirming_event.observation_id:
|
||||
_ensure_observation_link(
|
||||
session,
|
||||
event_id=primary.id,
|
||||
observation_id=suggestion.confirming_event.observation_id,
|
||||
relation=EventObservationRelation.EVIDENCE_FOR,
|
||||
confidence=suggestion.confidence,
|
||||
)
|
||||
if suggestion.primary_event.observation_id:
|
||||
_ensure_observation_link(
|
||||
session,
|
||||
event_id=confirming.id,
|
||||
observation_id=suggestion.primary_event.observation_id,
|
||||
relation=EventObservationRelation.EVIDENCE_FOR,
|
||||
confidence=suggestion.confidence,
|
||||
)
|
||||
|
||||
if confirming.status == FinancialEventStatus.CONFIRMED:
|
||||
primary.status = FinancialEventStatus.CONFIRMED
|
||||
if not primary.payment_account_id and confirming.payment_account_id:
|
||||
primary.payment_account_id = confirming.payment_account_id
|
||||
if not primary.source_account_id and confirming.source_account_id:
|
||||
primary.source_account_id = confirming.source_account_id
|
||||
if not primary.payer_user_id and confirming.payer_user_id:
|
||||
primary.payer_user_id = confirming.payer_user_id
|
||||
session.add(primary)
|
||||
_record_reconciliation_decision(
|
||||
session,
|
||||
household_id=household_id,
|
||||
suggestion=suggestion,
|
||||
decision="accepted",
|
||||
)
|
||||
return suggestion
|
||||
|
||||
|
||||
def ignore_reconciliation_suggestion(
|
||||
session: Session,
|
||||
*,
|
||||
household_id: str,
|
||||
suggestion_id: str,
|
||||
actor_user_id: str | None = None,
|
||||
reason: str | None = None,
|
||||
) -> tuple[ReconciliationSuggestion, models.ReconciliationDecision]:
|
||||
suggestions = list_reconciliation_suggestions(
|
||||
session,
|
||||
household_id=household_id,
|
||||
limit=500,
|
||||
include_ignored=True,
|
||||
)
|
||||
suggestion = next((item for item in suggestions if item.id == suggestion_id), None)
|
||||
if suggestion is None:
|
||||
raise LookupError("Reconciliation suggestion not found")
|
||||
decision = _record_reconciliation_decision(
|
||||
session,
|
||||
household_id=household_id,
|
||||
suggestion=suggestion,
|
||||
decision="ignored",
|
||||
actor_user_id=actor_user_id,
|
||||
reason=reason,
|
||||
)
|
||||
return suggestion, decision
|
||||
|
||||
|
||||
def list_reconciliation_decisions(
|
||||
session: Session,
|
||||
*,
|
||||
household_id: str,
|
||||
) -> list[models.ReconciliationDecision]:
|
||||
return list(
|
||||
session.scalars(
|
||||
select(models.ReconciliationDecision)
|
||||
.where(models.ReconciliationDecision.household_id == household_id)
|
||||
.order_by(models.ReconciliationDecision.updated_at.desc())
|
||||
).all()
|
||||
)
|
||||
|
||||
|
||||
def _suggest_event_pair(
|
||||
primary: models.FinancialEvent,
|
||||
confirming: models.FinancialEvent,
|
||||
*,
|
||||
kind: str = "evidence_confirmation",
|
||||
) -> ReconciliationSuggestion | None:
|
||||
if primary.currency.upper() != confirming.currency.upper():
|
||||
return None
|
||||
if abs(primary.amount - confirming.amount) > Decimal("0.01"):
|
||||
return None
|
||||
days = abs((primary.event_date - confirming.event_date).days)
|
||||
if days > _max_date_window(primary, confirming, kind):
|
||||
return None
|
||||
|
||||
reasons: list[str] = ["amount_exact"]
|
||||
if days == 0:
|
||||
reasons.append("same_date")
|
||||
else:
|
||||
reasons.append(f"date_within_{days}_days")
|
||||
|
||||
reference_score = _reference_score(primary, confirming)
|
||||
if reference_score:
|
||||
reasons.append("shared_reference")
|
||||
|
||||
party_a = _party_text(primary)
|
||||
party_b = _party_text(confirming)
|
||||
score = Decimal(
|
||||
str(
|
||||
score_event_match(
|
||||
amount_a=primary.amount,
|
||||
amount_b=confirming.amount,
|
||||
currency_a=primary.currency,
|
||||
currency_b=confirming.currency,
|
||||
date_a=primary.event_date,
|
||||
date_b=confirming.event_date,
|
||||
party_a=party_a,
|
||||
party_b=party_b,
|
||||
)
|
||||
)
|
||||
)
|
||||
party_overlap = _party_overlap_score(party_a, party_b)
|
||||
if party_overlap >= Decimal("0.60"):
|
||||
reasons.append("party_overlap")
|
||||
if _provider_bridge(primary, confirming):
|
||||
reasons.append("provider_bridge")
|
||||
score += Decimal("0.12")
|
||||
if kind == "statement_bank" and str(confirming.event_type) == "liability_payment":
|
||||
reasons.append("liability_payment")
|
||||
score += Decimal("0.10")
|
||||
score += reference_score
|
||||
if kind == "statement_bank":
|
||||
score += Decimal("0.05")
|
||||
|
||||
confidence = min(score, Decimal("0.99")).quantize(Decimal("0.01"))
|
||||
if confidence < Decimal("0.72"):
|
||||
return None
|
||||
|
||||
relation_type = (
|
||||
RelationType.SAME_ECONOMIC_EVENT_AS
|
||||
if kind == "evidence_confirmation"
|
||||
else RelationType.SETTLES
|
||||
)
|
||||
primary_summary = _event_summary(primary)
|
||||
confirming_summary = _event_summary(confirming)
|
||||
return ReconciliationSuggestion(
|
||||
id=_suggestion_id(kind, primary.id, confirming.id),
|
||||
kind=kind,
|
||||
relation_type=relation_type,
|
||||
primary_event=primary_summary,
|
||||
confirming_event=confirming_summary,
|
||||
confidence=confidence,
|
||||
reasons=reasons,
|
||||
action=(
|
||||
"Attach confirming evidence and mark primary event confirmed"
|
||||
if kind == "evidence_confirmation"
|
||||
else "Link statement event to cash movement"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _events_with_observations(session: Session, *, household_id: str) -> list[models.FinancialEvent]:
|
||||
return list(
|
||||
session.scalars(
|
||||
select(models.FinancialEvent)
|
||||
.where(models.FinancialEvent.household_id == household_id)
|
||||
.options(
|
||||
selectinload(models.FinancialEvent.observation_links).selectinload(
|
||||
models.EventObservationLink.observation
|
||||
)
|
||||
)
|
||||
.order_by(models.FinancialEvent.event_date.desc(), models.FinancialEvent.created_at.desc())
|
||||
).all()
|
||||
)
|
||||
|
||||
|
||||
def _existing_relation_pairs(session: Session, *, household_id: str) -> set[tuple[str, str]]:
|
||||
event_ids = select(models.FinancialEvent.id).where(models.FinancialEvent.household_id == household_id)
|
||||
rows = session.execute(
|
||||
select(models.EventRelation.from_event_id, models.EventRelation.to_event_id)
|
||||
.where(models.EventRelation.from_event_id.in_(event_ids))
|
||||
).all()
|
||||
return {(row[0], row[1]) for row in rows} | {(row[1], row[0]) for row in rows}
|
||||
|
||||
|
||||
def _decision_suggestion_ids(session: Session, *, household_id: str, decision: str) -> set[str]:
|
||||
return set(
|
||||
session.scalars(
|
||||
select(models.ReconciliationDecision.suggestion_id)
|
||||
.where(models.ReconciliationDecision.household_id == household_id)
|
||||
.where(models.ReconciliationDecision.decision == decision)
|
||||
).all()
|
||||
)
|
||||
|
||||
|
||||
def _record_reconciliation_decision(
|
||||
session: Session,
|
||||
*,
|
||||
household_id: str,
|
||||
suggestion: ReconciliationSuggestion,
|
||||
decision: str,
|
||||
actor_user_id: str | None = None,
|
||||
reason: str | None = None,
|
||||
) -> models.ReconciliationDecision:
|
||||
existing = session.scalars(
|
||||
select(models.ReconciliationDecision)
|
||||
.where(models.ReconciliationDecision.household_id == household_id)
|
||||
.where(models.ReconciliationDecision.suggestion_id == suggestion.id)
|
||||
.limit(1)
|
||||
).first()
|
||||
if existing:
|
||||
existing.decision = decision
|
||||
existing.kind = suggestion.kind
|
||||
existing.relation_type = suggestion.relation_type
|
||||
existing.primary_event_id = suggestion.primary_event.id
|
||||
existing.confirming_event_id = suggestion.confirming_event.id
|
||||
existing.confidence = suggestion.confidence
|
||||
existing.actor_user_id = actor_user_id
|
||||
existing.reason = reason
|
||||
session.add(existing)
|
||||
return existing
|
||||
row = models.ReconciliationDecision(
|
||||
household_id=household_id,
|
||||
suggestion_id=suggestion.id,
|
||||
decision=decision,
|
||||
kind=suggestion.kind,
|
||||
relation_type=suggestion.relation_type,
|
||||
primary_event_id=suggestion.primary_event.id,
|
||||
confirming_event_id=suggestion.confirming_event.id,
|
||||
confidence=suggestion.confidence,
|
||||
actor_user_id=actor_user_id,
|
||||
reason=reason,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return row
|
||||
|
||||
|
||||
def _first_observation(event: models.FinancialEvent) -> models.RawObservation | None:
|
||||
for link in event.observation_links:
|
||||
if link.observation:
|
||||
return link.observation
|
||||
return None
|
||||
|
||||
|
||||
def _first_source_type(event: models.FinancialEvent) -> SourceType | None:
|
||||
observation = _first_observation(event)
|
||||
return observation.source_type if observation else None
|
||||
|
||||
|
||||
def _event_summary(event: models.FinancialEvent) -> EventSummary:
|
||||
observation = _first_observation(event)
|
||||
return EventSummary(
|
||||
id=event.id,
|
||||
source_type=str(observation.source_type) if observation else None,
|
||||
event_type=str(event.event_type),
|
||||
status=str(event.status),
|
||||
event_date=event.event_date,
|
||||
amount=event.amount,
|
||||
currency=event.currency,
|
||||
merchant=event.merchant,
|
||||
counterparty=event.counterparty,
|
||||
description=event.description,
|
||||
source_reference=observation.source_reference if observation else None,
|
||||
provider_transaction_id=observation.provider_transaction_id if observation else None,
|
||||
observation_id=observation.id if observation else None,
|
||||
)
|
||||
|
||||
|
||||
def _suggestion_id(kind: str, primary_id: str, confirming_id: str) -> str:
|
||||
return f"{kind}:{primary_id}:{confirming_id}"
|
||||
|
||||
|
||||
def _party_text(event: models.FinancialEvent) -> str:
|
||||
parts = [event.merchant, event.counterparty, event.description]
|
||||
observation = _first_observation(event)
|
||||
if observation and observation.raw_payload_json:
|
||||
refs = observation.raw_payload_json.get("references")
|
||||
if isinstance(refs, dict):
|
||||
parts.extend(str(value) for value in refs.values() if value)
|
||||
return " ".join(part for part in parts if part)
|
||||
|
||||
|
||||
def _party_overlap_score(a: str, b: str) -> Decimal:
|
||||
left = set(normalize_counterparty(a).split())
|
||||
right = set(normalize_counterparty(b).split())
|
||||
if not left or not right:
|
||||
return Decimal("0")
|
||||
return Decimal(len(left & right)) / Decimal(max(len(left), len(right)))
|
||||
|
||||
|
||||
def _reference_score(a: models.FinancialEvent, b: models.FinancialEvent) -> Decimal:
|
||||
left = _references(a)
|
||||
right = _references(b)
|
||||
if not left or not right:
|
||||
return Decimal("0")
|
||||
return Decimal("0.22") if left & right else Decimal("0")
|
||||
|
||||
|
||||
def _references(event: models.FinancialEvent) -> set[str]:
|
||||
refs: set[str] = set()
|
||||
observation = _first_observation(event)
|
||||
if not observation:
|
||||
return refs
|
||||
for value in (observation.source_reference, observation.provider_transaction_id, observation.message_id):
|
||||
if value:
|
||||
refs.add(normalize_counterparty(value))
|
||||
payload = observation.raw_payload_json or {}
|
||||
raw_refs = payload.get("references")
|
||||
if isinstance(raw_refs, dict):
|
||||
refs.update(normalize_counterparty(str(value)) for value in raw_refs.values() if value)
|
||||
return {value for value in refs if value}
|
||||
|
||||
|
||||
def _provider_bridge(primary: models.FinancialEvent, confirming: models.FinancialEvent) -> bool:
|
||||
party = normalize_counterparty(_party_text(confirming))
|
||||
primary_source = _first_source_type(primary)
|
||||
confirming_source = _first_source_type(confirming)
|
||||
if primary_source in {SourceType.PAYPAL_EMAIL, SourceType.PAYPAL_CSV, SourceType.PAYPAL_API}:
|
||||
return "paypal" in party or confirming_source in {SourceType.PAYPAL_CSV, SourceType.PAYPAL_API}
|
||||
if primary_source in {SourceType.KLARNA_EMAIL, SourceType.KLARNA_CSV}:
|
||||
return "klarna" in party or confirming_source == SourceType.KLARNA_CSV
|
||||
return False
|
||||
|
||||
|
||||
def _max_date_window(primary: models.FinancialEvent, confirming: models.FinancialEvent, kind: str) -> int:
|
||||
primary_source = _first_source_type(primary)
|
||||
confirming_source = _first_source_type(confirming)
|
||||
if kind == "statement_bank":
|
||||
return 45
|
||||
if primary_source in {SourceType.KLARNA_EMAIL, SourceType.PAYPAL_EMAIL} and confirming_source in {
|
||||
SourceType.KLARNA_CSV,
|
||||
SourceType.PAYPAL_CSV,
|
||||
SourceType.PAYPAL_API,
|
||||
}:
|
||||
return 45
|
||||
return 21
|
||||
|
||||
|
||||
def _ensure_event_relation(
|
||||
session: Session,
|
||||
*,
|
||||
from_event_id: str,
|
||||
to_event_id: str,
|
||||
relation_type: RelationType,
|
||||
confidence: Decimal,
|
||||
) -> None:
|
||||
existing = session.scalars(
|
||||
select(models.EventRelation)
|
||||
.where(models.EventRelation.from_event_id == from_event_id)
|
||||
.where(models.EventRelation.to_event_id == to_event_id)
|
||||
.where(models.EventRelation.relation_type == relation_type)
|
||||
.limit(1)
|
||||
).first()
|
||||
if existing:
|
||||
return
|
||||
session.add(
|
||||
models.EventRelation(
|
||||
from_event_id=from_event_id,
|
||||
to_event_id=to_event_id,
|
||||
relation_type=relation_type,
|
||||
confidence=confidence,
|
||||
created_by="reconciliation",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _ensure_observation_link(
|
||||
session: Session,
|
||||
*,
|
||||
event_id: str,
|
||||
observation_id: str,
|
||||
relation: EventObservationRelation,
|
||||
confidence: Decimal,
|
||||
) -> None:
|
||||
existing = session.scalars(
|
||||
select(models.EventObservationLink)
|
||||
.where(models.EventObservationLink.event_id == event_id)
|
||||
.where(models.EventObservationLink.observation_id == observation_id)
|
||||
.where(models.EventObservationLink.relation == relation)
|
||||
.limit(1)
|
||||
).first()
|
||||
if existing:
|
||||
return
|
||||
session.add(
|
||||
models.EventObservationLink(
|
||||
event_id=event_id,
|
||||
observation_id=observation_id,
|
||||
relation=relation,
|
||||
confidence=confidence,
|
||||
)
|
||||
)
|
||||
166
apps/api/mousehold_api/services/settlement.py
Normal file
166
apps/api/mousehold_api/services/settlement.py
Normal file
@@ -0,0 +1,166 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
from mousehold_domain import AccountType, FinancialEventStatus, FinancialEventType
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, selectinload
|
||||
|
||||
from .. import models
|
||||
from ..schemas import SettlementSummary, SettlementTransfer, SettlementUserBalance
|
||||
|
||||
SETTLEMENT_STATUSES = {
|
||||
FinancialEventStatus.CONFIRMED,
|
||||
FinancialEventStatus.BOOKED,
|
||||
FinancialEventStatus.SETTLED,
|
||||
}
|
||||
SETTLEMENT_EVENT_TYPES = {
|
||||
FinancialEventType.EXPENSE,
|
||||
FinancialEventType.FEE,
|
||||
FinancialEventType.INTEREST,
|
||||
FinancialEventType.REFUND,
|
||||
FinancialEventType.REIMBURSEMENT,
|
||||
}
|
||||
|
||||
|
||||
def _money(value: Decimal | int | str = "0.00") -> Decimal:
|
||||
return Decimal(value).quantize(Decimal("0.01"))
|
||||
|
||||
|
||||
def calculate_settlement(
|
||||
session: Session,
|
||||
*,
|
||||
household_id: str,
|
||||
project_id: str | None = None,
|
||||
currency: str = "EUR",
|
||||
include_announced: bool = False,
|
||||
) -> SettlementSummary:
|
||||
accepted_statuses = set(SETTLEMENT_STATUSES)
|
||||
if include_announced:
|
||||
accepted_statuses.update({FinancialEventStatus.ANNOUNCED, FinancialEventStatus.OUTSTANDING})
|
||||
|
||||
query = (
|
||||
select(models.FinancialEvent)
|
||||
.where(models.FinancialEvent.household_id == household_id)
|
||||
.options(selectinload(models.FinancialEvent.split_lines))
|
||||
.order_by(models.FinancialEvent.event_date, models.FinancialEvent.created_at)
|
||||
)
|
||||
if project_id:
|
||||
query = query.where(models.FinancialEvent.project_id == project_id)
|
||||
events = list(session.scalars(query).all())
|
||||
|
||||
accounts_by_id = {
|
||||
account.id: account
|
||||
for account in session.scalars(
|
||||
select(models.Account).where(models.Account.household_id == household_id)
|
||||
).all()
|
||||
}
|
||||
paid: dict[str, Decimal] = {}
|
||||
share: dict[str, Decimal] = {}
|
||||
included: list[str] = []
|
||||
excluded: list[str] = []
|
||||
explanation: list[str] = []
|
||||
|
||||
for event in events:
|
||||
if (
|
||||
event.currency != currency
|
||||
or event.status not in accepted_statuses
|
||||
or event.event_type not in SETTLEMENT_EVENT_TYPES
|
||||
):
|
||||
excluded.append(event.id)
|
||||
continue
|
||||
payment_account = accounts_by_id.get(
|
||||
event.payment_account_id or event.source_account_id or ""
|
||||
)
|
||||
if (
|
||||
payment_account
|
||||
and payment_account.account_type == AccountType.SHARED_CHECKING
|
||||
and not payment_account.owner_user_id
|
||||
):
|
||||
excluded.append(event.id)
|
||||
explanation.append(f"{event.id} excluded because a neutral shared account paid it.")
|
||||
continue
|
||||
if not event.payer_user_id or not event.split_lines:
|
||||
excluded.append(event.id)
|
||||
explanation.append(f"{event.id} excluded because payer or split lines are missing.")
|
||||
continue
|
||||
|
||||
included.append(event.id)
|
||||
amount = _money(event.amount)
|
||||
multiplier = (
|
||||
Decimal("-1.00") if event.event_type == FinancialEventType.REFUND else Decimal("1.00")
|
||||
)
|
||||
paid[event.payer_user_id] = _money(
|
||||
paid.get(event.payer_user_id, Decimal("0.00")) + (amount * multiplier)
|
||||
)
|
||||
for split in event.split_lines:
|
||||
share[split.user_id] = _money(
|
||||
share.get(split.user_id, Decimal("0.00")) + (split.share_amount * multiplier)
|
||||
)
|
||||
|
||||
user_ids = sorted(set(paid) | set(share))
|
||||
balances = [
|
||||
SettlementUserBalance(
|
||||
user_id=user_id,
|
||||
paid=_money(paid.get(user_id, Decimal("0.00"))),
|
||||
share=_money(share.get(user_id, Decimal("0.00"))),
|
||||
net=_money(paid.get(user_id, Decimal("0.00")) - share.get(user_id, Decimal("0.00"))),
|
||||
)
|
||||
for user_id in user_ids
|
||||
]
|
||||
transfers = _minimize_transfers(balances, currency)
|
||||
explanation.append(
|
||||
f"Included {len(included)} confirmed/booked/settled events"
|
||||
+ (" plus announced/outstanding events." if include_announced else ".")
|
||||
)
|
||||
if not include_announced:
|
||||
explanation.append(
|
||||
"Announced and outstanding events are excluded from settlement by default."
|
||||
)
|
||||
|
||||
return SettlementSummary(
|
||||
household_id=household_id,
|
||||
project_id=project_id,
|
||||
currency=currency,
|
||||
included_event_ids=included,
|
||||
excluded_event_ids=excluded,
|
||||
balances=balances,
|
||||
suggested_transfers=transfers,
|
||||
explanation=explanation,
|
||||
)
|
||||
|
||||
|
||||
def _minimize_transfers(
|
||||
balances: list[SettlementUserBalance], currency: str
|
||||
) -> list[SettlementTransfer]:
|
||||
debtors = [
|
||||
{"user_id": balance.user_id, "amount": _money(-balance.net)}
|
||||
for balance in balances
|
||||
if balance.net < 0
|
||||
]
|
||||
creditors = [
|
||||
{"user_id": balance.user_id, "amount": _money(balance.net)}
|
||||
for balance in balances
|
||||
if balance.net > 0
|
||||
]
|
||||
transfers: list[SettlementTransfer] = []
|
||||
debtor_index = 0
|
||||
creditor_index = 0
|
||||
while debtor_index < len(debtors) and creditor_index < len(creditors):
|
||||
amount = min(debtors[debtor_index]["amount"], creditors[creditor_index]["amount"])
|
||||
if amount > 0:
|
||||
transfers.append(
|
||||
SettlementTransfer(
|
||||
from_user_id=debtors[debtor_index]["user_id"],
|
||||
to_user_id=creditors[creditor_index]["user_id"],
|
||||
amount=_money(amount),
|
||||
currency=currency,
|
||||
)
|
||||
)
|
||||
debtors[debtor_index]["amount"] = _money(debtors[debtor_index]["amount"] - amount)
|
||||
creditors[creditor_index]["amount"] = _money(creditors[creditor_index]["amount"] - amount)
|
||||
if debtors[debtor_index]["amount"] == 0:
|
||||
debtor_index += 1
|
||||
if creditors[creditor_index]["amount"] == 0:
|
||||
creditor_index += 1
|
||||
return transfers
|
||||
Reference in New Issue
Block a user