first ruggedy draft

apps/api/mousehold_api/services/ingestion.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+from decimal import Decimal
+
+from mousehold_domain import (
+    EventObservationRelation,
+    FinancialEventStatus,
+    FinancialEventType,
+    ObservationStatus,
+    SourceType,
+)
+from mousehold_matching import (
+    compute_normalized_transaction_fingerprint,
+    compute_raw_observation_hash,
+)
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from .. import models
+from ..schemas import ManualObservationCreate, ObservationImportCreate
+
+
+def infer_event_status(payload: ManualObservationCreate) -> FinancialEventStatus:
+    if payload.status:
+        return payload.status
+    if payload.source_type in {
+        SourceType.BANK_CSV,
+        SourceType.FINTS,
+        SourceType.PSD2_AGGREGATOR,
+        SourceType.PAYPAL_API,
+        SourceType.PAYPAL_CSV,
+        SourceType.KLARNA_CSV,
+    }:
+        return FinancialEventStatus.CONFIRMED
+    if (
+        payload.source_type in {SourceType.KLARNA_EMAIL, SourceType.PAYPAL_EMAIL}
+        and payload.due_date
+    ):
+        return FinancialEventStatus.OUTSTANDING
+    if payload.source_type in {
+        SourceType.ORDER_CONFIRMATION_EMAIL,
+        SourceType.IMAP_EMAIL,
+        SourceType.PDF_INVOICE,
+    }:
+        return FinancialEventStatus.ANNOUNCED
+    return FinancialEventStatus.CANDIDATE
+
+
+def infer_event_type(payload: ManualObservationCreate) -> FinancialEventType:
+    if payload.event_type != FinancialEventType.EXPENSE:
+        return payload.event_type
+    if payload.source_type not in {SourceType.BANK_CSV, SourceType.FINTS, SourceType.PSD2_AGGREGATOR}:
+        return payload.event_type
+    text = f"{payload.merchant or ''} {payload.counterparty or ''} {payload.description or ''}".lower()
+    liability_markers = (
+        "paypal",
+        "klarna",
+        "visa",
+        "mastercard",
+        "amex",
+        "american express",
+        "credit card",
+        "kreditkarte",
+        "kartenabrechnung",
+        "card repayment",
+    )
+    if any(marker in text for marker in liability_markers):
+        return FinancialEventType.LIABILITY_PAYMENT
+    return payload.event_type
+
+
+def create_observation_with_candidate(
+    session: Session,
+    payload: ManualObservationCreate | ObservationImportCreate,
+) -> tuple[models.RawObservation, models.FinancialEvent | None]:
+    raw_payload = payload.raw_payload_json or payload.model_dump(
+        mode="json", exclude={"split_lines"}
+    )
+    event_type = infer_event_type(payload)
+    event_status = infer_event_status(payload)
+    raw_hash = compute_raw_observation_hash(
+        source_type=str(payload.source_type),
+        source_reference=payload.source_reference
+        or payload.provider_transaction_id
+        or payload.message_id,
+        raw_text=payload.raw_text,
+        raw_payload=raw_payload,
+    )
+
+    duplicate = session.scalars(
+        select(models.RawObservation)
+        .where(models.RawObservation.household_id == payload.household_id)
+        .where(models.RawObservation.source_type == payload.source_type)
+        .where(models.RawObservation.raw_hash == raw_hash)
+        .limit(1)
+    ).first()
+
+    observation = models.RawObservation(
+        household_id=payload.household_id,
+        source_type=payload.source_type,
+        source_account_id=payload.source_account_id,
+        owner_user_id=payload.owner_user_id,
+        observed_at=payload.observed_at,
+        raw_hash=raw_hash,
+        source_reference=payload.source_reference,
+        provider_transaction_id=payload.provider_transaction_id,
+        message_id=payload.message_id,
+        raw_text=payload.raw_text,
+        raw_payload_json=raw_payload,
+        parsed_json={
+            "amount": str(payload.amount),
+            "currency": payload.currency,
+            "event_date": payload.event_date.isoformat(),
+            "booking_date": payload.booking_date.isoformat() if payload.booking_date else None,
+            "due_date": payload.due_date.isoformat() if payload.due_date else None,
+            "merchant": payload.merchant,
+            "counterparty": payload.counterparty,
+            "description": payload.description,
+            "event_type": str(event_type),
+            "status": str(event_status),
+        },
+        parser_name="manual-or-normalized-import",
+        parser_version="0.1.0",
+        confidence=Decimal("1.00"),
+        status=ObservationStatus.DUPLICATE_SOURCE
+        if duplicate
+        else ObservationStatus.CANDIDATE_CREATED,
+    )
+    session.add(observation)
+    session.flush()
+
+    if duplicate:
+        return observation, None
+
+    fingerprint = compute_normalized_transaction_fingerprint(
+        amount=payload.amount,
+        currency=payload.currency,
+        event_date=payload.event_date,
+        merchant=payload.merchant,
+        counterparty=payload.counterparty,
+        external_id=payload.provider_transaction_id,
+    )
+    event = models.FinancialEvent(
+        household_id=payload.household_id,
+        source_account_id=payload.source_account_id,
+        payment_account_id=payload.payment_account_id or payload.source_account_id,
+        payer_user_id=payload.payer_user_id or payload.owner_user_id,
+        event_type=event_type,
+        status=event_status,
+        event_date=payload.event_date,
+        booking_date=payload.booking_date,
+        due_date=payload.due_date,
+        amount=payload.amount,
+        currency=payload.currency,
+        merchant=payload.merchant,
+        counterparty=payload.counterparty,
+        description=payload.description,
+        category_id=payload.category_id,
+        project_id=payload.project_id,
+        created_by_user_id=payload.owner_user_id,
+        fingerprint=fingerprint,
+    )
+    session.add(event)
+    session.flush()
+
+    session.add(
+        models.EventObservationLink(
+            event_id=event.id,
+            observation_id=observation.id,
+            relation=EventObservationRelation.IMPORTED_FROM,
+            confidence=Decimal("1.00"),
+        )
+    )
+    for split in payload.split_lines:
+        session.add(
+            models.SplitLine(
+                event_id=event.id,
+                user_id=split.user_id,
+                share_amount=split.share_amount,
+                share_percent=split.share_percent,
+                reason=split.reason,
+            )
+        )
+    session.flush()
+    return observation, event
+
+
+def confirm_event(session: Session, event: models.FinancialEvent) -> models.FinancialEvent:
+    event.status = FinancialEventStatus.CONFIRMED
+    session.add(event)
+    session.flush()
+    return event