diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a040e6c --- /dev/null +++ b/.env.example @@ -0,0 +1,22 @@ +MOUSEHOLD_DATABASE_URL=sqlite:///./data/mousehold.db +MOUSEHOLD_CORS_ORIGINS=["http://localhost:5173","http://127.0.0.1:5173"] +MOUSEHOLD_LOCAL_AGENT_TOKEN=dev-local-agent-token +MOUSEHOLD_LOCAL_AGENT_CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173 + +# Local agent only. Do not put production secrets in shared env files. +MOUSEHOLD_IMAP_HOST=imap.example.com +MOUSEHOLD_IMAP_USERNAME=you@example.com +MOUSEHOLD_IMAP_PASSWORD=app-password +MOUSEHOLD_IMAP_FOLDER=INBOX +MOUSEHOLD_IMAP_SENDER_FILTERS=klarna,paypal,bestellung,order,invoice + +# Local agent FinTS connector. Prefer prompt over storing PIN in .env. +MOUSEHOLD_FINTS_BLZ=43060967 +MOUSEHOLD_FINTS_SERVER=https://fints.example-bank.de/fints +MOUSEHOLD_FINTS_USER_ID=your-bank-login-id +MOUSEHOLD_FINTS_CUSTOMER_ID= +MOUSEHOLD_FINTS_PIN= +MOUSEHOLD_FINTS_TAN_MECHANISM= +MOUSEHOLD_FINTS_TAN_MEDIUM= +MOUSEHOLD_FINTS_PRODUCT_ID=your-registered-fints-product-id +MOUSEHOLD_FINTS_PRODUCT_VERSION=0.1.0 diff --git a/.gitignore b/.gitignore index ccaad04..3131e06 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,10 @@ celerybeat.pid # Environments .env +.env.* +!.env.example +!apps/web/.env.development +!apps/web/.env.example .venv env/ venv/ @@ -234,6 +238,9 @@ web_modules/ # Optional stylelint cache .stylelintcache +# Local app state +data/ + # Microbundle cache .rpt2_cache/ .rts2_cache_cjs/ @@ -325,4 +332,3 @@ dist # Built Visual Studio Code Extensions *.vsix - diff --git a/README.md b/README.md index 653c270..6535db9 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,210 @@ -# mousehold +# Mousehold +Household finance ledger scaffold for an evidence-driven household budget and settlement system. + +The first milestone is a local proof of concept: + +- Python/FastAPI backend with SQLite by default. +- React/Vite web app. +- Local-agent scaffold for bank, CSV, statement, and IMAP ingestion. +- Raw observations become candidate, announced, outstanding, or confirmed financial events. +- Settlement and project summaries are calculated from confirmed ledger state. + +## Run + +```sh +python -m venv .venv +. .venv/bin/activate +pip install -r requirements-dev.txt +pip install -e . +mousehold-api +``` + +In another shell: + +```sh +npm install +npm run dev:web +``` + +Open `http://127.0.0.1:5173`. + +The API listens on `http://127.0.0.1:8000` and exposes OpenAPI at `/docs`. +If `8000` is already occupied: + +```sh +uvicorn mousehold_api.main:app --host 127.0.0.1 --port 8001 +cd apps/web +npm run dev -- --host 127.0.0.1 --port 5173 +``` + +## MVP workflow + +Use `Setup` to create a household, users, accounts, and connector profiles. The +top bar has a `Signed in` selector; choosing a user creates a local session +cookie for that browser. This is passwordless local-session support for the +self-hosted MVP, not a hosted multi-tenant auth system. + +Create FinTS, IMAP, or CSV/statement connector profiles in `Setup` before using +the `Bank`, `Mail`, or `Statements` pages. Those pages use the saved profile for +defaults and import attribution, while PIN/TAN, IMAP password, and selected CSV +files stay ephemeral. + +Use `Inbox` after manual entry or imports to review source observations. You can +keep an observation, confirm its linked event, ignore just the observation, +ignore the linked event too, or link the observation to an existing ledger event. +Use `Import runs` on the dashboard or connector pages to see imported, +duplicate, and event counts from each sync. + +Use `Setup -> Backup` to export the household as JSON or create a local SQLite +backup file. + +Browser-driven FinTS access is embedded in the API under `/agent/fints/...`. +Use the `FinTS` panel in the web app. PIN/TAN inputs go to the API process you +are running locally or self-hosting; normalized observations are imported into +the ledger only when you click import. + +For comdirect, choose the `comdirect` preset in the FinTS panel. Enter your +comdirect Zugangsnummer as the bank user ID, leave customer ID empty, and enter +your registered FinTS product ID. + +IMAP access is embedded under `/agent/imap/...` and available from the `Mail` +panel. Enter the IMAP host, login, app password, folder, and filters, then list +folders or scan. The scan uses a read-only folder selection and stages parsed +messages in the browser. Ledger observations are stored only when you click +import. + +Use the `Statements` panel for PayPal Activity Download CSV files and Klarna CSV +statements. It previews rows in the browser and imports them as `paypal_csv` or +`klarna_csv` confirmation evidence. + +Use the `Reconcile` panel after imports to find likely links between early mail +evidence, PayPal/Klarna statement rows, and bank/FinTS cash movement. Accepting a +suggestion links the evidence and can upgrade the original announced/outstanding +event to confirmed. + +## Local Agent + +Register a local agent after creating a household and user: + +```sh +mousehold-agent register \ + --household-id \ + --owner-user-id +``` + +Import demo observations: + +```sh +mousehold-agent sync-demo \ + --connection-id \ + --household-id \ + --owner-user-id \ + --source-account-id +``` + +Import bank or PayPal CSV: + +```sh +mousehold-agent sync-csv \ + --connection-id \ + --household-id \ + --owner-user-id \ + --source-account-id \ + --source-type bank_csv \ + --path ./transactions.csv +``` + +Discover FinTS accounts locally: + +```sh +mousehold-agent fints-accounts \ + --profile my-bank \ + --blz \ + --server \ + --bank-user-id \ + --with-balances +``` + +Display recent FinTS transactions locally: + +```sh +mousehold-agent fints-transactions \ + --profile my-bank \ + --blz \ + --server \ + --bank-user-id \ + --iban \ + --days 30 +``` + +Import FinTS transactions into the ledger: + +```sh +mousehold-agent sync-fints \ + --api-url http://127.0.0.1:8001 \ + --connection-id \ + --household-id \ + --owner-user-id \ + --source-account-id \ + --profile my-bank \ + --blz \ + --server \ + --bank-user-id \ + --iban \ + --days 30 \ + --display +``` + +The FinTS connector prompts for the PIN if `MOUSEHOLD_FINTS_PIN` or `--pin` is +not supplied. TAN mechanism and TAN medium selection are handled interactively +and cached under `~/.config/mousehold/fints/.json`; the cache does not +store the PIN. + +You can also run the FinTS browser bridge as a separate local-only process: + +```sh +mousehold-agent serve --host 127.0.0.1 --port 8765 +``` + +Set `VITE_LOCAL_AGENT_URL=http://127.0.0.1:8765` in `apps/web/.env.development` +if you choose this mode. + +CLI IMAP scanning can still use local environment variables: + +```sh +MOUSEHOLD_IMAP_HOST=imap.example.com +MOUSEHOLD_IMAP_USERNAME=you@example.com +MOUSEHOLD_IMAP_PASSWORD=app-password +MOUSEHOLD_IMAP_SENDER_FILTERS=klarna,paypal,bestellung,order,invoice +``` + +In embedded connector mode, the API process receives FinTS PIN/TAN and IMAP +password inputs while the request is running. In separate local-agent mode, +those inputs stay out of the API process. In both modes, connector metadata and +normalized observations are what get stored; credentials are not written to the +database. + +## Scope + +FinTS account and transaction retrieval is implemented and can run embedded in +the API or in the separate local agent. It still depends on your bank supporting +FinTS PIN/TAN access and on you providing the bank-specific endpoint and login +data. IMAP folder listing, scanning, email parsing, preview, and import are +implemented for the PoC. PayPal CSV and email evidence are the supported PayPal +PoC path. PSD2 aggregator connectors are still scaffolded. + +## Tests + +```sh +pytest +npm run lint:web +npm run build:web +``` + +The Playwright MVP smoke test expects the API and web app to be running: + +```sh +npx playwright install +npm run test:e2e +``` diff --git a/apps/api/mousehold_api/__init__.py b/apps/api/mousehold_api/__init__.py new file mode 100644 index 0000000..3dc1f76 --- /dev/null +++ b/apps/api/mousehold_api/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/apps/api/mousehold_api/auth.py b/apps/api/mousehold_api/auth.py new file mode 100644 index 0000000..e73bceb --- /dev/null +++ b/apps/api/mousehold_api/auth.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from hashlib import sha256 +from secrets import token_urlsafe + +from fastapi import Cookie, Depends, HTTPException, Response +from sqlalchemy import select +from sqlalchemy.orm import Session + +from . import models +from .db import get_session + +SESSION_COOKIE = "mousehold_session" +SESSION_DAYS = 30 + + +def hash_session_token(token: str) -> str: + return sha256(token.encode("utf-8")).hexdigest() + + +def create_local_session(session: Session, user: models.User) -> tuple[models.LocalSession, str]: + token = token_urlsafe(32) + now = datetime.now(UTC) + local_session = models.LocalSession( + user_id=user.id, + household_id=user.household_id, + token_hash=hash_session_token(token), + created_at=now, + last_seen_at=now, + expires_at=now + timedelta(days=SESSION_DAYS), + ) + session.add(local_session) + session.flush() + return local_session, token + + +def set_session_cookie(response: Response, token: str) -> None: + response.set_cookie( + SESSION_COOKIE, + token, + httponly=True, + samesite="lax", + secure=False, + max_age=SESSION_DAYS * 24 * 60 * 60, + path="/", + ) + + +def clear_session_cookie(response: Response) -> None: + response.delete_cookie(SESSION_COOKIE, path="/") + + +def get_optional_local_session( + session_token: str | None = Cookie(default=None, alias=SESSION_COOKIE), + session: Session = Depends(get_session), +) -> models.LocalSession | None: + if not session_token: + return None + now = datetime.now(UTC) + local_session = session.scalars( + select(models.LocalSession) + .where(models.LocalSession.token_hash == hash_session_token(session_token)) + .where(models.LocalSession.revoked_at.is_(None)) + .where(models.LocalSession.expires_at > now) + .limit(1) + ).first() + if local_session: + local_session.last_seen_at = now + session.add(local_session) + session.commit() + session.refresh(local_session) + return local_session + + +def require_local_session( + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> models.LocalSession: + if not local_session: + raise HTTPException(status_code=401, detail="Local session required") + return local_session + + +def require_household_access(household_id: str, local_session: models.LocalSession | None) -> None: + if local_session and local_session.household_id != household_id: + raise HTTPException(status_code=403, detail="Session does not belong to this household") diff --git a/apps/api/mousehold_api/config.py b/apps/api/mousehold_api/config.py new file mode 100644 index 0000000..04a9dbc --- /dev/null +++ b/apps/api/mousehold_api/config.py @@ -0,0 +1,18 @@ +from functools import lru_cache + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + app_name: str = "Mousehold API" + api_prefix: str = "" + database_url: str = "sqlite:///./data/mousehold.db" + cors_origins: list[str] = ["http://localhost:5173", "http://127.0.0.1:5173"] + local_agent_token: str = "dev-local-agent-token" + + model_config = SettingsConfigDict(env_prefix="MOUSEHOLD_", env_file=".env", extra="ignore") + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/apps/api/mousehold_api/db.py b/apps/api/mousehold_api/db.py new file mode 100644 index 0000000..0a8cf74 --- /dev/null +++ b/apps/api/mousehold_api/db.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from collections.abc import Generator +from pathlib import Path + +from sqlalchemy import create_engine +from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker +from sqlalchemy.pool import StaticPool + +from .config import get_settings + + +class Base(DeclarativeBase): + pass + + +def _connect_args(database_url: str) -> dict[str, object]: + if database_url.startswith("sqlite"): + return {"check_same_thread": False} + return {} + + +def _engine_kwargs(database_url: str) -> dict[str, object]: + if database_url in {"sqlite://", "sqlite:///:memory:"}: + return {"poolclass": StaticPool} + return {} + + +def _ensure_sqlite_parent(database_url: str) -> None: + if not database_url.startswith("sqlite:///"): + return + db_path = database_url.replace("sqlite:///", "", 1) + if db_path not in (":memory:", ""): + Path(db_path).parent.mkdir(parents=True, exist_ok=True) + + +settings = get_settings() +_ensure_sqlite_parent(settings.database_url) +engine = create_engine( + settings.database_url, + connect_args=_connect_args(settings.database_url), + **_engine_kwargs(settings.database_url), +) +SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, expire_on_commit=False) + + +def init_db() -> None: + from . import models # noqa: F401 + + Base.metadata.create_all(bind=engine) + + +def get_session() -> Generator[Session, None, None]: + session = SessionLocal() + try: + yield session + finally: + session.close() diff --git a/apps/api/mousehold_api/main.py b/apps/api/mousehold_api/main.py new file mode 100644 index 0000000..aa55833 --- /dev/null +++ b/apps/api/mousehold_api/main.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import uvicorn +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from mousehold_agent.server import create_router as create_agent_router + +from .config import get_settings +from .db import init_db +from .routers import ( + accounts, + auth, + backup, + connections, + events, + households, + observations, + projects, + reconciliation, + recurring, + settlement, + taxonomy, +) + + +def create_app() -> FastAPI: + settings = get_settings() + app = FastAPI(title=settings.app_name, version="0.1.0") + app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + @app.on_event("startup") + def startup() -> None: + init_db() + + @app.get("/health") + def health() -> dict[str, str]: + return {"status": "ok"} + + app.include_router(households.router, prefix=settings.api_prefix) + app.include_router(auth.router, prefix=settings.api_prefix) + app.include_router(backup.router, prefix=settings.api_prefix) + app.include_router(accounts.router, prefix=settings.api_prefix) + app.include_router(observations.router, prefix=settings.api_prefix) + app.include_router(events.router, prefix=settings.api_prefix) + app.include_router(settlement.router, prefix=settings.api_prefix) + app.include_router(reconciliation.router, prefix=settings.api_prefix) + app.include_router(projects.router, prefix=settings.api_prefix) + app.include_router(recurring.router, prefix=settings.api_prefix) + app.include_router(taxonomy.router, prefix=settings.api_prefix) + app.include_router(connections.router, prefix=settings.api_prefix) + app.include_router(create_agent_router(), prefix=f"{settings.api_prefix.rstrip('/')}/agent") + return app + + +app = create_app() + + +def run() -> None: + uvicorn.run("mousehold_api.main:app", host="127.0.0.1", port=8000, reload=True) diff --git a/apps/api/mousehold_api/models.py b/apps/api/mousehold_api/models.py new file mode 100644 index 0000000..431404a --- /dev/null +++ b/apps/api/mousehold_api/models.py @@ -0,0 +1,643 @@ +from __future__ import annotations + +from datetime import UTC, date, datetime +from decimal import Decimal +from uuid import uuid4 + +from mousehold_domain import ( + AccountType, + EventObservationRelation, + FinancialEventStatus, + FinancialEventType, + ImportConnectionStatus, + ImportConnectionType, + ObservationStatus, + PaymentLineStatus, + PaymentStatus, + PlanningStatus, + ProjectStatus, + ReconciliationStatus, + RecurringCommitmentStatus, + RecurringInterval, + RelationType, + SettlementStatus, + SourceType, + VisibilityMode, +) +from sqlalchemy import ( + Boolean, + Date, + DateTime, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, +) +from sqlalchemy import Enum as SqlEnum +from sqlalchemy.dialects.sqlite import JSON +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .db import Base + + +def new_id() -> str: + return str(uuid4()) + + +def utcnow() -> datetime: + return datetime.now(UTC) + + +Money = Numeric(14, 2) + + +class Household(Base): + __tablename__ = "households" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + name: Mapped[str] = mapped_column(String(200), nullable=False) + default_currency: Mapped[str] = mapped_column(String(3), default="EUR", nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + + users: Mapped[list[User]] = relationship( + back_populates="household", cascade="all, delete-orphan" + ) + accounts: Mapped[list[Account]] = relationship( + back_populates="household", cascade="all, delete-orphan" + ) + + +class User(Base): + __tablename__ = "users" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + display_name: Mapped[str] = mapped_column(String(200), nullable=False) + email: Mapped[str | None] = mapped_column(String(320), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + + household: Mapped[Household] = relationship(back_populates="users") + login_identities: Mapped[list[LoginIdentity]] = relationship( + back_populates="user", cascade="all, delete-orphan" + ) + + +class LoginIdentity(Base): + __tablename__ = "login_identities" + __table_args__ = ( + UniqueConstraint("provider", "provider_subject", name="uq_login_provider_subject"), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + user_id: Mapped[str] = mapped_column(ForeignKey("users.id"), nullable=False, index=True) + provider: Mapped[str] = mapped_column(String(80), nullable=False) + provider_subject: Mapped[str] = mapped_column(String(320), nullable=False) + email: Mapped[str | None] = mapped_column(String(320), nullable=True) + display_name: Mapped[str | None] = mapped_column(String(200), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + + user: Mapped[User] = relationship(back_populates="login_identities") + + +class LocalSession(Base): + __tablename__ = "local_sessions" + __table_args__ = ( + Index("ix_local_session_token", "token_hash"), + Index("ix_local_session_user", "user_id", "expires_at"), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + user_id: Mapped[str] = mapped_column(ForeignKey("users.id"), nullable=False, index=True) + household_id: Mapped[str] = mapped_column(ForeignKey("households.id"), nullable=False, index=True) + token_hash: Mapped[str] = mapped_column(String(64), nullable=False, unique=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class Account(Base): + __tablename__ = "accounts" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + owner_user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id"), nullable=True, index=True + ) + name: Mapped[str] = mapped_column(String(200), nullable=False) + account_type: Mapped[AccountType] = mapped_column(SqlEnum(AccountType), nullable=False) + institution_name: Mapped[str | None] = mapped_column(String(200), nullable=True) + currency: Mapped[str] = mapped_column(String(3), default="EUR", nullable=False) + visibility_mode: Mapped[VisibilityMode] = mapped_column(SqlEnum(VisibilityMode), nullable=False) + is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + + household: Mapped[Household] = relationship(back_populates="accounts") + + +class RawObservation(Base): + __tablename__ = "raw_observations" + __table_args__ = ( + Index("ix_observation_dedupe", "household_id", "source_type", "raw_hash"), + Index("ix_observation_external_refs", "provider_transaction_id", "message_id"), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + source_type: Mapped[SourceType] = mapped_column(SqlEnum(SourceType), nullable=False, index=True) + source_account_id: Mapped[str | None] = mapped_column( + ForeignKey("accounts.id"), nullable=True, index=True + ) + owner_user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id"), nullable=True, index=True + ) + observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + received_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + raw_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + source_reference: Mapped[str | None] = mapped_column(String(500), nullable=True) + provider_transaction_id: Mapped[str | None] = mapped_column( + String(200), nullable=True, index=True + ) + message_id: Mapped[str | None] = mapped_column(String(500), nullable=True, index=True) + raw_text: Mapped[str | None] = mapped_column(Text, nullable=True) + raw_file_path: Mapped[str | None] = mapped_column(String(1000), nullable=True) + raw_payload_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + parsed_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + parser_name: Mapped[str | None] = mapped_column(String(120), nullable=True) + parser_version: Mapped[str | None] = mapped_column(String(40), nullable=True) + confidence: Mapped[Decimal] = mapped_column(Money, default=Decimal("1.00"), nullable=False) + status: Mapped[ObservationStatus] = mapped_column( + SqlEnum(ObservationStatus), default=ObservationStatus.NEW, nullable=False + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + + event_links: Mapped[list[EventObservationLink]] = relationship( + back_populates="observation", cascade="all, delete-orphan" + ) + + +class FinancialEvent(Base): + __tablename__ = "financial_events" + __table_args__ = ( + Index("ix_events_household_status_date", "household_id", "status", "event_date"), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + source_account_id: Mapped[str | None] = mapped_column( + ForeignKey("accounts.id"), nullable=True, index=True + ) + payment_account_id: Mapped[str | None] = mapped_column( + ForeignKey("accounts.id"), nullable=True, index=True + ) + payer_user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id"), nullable=True, index=True + ) + event_type: Mapped[FinancialEventType] = mapped_column( + SqlEnum(FinancialEventType), nullable=False + ) + status: Mapped[FinancialEventStatus] = mapped_column( + SqlEnum(FinancialEventStatus), default=FinancialEventStatus.CANDIDATE, nullable=False + ) + event_date: Mapped[date] = mapped_column(Date, nullable=False) + booking_date: Mapped[date | None] = mapped_column(Date, nullable=True) + due_date: Mapped[date | None] = mapped_column(Date, nullable=True) + amount: Mapped[Decimal] = mapped_column(Money, nullable=False) + currency: Mapped[str] = mapped_column(String(3), default="EUR", nullable=False) + merchant: Mapped[str | None] = mapped_column(String(300), nullable=True) + counterparty: Mapped[str | None] = mapped_column(String(300), nullable=True) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + category_id: Mapped[str | None] = mapped_column( + ForeignKey("categories.id"), nullable=True, index=True + ) + project_id: Mapped[str | None] = mapped_column( + ForeignKey("projects.id"), nullable=True, index=True + ) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id"), nullable=True, index=True + ) + fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False + ) + + observation_links: Mapped[list[EventObservationLink]] = relationship( + back_populates="event", cascade="all, delete-orphan" + ) + split_lines: Mapped[list[SplitLine]] = relationship( + back_populates="event", cascade="all, delete-orphan" + ) + + +class EventObservationLink(Base): + __tablename__ = "event_observation_links" + __table_args__ = ( + UniqueConstraint( + "event_id", "observation_id", "relation", name="uq_event_observation_relation" + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + event_id: Mapped[str] = mapped_column( + ForeignKey("financial_events.id"), nullable=False, index=True + ) + observation_id: Mapped[str] = mapped_column( + ForeignKey("raw_observations.id"), nullable=False, index=True + ) + relation: Mapped[EventObservationRelation] = mapped_column( + SqlEnum(EventObservationRelation), nullable=False + ) + confidence: Mapped[Decimal] = mapped_column(Money, default=Decimal("1.00"), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + + event: Mapped[FinancialEvent] = relationship(back_populates="observation_links") + observation: Mapped[RawObservation] = relationship(back_populates="event_links") + + +class EventRelation(Base): + __tablename__ = "event_relations" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + from_event_id: Mapped[str] = mapped_column( + ForeignKey("financial_events.id"), nullable=False, index=True + ) + to_event_id: Mapped[str] = mapped_column( + ForeignKey("financial_events.id"), nullable=False, index=True + ) + relation_type: Mapped[RelationType] = mapped_column(SqlEnum(RelationType), nullable=False) + confidence: Mapped[Decimal] = mapped_column(Money, default=Decimal("1.00"), nullable=False) + created_by: Mapped[str] = mapped_column(String(40), default="system", nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + + +class ReconciliationDecision(Base): + __tablename__ = "reconciliation_decisions" + __table_args__ = ( + UniqueConstraint("household_id", "suggestion_id", name="uq_reconciliation_decision"), + Index("ix_reconciliation_decision_household", "household_id", "decision"), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + suggestion_id: Mapped[str] = mapped_column(String(160), nullable=False) + decision: Mapped[str] = mapped_column(String(40), nullable=False) + kind: Mapped[str] = mapped_column(String(80), nullable=False) + relation_type: Mapped[RelationType] = mapped_column(SqlEnum(RelationType), nullable=False) + primary_event_id: Mapped[str] = mapped_column( + ForeignKey("financial_events.id"), nullable=False, index=True + ) + confirming_event_id: Mapped[str] = mapped_column( + ForeignKey("financial_events.id"), nullable=False, index=True + ) + confidence: Mapped[Decimal] = mapped_column(Money, default=Decimal("1.00"), nullable=False) + actor_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id"), nullable=True) + reason: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False + ) + + +class SplitLine(Base): + __tablename__ = "split_lines" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + event_id: Mapped[str] = mapped_column( + ForeignKey("financial_events.id"), nullable=False, index=True + ) + user_id: Mapped[str] = mapped_column(ForeignKey("users.id"), nullable=False, index=True) + share_amount: Mapped[Decimal] = mapped_column(Money, nullable=False) + share_percent: Mapped[Decimal | None] = mapped_column(Money, nullable=True) + reason: Mapped[str] = mapped_column(String(80), default="manual", nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + + event: Mapped[FinancialEvent] = relationship(back_populates="split_lines") + + +class Category(Base): + __tablename__ = "categories" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + parent_id: Mapped[str | None] = mapped_column(ForeignKey("categories.id"), nullable=True) + name: Mapped[str] = mapped_column(String(160), nullable=False) + path: Mapped[str] = mapped_column(String(500), nullable=False) + is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + + +class Tag(Base): + __tablename__ = "tags" + __table_args__ = (UniqueConstraint("household_id", "name", name="uq_tag_household_name"),) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + name: Mapped[str] = mapped_column(String(120), nullable=False) + + +class EventTag(Base): + __tablename__ = "event_tags" + __table_args__ = (UniqueConstraint("event_id", "tag_id", name="uq_event_tag"),) + + event_id: Mapped[str] = mapped_column(ForeignKey("financial_events.id"), primary_key=True) + tag_id: Mapped[str] = mapped_column(ForeignKey("tags.id"), primary_key=True) + + +class Project(Base): + __tablename__ = "projects" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + name: Mapped[str] = mapped_column(String(220), nullable=False) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + start_date: Mapped[date | None] = mapped_column(Date, nullable=True) + end_date: Mapped[date | None] = mapped_column(Date, nullable=True) + default_split_rule_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + status: Mapped[ProjectStatus] = mapped_column( + SqlEnum(ProjectStatus), default=ProjectStatus.PLANNING, nullable=False + ) + currency: Mapped[str] = mapped_column(String(3), default="EUR", nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False + ) + + +class ProjectParticipant(Base): + __tablename__ = "project_participants" + __table_args__ = (UniqueConstraint("project_id", "user_id", name="uq_project_participant"),) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + project_id: Mapped[str] = mapped_column(ForeignKey("projects.id"), nullable=False, index=True) + user_id: Mapped[str] = mapped_column(ForeignKey("users.id"), nullable=False, index=True) + role: Mapped[str] = mapped_column(String(80), default="participant", nullable=False) + default_share_percent: Mapped[Decimal | None] = mapped_column(Money, nullable=True) + + +class ProjectBudgetLine(Base): + __tablename__ = "project_budget_lines" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + project_id: Mapped[str] = mapped_column(ForeignKey("projects.id"), nullable=False, index=True) + category_id: Mapped[str | None] = mapped_column(ForeignKey("categories.id"), nullable=True) + name: Mapped[str] = mapped_column(String(180), nullable=False) + budgeted_amount: Mapped[Decimal] = mapped_column(Money, default=Decimal("0.00"), nullable=False) + committed_amount_cached: Mapped[Decimal | None] = mapped_column(Money, nullable=True) + paid_amount_cached: Mapped[Decimal | None] = mapped_column(Money, nullable=True) + sort_order: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + notes: Mapped[str | None] = mapped_column(Text, nullable=True) + + +class ProjectPlannedItem(Base): + __tablename__ = "project_planned_items" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + project_id: Mapped[str] = mapped_column(ForeignKey("projects.id"), nullable=False, index=True) + budget_line_id: Mapped[str | None] = mapped_column( + ForeignKey("project_budget_lines.id"), nullable=True, index=True + ) + description: Mapped[str] = mapped_column(String(300), nullable=False) + estimated_amount: Mapped[Decimal] = mapped_column( + Money, default=Decimal("0.00"), nullable=False + ) + committed_amount: Mapped[Decimal | None] = mapped_column(Money, nullable=True) + expected_date: Mapped[date | None] = mapped_column(Date, nullable=True) + due_date: Mapped[date | None] = mapped_column(Date, nullable=True) + expected_payer_account_id: Mapped[str | None] = mapped_column( + ForeignKey("accounts.id"), nullable=True + ) + split_rule_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + planning_status: Mapped[PlanningStatus] = mapped_column( + SqlEnum(PlanningStatus), default=PlanningStatus.IDEA, nullable=False + ) + payment_status: Mapped[PaymentStatus] = mapped_column( + SqlEnum(PaymentStatus), default=PaymentStatus.UNPAID, nullable=False + ) + reconciliation_status: Mapped[ReconciliationStatus] = mapped_column( + SqlEnum(ReconciliationStatus), default=ReconciliationStatus.NO_EVIDENCE, nullable=False + ) + settlement_status: Mapped[SettlementStatus] = mapped_column( + SqlEnum(SettlementStatus), default=SettlementStatus.UNSETTLED, nullable=False + ) + notes: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False + ) + + +class PaymentLine(Base): + __tablename__ = "payment_lines" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + planned_item_id: Mapped[str | None] = mapped_column( + ForeignKey("project_planned_items.id"), nullable=True, index=True + ) + event_id: Mapped[str | None] = mapped_column( + ForeignKey("financial_events.id"), nullable=True, index=True + ) + amount: Mapped[Decimal] = mapped_column(Money, nullable=False) + currency: Mapped[str] = mapped_column(String(3), default="EUR", nullable=False) + date: Mapped[date] = mapped_column(Date, nullable=False) + paid_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("users.id"), nullable=True, index=True + ) + payment_account_id: Mapped[str | None] = mapped_column( + ForeignKey("accounts.id"), nullable=True, index=True + ) + source_observation_id: Mapped[str | None] = mapped_column( + ForeignKey("raw_observations.id"), nullable=True + ) + status: Mapped[PaymentLineStatus] = mapped_column( + SqlEnum(PaymentLineStatus), default=PaymentLineStatus.EXPECTED, nullable=False + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + + +class RecurringCommitment(Base): + __tablename__ = "recurring_commitments" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + provider: Mapped[str] = mapped_column(String(200), nullable=False) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + amount_expected: Mapped[Decimal] = mapped_column(Money, nullable=False) + amount_min: Mapped[Decimal | None] = mapped_column(Money, nullable=True) + amount_max: Mapped[Decimal | None] = mapped_column(Money, nullable=True) + currency: Mapped[str] = mapped_column(String(3), default="EUR", nullable=False) + interval: Mapped[RecurringInterval] = mapped_column(SqlEnum(RecurringInterval), nullable=False) + next_due_date: Mapped[date] = mapped_column(Date, nullable=False) + payer_account_id: Mapped[str | None] = mapped_column(ForeignKey("accounts.id"), nullable=True) + split_rule_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + category_id: Mapped[str | None] = mapped_column(ForeignKey("categories.id"), nullable=True) + project_id: Mapped[str | None] = mapped_column(ForeignKey("projects.id"), nullable=True) + notice_period: Mapped[str | None] = mapped_column(String(120), nullable=True) + renewal_date: Mapped[date | None] = mapped_column(Date, nullable=True) + status: Mapped[RecurringCommitmentStatus] = mapped_column( + SqlEnum(RecurringCommitmentStatus), default=RecurringCommitmentStatus.ACTIVE, nullable=False + ) + matching_rule_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False + ) + + +class PromisedPayment(Base): + __tablename__ = "promised_payments" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + related_event_id: Mapped[str | None] = mapped_column( + ForeignKey("financial_events.id"), nullable=True, index=True + ) + provider: Mapped[str] = mapped_column(String(120), nullable=False) + amount: Mapped[Decimal] = mapped_column(Money, nullable=False) + currency: Mapped[str] = mapped_column(String(3), default="EUR", nullable=False) + created_date: Mapped[date] = mapped_column(Date, nullable=False) + due_date: Mapped[date] = mapped_column(Date, nullable=False, index=True) + expected_funding_account_id: Mapped[str | None] = mapped_column( + ForeignKey("accounts.id"), nullable=True + ) + responsible_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id"), nullable=True) + status: Mapped[str] = mapped_column(String(40), default="expected", nullable=False) + source_observation_id: Mapped[str | None] = mapped_column( + ForeignKey("raw_observations.id"), nullable=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False + ) + + +class ImportConnection(Base): + __tablename__ = "import_connections" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + owner_user_id: Mapped[str] = mapped_column(ForeignKey("users.id"), nullable=False, index=True) + connection_type: Mapped[ImportConnectionType] = mapped_column( + SqlEnum(ImportConnectionType), nullable=False + ) + display_name: Mapped[str] = mapped_column(String(200), nullable=False) + status: Mapped[ImportConnectionStatus] = mapped_column( + SqlEnum(ImportConnectionStatus), default=ImportConnectionStatus.NEEDS_AUTH, nullable=False + ) + last_sync_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + secret_storage_ref: Mapped[str | None] = mapped_column(String(500), nullable=True) + runs_locally: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + connector_config_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False + ) + + +class ImportRun(Base): + __tablename__ = "import_runs" + __table_args__ = ( + Index("ix_import_runs_household_started", "household_id", "started_at"), + Index("ix_import_runs_connection_started", "connection_id", "started_at"), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + connection_id: Mapped[str | None] = mapped_column( + ForeignKey("import_connections.id"), nullable=True, index=True + ) + actor_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id"), nullable=True) + source_type: Mapped[str | None] = mapped_column(String(80), nullable=True) + status: Mapped[str] = mapped_column(String(40), default="running", nullable=False) + imported_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + duplicate_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + created_event_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + skipped_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + summary_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class AuditLog(Base): + __tablename__ = "audit_log" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + household_id: Mapped[str] = mapped_column( + ForeignKey("households.id"), nullable=False, index=True + ) + actor_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id"), nullable=True) + action: Mapped[str] = mapped_column(String(120), nullable=False) + entity_type: Mapped[str] = mapped_column(String(120), nullable=False) + entity_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) diff --git a/apps/api/mousehold_api/routers/__init__.py b/apps/api/mousehold_api/routers/__init__.py new file mode 100644 index 0000000..c5cfac8 --- /dev/null +++ b/apps/api/mousehold_api/routers/__init__.py @@ -0,0 +1 @@ +"""FastAPI routers.""" diff --git a/apps/api/mousehold_api/routers/accounts.py b/apps/api/mousehold_api/routers/accounts.py new file mode 100644 index 0000000..eeb7761 --- /dev/null +++ b/apps/api/mousehold_api/routers/accounts.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .. import models +from ..db import get_session +from ..schemas import AccountCreate, AccountRead, AccountUpdate + +router = APIRouter(tags=["accounts"]) + + +@router.post("/accounts", response_model=AccountRead) +def create_account( + payload: AccountCreate, session: Session = Depends(get_session) +) -> models.Account: + if not session.get(models.Household, payload.household_id): + raise HTTPException(status_code=404, detail="Household not found") + account = models.Account(**payload.model_dump()) + session.add(account) + session.commit() + session.refresh(account) + return account + + +@router.get("/accounts", response_model=list[AccountRead]) +def list_accounts( + household_id: str, session: Session = Depends(get_session) +) -> list[models.Account]: + return list( + session.scalars( + select(models.Account) + .where(models.Account.household_id == household_id) + .order_by(models.Account.name) + ).all() + ) + + +@router.patch("/accounts/{account_id}", response_model=AccountRead) +def update_account( + account_id: str, + payload: AccountUpdate, + session: Session = Depends(get_session), +) -> models.Account: + account = session.get(models.Account, account_id) + if not account: + raise HTTPException(status_code=404, detail="Account not found") + for field, value in payload.model_dump(exclude_unset=True).items(): + if field == "currency" and value is not None: + value = value.upper() + setattr(account, field, value) + session.add(account) + session.commit() + session.refresh(account) + return account + + +@router.delete("/accounts/{account_id}") +def delete_account(account_id: str, session: Session = Depends(get_session)) -> dict[str, bool]: + account = session.get(models.Account, account_id) + if not account: + raise HTTPException(status_code=404, detail="Account not found") + session.delete(account) + session.commit() + return {"deleted": True} diff --git a/apps/api/mousehold_api/routers/auth.py b/apps/api/mousehold_api/routers/auth.py new file mode 100644 index 0000000..25d60cb --- /dev/null +++ b/apps/api/mousehold_api/routers/auth.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +from fastapi import APIRouter, Depends, HTTPException, Response +from sqlalchemy.orm import Session + +from .. import models +from ..auth import ( + clear_session_cookie, + create_local_session, + get_optional_local_session, + set_session_cookie, +) +from ..db import get_session +from ..schemas import AuthSessionRead, HouseholdRead, LocalLoginRequest, LocalSessionRead, UserRead + +router = APIRouter(tags=["auth"]) + + +def _auth_session_read( + session: Session, + local_session: models.LocalSession, +) -> AuthSessionRead: + user = session.get(models.User, local_session.user_id) + household = session.get(models.Household, local_session.household_id) + if not user or not household: + raise HTTPException(status_code=404, detail="Session user or household not found") + return AuthSessionRead( + session=LocalSessionRead.model_validate(local_session), + user=UserRead.model_validate(user), + household=HouseholdRead.model_validate(household), + ) + + +@router.post("/auth/login", response_model=AuthSessionRead) +def local_login( + payload: LocalLoginRequest, + response: Response, + session: Session = Depends(get_session), +) -> AuthSessionRead: + user = session.get(models.User, payload.user_id) + if not user: + raise HTTPException(status_code=404, detail="User not found") + local_session, token = create_local_session(session, user) + session.commit() + session.refresh(local_session) + set_session_cookie(response, token) + return _auth_session_read(session, local_session) + + +@router.get("/auth/me", response_model=AuthSessionRead | None) +def current_local_session( + local_session: models.LocalSession | None = Depends(get_optional_local_session), + session: Session = Depends(get_session), +) -> AuthSessionRead | None: + if not local_session: + return None + return _auth_session_read(session, local_session) + + +@router.post("/auth/logout") +def logout( + response: Response, + local_session: models.LocalSession | None = Depends(get_optional_local_session), + session: Session = Depends(get_session), +) -> dict[str, bool]: + if local_session: + local_session.revoked_at = datetime.now(UTC) + session.add(local_session) + session.commit() + clear_session_cookie(response) + return {"logged_out": True} diff --git a/apps/api/mousehold_api/routers/backup.py b/apps/api/mousehold_api/routers/backup.py new file mode 100644 index 0000000..98efea1 --- /dev/null +++ b/apps/api/mousehold_api/routers/backup.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from shutil import copy2 +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import or_, select +from sqlalchemy.orm import Session + +from .. import models +from ..auth import get_optional_local_session, require_household_access +from ..config import get_settings +from ..db import get_session +from ..schemas import BackupResult, HouseholdExport + +router = APIRouter(tags=["backup"]) + + +def _row(model: object) -> dict[str, Any]: + table = model.__table__ + return {column.name: getattr(model, column.name) for column in table.columns} + + +def _rows(items: list[object]) -> list[dict[str, Any]]: + return [_row(item) for item in items] + + +@router.get("/backup/export", response_model=HouseholdExport) +def export_household( + household_id: str, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> HouseholdExport: + require_household_access(household_id, local_session) + household = session.get(models.Household, household_id) + if not household: + raise HTTPException(status_code=404, detail="Household not found") + + users = list(session.scalars(select(models.User).where(models.User.household_id == household_id)).all()) + accounts = list( + session.scalars(select(models.Account).where(models.Account.household_id == household_id)).all() + ) + observations = list( + session.scalars( + select(models.RawObservation).where(models.RawObservation.household_id == household_id) + ).all() + ) + events = list( + session.scalars( + select(models.FinancialEvent).where(models.FinancialEvent.household_id == household_id) + ).all() + ) + categories = list( + session.scalars(select(models.Category).where(models.Category.household_id == household_id)).all() + ) + tags = list(session.scalars(select(models.Tag).where(models.Tag.household_id == household_id)).all()) + projects = list( + session.scalars(select(models.Project).where(models.Project.household_id == household_id)).all() + ) + recurring = list( + session.scalars( + select(models.RecurringCommitment).where(models.RecurringCommitment.household_id == household_id) + ).all() + ) + promised = list( + session.scalars( + select(models.PromisedPayment).where(models.PromisedPayment.household_id == household_id) + ).all() + ) + connections = list( + session.scalars( + select(models.ImportConnection).where(models.ImportConnection.household_id == household_id) + ).all() + ) + import_runs = list( + session.scalars(select(models.ImportRun).where(models.ImportRun.household_id == household_id)).all() + ) + audit_log = list( + session.scalars(select(models.AuditLog).where(models.AuditLog.household_id == household_id)).all() + ) + + event_ids = [event.id for event in events] + observation_ids = [observation.id for observation in observations] + project_ids = [project.id for project in projects] + event_links = ( + list( + session.scalars( + select(models.EventObservationLink).where( + or_( + models.EventObservationLink.event_id.in_(event_ids), + models.EventObservationLink.observation_id.in_(observation_ids), + ) + ) + ).all() + ) + if event_ids or observation_ids + else [] + ) + event_relations = ( + list( + session.scalars( + select(models.EventRelation).where( + or_( + models.EventRelation.from_event_id.in_(event_ids), + models.EventRelation.to_event_id.in_(event_ids), + ) + ) + ).all() + ) + if event_ids + else [] + ) + split_lines = ( + list( + session.scalars(select(models.SplitLine).where(models.SplitLine.event_id.in_(event_ids))).all() + ) + if event_ids + else [] + ) + event_tags = ( + list(session.scalars(select(models.EventTag).where(models.EventTag.event_id.in_(event_ids))).all()) + if event_ids + else [] + ) + project_participants = ( + list( + session.scalars( + select(models.ProjectParticipant).where(models.ProjectParticipant.project_id.in_(project_ids)) + ).all() + ) + if project_ids + else [] + ) + budget_lines = ( + list( + session.scalars( + select(models.ProjectBudgetLine).where(models.ProjectBudgetLine.project_id.in_(project_ids)) + ).all() + ) + if project_ids + else [] + ) + planned_items = ( + list( + session.scalars( + select(models.ProjectPlannedItem).where(models.ProjectPlannedItem.project_id.in_(project_ids)) + ).all() + ) + if project_ids + else [] + ) + planned_item_ids = [item.id for item in planned_items] + payment_lines = ( + list( + session.scalars( + select(models.PaymentLine).where( + or_( + models.PaymentLine.event_id.in_(event_ids), + models.PaymentLine.planned_item_id.in_(planned_item_ids), + ) + ) + ).all() + ) + if event_ids or planned_item_ids + else [] + ) + + data = { + "users": _rows(users), + "accounts": _rows(accounts), + "raw_observations": _rows(observations), + "financial_events": _rows(events), + "event_observation_links": _rows(event_links), + "event_relations": _rows(event_relations), + "split_lines": _rows(split_lines), + "categories": _rows(categories), + "tags": _rows(tags), + "event_tags": _rows(event_tags), + "projects": _rows(projects), + "project_participants": _rows(project_participants), + "project_budget_lines": _rows(budget_lines), + "project_planned_items": _rows(planned_items), + "payment_lines": _rows(payment_lines), + "recurring_commitments": _rows(recurring), + "promised_payments": _rows(promised), + "import_connections": _rows(connections), + "import_runs": _rows(import_runs), + "audit_log": _rows(audit_log), + } + return HouseholdExport( + household=_row(household), + exported_at=datetime.now(UTC), + counts={key: len(value) for key, value in data.items()}, + data=data, + ) + + +@router.post("/backup/sqlite", response_model=BackupResult) +def backup_sqlite() -> BackupResult: + settings = get_settings() + if not settings.database_url.startswith("sqlite:///"): + raise HTTPException(status_code=400, detail="SQLite backup is only available for sqlite:/// databases") + db_path_text = settings.database_url.replace("sqlite:///", "", 1) + if db_path_text in {"", ":memory:"}: + raise HTTPException(status_code=400, detail="In-memory SQLite databases cannot be backed up to a file") + db_path = Path(db_path_text) + if not db_path.is_absolute(): + db_path = Path.cwd() / db_path + if not db_path.exists(): + raise HTTPException(status_code=404, detail="SQLite database file not found") + backup_dir = db_path.parent / "backups" + backup_dir.mkdir(parents=True, exist_ok=True) + backup_path = backup_dir / f"{db_path.stem}-{datetime.now(UTC).strftime('%Y%m%d-%H%M%S')}.db" + copy2(db_path, backup_path) + return BackupResult( + file_path=str(backup_path), + size_bytes=backup_path.stat().st_size, + created_at=datetime.now(UTC), + ) diff --git a/apps/api/mousehold_api/routers/connections.py b/apps/api/mousehold_api/routers/connections.py new file mode 100644 index 0000000..5904731 --- /dev/null +++ b/apps/api/mousehold_api/routers/connections.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +from fastapi import APIRouter, Depends, HTTPException +from mousehold_domain import ImportConnectionStatus, ImportConnectionType +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .. import models +from ..db import get_session +from ..schemas import ( + ImportConnectionCreate, + ImportConnectionRead, + ImportConnectionUpdate, + LocalAgentHeartbeat, + LocalAgentRegister, +) + +router = APIRouter(tags=["connections"]) + + +@router.get("/connections/supported") +def supported_connections() -> dict[str, object]: + return { + "connectors": [ + { + "type": ImportConnectionType.FINTS, + "runs_locally": True, + "status": "local-agent-ready", + "notes": "PIN/TAN auth runs in the local agent; normalized observations can be synced to the API.", + }, + { + "type": ImportConnectionType.PAYPAL_API, + "runs_locally": True, + "status": "experimental-off-ui", + "notes": "PayPal CSV statements and email evidence are the active PoC path.", + }, + { + "type": ImportConnectionType.IMAP, + "runs_locally": True, + "status": "scaffold", + "notes": "Local agent can filter Klarna, PayPal, invoices, and order confirmations via IMAP.", + }, + { + "type": ImportConnectionType.CSV_FOLDER, + "runs_locally": True, + "status": "ready-shape", + "notes": "CSV rows can be normalized into /observations/sync payloads.", + }, + ] + } + + +@router.post("/connections", response_model=ImportConnectionRead) +def create_connection( + payload: ImportConnectionCreate, + session: Session = Depends(get_session), +) -> models.ImportConnection: + connection = models.ImportConnection(**payload.model_dump()) + session.add(connection) + session.commit() + session.refresh(connection) + return connection + + +@router.get("/connections", response_model=list[ImportConnectionRead]) +def list_connections( + household_id: str, + session: Session = Depends(get_session), +) -> list[models.ImportConnection]: + return list( + session.scalars( + select(models.ImportConnection) + .where(models.ImportConnection.household_id == household_id) + .order_by(models.ImportConnection.created_at.desc()) + ).all() + ) + + +@router.patch("/connections/{connection_id}", response_model=ImportConnectionRead) +def update_connection( + connection_id: str, + payload: ImportConnectionUpdate, + session: Session = Depends(get_session), +) -> models.ImportConnection: + connection = session.get(models.ImportConnection, connection_id) + if not connection: + raise HTTPException(status_code=404, detail="Connection not found") + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(connection, field, value) + session.add(connection) + session.commit() + session.refresh(connection) + return connection + + +@router.delete("/connections/{connection_id}") +def delete_connection(connection_id: str, session: Session = Depends(get_session)) -> dict[str, bool]: + connection = session.get(models.ImportConnection, connection_id) + if not connection: + raise HTTPException(status_code=404, detail="Connection not found") + session.delete(connection) + session.commit() + return {"deleted": True} + + +@router.post("/local-agent/register", response_model=ImportConnectionRead) +def register_local_agent( + payload: LocalAgentRegister, + session: Session = Depends(get_session), +) -> models.ImportConnection: + connection = models.ImportConnection( + household_id=payload.household_id, + owner_user_id=payload.owner_user_id, + connection_type=ImportConnectionType.LOCAL_AGENT, + display_name=payload.display_name, + status=ImportConnectionStatus.ACTIVE, + runs_locally=True, + connector_config_json={ + "capabilities": ["bank_csv", "paypal_csv", "imap", "fints_placeholder"] + }, + ) + session.add(connection) + session.commit() + session.refresh(connection) + return connection + + +@router.post("/local-agent/heartbeat", response_model=ImportConnectionRead) +def local_agent_heartbeat( + payload: LocalAgentHeartbeat, + session: Session = Depends(get_session), +) -> models.ImportConnection: + connection = session.get(models.ImportConnection, payload.connection_id) + if not connection: + raise HTTPException(status_code=404, detail="Connection not found") + connection.status = payload.status + connection.last_error = payload.last_error + connection.last_sync_at = datetime.now(UTC) + session.add(connection) + session.commit() + session.refresh(connection) + return connection diff --git a/apps/api/mousehold_api/routers/events.py b/apps/api/mousehold_api/routers/events.py new file mode 100644 index 0000000..c7295e4 --- /dev/null +++ b/apps/api/mousehold_api/routers/events.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from mousehold_domain import FinancialEventStatus, RelationType +from mousehold_matching import compute_normalized_transaction_fingerprint +from sqlalchemy import or_, select +from sqlalchemy.orm import Session, selectinload + +from .. import models +from ..db import get_session +from ..schemas import ( + EventCreate, + EventDetailRead, + EventObservationLinkRead, + EventRead, + EventRelationCreate, + EventRelationRead, + EventTagsUpdate, + EventUpdate, + SplitLineCreate, + SplitLineRead, + TagRead, +) +from ..services.ingestion import confirm_event + +router = APIRouter(tags=["events"]) + + +def _get_event(session: Session, event_id: str) -> models.FinancialEvent: + event = session.scalars( + select(models.FinancialEvent) + .where(models.FinancialEvent.id == event_id) + .options(selectinload(models.FinancialEvent.split_lines)) + ).first() + if not event: + raise HTTPException(status_code=404, detail="Event not found") + _attach_event_tags(session, [event]) + return event + + +def _attach_event_tags(session: Session, events: list[models.FinancialEvent]) -> None: + if not events: + return + event_ids = [event.id for event in events] + rows = session.execute( + select(models.EventTag.event_id, models.EventTag.tag_id).where( + models.EventTag.event_id.in_(event_ids) + ) + ).all() + tags_by_event: dict[str, list[str]] = {event_id: [] for event_id in event_ids} + for event_id, tag_id in rows: + tags_by_event.setdefault(event_id, []).append(tag_id) + for event in events: + event.tag_ids = tags_by_event.get(event.id, []) + + +@router.post("/events", response_model=EventRead) +def create_event( + payload: EventCreate, session: Session = Depends(get_session) +) -> models.FinancialEvent: + fingerprint = compute_normalized_transaction_fingerprint( + amount=payload.amount, + currency=payload.currency, + event_date=payload.event_date, + merchant=payload.merchant, + counterparty=payload.counterparty, + ) + event = models.FinancialEvent( + **payload.model_dump(exclude={"split_lines"}), + fingerprint=fingerprint, + ) + session.add(event) + session.flush() + for split in payload.split_lines: + session.add(models.SplitLine(event_id=event.id, **split.model_dump())) + session.commit() + return _get_event(session, event.id) + + +@router.get("/events", response_model=list[EventRead]) +def list_events( + household_id: str, + status: FinancialEventStatus | None = None, + hide_related_confirmations: bool = False, + session: Session = Depends(get_session), +) -> list[models.FinancialEvent]: + query = ( + select(models.FinancialEvent) + .where(models.FinancialEvent.household_id == household_id) + .options(selectinload(models.FinancialEvent.split_lines)) + .order_by(models.FinancialEvent.event_date.desc(), models.FinancialEvent.created_at.desc()) + ) + if status: + query = query.where(models.FinancialEvent.status == status) + if hide_related_confirmations: + household_event_ids = select(models.FinancialEvent.id).where( + models.FinancialEvent.household_id == household_id + ) + hidden_ids = select(models.EventRelation.to_event_id).where( + models.EventRelation.from_event_id.in_(household_event_ids), + models.EventRelation.relation_type.in_( + [RelationType.SAME_ECONOMIC_EVENT_AS, RelationType.DUPLICATE_OF] + ), + ) + query = query.where(models.FinancialEvent.id.not_in(hidden_ids)) + events = list(session.scalars(query).all()) + _attach_event_tags(session, events) + return events + + +@router.get("/events/{event_id}", response_model=EventRead) +def get_event(event_id: str, session: Session = Depends(get_session)) -> models.FinancialEvent: + return _get_event(session, event_id) + + +@router.get("/events/{event_id}/details", response_model=EventDetailRead) +def get_event_details(event_id: str, session: Session = Depends(get_session)) -> EventDetailRead: + event = _get_event(session, event_id) + observation_links = list( + session.scalars( + select(models.EventObservationLink) + .where(models.EventObservationLink.event_id == event_id) + .order_by(models.EventObservationLink.created_at) + ).all() + ) + observations: list[models.RawObservation] = [] + if observation_links: + observations = list( + session.scalars( + select(models.RawObservation).where( + models.RawObservation.id.in_([link.observation_id for link in observation_links]) + ) + ).all() + ) + relations = list( + session.scalars( + select(models.EventRelation) + .where( + or_( + models.EventRelation.from_event_id == event_id, + models.EventRelation.to_event_id == event_id, + ) + ) + .order_by(models.EventRelation.created_at) + ).all() + ) + related_event_ids = { + relation.to_event_id if relation.from_event_id == event_id else relation.from_event_id + for relation in relations + } + related_events: list[models.FinancialEvent] = [] + if related_event_ids: + related_events = list( + session.scalars( + select(models.FinancialEvent) + .where(models.FinancialEvent.id.in_(related_event_ids)) + .options(selectinload(models.FinancialEvent.split_lines)) + .order_by(models.FinancialEvent.event_date.desc()) + ).all() + ) + _attach_event_tags(session, related_events) + category = session.get(models.Category, event.category_id) if event.category_id else None + tags = ( + list(session.scalars(select(models.Tag).where(models.Tag.id.in_(event.tag_ids))).all()) + if event.tag_ids + else [] + ) + return EventDetailRead( + event=event, + category=category, + tags=tags, + observation_links=[EventObservationLinkRead.model_validate(link) for link in observation_links], + observations=observations, + relations=[EventRelationRead.model_validate(relation) for relation in relations], + related_events=related_events, + ) + + +@router.patch("/events/{event_id}", response_model=EventRead) +def update_event( + event_id: str, + payload: EventUpdate, + session: Session = Depends(get_session), +) -> models.FinancialEvent: + event = _get_event(session, event_id) + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(event, field, value) + event.fingerprint = compute_normalized_transaction_fingerprint( + amount=event.amount, + currency=event.currency, + event_date=event.event_date, + merchant=event.merchant, + counterparty=event.counterparty, + ) + session.add(event) + session.commit() + return _get_event(session, event_id) + + +@router.patch("/events/{event_id}/tags", response_model=list[TagRead]) +def update_event_tags( + event_id: str, + payload: EventTagsUpdate, + session: Session = Depends(get_session), +) -> list[models.Tag]: + event = _get_event(session, event_id) + tags = ( + list( + session.scalars( + select(models.Tag).where( + models.Tag.id.in_(payload.tag_ids), + models.Tag.household_id == event.household_id, + ) + ).all() + ) + if payload.tag_ids + else [] + ) + if len(tags) != len(set(payload.tag_ids)): + raise HTTPException(status_code=400, detail="One or more tags do not belong to this household") + session.query(models.EventTag).filter(models.EventTag.event_id == event_id).delete( + synchronize_session=False + ) + session.add_all([models.EventTag(event_id=event_id, tag_id=tag.id) for tag in tags]) + session.commit() + return tags + + +@router.post("/events/{event_id}/confirm", response_model=EventRead) +def confirm_event_endpoint( + event_id: str, session: Session = Depends(get_session) +) -> models.FinancialEvent: + event = confirm_event(session, _get_event(session, event_id)) + session.commit() + return _get_event(session, event.id) + + +@router.post("/events/{event_id}/relations", response_model=EventRelationRead) +def add_event_relation( + event_id: str, + payload: EventRelationCreate, + session: Session = Depends(get_session), +) -> models.EventRelation: + if not session.get(models.FinancialEvent, event_id) or not session.get( + models.FinancialEvent, payload.to_event_id + ): + raise HTTPException(status_code=404, detail="Event not found") + relation = models.EventRelation(from_event_id=event_id, **payload.model_dump()) + session.add(relation) + session.commit() + session.refresh(relation) + return relation + + +@router.post("/events/{event_id}/splits", response_model=list[SplitLineRead]) +def replace_event_splits( + event_id: str, + payload: list[SplitLineCreate], + session: Session = Depends(get_session), +) -> list[models.SplitLine]: + event = _get_event(session, event_id) + for split in list(event.split_lines): + session.delete(split) + session.flush() + new_splits = [models.SplitLine(event_id=event_id, **split.model_dump()) for split in payload] + session.add_all(new_splits) + session.commit() + return list( + session.scalars(select(models.SplitLine).where(models.SplitLine.event_id == event_id)).all() + ) diff --git a/apps/api/mousehold_api/routers/households.py b/apps/api/mousehold_api/routers/households.py new file mode 100644 index 0000000..550bccd --- /dev/null +++ b/apps/api/mousehold_api/routers/households.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .. import models +from ..db import get_session +from ..schemas import ( + HouseholdCreate, + HouseholdRead, + HouseholdUpdate, + LoginIdentityCreate, + LoginIdentityRead, + UserCreate, + UserRead, + UserUpdate, +) + +router = APIRouter(tags=["households"]) + + +@router.post("/households", response_model=HouseholdRead) +def create_household( + payload: HouseholdCreate, session: Session = Depends(get_session) +) -> models.Household: + household = models.Household( + name=payload.name, default_currency=payload.default_currency.upper() + ) + session.add(household) + session.commit() + session.refresh(household) + return household + + +@router.get("/households", response_model=list[HouseholdRead]) +def list_households(session: Session = Depends(get_session)) -> list[models.Household]: + return list( + session.scalars(select(models.Household).order_by(models.Household.created_at)).all() + ) + + +@router.get("/households/{household_id}", response_model=HouseholdRead) +def get_household(household_id: str, session: Session = Depends(get_session)) -> models.Household: + household = session.get(models.Household, household_id) + if not household: + raise HTTPException(status_code=404, detail="Household not found") + return household + + +@router.patch("/households/{household_id}", response_model=HouseholdRead) +def update_household( + household_id: str, + payload: HouseholdUpdate, + session: Session = Depends(get_session), +) -> models.Household: + household = session.get(models.Household, household_id) + if not household: + raise HTTPException(status_code=404, detail="Household not found") + if payload.name is not None: + household.name = payload.name + if payload.default_currency is not None: + household.default_currency = payload.default_currency.upper() + session.add(household) + session.commit() + session.refresh(household) + return household + + +@router.delete("/households/{household_id}") +def delete_household(household_id: str, session: Session = Depends(get_session)) -> dict[str, bool]: + household = session.get(models.Household, household_id) + if not household: + raise HTTPException(status_code=404, detail="Household not found") + session.delete(household) + session.commit() + return {"deleted": True} + + +@router.post("/users", response_model=UserRead) +def create_user(payload: UserCreate, session: Session = Depends(get_session)) -> models.User: + if not session.get(models.Household, payload.household_id): + raise HTTPException(status_code=404, detail="Household not found") + user = models.User( + household_id=payload.household_id, + display_name=payload.display_name, + email=payload.email, + ) + session.add(user) + session.commit() + session.refresh(user) + return user + + +@router.get("/users", response_model=list[UserRead]) +def list_users(household_id: str, session: Session = Depends(get_session)) -> list[models.User]: + return list( + session.scalars( + select(models.User) + .where(models.User.household_id == household_id) + .order_by(models.User.display_name) + ).all() + ) + + +@router.patch("/users/{user_id}", response_model=UserRead) +def update_user( + user_id: str, + payload: UserUpdate, + session: Session = Depends(get_session), +) -> models.User: + user = session.get(models.User, user_id) + if not user: + raise HTTPException(status_code=404, detail="User not found") + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(user, field, value) + session.add(user) + session.commit() + session.refresh(user) + return user + + +@router.delete("/users/{user_id}") +def delete_user(user_id: str, session: Session = Depends(get_session)) -> dict[str, bool]: + user = session.get(models.User, user_id) + if not user: + raise HTTPException(status_code=404, detail="User not found") + session.delete(user) + session.commit() + return {"deleted": True} + + +@router.post("/login-identities", response_model=LoginIdentityRead) +def create_login_identity( + payload: LoginIdentityCreate, + session: Session = Depends(get_session), +) -> models.LoginIdentity: + if not session.get(models.User, payload.user_id): + raise HTTPException(status_code=404, detail="User not found") + identity = models.LoginIdentity(**payload.model_dump()) + session.add(identity) + session.commit() + session.refresh(identity) + return identity + + +@router.get("/login-identities", response_model=list[LoginIdentityRead]) +def list_login_identities( + user_id: str, session: Session = Depends(get_session) +) -> list[models.LoginIdentity]: + return list( + session.scalars( + select(models.LoginIdentity) + .where(models.LoginIdentity.user_id == user_id) + .order_by(models.LoginIdentity.created_at) + ).all() + ) diff --git a/apps/api/mousehold_api/routers/observations.py b/apps/api/mousehold_api/routers/observations.py new file mode 100644 index 0000000..d7b4767 --- /dev/null +++ b/apps/api/mousehold_api/routers/observations.py @@ -0,0 +1,644 @@ +from __future__ import annotations + +from datetime import UTC, date, datetime +from decimal import Decimal + +from fastapi import APIRouter, Depends, HTTPException +from mousehold_domain import ( + EventObservationRelation, + FinancialEventStatus, + FinancialEventType, + ImportConnectionStatus, + ObservationStatus, +) +from mousehold_matching import compute_normalized_transaction_fingerprint +from sqlalchemy import select +from sqlalchemy.orm import Session, selectinload + +from .. import models +from ..auth import get_optional_local_session, require_household_access +from ..db import get_session +from ..schemas import ( + EventRead, + ImportRunDetailRead, + ImportRunRead, + ManualObservationCreate, + ObservationAcceptRequest, + ObservationBatchRequest, + ObservationBatchResponse, + ObservationIgnoreRequest, + ObservationImportCreate, + ObservationLinkRequest, + ObservationReviewItem, + ObservationUpdateRequest, + RawObservationRead, + SyncObservationsRequest, + SyncObservationsResponse, +) +from ..services.ingestion import create_observation_with_candidate +from .events import _attach_event_tags + +router = APIRouter(tags=["observations"]) + + +def _get_observation(session: Session, observation_id: str) -> models.RawObservation: + observation = session.get(models.RawObservation, observation_id) + if not observation: + raise HTTPException(status_code=404, detail="Observation not found") + return observation + + +def _events_for_observation( + session: Session, observation_id: str +) -> list[models.FinancialEvent]: + events = list( + session.scalars( + select(models.FinancialEvent) + .join( + models.EventObservationLink, + models.EventObservationLink.event_id == models.FinancialEvent.id, + ) + .where(models.EventObservationLink.observation_id == observation_id) + .options(selectinload(models.FinancialEvent.split_lines)) + .order_by(models.FinancialEvent.created_at.desc()) + ).all() + ) + _attach_event_tags(session, events) + return events + + +def _review_item(session: Session, observation: models.RawObservation) -> ObservationReviewItem: + events = _events_for_observation(session, observation.id) + if observation.status == ObservationStatus.DUPLICATE_SOURCE: + suggested_action = "duplicate" + elif observation.status == ObservationStatus.IGNORED: + suggested_action = "ignored" + elif events: + suggested_action = "review_event" + else: + suggested_action = "create_event" + action_required = observation.status not in { + ObservationStatus.IGNORED, + ObservationStatus.DUPLICATE_SOURCE, + } + return ObservationReviewItem( + observation=RawObservationRead.model_validate(observation), + events=[EventRead.model_validate(event) for event in events], + action_required=action_required, + suggested_action=suggested_action, + ) + + +def _date_from_parsed(value: object, field: str) -> date | None: + if not value: + return None + try: + return date.fromisoformat(str(value)) + except ValueError as exc: + raise HTTPException(status_code=400, detail=f"Observation has invalid {field}") from exc + + +def _create_event_from_observation( + session: Session, + observation: models.RawObservation, + event_status: FinancialEventStatus | None, +) -> models.FinancialEvent: + parsed = observation.parsed_json or {} + event_date = _date_from_parsed(parsed.get("event_date"), "event_date") + if not event_date: + raise HTTPException(status_code=400, detail="Observation does not contain an event date") + amount = parsed.get("amount") + if amount is None: + raise HTTPException(status_code=400, detail="Observation does not contain an amount") + event_type = FinancialEventType(str(parsed.get("event_type") or FinancialEventType.EXPENSE)) + status = event_status or FinancialEventStatus(str(parsed.get("status") or FinancialEventStatus.CANDIDATE)) + fingerprint = compute_normalized_transaction_fingerprint( + amount=Decimal(str(amount)), + currency=str(parsed.get("currency") or "EUR"), + event_date=event_date, + merchant=parsed.get("merchant"), + counterparty=parsed.get("counterparty"), + external_id=observation.provider_transaction_id, + ) + event = models.FinancialEvent( + household_id=observation.household_id, + source_account_id=observation.source_account_id, + payment_account_id=observation.source_account_id, + payer_user_id=observation.owner_user_id, + event_type=event_type, + status=status, + event_date=event_date, + booking_date=_date_from_parsed(parsed.get("booking_date"), "booking_date"), + due_date=_date_from_parsed(parsed.get("due_date"), "due_date"), + amount=Decimal(str(amount)), + currency=str(parsed.get("currency") or "EUR"), + merchant=parsed.get("merchant"), + counterparty=parsed.get("counterparty"), + description=parsed.get("description"), + created_by_user_id=observation.owner_user_id, + fingerprint=fingerprint, + ) + session.add(event) + session.flush() + session.add( + models.EventObservationLink( + event_id=event.id, + observation_id=observation.id, + relation=EventObservationRelation.IMPORTED_FROM, + confidence=Decimal("1.00"), + ) + ) + session.flush() + return event + + +def _require_account(session: Session, household_id: str, account_id: str | None) -> None: + if not account_id: + return + account = session.get(models.Account, account_id) + if not account or account.household_id != household_id: + raise HTTPException(status_code=404, detail="Account not found") + + +def _require_user(session: Session, household_id: str, user_id: str | None) -> None: + if not user_id: + return + user = session.get(models.User, user_id) + if not user or user.household_id != household_id: + raise HTTPException(status_code=404, detail="User not found") + + +def _parsed_update_payload(payload: ObservationUpdateRequest) -> dict[str, object]: + parsed_fields = { + "amount", + "currency", + "event_date", + "booking_date", + "due_date", + "merchant", + "counterparty", + "description", + "event_type", + "status", + } + dumped = payload.model_dump(exclude_unset=True, mode="json") + return {key: value for key, value in dumped.items() if key in parsed_fields} + + +def _sync_event_from_observation( + event: models.FinancialEvent, + observation: models.RawObservation, + parsed_update: dict[str, object], +) -> None: + parsed = observation.parsed_json or {} + if "amount" in parsed_update: + event.amount = Decimal(str(parsed["amount"])) + if "currency" in parsed_update: + event.currency = str(parsed["currency"] or "EUR").upper() + if "event_date" in parsed_update: + parsed_date = _date_from_parsed(parsed.get("event_date"), "event_date") + if parsed_date: + event.event_date = parsed_date + if "booking_date" in parsed_update: + event.booking_date = _date_from_parsed(parsed.get("booking_date"), "booking_date") + if "due_date" in parsed_update: + event.due_date = _date_from_parsed(parsed.get("due_date"), "due_date") + if "merchant" in parsed_update: + event.merchant = parsed.get("merchant") + if "counterparty" in parsed_update: + event.counterparty = parsed.get("counterparty") + if "description" in parsed_update: + event.description = parsed.get("description") + if "event_type" in parsed_update and parsed.get("event_type"): + event.event_type = FinancialEventType(str(parsed["event_type"])) + if "status" in parsed_update and parsed.get("status"): + event.status = FinancialEventStatus(str(parsed["status"])) + event.fingerprint = compute_normalized_transaction_fingerprint( + amount=event.amount, + currency=event.currency, + event_date=event.event_date, + merchant=event.merchant, + counterparty=event.counterparty, + external_id=observation.provider_transaction_id, + ) + + +def _apply_observation_update( + session: Session, + observation: models.RawObservation, + payload: ObservationUpdateRequest, +) -> None: + if "source_account_id" in payload.model_fields_set: + _require_account(session, observation.household_id, payload.source_account_id) + observation.source_account_id = payload.source_account_id + if "owner_user_id" in payload.model_fields_set: + _require_user(session, observation.household_id, payload.owner_user_id) + observation.owner_user_id = payload.owner_user_id + if "raw_text" in payload.model_fields_set: + observation.raw_text = payload.raw_text + parsed_update = _parsed_update_payload(payload) + if parsed_update: + parsed = dict(observation.parsed_json or {}) + parsed.update(parsed_update) + if "currency" in parsed and parsed["currency"]: + parsed["currency"] = str(parsed["currency"]).upper() + observation.parsed_json = parsed + session.add(observation) + session.flush() + if payload.sync_linked_events: + for event in _events_for_observation(session, observation.id): + _sync_event_from_observation(event, observation, parsed_update) + if "source_account_id" in payload.model_fields_set: + event.source_account_id = observation.source_account_id + event.payment_account_id = observation.source_account_id + if "owner_user_id" in payload.model_fields_set: + event.payer_user_id = observation.owner_user_id + session.add(event) + session.flush() + + +def _accept_observation( + session: Session, + observation: models.RawObservation, + event_status: FinancialEventStatus | None, + create_event_if_missing: bool = True, +) -> None: + events = _events_for_observation(session, observation.id) + if not events and create_event_if_missing: + events = [_create_event_from_observation(session, observation, event_status)] + for event in events: + if event_status: + event.status = event_status + session.add(event) + if events and observation.status not in { + ObservationStatus.DUPLICATE_SOURCE, + ObservationStatus.IGNORED, + }: + observation.status = ObservationStatus.CANDIDATE_CREATED + session.add(observation) + session.flush() + + +def _ignore_observation( + session: Session, + observation: models.RawObservation, + ignore_linked_events: bool, + actor_user_id: str | None, + reason: str | None, +) -> None: + observation.status = ObservationStatus.IGNORED + session.add(observation) + if ignore_linked_events: + for event in _events_for_observation(session, observation.id): + event.status = FinancialEventStatus.IGNORED + session.add(event) + session.add( + models.AuditLog( + household_id=observation.household_id, + actor_user_id=actor_user_id, + action="observation.ignore", + entity_type="raw_observation", + entity_id=observation.id, + details_json={"reason": reason}, + ) + ) + session.flush() + + +@router.post("/observations/manual", response_model=RawObservationRead) +def create_manual_observation( + payload: ManualObservationCreate, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> models.RawObservation: + require_household_access(payload.household_id, local_session) + observation, _event = create_observation_with_candidate(session, payload) + session.commit() + session.refresh(observation) + return observation + + +@router.post("/observations/import", response_model=RawObservationRead) +def import_observation( + payload: ObservationImportCreate, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> models.RawObservation: + require_household_access(payload.household_id, local_session) + observation, _event = create_observation_with_candidate(session, payload) + session.commit() + session.refresh(observation) + return observation + + +@router.post("/observations/sync", response_model=SyncObservationsResponse) +def sync_observations( + payload: SyncObservationsRequest, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> SyncObservationsResponse: + imported_ids: list[str] = [] + event_ids: list[str] = [] + duplicate_ids: list[str] = [] + connection = session.get(models.ImportConnection, payload.connection_id) + household_id = ( + payload.observations[0].household_id + if payload.observations + else connection.household_id if connection else None + ) + if not household_id: + raise HTTPException(status_code=400, detail="No household available for import") + require_household_access(household_id, local_session) + run = models.ImportRun( + household_id=household_id, + connection_id=connection.id if connection else None, + actor_user_id=local_session.user_id if local_session else None, + source_type=str(payload.observations[0].source_type) if payload.observations else None, + status="running", + summary_json={"requested_count": len(payload.observations)}, + ) + session.add(run) + session.flush() + for observation_payload in payload.observations: + if observation_payload.household_id != household_id: + raise HTTPException(status_code=400, detail="Import payload mixes households") + observation, event = create_observation_with_candidate(session, observation_payload) + imported_ids.append(observation.id) + if observation.status == ObservationStatus.DUPLICATE_SOURCE: + duplicate_ids.append(observation.id) + if event: + event_ids.append(event.id) + if connection: + connection.status = ImportConnectionStatus.ACTIVE + connection.last_sync_at = datetime.now(UTC) + connection.last_error = None + config = dict(connection.connector_config_json or {}) + config["last_imported_count"] = len(imported_ids) - len(duplicate_ids) + config["last_duplicate_count"] = len(duplicate_ids) + config["last_created_event_count"] = len(event_ids) + config["last_import_run_id"] = run.id + connection.connector_config_json = config + session.add(connection) + run.status = "done" + run.imported_count = len(imported_ids) - len(duplicate_ids) + run.duplicate_count = len(duplicate_ids) + run.created_event_count = len(event_ids) + run.finished_at = datetime.now(UTC) + run.summary_json = { + "requested_count": len(payload.observations), + "imported_observation_ids": imported_ids, + "created_event_ids": event_ids, + "duplicate_observation_ids": duplicate_ids, + } + session.add(run) + session.commit() + session.refresh(run) + return SyncObservationsResponse( + imported_observation_ids=imported_ids, + created_event_ids=event_ids, + duplicate_observation_ids=duplicate_ids, + connection_id=connection.id if connection else None, + imported_count=len(imported_ids) - len(duplicate_ids), + duplicate_count=len(duplicate_ids), + created_event_count=len(event_ids), + run=ImportRunRead.model_validate(run), + ) + + +@router.get("/observations/inbox", response_model=list[RawObservationRead]) +def get_observation_inbox( + household_id: str, + status: ObservationStatus | None = None, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> list[models.RawObservation]: + require_household_access(household_id, local_session) + query = ( + select(models.RawObservation) + .where(models.RawObservation.household_id == household_id) + .order_by(models.RawObservation.received_at.desc()) + ) + if status: + query = query.where(models.RawObservation.status == status) + return list(session.scalars(query).all()) + + +@router.get("/observations/review", response_model=list[ObservationReviewItem]) +def get_observation_review_items( + household_id: str, + status: ObservationStatus | None = None, + limit: int = 100, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> list[ObservationReviewItem]: + require_household_access(household_id, local_session) + query = ( + select(models.RawObservation) + .where(models.RawObservation.household_id == household_id) + .order_by(models.RawObservation.received_at.desc()) + .limit(limit) + ) + if status: + query = query.where(models.RawObservation.status == status) + return [_review_item(session, observation) for observation in session.scalars(query).all()] + + +@router.patch("/observations/{observation_id}", response_model=ObservationReviewItem) +def update_observation( + observation_id: str, + payload: ObservationUpdateRequest, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> ObservationReviewItem: + observation = _get_observation(session, observation_id) + require_household_access(observation.household_id, local_session) + _apply_observation_update(session, observation, payload) + session.commit() + session.refresh(observation) + return _review_item(session, observation) + + +@router.post("/observations/{observation_id}/accept", response_model=ObservationReviewItem) +def accept_observation( + observation_id: str, + payload: ObservationAcceptRequest, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> ObservationReviewItem: + observation = _get_observation(session, observation_id) + require_household_access(observation.household_id, local_session) + _accept_observation( + session, + observation, + event_status=payload.event_status, + create_event_if_missing=payload.create_event_if_missing, + ) + session.commit() + session.refresh(observation) + return _review_item(session, observation) + + +@router.post("/observations/{observation_id}/ignore", response_model=ObservationReviewItem) +def ignore_observation( + observation_id: str, + payload: ObservationIgnoreRequest, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> ObservationReviewItem: + observation = _get_observation(session, observation_id) + require_household_access(observation.household_id, local_session) + _ignore_observation( + session, + observation, + ignore_linked_events=payload.ignore_linked_events, + actor_user_id=local_session.user_id if local_session else None, + reason=payload.reason, + ) + session.commit() + session.refresh(observation) + return _review_item(session, observation) + + +@router.post("/observations/{observation_id}/link", response_model=ObservationReviewItem) +def link_observation( + observation_id: str, + payload: ObservationLinkRequest, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> ObservationReviewItem: + observation = _get_observation(session, observation_id) + require_household_access(observation.household_id, local_session) + event = session.get(models.FinancialEvent, payload.event_id) + if not event or event.household_id != observation.household_id: + raise HTTPException(status_code=404, detail="Event not found") + exists = session.scalars( + select(models.EventObservationLink) + .where(models.EventObservationLink.event_id == event.id) + .where(models.EventObservationLink.observation_id == observation.id) + .where(models.EventObservationLink.relation == payload.relation) + .limit(1) + ).first() + if not exists: + session.add( + models.EventObservationLink( + event_id=event.id, + observation_id=observation.id, + relation=payload.relation, + confidence=Decimal("1.00"), + ) + ) + if observation.status == ObservationStatus.NEW: + observation.status = ObservationStatus.PARSED + session.add(observation) + session.commit() + session.refresh(observation) + return _review_item(session, observation) + + +@router.post("/observations/batch", response_model=ObservationBatchResponse) +def batch_review_observations( + payload: ObservationBatchRequest, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> ObservationBatchResponse: + if payload.action not in {"accept", "confirm", "ignore"}: + raise HTTPException(status_code=400, detail="Unsupported batch action") + if not payload.observation_ids: + return ObservationBatchResponse(updated_count=0, items=[]) + items: list[ObservationReviewItem] = [] + household_id: str | None = None + for observation_id in payload.observation_ids: + observation = _get_observation(session, observation_id) + if household_id is None: + household_id = observation.household_id + require_household_access(household_id, local_session) + elif observation.household_id != household_id: + raise HTTPException(status_code=400, detail="Batch payload mixes households") + if payload.action == "ignore": + _ignore_observation( + session, + observation, + ignore_linked_events=payload.ignore_linked_events, + actor_user_id=local_session.user_id if local_session else None, + reason="Batch review", + ) + else: + _accept_observation( + session, + observation, + event_status=FinancialEventStatus.CONFIRMED + if payload.action == "confirm" + else payload.event_status, + create_event_if_missing=True, + ) + items.append(_review_item(session, observation)) + session.commit() + return ObservationBatchResponse(updated_count=len(items), items=items) + + +@router.get("/import-runs", response_model=list[ImportRunRead]) +def list_import_runs( + household_id: str, + connection_id: str | None = None, + limit: int = 25, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> list[models.ImportRun]: + require_household_access(household_id, local_session) + query = ( + select(models.ImportRun) + .where(models.ImportRun.household_id == household_id) + .order_by(models.ImportRun.started_at.desc()) + .limit(limit) + ) + if connection_id: + query = query.where(models.ImportRun.connection_id == connection_id) + return list(session.scalars(query).all()) + + +@router.get("/import-runs/{run_id}", response_model=ImportRunDetailRead) +def get_import_run_detail( + run_id: str, + session: Session = Depends(get_session), + local_session: models.LocalSession | None = Depends(get_optional_local_session), +) -> ImportRunDetailRead: + run = session.get(models.ImportRun, run_id) + if not run: + raise HTTPException(status_code=404, detail="Import run not found") + require_household_access(run.household_id, local_session) + summary = run.summary_json or {} + observation_ids = list( + dict.fromkeys( + [ + *summary.get("imported_observation_ids", []), + *summary.get("duplicate_observation_ids", []), + ] + ) + ) + event_ids = summary.get("created_event_ids", []) + observations = ( + list( + session.scalars( + select(models.RawObservation).where(models.RawObservation.id.in_(observation_ids)) + ).all() + ) + if observation_ids + else [] + ) + events = ( + list( + session.scalars( + select(models.FinancialEvent) + .where(models.FinancialEvent.id.in_(event_ids)) + .options(selectinload(models.FinancialEvent.split_lines)) + ).all() + ) + if event_ids + else [] + ) + _attach_event_tags(session, events) + return ImportRunDetailRead( + run=ImportRunRead.model_validate(run), + observations=[RawObservationRead.model_validate(observation) for observation in observations], + events=[EventRead.model_validate(event) for event in events], + ) diff --git a/apps/api/mousehold_api/routers/projects.py b/apps/api/mousehold_api/routers/projects.py new file mode 100644 index 0000000..4a92aa7 --- /dev/null +++ b/apps/api/mousehold_api/routers/projects.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .. import models +from ..db import get_session +from ..schemas import ( + PaymentLineCreate, + PaymentLineRead, + ProjectBudgetLineCreate, + ProjectBudgetLineRead, + ProjectCreate, + ProjectParticipantCreate, + ProjectParticipantRead, + ProjectPlannedItemCreate, + ProjectPlannedItemRead, + ProjectRead, + ProjectSummary, + SettlementSummary, +) +from ..services.projects import calculate_project_summary +from ..services.settlement import calculate_settlement + +router = APIRouter(tags=["projects"]) + + +def _get_project(session: Session, project_id: str) -> models.Project: + project = session.get(models.Project, project_id) + if not project: + raise HTTPException(status_code=404, detail="Project not found") + return project + + +@router.post("/projects", response_model=ProjectRead) +def create_project( + payload: ProjectCreate, session: Session = Depends(get_session) +) -> models.Project: + project = models.Project(**payload.model_dump(exclude={"participant_user_ids"})) + session.add(project) + session.flush() + for user_id in payload.participant_user_ids: + session.add(models.ProjectParticipant(project_id=project.id, user_id=user_id)) + session.commit() + session.refresh(project) + return project + + +@router.get("/projects", response_model=list[ProjectRead]) +def list_projects( + household_id: str, session: Session = Depends(get_session) +) -> list[models.Project]: + return list( + session.scalars( + select(models.Project) + .where(models.Project.household_id == household_id) + .order_by(models.Project.created_at.desc()) + ).all() + ) + + +@router.get("/projects/{project_id}", response_model=ProjectRead) +def get_project(project_id: str, session: Session = Depends(get_session)) -> models.Project: + return _get_project(session, project_id) + + +@router.post("/projects/{project_id}/participants", response_model=ProjectParticipantRead) +def add_project_participant( + project_id: str, + payload: ProjectParticipantCreate, + session: Session = Depends(get_session), +) -> models.ProjectParticipant: + _get_project(session, project_id) + participant = models.ProjectParticipant(project_id=project_id, **payload.model_dump()) + session.add(participant) + session.commit() + session.refresh(participant) + return participant + + +@router.post("/projects/{project_id}/budget-lines", response_model=ProjectBudgetLineRead) +def add_project_budget_line( + project_id: str, + payload: ProjectBudgetLineCreate, + session: Session = Depends(get_session), +) -> models.ProjectBudgetLine: + _get_project(session, project_id) + budget_line = models.ProjectBudgetLine(project_id=project_id, **payload.model_dump()) + session.add(budget_line) + session.commit() + session.refresh(budget_line) + return budget_line + + +@router.post("/projects/{project_id}/planned-items", response_model=ProjectPlannedItemRead) +def add_project_planned_item( + project_id: str, + payload: ProjectPlannedItemCreate, + session: Session = Depends(get_session), +) -> models.ProjectPlannedItem: + _get_project(session, project_id) + planned_item = models.ProjectPlannedItem(project_id=project_id, **payload.model_dump()) + session.add(planned_item) + session.commit() + session.refresh(planned_item) + return planned_item + + +@router.post("/projects/{project_id}/payment-lines", response_model=PaymentLineRead) +def add_project_payment_line( + project_id: str, + payload: PaymentLineCreate, + session: Session = Depends(get_session), +) -> models.PaymentLine: + _get_project(session, project_id) + if payload.planned_item_id and not session.get( + models.ProjectPlannedItem, payload.planned_item_id + ): + raise HTTPException(status_code=404, detail="Planned item not found") + payment_line = models.PaymentLine(**payload.model_dump()) + session.add(payment_line) + session.commit() + session.refresh(payment_line) + return payment_line + + +@router.get("/projects/{project_id}/summary", response_model=ProjectSummary) +def get_project_summary(project_id: str, session: Session = Depends(get_session)) -> ProjectSummary: + try: + return calculate_project_summary(session, project_id) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/projects/{project_id}/settlement", response_model=SettlementSummary) +def get_project_settlement( + project_id: str, + currency: str = "EUR", + include_announced: bool = False, + session: Session = Depends(get_session), +) -> SettlementSummary: + project = _get_project(session, project_id) + return calculate_settlement( + session, + household_id=project.household_id, + project_id=project_id, + currency=currency, + include_announced=include_announced, + ) diff --git a/apps/api/mousehold_api/routers/reconciliation.py b/apps/api/mousehold_api/routers/reconciliation.py new file mode 100644 index 0000000..3f0c7ae --- /dev/null +++ b/apps/api/mousehold_api/routers/reconciliation.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session + +from ..db import get_session +from ..schemas import ( + ReconciliationAcceptResponse, + ReconciliationDecisionRead, + ReconciliationIgnoreRequest, + ReconciliationIgnoreResponse, + ReconciliationSuggestionRead, +) +from ..services.reconciliation import ( + accept_reconciliation_suggestion, + ignore_reconciliation_suggestion, + list_reconciliation_decisions, + list_reconciliation_suggestions, +) + +router = APIRouter(tags=["reconciliation"]) + + +@router.get("/reconciliation/suggestions", response_model=list[ReconciliationSuggestionRead]) +def get_reconciliation_suggestions( + household_id: str, + limit: int = 50, + session: Session = Depends(get_session), +) -> list[object]: + return list_reconciliation_suggestions(session, household_id=household_id, limit=limit) + + +@router.post( + "/reconciliation/suggestions/{suggestion_id}/accept", + response_model=ReconciliationAcceptResponse, +) +def accept_suggestion( + suggestion_id: str, + household_id: str, + session: Session = Depends(get_session), +) -> ReconciliationAcceptResponse: + try: + suggestion = accept_reconciliation_suggestion( + session, + household_id=household_id, + suggestion_id=suggestion_id, + ) + except LookupError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + session.commit() + return ReconciliationAcceptResponse(accepted=suggestion) + + +@router.post( + "/reconciliation/suggestions/{suggestion_id}/ignore", + response_model=ReconciliationIgnoreResponse, +) +def ignore_suggestion( + suggestion_id: str, + household_id: str, + payload: ReconciliationIgnoreRequest | None = None, + session: Session = Depends(get_session), +) -> ReconciliationIgnoreResponse: + try: + suggestion, decision = ignore_reconciliation_suggestion( + session, + household_id=household_id, + suggestion_id=suggestion_id, + actor_user_id=payload.actor_user_id if payload else None, + reason=payload.reason if payload else None, + ) + except LookupError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + session.commit() + session.refresh(decision) + return ReconciliationIgnoreResponse(ignored=suggestion, decision=decision) + + +@router.get("/reconciliation/decisions", response_model=list[ReconciliationDecisionRead]) +def get_reconciliation_decisions( + household_id: str, + session: Session = Depends(get_session), +) -> list[object]: + return list_reconciliation_decisions(session, household_id=household_id) diff --git a/apps/api/mousehold_api/routers/recurring.py b/apps/api/mousehold_api/routers/recurring.py new file mode 100644 index 0000000..ca85ccb --- /dev/null +++ b/apps/api/mousehold_api/routers/recurring.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .. import models +from ..db import get_session +from ..schemas import ( + RecurringCommitmentCreate, + RecurringCommitmentRead, + RecurringCommitmentUpdate, +) + +router = APIRouter(tags=["recurring"]) + + +@router.post("/recurring-commitments", response_model=RecurringCommitmentRead) +def create_recurring_commitment( + payload: RecurringCommitmentCreate, + session: Session = Depends(get_session), +) -> models.RecurringCommitment: + if not session.get(models.Household, payload.household_id): + raise HTTPException(status_code=404, detail="Household not found") + commitment = models.RecurringCommitment(**payload.model_dump()) + session.add(commitment) + session.commit() + session.refresh(commitment) + return commitment + + +@router.get("/recurring-commitments", response_model=list[RecurringCommitmentRead]) +def list_recurring_commitments( + household_id: str, + session: Session = Depends(get_session), +) -> list[models.RecurringCommitment]: + return list( + session.scalars( + select(models.RecurringCommitment) + .where(models.RecurringCommitment.household_id == household_id) + .order_by(models.RecurringCommitment.next_due_date) + ).all() + ) + + +@router.patch("/recurring-commitments/{commitment_id}", response_model=RecurringCommitmentRead) +def update_recurring_commitment( + commitment_id: str, + payload: RecurringCommitmentUpdate, + session: Session = Depends(get_session), +) -> models.RecurringCommitment: + commitment = session.get(models.RecurringCommitment, commitment_id) + if not commitment: + raise HTTPException(status_code=404, detail="Recurring commitment not found") + for field, value in payload.model_dump(exclude_unset=True).items(): + if field == "currency" and value is not None: + value = value.upper() + setattr(commitment, field, value) + session.add(commitment) + session.commit() + session.refresh(commitment) + return commitment + + +@router.delete("/recurring-commitments/{commitment_id}") +def delete_recurring_commitment( + commitment_id: str, + session: Session = Depends(get_session), +) -> dict[str, bool]: + commitment = session.get(models.RecurringCommitment, commitment_id) + if not commitment: + raise HTTPException(status_code=404, detail="Recurring commitment not found") + session.delete(commitment) + session.commit() + return {"deleted": True} diff --git a/apps/api/mousehold_api/routers/settlement.py b/apps/api/mousehold_api/routers/settlement.py new file mode 100644 index 0000000..cd1ae25 --- /dev/null +++ b/apps/api/mousehold_api/routers/settlement.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from ..db import get_session +from ..schemas import SettlementSummary +from ..services.settlement import calculate_settlement + +router = APIRouter(tags=["settlement"]) + + +@router.get("/settlement/summary", response_model=SettlementSummary) +def get_settlement_summary( + household_id: str, + project_id: str | None = None, + currency: str = "EUR", + include_announced: bool = False, + session: Session = Depends(get_session), +) -> SettlementSummary: + return calculate_settlement( + session, + household_id=household_id, + project_id=project_id, + currency=currency, + include_announced=include_announced, + ) diff --git a/apps/api/mousehold_api/routers/taxonomy.py b/apps/api/mousehold_api/routers/taxonomy.py new file mode 100644 index 0000000..6b16c45 --- /dev/null +++ b/apps/api/mousehold_api/routers/taxonomy.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .. import models +from ..db import get_session +from ..schemas import ( + CategoryCreate, + CategoryRead, + CategoryUpdate, + TagCreate, + TagRead, + TagUpdate, +) + +router = APIRouter(tags=["taxonomy"]) + + +def _require_household(session: Session, household_id: str) -> None: + if not session.get(models.Household, household_id): + raise HTTPException(status_code=404, detail="Household not found") + + +def _category_path( + session: Session, + payload: CategoryCreate | CategoryUpdate, + category: models.Category | None = None, +) -> str: + if payload.path: + return payload.path + name = payload.name if payload.name is not None else category.name if category else "" + parent_id = payload.parent_id if payload.parent_id is not None else category.parent_id if category else None + if parent_id: + parent = session.get(models.Category, parent_id) + if not parent: + raise HTTPException(status_code=404, detail="Parent category not found") + return f"{parent.path}/{name}" + return name + + +@router.post("/categories", response_model=CategoryRead) +def create_category( + payload: CategoryCreate, + session: Session = Depends(get_session), +) -> models.Category: + _require_household(session, payload.household_id) + if payload.parent_id: + parent = session.get(models.Category, payload.parent_id) + if not parent or parent.household_id != payload.household_id: + raise HTTPException(status_code=404, detail="Parent category not found") + category = models.Category( + household_id=payload.household_id, + parent_id=payload.parent_id, + name=payload.name, + path=_category_path(session, payload), + is_active=payload.is_active, + ) + session.add(category) + session.commit() + session.refresh(category) + return category + + +@router.get("/categories", response_model=list[CategoryRead]) +def list_categories( + household_id: str, + include_inactive: bool = False, + session: Session = Depends(get_session), +) -> list[models.Category]: + query = ( + select(models.Category) + .where(models.Category.household_id == household_id) + .order_by(models.Category.path) + ) + if not include_inactive: + query = query.where(models.Category.is_active.is_(True)) + return list(session.scalars(query).all()) + + +@router.patch("/categories/{category_id}", response_model=CategoryRead) +def update_category( + category_id: str, + payload: CategoryUpdate, + session: Session = Depends(get_session), +) -> models.Category: + category = session.get(models.Category, category_id) + if not category: + raise HTTPException(status_code=404, detail="Category not found") + if payload.parent_id: + parent = session.get(models.Category, payload.parent_id) + if not parent or parent.household_id != category.household_id: + raise HTTPException(status_code=404, detail="Parent category not found") + for field, value in payload.model_dump(exclude_unset=True, exclude={"path"}).items(): + setattr(category, field, value) + if {"name", "parent_id", "path"} & payload.model_fields_set: + category.path = _category_path(session, payload, category) + session.add(category) + session.commit() + session.refresh(category) + return category + + +@router.delete("/categories/{category_id}") +def delete_category( + category_id: str, + session: Session = Depends(get_session), +) -> dict[str, bool]: + category = session.get(models.Category, category_id) + if not category: + raise HTTPException(status_code=404, detail="Category not found") + for event in session.scalars( + select(models.FinancialEvent).where(models.FinancialEvent.category_id == category_id) + ): + event.category_id = None + for commitment in session.scalars( + select(models.RecurringCommitment).where(models.RecurringCommitment.category_id == category_id) + ): + commitment.category_id = None + for budget_line in session.scalars( + select(models.ProjectBudgetLine).where(models.ProjectBudgetLine.category_id == category_id) + ): + budget_line.category_id = None + for child in session.scalars(select(models.Category).where(models.Category.parent_id == category_id)): + child.parent_id = None + child.path = child.name + session.delete(category) + session.commit() + return {"deleted": True} + + +@router.post("/tags", response_model=TagRead) +def create_tag( + payload: TagCreate, + session: Session = Depends(get_session), +) -> models.Tag: + _require_household(session, payload.household_id) + tag = models.Tag(household_id=payload.household_id, name=payload.name) + session.add(tag) + session.commit() + session.refresh(tag) + return tag + + +@router.get("/tags", response_model=list[TagRead]) +def list_tags( + household_id: str, + session: Session = Depends(get_session), +) -> list[models.Tag]: + return list( + session.scalars( + select(models.Tag).where(models.Tag.household_id == household_id).order_by(models.Tag.name) + ).all() + ) + + +@router.patch("/tags/{tag_id}", response_model=TagRead) +def update_tag( + tag_id: str, + payload: TagUpdate, + session: Session = Depends(get_session), +) -> models.Tag: + tag = session.get(models.Tag, tag_id) + if not tag: + raise HTTPException(status_code=404, detail="Tag not found") + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(tag, field, value) + session.add(tag) + session.commit() + session.refresh(tag) + return tag + + +@router.delete("/tags/{tag_id}") +def delete_tag( + tag_id: str, + session: Session = Depends(get_session), +) -> dict[str, bool]: + tag = session.get(models.Tag, tag_id) + if not tag: + raise HTTPException(status_code=404, detail="Tag not found") + session.query(models.EventTag).filter(models.EventTag.tag_id == tag_id).delete( + synchronize_session=False + ) + session.delete(tag) + session.commit() + return {"deleted": True} diff --git a/apps/api/mousehold_api/schemas.py b/apps/api/mousehold_api/schemas.py new file mode 100644 index 0000000..1fce0c7 --- /dev/null +++ b/apps/api/mousehold_api/schemas.py @@ -0,0 +1,782 @@ +from __future__ import annotations + +from datetime import date, datetime +from decimal import Decimal +from typing import Any + +from mousehold_domain import ( + AccountType, + EventObservationRelation, + FinancialEventStatus, + FinancialEventType, + ImportConnectionStatus, + ImportConnectionType, + ObservationStatus, + PaymentLineStatus, + PaymentStatus, + PlanningStatus, + ProjectStatus, + ReconciliationStatus, + RecurringCommitmentStatus, + RecurringInterval, + RelationType, + SettlementStatus, + SourceType, + VisibilityMode, +) +from pydantic import BaseModel, ConfigDict, Field + + +class ApiModel(BaseModel): + model_config = ConfigDict(from_attributes=True) + + +class HouseholdCreate(ApiModel): + name: str + default_currency: str = "EUR" + + +class HouseholdUpdate(ApiModel): + name: str | None = None + default_currency: str | None = None + + +class HouseholdRead(ApiModel): + id: str + name: str + default_currency: str + created_at: datetime + + +class UserCreate(ApiModel): + household_id: str + display_name: str + email: str | None = None + + +class UserUpdate(ApiModel): + display_name: str | None = None + email: str | None = None + + +class UserRead(ApiModel): + id: str + household_id: str + display_name: str + email: str | None + created_at: datetime + + +class LoginIdentityCreate(ApiModel): + user_id: str + provider: str + provider_subject: str + email: str | None = None + display_name: str | None = None + + +class LoginIdentityRead(ApiModel): + id: str + user_id: str + provider: str + provider_subject: str + email: str | None + display_name: str | None + created_at: datetime + + +class LocalLoginRequest(ApiModel): + user_id: str + + +class LocalSessionRead(ApiModel): + id: str + user_id: str + household_id: str + created_at: datetime + last_seen_at: datetime + expires_at: datetime + + +class AuthSessionRead(ApiModel): + session: LocalSessionRead + user: UserRead + household: HouseholdRead + + +class AccountCreate(ApiModel): + household_id: str + owner_user_id: str | None = None + name: str + account_type: AccountType + institution_name: str | None = None + currency: str = "EUR" + visibility_mode: VisibilityMode = VisibilityMode.PRIVATE + is_active: bool = True + + +class AccountUpdate(ApiModel): + owner_user_id: str | None = None + name: str | None = None + account_type: AccountType | None = None + institution_name: str | None = None + currency: str | None = None + visibility_mode: VisibilityMode | None = None + is_active: bool | None = None + + +class AccountRead(ApiModel): + id: str + household_id: str + owner_user_id: str | None + name: str + account_type: AccountType + institution_name: str | None + currency: str + visibility_mode: VisibilityMode + is_active: bool + created_at: datetime + + +class SplitLineCreate(ApiModel): + user_id: str + share_amount: Decimal + share_percent: Decimal | None = None + reason: str = "manual" + + +class SplitLineRead(ApiModel): + id: str + event_id: str + user_id: str + share_amount: Decimal + share_percent: Decimal | None + reason: str + created_at: datetime + + +class ManualObservationCreate(ApiModel): + household_id: str + source_type: SourceType = SourceType.MANUAL + source_account_id: str | None = None + owner_user_id: str | None = None + observed_at: datetime | None = None + source_reference: str | None = None + provider_transaction_id: str | None = None + message_id: str | None = None + raw_text: str | None = None + raw_payload_json: dict[str, Any] | None = None + amount: Decimal + currency: str = "EUR" + event_date: date + booking_date: date | None = None + due_date: date | None = None + merchant: str | None = None + counterparty: str | None = None + description: str | None = None + event_type: FinancialEventType = FinancialEventType.EXPENSE + status: FinancialEventStatus | None = None + payer_user_id: str | None = None + payment_account_id: str | None = None + category_id: str | None = None + project_id: str | None = None + split_lines: list[SplitLineCreate] = Field(default_factory=list) + + +class ObservationImportCreate(ManualObservationCreate): + source_type: SourceType + + +class RawObservationRead(ApiModel): + id: str + household_id: str + source_type: SourceType + source_account_id: str | None + owner_user_id: str | None + observed_at: datetime | None + received_at: datetime + raw_hash: str + source_reference: str | None + provider_transaction_id: str | None + message_id: str | None + raw_text: str | None + raw_file_path: str | None + raw_payload_json: dict[str, Any] | None + parsed_json: dict[str, Any] | None + parser_name: str | None + parser_version: str | None + confidence: Decimal + status: ObservationStatus + created_at: datetime + + +class EventObservationLinkRead(ApiModel): + id: str + event_id: str + observation_id: str + relation: EventObservationRelation + confidence: Decimal + created_at: datetime + + +class EventCreate(ApiModel): + household_id: str + source_account_id: str | None = None + payment_account_id: str | None = None + payer_user_id: str | None = None + event_type: FinancialEventType = FinancialEventType.EXPENSE + status: FinancialEventStatus = FinancialEventStatus.CANDIDATE + event_date: date + booking_date: date | None = None + due_date: date | None = None + amount: Decimal + currency: str = "EUR" + merchant: str | None = None + counterparty: str | None = None + description: str | None = None + category_id: str | None = None + project_id: str | None = None + created_by_user_id: str | None = None + split_lines: list[SplitLineCreate] = Field(default_factory=list) + + +class EventUpdate(ApiModel): + source_account_id: str | None = None + payment_account_id: str | None = None + payer_user_id: str | None = None + event_type: FinancialEventType | None = None + status: FinancialEventStatus | None = None + event_date: date | None = None + booking_date: date | None = None + due_date: date | None = None + amount: Decimal | None = None + currency: str | None = None + merchant: str | None = None + counterparty: str | None = None + description: str | None = None + category_id: str | None = None + project_id: str | None = None + + +class EventRead(ApiModel): + id: str + household_id: str + source_account_id: str | None + payment_account_id: str | None + payer_user_id: str | None + event_type: FinancialEventType + status: FinancialEventStatus + event_date: date + booking_date: date | None + due_date: date | None + amount: Decimal + currency: str + merchant: str | None + counterparty: str | None + description: str | None + category_id: str | None + project_id: str | None + created_by_user_id: str | None + fingerprint: str | None + created_at: datetime + updated_at: datetime + split_lines: list[SplitLineRead] = Field(default_factory=list) + tag_ids: list[str] = Field(default_factory=list) + + +class EventRelationCreate(ApiModel): + to_event_id: str + relation_type: RelationType + confidence: Decimal = Decimal("1.00") + created_by: str = "user" + + +class EventRelationRead(ApiModel): + id: str + from_event_id: str + to_event_id: str + relation_type: RelationType + confidence: Decimal + created_by: str + created_at: datetime + + +class CategoryCreate(ApiModel): + household_id: str + name: str + parent_id: str | None = None + path: str | None = None + is_active: bool = True + + +class CategoryUpdate(ApiModel): + name: str | None = None + parent_id: str | None = None + path: str | None = None + is_active: bool | None = None + + +class CategoryRead(ApiModel): + id: str + household_id: str + parent_id: str | None + name: str + path: str + is_active: bool + + +class TagCreate(ApiModel): + household_id: str + name: str + + +class TagUpdate(ApiModel): + name: str | None = None + + +class TagRead(ApiModel): + id: str + household_id: str + name: str + + +class EventTagsUpdate(ApiModel): + tag_ids: list[str] = Field(default_factory=list) + + +class EventDetailRead(ApiModel): + event: EventRead + category: CategoryRead | None = None + tags: list[TagRead] = Field(default_factory=list) + observation_links: list[EventObservationLinkRead] = Field(default_factory=list) + observations: list[RawObservationRead] = Field(default_factory=list) + relations: list[EventRelationRead] = Field(default_factory=list) + related_events: list[EventRead] = Field(default_factory=list) + + +class ObservationReviewItem(ApiModel): + observation: RawObservationRead + events: list[EventRead] = Field(default_factory=list) + action_required: bool = True + suggested_action: str = "review" + + +class ObservationAcceptRequest(ApiModel): + event_status: FinancialEventStatus | None = None + create_event_if_missing: bool = True + + +class ObservationUpdateRequest(ApiModel): + source_account_id: str | None = None + owner_user_id: str | None = None + raw_text: str | None = None + amount: Decimal | None = None + currency: str | None = None + event_date: date | None = None + booking_date: date | None = None + due_date: date | None = None + merchant: str | None = None + counterparty: str | None = None + description: str | None = None + event_type: FinancialEventType | None = None + status: FinancialEventStatus | None = None + sync_linked_events: bool = True + + +class ObservationIgnoreRequest(ApiModel): + ignore_linked_events: bool = False + reason: str | None = None + + +class ObservationLinkRequest(ApiModel): + event_id: str + relation: EventObservationRelation = EventObservationRelation.EVIDENCE_FOR + + +class ObservationBatchRequest(ApiModel): + observation_ids: list[str] + action: str + event_status: FinancialEventStatus | None = None + ignore_linked_events: bool = False + + +class ObservationBatchResponse(ApiModel): + updated_count: int + items: list[ObservationReviewItem] + + +class SettlementTransfer(ApiModel): + from_user_id: str + to_user_id: str + amount: Decimal + currency: str + + +class SettlementUserBalance(ApiModel): + user_id: str + paid: Decimal + share: Decimal + net: Decimal + + +class SettlementSummary(ApiModel): + household_id: str + project_id: str | None + currency: str + included_event_ids: list[str] + excluded_event_ids: list[str] + balances: list[SettlementUserBalance] + suggested_transfers: list[SettlementTransfer] + explanation: list[str] + + +class ProjectCreate(ApiModel): + household_id: str + name: str + description: str | None = None + start_date: date | None = None + end_date: date | None = None + default_split_rule_id: str | None = None + status: ProjectStatus = ProjectStatus.PLANNING + currency: str = "EUR" + participant_user_ids: list[str] = Field(default_factory=list) + + +class ProjectRead(ApiModel): + id: str + household_id: str + name: str + description: str | None + start_date: date | None + end_date: date | None + default_split_rule_id: str | None + status: ProjectStatus + currency: str + created_at: datetime + updated_at: datetime + + +class ProjectParticipantCreate(ApiModel): + user_id: str + role: str = "participant" + default_share_percent: Decimal | None = None + + +class ProjectParticipantRead(ApiModel): + id: str + project_id: str + user_id: str + role: str + default_share_percent: Decimal | None + + +class ProjectBudgetLineCreate(ApiModel): + category_id: str | None = None + name: str + budgeted_amount: Decimal = Decimal("0.00") + sort_order: int = 0 + notes: str | None = None + + +class ProjectBudgetLineRead(ApiModel): + id: str + project_id: str + category_id: str | None + name: str + budgeted_amount: Decimal + committed_amount_cached: Decimal | None + paid_amount_cached: Decimal | None + sort_order: int + notes: str | None + + +class ProjectPlannedItemCreate(ApiModel): + budget_line_id: str | None = None + description: str + estimated_amount: Decimal = Decimal("0.00") + committed_amount: Decimal | None = None + expected_date: date | None = None + due_date: date | None = None + expected_payer_account_id: str | None = None + split_rule_id: str | None = None + planning_status: PlanningStatus = PlanningStatus.IDEA + payment_status: PaymentStatus = PaymentStatus.UNPAID + reconciliation_status: ReconciliationStatus = ReconciliationStatus.NO_EVIDENCE + settlement_status: SettlementStatus = SettlementStatus.UNSETTLED + notes: str | None = None + + +class ProjectPlannedItemRead(ApiModel): + id: str + project_id: str + budget_line_id: str | None + description: str + estimated_amount: Decimal + committed_amount: Decimal | None + expected_date: date | None + due_date: date | None + expected_payer_account_id: str | None + split_rule_id: str | None + planning_status: PlanningStatus + payment_status: PaymentStatus + reconciliation_status: ReconciliationStatus + settlement_status: SettlementStatus + notes: str | None + created_at: datetime + updated_at: datetime + + +class PaymentLineCreate(ApiModel): + planned_item_id: str | None = None + event_id: str | None = None + amount: Decimal + currency: str = "EUR" + date: date + paid_by_user_id: str | None = None + payment_account_id: str | None = None + source_observation_id: str | None = None + status: PaymentLineStatus = PaymentLineStatus.EXPECTED + + +class PaymentLineRead(ApiModel): + id: str + planned_item_id: str | None + event_id: str | None + amount: Decimal + currency: str + date: date + paid_by_user_id: str | None + payment_account_id: str | None + source_observation_id: str | None + status: PaymentLineStatus + created_at: datetime + + +class ProjectSummary(ApiModel): + project_id: str + currency: str + budgeted_total: Decimal + committed_total: Decimal + paid_total: Decimal + reconciled_total: Decimal + settled_total: Decimal + due_later_total: Decimal + remaining_uncommitted_budget: Decimal + open_actions: list[str] + + +class ImportConnectionCreate(ApiModel): + household_id: str + owner_user_id: str + connection_type: ImportConnectionType + display_name: str + status: ImportConnectionStatus = ImportConnectionStatus.NEEDS_AUTH + secret_storage_ref: str | None = None + runs_locally: bool = True + connector_config_json: dict[str, Any] | None = None + + +class ImportConnectionUpdate(ApiModel): + owner_user_id: str | None = None + connection_type: ImportConnectionType | None = None + display_name: str | None = None + status: ImportConnectionStatus | None = None + secret_storage_ref: str | None = None + runs_locally: bool | None = None + connector_config_json: dict[str, Any] | None = None + + +class ImportConnectionRead(ApiModel): + id: str + household_id: str + owner_user_id: str + connection_type: ImportConnectionType + display_name: str + status: ImportConnectionStatus + last_sync_at: datetime | None + last_error: str | None + secret_storage_ref: str | None + runs_locally: bool + connector_config_json: dict[str, Any] | None + created_at: datetime + updated_at: datetime + + +class ImportRunRead(ApiModel): + id: str + household_id: str + connection_id: str | None + actor_user_id: str | None + source_type: str | None + status: str + imported_count: int + duplicate_count: int + created_event_count: int + skipped_count: int + error: str | None + summary_json: dict[str, Any] | None + started_at: datetime + finished_at: datetime | None + + +class ImportRunDetailRead(ApiModel): + run: ImportRunRead + observations: list[RawObservationRead] = Field(default_factory=list) + events: list[EventRead] = Field(default_factory=list) + + +class LocalAgentRegister(ApiModel): + household_id: str + owner_user_id: str + display_name: str = "Local connector agent" + + +class LocalAgentHeartbeat(ApiModel): + connection_id: str + status: ImportConnectionStatus = ImportConnectionStatus.ACTIVE + last_error: str | None = None + + +class SyncObservationsRequest(ApiModel): + connection_id: str + observations: list[ObservationImportCreate] + + +class SyncObservationsResponse(ApiModel): + imported_observation_ids: list[str] + created_event_ids: list[str] + duplicate_observation_ids: list[str] + connection_id: str | None = None + imported_count: int = 0 + duplicate_count: int = 0 + created_event_count: int = 0 + run: ImportRunRead | None = None + + +class BackupResult(ApiModel): + file_path: str + size_bytes: int + created_at: datetime + + +class HouseholdExport(ApiModel): + household: dict[str, Any] + exported_at: datetime + counts: dict[str, int] + data: dict[str, list[dict[str, Any]]] + + +class RecurringCommitmentCreate(ApiModel): + household_id: str + provider: str + description: str | None = None + amount_expected: Decimal + amount_min: Decimal | None = None + amount_max: Decimal | None = None + currency: str = "EUR" + interval: RecurringInterval + next_due_date: date + payer_account_id: str | None = None + category_id: str | None = None + project_id: str | None = None + notice_period: str | None = None + renewal_date: date | None = None + status: RecurringCommitmentStatus = RecurringCommitmentStatus.ACTIVE + matching_rule_json: dict[str, Any] | None = None + + +class RecurringCommitmentUpdate(ApiModel): + provider: str | None = None + description: str | None = None + amount_expected: Decimal | None = None + amount_min: Decimal | None = None + amount_max: Decimal | None = None + currency: str | None = None + interval: RecurringInterval | None = None + next_due_date: date | None = None + payer_account_id: str | None = None + category_id: str | None = None + project_id: str | None = None + notice_period: str | None = None + renewal_date: date | None = None + status: RecurringCommitmentStatus | None = None + matching_rule_json: dict[str, Any] | None = None + + +class RecurringCommitmentRead(ApiModel): + id: str + household_id: str + provider: str + description: str | None + amount_expected: Decimal + amount_min: Decimal | None + amount_max: Decimal | None + currency: str + interval: RecurringInterval + next_due_date: date + payer_account_id: str | None + category_id: str | None + project_id: str | None + notice_period: str | None + renewal_date: date | None + status: RecurringCommitmentStatus + matching_rule_json: dict[str, Any] | None + created_at: datetime + updated_at: datetime + + +class ReconciliationEventSummary(ApiModel): + id: str + source_type: str | None + event_type: str + status: str + event_date: date + amount: Decimal + currency: str + merchant: str | None + counterparty: str | None + description: str | None + source_reference: str | None + provider_transaction_id: str | None + observation_id: str | None + + +class ReconciliationSuggestionRead(ApiModel): + id: str + kind: str + relation_type: RelationType + primary_event: ReconciliationEventSummary + confirming_event: ReconciliationEventSummary + confidence: Decimal + reasons: list[str] + action: str + + +class ReconciliationAcceptResponse(ApiModel): + accepted: ReconciliationSuggestionRead + + +class ReconciliationIgnoreRequest(ApiModel): + actor_user_id: str | None = None + reason: str | None = None + + +class ReconciliationDecisionRead(ApiModel): + id: str + household_id: str + suggestion_id: str + decision: str + kind: str + relation_type: RelationType + primary_event_id: str + confirming_event_id: str + confidence: Decimal + actor_user_id: str | None + reason: str | None + created_at: datetime + updated_at: datetime + + +class ReconciliationIgnoreResponse(ApiModel): + ignored: ReconciliationSuggestionRead + decision: ReconciliationDecisionRead diff --git a/apps/api/mousehold_api/services/__init__.py b/apps/api/mousehold_api/services/__init__.py new file mode 100644 index 0000000..8419019 --- /dev/null +++ b/apps/api/mousehold_api/services/__init__.py @@ -0,0 +1 @@ +"""Application services for ledger ingestion, reconciliation, and summaries.""" diff --git a/apps/api/mousehold_api/services/ingestion.py b/apps/api/mousehold_api/services/ingestion.py new file mode 100644 index 0000000..f6291eb --- /dev/null +++ b/apps/api/mousehold_api/services/ingestion.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +from decimal import Decimal + +from mousehold_domain import ( + EventObservationRelation, + FinancialEventStatus, + FinancialEventType, + ObservationStatus, + SourceType, +) +from mousehold_matching import ( + compute_normalized_transaction_fingerprint, + compute_raw_observation_hash, +) +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .. import models +from ..schemas import ManualObservationCreate, ObservationImportCreate + + +def infer_event_status(payload: ManualObservationCreate) -> FinancialEventStatus: + if payload.status: + return payload.status + if payload.source_type in { + SourceType.BANK_CSV, + SourceType.FINTS, + SourceType.PSD2_AGGREGATOR, + SourceType.PAYPAL_API, + SourceType.PAYPAL_CSV, + SourceType.KLARNA_CSV, + }: + return FinancialEventStatus.CONFIRMED + if ( + payload.source_type in {SourceType.KLARNA_EMAIL, SourceType.PAYPAL_EMAIL} + and payload.due_date + ): + return FinancialEventStatus.OUTSTANDING + if payload.source_type in { + SourceType.ORDER_CONFIRMATION_EMAIL, + SourceType.IMAP_EMAIL, + SourceType.PDF_INVOICE, + }: + return FinancialEventStatus.ANNOUNCED + return FinancialEventStatus.CANDIDATE + + +def infer_event_type(payload: ManualObservationCreate) -> FinancialEventType: + if payload.event_type != FinancialEventType.EXPENSE: + return payload.event_type + if payload.source_type not in {SourceType.BANK_CSV, SourceType.FINTS, SourceType.PSD2_AGGREGATOR}: + return payload.event_type + text = f"{payload.merchant or ''} {payload.counterparty or ''} {payload.description or ''}".lower() + liability_markers = ( + "paypal", + "klarna", + "visa", + "mastercard", + "amex", + "american express", + "credit card", + "kreditkarte", + "kartenabrechnung", + "card repayment", + ) + if any(marker in text for marker in liability_markers): + return FinancialEventType.LIABILITY_PAYMENT + return payload.event_type + + +def create_observation_with_candidate( + session: Session, + payload: ManualObservationCreate | ObservationImportCreate, +) -> tuple[models.RawObservation, models.FinancialEvent | None]: + raw_payload = payload.raw_payload_json or payload.model_dump( + mode="json", exclude={"split_lines"} + ) + event_type = infer_event_type(payload) + event_status = infer_event_status(payload) + raw_hash = compute_raw_observation_hash( + source_type=str(payload.source_type), + source_reference=payload.source_reference + or payload.provider_transaction_id + or payload.message_id, + raw_text=payload.raw_text, + raw_payload=raw_payload, + ) + + duplicate = session.scalars( + select(models.RawObservation) + .where(models.RawObservation.household_id == payload.household_id) + .where(models.RawObservation.source_type == payload.source_type) + .where(models.RawObservation.raw_hash == raw_hash) + .limit(1) + ).first() + + observation = models.RawObservation( + household_id=payload.household_id, + source_type=payload.source_type, + source_account_id=payload.source_account_id, + owner_user_id=payload.owner_user_id, + observed_at=payload.observed_at, + raw_hash=raw_hash, + source_reference=payload.source_reference, + provider_transaction_id=payload.provider_transaction_id, + message_id=payload.message_id, + raw_text=payload.raw_text, + raw_payload_json=raw_payload, + parsed_json={ + "amount": str(payload.amount), + "currency": payload.currency, + "event_date": payload.event_date.isoformat(), + "booking_date": payload.booking_date.isoformat() if payload.booking_date else None, + "due_date": payload.due_date.isoformat() if payload.due_date else None, + "merchant": payload.merchant, + "counterparty": payload.counterparty, + "description": payload.description, + "event_type": str(event_type), + "status": str(event_status), + }, + parser_name="manual-or-normalized-import", + parser_version="0.1.0", + confidence=Decimal("1.00"), + status=ObservationStatus.DUPLICATE_SOURCE + if duplicate + else ObservationStatus.CANDIDATE_CREATED, + ) + session.add(observation) + session.flush() + + if duplicate: + return observation, None + + fingerprint = compute_normalized_transaction_fingerprint( + amount=payload.amount, + currency=payload.currency, + event_date=payload.event_date, + merchant=payload.merchant, + counterparty=payload.counterparty, + external_id=payload.provider_transaction_id, + ) + event = models.FinancialEvent( + household_id=payload.household_id, + source_account_id=payload.source_account_id, + payment_account_id=payload.payment_account_id or payload.source_account_id, + payer_user_id=payload.payer_user_id or payload.owner_user_id, + event_type=event_type, + status=event_status, + event_date=payload.event_date, + booking_date=payload.booking_date, + due_date=payload.due_date, + amount=payload.amount, + currency=payload.currency, + merchant=payload.merchant, + counterparty=payload.counterparty, + description=payload.description, + category_id=payload.category_id, + project_id=payload.project_id, + created_by_user_id=payload.owner_user_id, + fingerprint=fingerprint, + ) + session.add(event) + session.flush() + + session.add( + models.EventObservationLink( + event_id=event.id, + observation_id=observation.id, + relation=EventObservationRelation.IMPORTED_FROM, + confidence=Decimal("1.00"), + ) + ) + for split in payload.split_lines: + session.add( + models.SplitLine( + event_id=event.id, + user_id=split.user_id, + share_amount=split.share_amount, + share_percent=split.share_percent, + reason=split.reason, + ) + ) + session.flush() + return observation, event + + +def confirm_event(session: Session, event: models.FinancialEvent) -> models.FinancialEvent: + event.status = FinancialEventStatus.CONFIRMED + session.add(event) + session.flush() + return event diff --git a/apps/api/mousehold_api/services/projects.py b/apps/api/mousehold_api/services/projects.py new file mode 100644 index 0000000..10aadd4 --- /dev/null +++ b/apps/api/mousehold_api/services/projects.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from decimal import Decimal + +from mousehold_domain import ( + FinancialEventStatus, + PaymentLineStatus, + PaymentStatus, + PlanningStatus, + ReconciliationStatus, + SettlementStatus, +) +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .. import models +from ..schemas import ProjectSummary + + +def _money(value: Decimal | int | str = "0.00") -> Decimal: + return Decimal(value).quantize(Decimal("0.01")) + + +def calculate_project_summary(session: Session, project_id: str) -> ProjectSummary: + project = session.get(models.Project, project_id) + if not project: + raise ValueError("Project not found") + + budget_lines = list( + session.scalars( + select(models.ProjectBudgetLine).where( + models.ProjectBudgetLine.project_id == project_id + ) + ).all() + ) + planned_items = list( + session.scalars( + select(models.ProjectPlannedItem).where( + models.ProjectPlannedItem.project_id == project_id + ) + ).all() + ) + payment_lines = list( + session.scalars( + select(models.PaymentLine) + .join( + models.ProjectPlannedItem, + models.PaymentLine.planned_item_id == models.ProjectPlannedItem.id, + ) + .where(models.ProjectPlannedItem.project_id == project_id) + ).all() + ) + linked_events = list( + session.scalars( + select(models.FinancialEvent).where(models.FinancialEvent.project_id == project_id) + ).all() + ) + + budgeted_total = _money(sum((line.budgeted_amount for line in budget_lines), Decimal("0.00"))) + committed_total = _money( + sum( + ( + item.committed_amount + if item.committed_amount is not None + else item.estimated_amount + if item.planning_status == PlanningStatus.COMMITTED + else Decimal("0.00") + ) + for item in planned_items + ) + ) + paid_total = _money( + sum( + ( + line.amount + for line in payment_lines + if line.status in {PaymentLineStatus.PAID, PaymentLineStatus.RECONCILED} + ), + Decimal("0.00"), + ) + + sum( + ( + event.amount + for event in linked_events + if event.status + in { + FinancialEventStatus.CONFIRMED, + FinancialEventStatus.BOOKED, + FinancialEventStatus.SETTLED, + } + ), + Decimal("0.00"), + ) + ) + reconciled_total = _money( + sum( + (line.amount for line in payment_lines if line.status == PaymentLineStatus.RECONCILED), + Decimal("0.00"), + ) + ) + settled_total = _money( + sum( + ( + item.committed_amount or item.estimated_amount + for item in planned_items + if item.settlement_status == SettlementStatus.SETTLED + ), + Decimal("0.00"), + ) + ) + due_later_total = _money(max(Decimal("0.00"), committed_total - paid_total)) + remaining_uncommitted_budget = _money(budgeted_total - committed_total) + + open_actions: list[str] = [] + for item in planned_items: + expected = item.committed_amount or item.estimated_amount + if item.planning_status == PlanningStatus.COMMITTED and item.payment_status in { + PaymentStatus.UNPAID, + PaymentStatus.PARTIALLY_PAID, + }: + open_actions.append(f"{item.description} still has payment due.") + if item.reconciliation_status == ReconciliationStatus.NO_EVIDENCE and expected > 0: + open_actions.append(f"{item.description} needs evidence.") + + return ProjectSummary( + project_id=project.id, + currency=project.currency, + budgeted_total=budgeted_total, + committed_total=committed_total, + paid_total=paid_total, + reconciled_total=reconciled_total, + settled_total=settled_total, + due_later_total=due_later_total, + remaining_uncommitted_budget=remaining_uncommitted_budget, + open_actions=open_actions, + ) diff --git a/apps/api/mousehold_api/services/reconciliation.py b/apps/api/mousehold_api/services/reconciliation.py new file mode 100644 index 0000000..11c7905 --- /dev/null +++ b/apps/api/mousehold_api/services/reconciliation.py @@ -0,0 +1,532 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date +from decimal import Decimal + +from mousehold_domain import ( + EventObservationRelation, + FinancialEventStatus, + RelationType, + SourceType, +) +from mousehold_matching import score_event_match +from mousehold_matching.fingerprints import normalize_counterparty +from sqlalchemy import select +from sqlalchemy.orm import Session, selectinload + +from .. import models + +ANNOUNCEMENT_SOURCES = { + SourceType.PAYPAL_EMAIL, + SourceType.KLARNA_EMAIL, + SourceType.ORDER_CONFIRMATION_EMAIL, + SourceType.IMAP_EMAIL, +} +CONFIRMATION_SOURCES = { + SourceType.PAYPAL_API, + SourceType.PAYPAL_CSV, + SourceType.KLARNA_CSV, + SourceType.BANK_CSV, + SourceType.FINTS, + SourceType.PSD2_AGGREGATOR, +} +BANK_CONFIRMATION_SOURCES = {SourceType.BANK_CSV, SourceType.FINTS, SourceType.PSD2_AGGREGATOR} + + +@dataclass(frozen=True) +class EventSummary: + id: str + source_type: str | None + event_type: str + status: str + event_date: date + amount: Decimal + currency: str + merchant: str | None + counterparty: str | None + description: str | None + source_reference: str | None + provider_transaction_id: str | None + observation_id: str | None + + +@dataclass(frozen=True) +class ReconciliationSuggestion: + id: str + kind: str + relation_type: RelationType + primary_event: EventSummary + confirming_event: EventSummary + confidence: Decimal + reasons: list[str] + action: str + + +def list_reconciliation_suggestions( + session: Session, + *, + household_id: str, + limit: int = 50, + include_ignored: bool = False, +) -> list[ReconciliationSuggestion]: + events = _events_with_observations(session, household_id=household_id) + suggestions: list[ReconciliationSuggestion] = [] + existing_pairs = _existing_relation_pairs(session, household_id=household_id) + ignored_suggestion_ids = ( + set() if include_ignored else _decision_suggestion_ids(session, household_id=household_id, decision="ignored") + ) + + for primary in events: + primary_source = _first_source_type(primary) + if primary_source not in ANNOUNCEMENT_SOURCES: + continue + if primary.status in { + FinancialEventStatus.CONFIRMED, + FinancialEventStatus.CANCELLED, + FinancialEventStatus.IGNORED, + }: + continue + for confirming in events: + if primary.id == confirming.id or (primary.id, confirming.id) in existing_pairs: + continue + confirming_source = _first_source_type(confirming) + if confirming_source not in CONFIRMATION_SOURCES: + continue + suggestion = _suggest_event_pair(primary, confirming) + if suggestion: + suggestions.append(suggestion) + + for source in events: + source_type = _first_source_type(source) + if source_type not in {SourceType.PAYPAL_CSV, SourceType.PAYPAL_API, SourceType.KLARNA_CSV}: + continue + for bank_event in events: + if source.id == bank_event.id or (source.id, bank_event.id) in existing_pairs: + continue + if _first_source_type(bank_event) not in BANK_CONFIRMATION_SOURCES: + continue + suggestion = _suggest_event_pair(source, bank_event, kind="statement_bank") + if suggestion: + suggestions.append(suggestion) + + unique: dict[str, ReconciliationSuggestion] = {} + for suggestion in sorted(suggestions, key=lambda item: item.confidence, reverse=True): + if suggestion.id not in ignored_suggestion_ids: + unique.setdefault(suggestion.id, suggestion) + return list(unique.values())[:limit] + + +def accept_reconciliation_suggestion( + session: Session, + *, + household_id: str, + suggestion_id: str, +) -> ReconciliationSuggestion: + suggestions = list_reconciliation_suggestions(session, household_id=household_id, limit=500) + suggestion = next((item for item in suggestions if item.id == suggestion_id), None) + if suggestion is None: + raise LookupError("Reconciliation suggestion not found") + + primary = session.get(models.FinancialEvent, suggestion.primary_event.id) + confirming = session.get(models.FinancialEvent, suggestion.confirming_event.id) + if not primary or not confirming: + raise LookupError("Reconciliation event not found") + if primary.household_id != household_id or confirming.household_id != household_id: + raise LookupError("Reconciliation event not found") + + _ensure_event_relation( + session, + from_event_id=primary.id, + to_event_id=confirming.id, + relation_type=suggestion.relation_type, + confidence=suggestion.confidence, + ) + if suggestion.confirming_event.observation_id: + _ensure_observation_link( + session, + event_id=primary.id, + observation_id=suggestion.confirming_event.observation_id, + relation=EventObservationRelation.EVIDENCE_FOR, + confidence=suggestion.confidence, + ) + if suggestion.primary_event.observation_id: + _ensure_observation_link( + session, + event_id=confirming.id, + observation_id=suggestion.primary_event.observation_id, + relation=EventObservationRelation.EVIDENCE_FOR, + confidence=suggestion.confidence, + ) + + if confirming.status == FinancialEventStatus.CONFIRMED: + primary.status = FinancialEventStatus.CONFIRMED + if not primary.payment_account_id and confirming.payment_account_id: + primary.payment_account_id = confirming.payment_account_id + if not primary.source_account_id and confirming.source_account_id: + primary.source_account_id = confirming.source_account_id + if not primary.payer_user_id and confirming.payer_user_id: + primary.payer_user_id = confirming.payer_user_id + session.add(primary) + _record_reconciliation_decision( + session, + household_id=household_id, + suggestion=suggestion, + decision="accepted", + ) + return suggestion + + +def ignore_reconciliation_suggestion( + session: Session, + *, + household_id: str, + suggestion_id: str, + actor_user_id: str | None = None, + reason: str | None = None, +) -> tuple[ReconciliationSuggestion, models.ReconciliationDecision]: + suggestions = list_reconciliation_suggestions( + session, + household_id=household_id, + limit=500, + include_ignored=True, + ) + suggestion = next((item for item in suggestions if item.id == suggestion_id), None) + if suggestion is None: + raise LookupError("Reconciliation suggestion not found") + decision = _record_reconciliation_decision( + session, + household_id=household_id, + suggestion=suggestion, + decision="ignored", + actor_user_id=actor_user_id, + reason=reason, + ) + return suggestion, decision + + +def list_reconciliation_decisions( + session: Session, + *, + household_id: str, +) -> list[models.ReconciliationDecision]: + return list( + session.scalars( + select(models.ReconciliationDecision) + .where(models.ReconciliationDecision.household_id == household_id) + .order_by(models.ReconciliationDecision.updated_at.desc()) + ).all() + ) + + +def _suggest_event_pair( + primary: models.FinancialEvent, + confirming: models.FinancialEvent, + *, + kind: str = "evidence_confirmation", +) -> ReconciliationSuggestion | None: + if primary.currency.upper() != confirming.currency.upper(): + return None + if abs(primary.amount - confirming.amount) > Decimal("0.01"): + return None + days = abs((primary.event_date - confirming.event_date).days) + if days > _max_date_window(primary, confirming, kind): + return None + + reasons: list[str] = ["amount_exact"] + if days == 0: + reasons.append("same_date") + else: + reasons.append(f"date_within_{days}_days") + + reference_score = _reference_score(primary, confirming) + if reference_score: + reasons.append("shared_reference") + + party_a = _party_text(primary) + party_b = _party_text(confirming) + score = Decimal( + str( + score_event_match( + amount_a=primary.amount, + amount_b=confirming.amount, + currency_a=primary.currency, + currency_b=confirming.currency, + date_a=primary.event_date, + date_b=confirming.event_date, + party_a=party_a, + party_b=party_b, + ) + ) + ) + party_overlap = _party_overlap_score(party_a, party_b) + if party_overlap >= Decimal("0.60"): + reasons.append("party_overlap") + if _provider_bridge(primary, confirming): + reasons.append("provider_bridge") + score += Decimal("0.12") + if kind == "statement_bank" and str(confirming.event_type) == "liability_payment": + reasons.append("liability_payment") + score += Decimal("0.10") + score += reference_score + if kind == "statement_bank": + score += Decimal("0.05") + + confidence = min(score, Decimal("0.99")).quantize(Decimal("0.01")) + if confidence < Decimal("0.72"): + return None + + relation_type = ( + RelationType.SAME_ECONOMIC_EVENT_AS + if kind == "evidence_confirmation" + else RelationType.SETTLES + ) + primary_summary = _event_summary(primary) + confirming_summary = _event_summary(confirming) + return ReconciliationSuggestion( + id=_suggestion_id(kind, primary.id, confirming.id), + kind=kind, + relation_type=relation_type, + primary_event=primary_summary, + confirming_event=confirming_summary, + confidence=confidence, + reasons=reasons, + action=( + "Attach confirming evidence and mark primary event confirmed" + if kind == "evidence_confirmation" + else "Link statement event to cash movement" + ), + ) + + +def _events_with_observations(session: Session, *, household_id: str) -> list[models.FinancialEvent]: + return list( + session.scalars( + select(models.FinancialEvent) + .where(models.FinancialEvent.household_id == household_id) + .options( + selectinload(models.FinancialEvent.observation_links).selectinload( + models.EventObservationLink.observation + ) + ) + .order_by(models.FinancialEvent.event_date.desc(), models.FinancialEvent.created_at.desc()) + ).all() + ) + + +def _existing_relation_pairs(session: Session, *, household_id: str) -> set[tuple[str, str]]: + event_ids = select(models.FinancialEvent.id).where(models.FinancialEvent.household_id == household_id) + rows = session.execute( + select(models.EventRelation.from_event_id, models.EventRelation.to_event_id) + .where(models.EventRelation.from_event_id.in_(event_ids)) + ).all() + return {(row[0], row[1]) for row in rows} | {(row[1], row[0]) for row in rows} + + +def _decision_suggestion_ids(session: Session, *, household_id: str, decision: str) -> set[str]: + return set( + session.scalars( + select(models.ReconciliationDecision.suggestion_id) + .where(models.ReconciliationDecision.household_id == household_id) + .where(models.ReconciliationDecision.decision == decision) + ).all() + ) + + +def _record_reconciliation_decision( + session: Session, + *, + household_id: str, + suggestion: ReconciliationSuggestion, + decision: str, + actor_user_id: str | None = None, + reason: str | None = None, +) -> models.ReconciliationDecision: + existing = session.scalars( + select(models.ReconciliationDecision) + .where(models.ReconciliationDecision.household_id == household_id) + .where(models.ReconciliationDecision.suggestion_id == suggestion.id) + .limit(1) + ).first() + if existing: + existing.decision = decision + existing.kind = suggestion.kind + existing.relation_type = suggestion.relation_type + existing.primary_event_id = suggestion.primary_event.id + existing.confirming_event_id = suggestion.confirming_event.id + existing.confidence = suggestion.confidence + existing.actor_user_id = actor_user_id + existing.reason = reason + session.add(existing) + return existing + row = models.ReconciliationDecision( + household_id=household_id, + suggestion_id=suggestion.id, + decision=decision, + kind=suggestion.kind, + relation_type=suggestion.relation_type, + primary_event_id=suggestion.primary_event.id, + confirming_event_id=suggestion.confirming_event.id, + confidence=suggestion.confidence, + actor_user_id=actor_user_id, + reason=reason, + ) + session.add(row) + session.flush() + return row + + +def _first_observation(event: models.FinancialEvent) -> models.RawObservation | None: + for link in event.observation_links: + if link.observation: + return link.observation + return None + + +def _first_source_type(event: models.FinancialEvent) -> SourceType | None: + observation = _first_observation(event) + return observation.source_type if observation else None + + +def _event_summary(event: models.FinancialEvent) -> EventSummary: + observation = _first_observation(event) + return EventSummary( + id=event.id, + source_type=str(observation.source_type) if observation else None, + event_type=str(event.event_type), + status=str(event.status), + event_date=event.event_date, + amount=event.amount, + currency=event.currency, + merchant=event.merchant, + counterparty=event.counterparty, + description=event.description, + source_reference=observation.source_reference if observation else None, + provider_transaction_id=observation.provider_transaction_id if observation else None, + observation_id=observation.id if observation else None, + ) + + +def _suggestion_id(kind: str, primary_id: str, confirming_id: str) -> str: + return f"{kind}:{primary_id}:{confirming_id}" + + +def _party_text(event: models.FinancialEvent) -> str: + parts = [event.merchant, event.counterparty, event.description] + observation = _first_observation(event) + if observation and observation.raw_payload_json: + refs = observation.raw_payload_json.get("references") + if isinstance(refs, dict): + parts.extend(str(value) for value in refs.values() if value) + return " ".join(part for part in parts if part) + + +def _party_overlap_score(a: str, b: str) -> Decimal: + left = set(normalize_counterparty(a).split()) + right = set(normalize_counterparty(b).split()) + if not left or not right: + return Decimal("0") + return Decimal(len(left & right)) / Decimal(max(len(left), len(right))) + + +def _reference_score(a: models.FinancialEvent, b: models.FinancialEvent) -> Decimal: + left = _references(a) + right = _references(b) + if not left or not right: + return Decimal("0") + return Decimal("0.22") if left & right else Decimal("0") + + +def _references(event: models.FinancialEvent) -> set[str]: + refs: set[str] = set() + observation = _first_observation(event) + if not observation: + return refs + for value in (observation.source_reference, observation.provider_transaction_id, observation.message_id): + if value: + refs.add(normalize_counterparty(value)) + payload = observation.raw_payload_json or {} + raw_refs = payload.get("references") + if isinstance(raw_refs, dict): + refs.update(normalize_counterparty(str(value)) for value in raw_refs.values() if value) + return {value for value in refs if value} + + +def _provider_bridge(primary: models.FinancialEvent, confirming: models.FinancialEvent) -> bool: + party = normalize_counterparty(_party_text(confirming)) + primary_source = _first_source_type(primary) + confirming_source = _first_source_type(confirming) + if primary_source in {SourceType.PAYPAL_EMAIL, SourceType.PAYPAL_CSV, SourceType.PAYPAL_API}: + return "paypal" in party or confirming_source in {SourceType.PAYPAL_CSV, SourceType.PAYPAL_API} + if primary_source in {SourceType.KLARNA_EMAIL, SourceType.KLARNA_CSV}: + return "klarna" in party or confirming_source == SourceType.KLARNA_CSV + return False + + +def _max_date_window(primary: models.FinancialEvent, confirming: models.FinancialEvent, kind: str) -> int: + primary_source = _first_source_type(primary) + confirming_source = _first_source_type(confirming) + if kind == "statement_bank": + return 45 + if primary_source in {SourceType.KLARNA_EMAIL, SourceType.PAYPAL_EMAIL} and confirming_source in { + SourceType.KLARNA_CSV, + SourceType.PAYPAL_CSV, + SourceType.PAYPAL_API, + }: + return 45 + return 21 + + +def _ensure_event_relation( + session: Session, + *, + from_event_id: str, + to_event_id: str, + relation_type: RelationType, + confidence: Decimal, +) -> None: + existing = session.scalars( + select(models.EventRelation) + .where(models.EventRelation.from_event_id == from_event_id) + .where(models.EventRelation.to_event_id == to_event_id) + .where(models.EventRelation.relation_type == relation_type) + .limit(1) + ).first() + if existing: + return + session.add( + models.EventRelation( + from_event_id=from_event_id, + to_event_id=to_event_id, + relation_type=relation_type, + confidence=confidence, + created_by="reconciliation", + ) + ) + + +def _ensure_observation_link( + session: Session, + *, + event_id: str, + observation_id: str, + relation: EventObservationRelation, + confidence: Decimal, +) -> None: + existing = session.scalars( + select(models.EventObservationLink) + .where(models.EventObservationLink.event_id == event_id) + .where(models.EventObservationLink.observation_id == observation_id) + .where(models.EventObservationLink.relation == relation) + .limit(1) + ).first() + if existing: + return + session.add( + models.EventObservationLink( + event_id=event_id, + observation_id=observation_id, + relation=relation, + confidence=confidence, + ) + ) diff --git a/apps/api/mousehold_api/services/settlement.py b/apps/api/mousehold_api/services/settlement.py new file mode 100644 index 0000000..a6a99cb --- /dev/null +++ b/apps/api/mousehold_api/services/settlement.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +from decimal import Decimal + +from mousehold_domain import AccountType, FinancialEventStatus, FinancialEventType +from sqlalchemy import select +from sqlalchemy.orm import Session, selectinload + +from .. import models +from ..schemas import SettlementSummary, SettlementTransfer, SettlementUserBalance + +SETTLEMENT_STATUSES = { + FinancialEventStatus.CONFIRMED, + FinancialEventStatus.BOOKED, + FinancialEventStatus.SETTLED, +} +SETTLEMENT_EVENT_TYPES = { + FinancialEventType.EXPENSE, + FinancialEventType.FEE, + FinancialEventType.INTEREST, + FinancialEventType.REFUND, + FinancialEventType.REIMBURSEMENT, +} + + +def _money(value: Decimal | int | str = "0.00") -> Decimal: + return Decimal(value).quantize(Decimal("0.01")) + + +def calculate_settlement( + session: Session, + *, + household_id: str, + project_id: str | None = None, + currency: str = "EUR", + include_announced: bool = False, +) -> SettlementSummary: + accepted_statuses = set(SETTLEMENT_STATUSES) + if include_announced: + accepted_statuses.update({FinancialEventStatus.ANNOUNCED, FinancialEventStatus.OUTSTANDING}) + + query = ( + select(models.FinancialEvent) + .where(models.FinancialEvent.household_id == household_id) + .options(selectinload(models.FinancialEvent.split_lines)) + .order_by(models.FinancialEvent.event_date, models.FinancialEvent.created_at) + ) + if project_id: + query = query.where(models.FinancialEvent.project_id == project_id) + events = list(session.scalars(query).all()) + + accounts_by_id = { + account.id: account + for account in session.scalars( + select(models.Account).where(models.Account.household_id == household_id) + ).all() + } + paid: dict[str, Decimal] = {} + share: dict[str, Decimal] = {} + included: list[str] = [] + excluded: list[str] = [] + explanation: list[str] = [] + + for event in events: + if ( + event.currency != currency + or event.status not in accepted_statuses + or event.event_type not in SETTLEMENT_EVENT_TYPES + ): + excluded.append(event.id) + continue + payment_account = accounts_by_id.get( + event.payment_account_id or event.source_account_id or "" + ) + if ( + payment_account + and payment_account.account_type == AccountType.SHARED_CHECKING + and not payment_account.owner_user_id + ): + excluded.append(event.id) + explanation.append(f"{event.id} excluded because a neutral shared account paid it.") + continue + if not event.payer_user_id or not event.split_lines: + excluded.append(event.id) + explanation.append(f"{event.id} excluded because payer or split lines are missing.") + continue + + included.append(event.id) + amount = _money(event.amount) + multiplier = ( + Decimal("-1.00") if event.event_type == FinancialEventType.REFUND else Decimal("1.00") + ) + paid[event.payer_user_id] = _money( + paid.get(event.payer_user_id, Decimal("0.00")) + (amount * multiplier) + ) + for split in event.split_lines: + share[split.user_id] = _money( + share.get(split.user_id, Decimal("0.00")) + (split.share_amount * multiplier) + ) + + user_ids = sorted(set(paid) | set(share)) + balances = [ + SettlementUserBalance( + user_id=user_id, + paid=_money(paid.get(user_id, Decimal("0.00"))), + share=_money(share.get(user_id, Decimal("0.00"))), + net=_money(paid.get(user_id, Decimal("0.00")) - share.get(user_id, Decimal("0.00"))), + ) + for user_id in user_ids + ] + transfers = _minimize_transfers(balances, currency) + explanation.append( + f"Included {len(included)} confirmed/booked/settled events" + + (" plus announced/outstanding events." if include_announced else ".") + ) + if not include_announced: + explanation.append( + "Announced and outstanding events are excluded from settlement by default." + ) + + return SettlementSummary( + household_id=household_id, + project_id=project_id, + currency=currency, + included_event_ids=included, + excluded_event_ids=excluded, + balances=balances, + suggested_transfers=transfers, + explanation=explanation, + ) + + +def _minimize_transfers( + balances: list[SettlementUserBalance], currency: str +) -> list[SettlementTransfer]: + debtors = [ + {"user_id": balance.user_id, "amount": _money(-balance.net)} + for balance in balances + if balance.net < 0 + ] + creditors = [ + {"user_id": balance.user_id, "amount": _money(balance.net)} + for balance in balances + if balance.net > 0 + ] + transfers: list[SettlementTransfer] = [] + debtor_index = 0 + creditor_index = 0 + while debtor_index < len(debtors) and creditor_index < len(creditors): + amount = min(debtors[debtor_index]["amount"], creditors[creditor_index]["amount"]) + if amount > 0: + transfers.append( + SettlementTransfer( + from_user_id=debtors[debtor_index]["user_id"], + to_user_id=creditors[creditor_index]["user_id"], + amount=_money(amount), + currency=currency, + ) + ) + debtors[debtor_index]["amount"] = _money(debtors[debtor_index]["amount"] - amount) + creditors[creditor_index]["amount"] = _money(creditors[creditor_index]["amount"] - amount) + if debtors[debtor_index]["amount"] == 0: + debtor_index += 1 + if creditors[creditor_index]["amount"] == 0: + creditor_index += 1 + return transfers diff --git a/apps/local-agent/mousehold_agent/__init__.py b/apps/local-agent/mousehold_agent/__init__.py new file mode 100644 index 0000000..3dc1f76 --- /dev/null +++ b/apps/local-agent/mousehold_agent/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/apps/local-agent/mousehold_agent/cli.py b/apps/local-agent/mousehold_agent/cli.py new file mode 100644 index 0000000..d6c49d1 --- /dev/null +++ b/apps/local-agent/mousehold_agent/cli.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + +from .csv_importer import read_transaction_csv +from .fints_connector import ( + account_to_dict, + config_from_args, + date_range_from_args, + display_accounts, + display_transactions, + fetch_transactions, + list_accounts, +) +from .http import post_json +from .imap_importer import fetch_imap_observations +from .server import run as run_local_agent_server + + +def main() -> None: + parser = argparse.ArgumentParser(prog="mousehold-agent") + parser.add_argument("--api-url", default="http://127.0.0.1:8000") + subcommands = parser.add_subparsers(dest="command", required=True) + + register = subcommands.add_parser("register") + register.add_argument("--household-id", required=True) + register.add_argument("--owner-user-id", required=True) + register.add_argument("--display-name", default="Local connector agent") + + csv_sync = subcommands.add_parser("sync-csv") + csv_sync.add_argument("--connection-id", required=True) + csv_sync.add_argument("--household-id", required=True) + csv_sync.add_argument("--source-type", choices=["bank_csv", "paypal_csv"], required=True) + csv_sync.add_argument("--path", type=Path, required=True) + csv_sync.add_argument("--source-account-id") + csv_sync.add_argument("--owner-user-id") + + imap_sync = subcommands.add_parser("sync-imap") + imap_sync.add_argument("--connection-id", required=True) + imap_sync.add_argument("--household-id", required=True) + imap_sync.add_argument("--owner-user-id") + imap_sync.add_argument("--source-account-id") + imap_sync.add_argument("--limit", type=int, default=25) + + demo = subcommands.add_parser("sync-demo") + demo.add_argument("--connection-id", required=True) + demo.add_argument("--household-id", required=True) + demo.add_argument("--owner-user-id") + demo.add_argument("--source-account-id") + + serve = subcommands.add_parser("serve") + serve.add_argument("--host", default="127.0.0.1") + serve.add_argument("--port", type=int, default=8765) + + fints_accounts = subcommands.add_parser("fints-accounts") + _add_fints_common_args(fints_accounts) + fints_accounts.add_argument("--with-balances", action="store_true") + fints_accounts.add_argument("--json", action="store_true") + + fints_transactions = subcommands.add_parser("fints-transactions") + _add_fints_common_args(fints_transactions) + _add_fints_transaction_args(fints_transactions) + fints_transactions.add_argument("--json", action="store_true") + + fints_sync = subcommands.add_parser("sync-fints") + _add_fints_common_args(fints_sync) + _add_fints_transaction_args(fints_sync) + fints_sync.add_argument("--connection-id", required=True) + fints_sync.add_argument("--household-id", required=True) + fints_sync.add_argument("--owner-user-id") + fints_sync.add_argument("--source-account-id") + fints_sync.add_argument("--display", action="store_true") + + args = parser.parse_args() + if args.command == "register": + result = post_json( + f"{args.api_url}/local-agent/register", + { + "household_id": args.household_id, + "owner_user_id": args.owner_user_id, + "display_name": args.display_name, + }, + ) + elif args.command == "sync-csv": + observations = read_transaction_csv( + args.path, + household_id=args.household_id, + source_type=args.source_type, + source_account_id=args.source_account_id, + owner_user_id=args.owner_user_id, + ) + result = _sync(args.api_url, args.connection_id, observations) + elif args.command == "sync-imap": + observations = fetch_imap_observations( + household_id=args.household_id, + owner_user_id=args.owner_user_id, + source_account_id=args.source_account_id, + limit=args.limit, + ) + result = _sync(args.api_url, args.connection_id, observations) + elif args.command == "sync-demo": + result = _sync( + args.api_url, + args.connection_id, + _demo_observations(args.household_id, args.owner_user_id, args.source_account_id), + ) + elif args.command == "serve": + run_local_agent_server(host=args.host, port=args.port) + return + elif args.command == "fints-accounts": + accounts = list_accounts(config_from_args(args), include_balances=args.with_balances) + if args.json: + result = accounts + else: + display_accounts(accounts) + return + elif args.command == "fints-transactions": + start_date, end_date = date_range_from_args(args) + fetched = fetch_transactions( + config_from_args(args), + iban=args.iban, + start_date=start_date, + end_date=end_date, + include_pending=args.include_pending, + ) + if args.json: + result = { + "account": account_to_dict(fetched.account), + "observations": fetched.observations, + } + else: + display_transactions(fetched.observations) + return + elif args.command == "sync-fints": + start_date, end_date = date_range_from_args(args) + fetched = fetch_transactions( + config_from_args(args), + iban=args.iban, + start_date=start_date, + end_date=end_date, + include_pending=args.include_pending, + household_id=args.household_id, + source_account_id=args.source_account_id, + owner_user_id=args.owner_user_id, + ) + if args.display: + display_transactions(fetched.observations) + result = _sync(args.api_url, args.connection_id, fetched.observations) + else: + raise AssertionError(args.command) + print(json.dumps(result, indent=2, sort_keys=True)) + + +def _sync(api_url: str, connection_id: str, observations: list[dict[str, Any]]) -> dict[str, Any]: + return post_json( + f"{api_url}/observations/sync", + { + "connection_id": connection_id, + "observations": observations, + }, + ) + + +def _add_fints_common_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--profile", default="default") + parser.add_argument("--state-dir", default="~/.config/mousehold/fints") + parser.add_argument("--no-state", action="store_true") + parser.add_argument("--no-bootstrap", action="store_true") + parser.add_argument("--blz", help="Bank identifier/BLZ. Env fallback: MOUSEHOLD_FINTS_BLZ") + parser.add_argument("--server", help="FinTS endpoint URL. Env fallback: MOUSEHOLD_FINTS_SERVER") + parser.add_argument("--bank-user-id", help="FinTS login/user ID. Env fallback: MOUSEHOLD_FINTS_USER_ID") + parser.add_argument("--customer-id", help="Optional customer ID. Env fallback: MOUSEHOLD_FINTS_CUSTOMER_ID") + parser.add_argument("--pin", help="FinTS PIN. Prefer prompt or MOUSEHOLD_FINTS_PIN over shell history.") + parser.add_argument("--pin-env", default="MOUSEHOLD_FINTS_PIN") + parser.add_argument("--tan-mechanism", help="Optional TAN mechanism. Env fallback: MOUSEHOLD_FINTS_TAN_MECHANISM") + parser.add_argument("--tan-medium", help="Optional TAN medium name. Env fallback: MOUSEHOLD_FINTS_TAN_MEDIUM") + parser.add_argument("--product-id", help="Registered FinTS product ID. Env fallback: MOUSEHOLD_FINTS_PRODUCT_ID") + parser.add_argument( + "--product-version", + help="Optional product version. Env fallback: MOUSEHOLD_FINTS_PRODUCT_VERSION", + ) + + +def _add_fints_transaction_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--iban", help="IBAN to fetch. Defaults to the first account returned by the bank.") + parser.add_argument("--days", type=int, default=30) + parser.add_argument("--start-date", help="YYYY-MM-DD") + parser.add_argument("--end-date", help="YYYY-MM-DD") + parser.add_argument("--include-pending", action="store_true") + + +def _demo_observations( + household_id: str, + owner_user_id: str | None, + source_account_id: str | None, +) -> list[dict[str, Any]]: + today = date.today().isoformat() + return [ + { + "household_id": household_id, + "source_type": "bank_csv", + "source_account_id": source_account_id, + "owner_user_id": owner_user_id, + "source_reference": "demo-bank-001", + "provider_transaction_id": "demo-bank-001", + "amount": "42.50", + "currency": "EUR", + "event_date": today, + "merchant": "REWE", + "description": "Groceries", + "event_type": "expense", + "status": "confirmed", + }, + { + "household_id": household_id, + "source_type": "paypal_email", + "source_account_id": source_account_id, + "owner_user_id": owner_user_id, + "source_reference": "demo-paypal-email-001", + "message_id": "demo-paypal-email-001", + "amount": "18.99", + "currency": "EUR", + "event_date": today, + "due_date": today, + "merchant": "PayPal Merchant", + "description": "PayPal purchase announced by email", + "event_type": "promised_payment", + "status": "outstanding", + }, + { + "household_id": household_id, + "source_type": "klarna_email", + "source_account_id": source_account_id, + "owner_user_id": owner_user_id, + "source_reference": "demo-klarna-email-001", + "message_id": "demo-klarna-email-001", + "amount": "84.00", + "currency": "EUR", + "event_date": today, + "due_date": today, + "merchant": "Klarna Shop", + "description": "Klarna pay-later obligation", + "event_type": "promised_payment", + "status": "outstanding", + }, + ] diff --git a/apps/local-agent/mousehold_agent/csv_importer.py b/apps/local-agent/mousehold_agent/csv_importer.py new file mode 100644 index 0000000..8d3e502 --- /dev/null +++ b/apps/local-agent/mousehold_agent/csv_importer.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import csv +from datetime import date +from decimal import Decimal +from pathlib import Path +from typing import Any + + +def read_transaction_csv( + path: Path, + *, + household_id: str, + source_type: str, + source_account_id: str | None, + owner_user_id: str | None, +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + with path.open(newline="", encoding="utf-8-sig") as handle: + reader = csv.DictReader(handle) + for row in reader: + event_date = _pick( + row, "event_date", "date", "booking_date", "Buchungstag", "Valutadatum" + ) + amount = _pick(row, "amount", "Betrag", "value") + if not event_date or not amount: + continue + merchant = _pick( + row, + "merchant", + "counterparty", + "Auftraggeber/Empfänger", + "Name Zahlungsbeteiligter", + ) + description = _pick(row, "description", "purpose", "Verwendungszweck", "Buchungstext") + currency = _pick(row, "currency", "Waehrung", "Währung") or "EUR" + provider_transaction_id = _pick( + row, "provider_transaction_id", "transaction_id", "Umsatz ID" + ) + normalized = { + "household_id": household_id, + "source_type": source_type, + "source_account_id": source_account_id, + "owner_user_id": owner_user_id, + "source_reference": provider_transaction_id, + "provider_transaction_id": provider_transaction_id, + "raw_payload_json": row, + "amount": str(_parse_amount(amount)), + "currency": currency, + "event_date": _parse_date(event_date).isoformat(), + "merchant": merchant, + "description": description, + "event_type": "expense" if _parse_amount(amount) < 0 else "income", + "status": "confirmed", + } + normalized["amount"] = str(abs(_parse_amount(amount))) + rows.append(normalized) + return rows + + +def _pick(row: dict[str, str], *keys: str) -> str | None: + lowered = {key.lower(): value for key, value in row.items()} + for key in keys: + value = row.get(key) or lowered.get(key.lower()) + if value: + return value.strip() + return None + + +def _parse_amount(value: str) -> Decimal: + normalized = value.strip().replace(".", "").replace(",", ".") + return Decimal(normalized) + + +def _parse_date(value: str) -> date: + stripped = value.strip() + for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%d/%m/%Y"): + try: + return ( + date.fromisoformat(stripped) + if fmt == "%Y-%m-%d" + else __import__("datetime").datetime.strptime(stripped, fmt).date() + ) + except ValueError: + continue + raise ValueError(f"Unsupported date format: {value}") diff --git a/apps/local-agent/mousehold_agent/fints_connector.py b/apps/local-agent/mousehold_agent/fints_connector.py new file mode 100644 index 0000000..a441736 --- /dev/null +++ b/apps/local-agent/mousehold_agent/fints_connector.py @@ -0,0 +1,582 @@ +from __future__ import annotations + +import base64 +import getpass +import hashlib +import json +import os +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from decimal import Decimal +from html.parser import HTMLParser +from pathlib import Path +from typing import Any + +from fints.client import FinTS3PinTanClient, NeedTANResponse +from fints.models import SEPAAccount +from fints.utils import minimal_interactive_cli_bootstrap + +DEFAULT_PROFILE = "default" +DEFAULT_STATE_DIR = Path.home() / ".config" / "mousehold" / "fints" +PRODUCT_ID_HELP = ( + "FinTS product_id is required by python-fints >=4. Register one with Deutsche Kreditwirtschaft/ZKA " + "and provide it as MOUSEHOLD_FINTS_PRODUCT_ID, --product-id, or in the FinTS web form." +) + + +class FinTSConfigurationError(RuntimeError): + pass + + +@dataclass(frozen=True) +class FinTSConfig: + bank_identifier: str + user_id: str + pin: str + server: str + product_id: str + customer_id: str | None = None + product_version: str | None = None + tan_mechanism: str | None = None + tan_medium: str | None = None + profile: str = DEFAULT_PROFILE + state_dir: Path = DEFAULT_STATE_DIR + store_state: bool = True + interactive_bootstrap: bool = True + + +@dataclass(frozen=True) +class FinTSFetchResult: + account: SEPAAccount + transactions: list[Any] + observations: list[dict[str, Any]] + + +class _HTMLTextExtractor(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.parts: list[str] = [] + + def handle_data(self, data: str) -> None: + stripped = data.strip() + if stripped: + self.parts.append(stripped) + + def text(self) -> str: + return " ".join(self.parts) + + +class FinTSProfileStore: + def __init__(self, state_dir: Path, profile: str) -> None: + self.state_dir = state_dir + self.profile = profile + self.path = state_dir / f"{profile}.json" + + def load_client_data(self) -> bytes | None: + if not self.path.exists(): + return None + data = json.loads(self.path.read_text(encoding="utf-8")) + client_data = data.get("client_data") + if not client_data: + return None + return base64.b64decode(client_data.encode("ascii")) + + def save_client_data(self, client: FinTS3PinTanClient, config: FinTSConfig) -> None: + self.state_dir.mkdir(parents=True, exist_ok=True) + payload = { + "profile": self.profile, + "bank_identifier": config.bank_identifier, + "server": config.server, + "user_id": config.user_id, + "customer_id": config.customer_id, + "updated_at": datetime.now().isoformat(timespec="seconds"), + "client_data": base64.b64encode(client.deconstruct()).decode("ascii"), + } + tmp_path = self.path.with_suffix(".tmp") + tmp_path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + os.chmod(tmp_path, 0o600) + tmp_path.replace(self.path) + os.chmod(self.path, 0o600) + + +def config_from_args(args: Any) -> FinTSConfig: + pin = getattr(args, "pin", None) or os.environ.get(getattr(args, "pin_env", "MOUSEHOLD_FINTS_PIN")) + if not pin: + pin = getpass.getpass("FinTS PIN: ") + + product_id = _required_arg(args, "product_id", "MOUSEHOLD_FINTS_PRODUCT_ID", PRODUCT_ID_HELP) + product_version = getattr(args, "product_version", None) or os.environ.get("MOUSEHOLD_FINTS_PRODUCT_VERSION") + + return FinTSConfig( + bank_identifier=_required_arg(args, "blz", "MOUSEHOLD_FINTS_BLZ"), + user_id=_required_arg(args, "bank_user_id", "MOUSEHOLD_FINTS_USER_ID"), + pin=pin, + server=_required_arg(args, "server", "MOUSEHOLD_FINTS_SERVER"), + customer_id=getattr(args, "customer_id", None) or os.environ.get("MOUSEHOLD_FINTS_CUSTOMER_ID"), + product_id=product_id, + product_version=product_version, + tan_mechanism=getattr(args, "tan_mechanism", None) or os.environ.get("MOUSEHOLD_FINTS_TAN_MECHANISM"), + tan_medium=getattr(args, "tan_medium", None) or os.environ.get("MOUSEHOLD_FINTS_TAN_MEDIUM"), + profile=getattr(args, "profile", DEFAULT_PROFILE), + state_dir=Path(getattr(args, "state_dir", DEFAULT_STATE_DIR)).expanduser(), + store_state=not getattr(args, "no_state", False), + interactive_bootstrap=not getattr(args, "no_bootstrap", False), + ) + + +def create_client(config: FinTSConfig) -> FinTS3PinTanClient: + if not config.product_id: + raise FinTSConfigurationError(PRODUCT_ID_HELP) + + kwargs: dict[str, Any] = {"product_id": config.product_id} + if config.product_version: + kwargs["product_version"] = config.product_version + + store = FinTSProfileStore(config.state_dir, config.profile) + if config.store_state and (client_data := store.load_client_data()): + kwargs["from_data"] = client_data + + return FinTS3PinTanClient( + config.bank_identifier, + config.user_id, + config.pin, + config.server, + customer_id=config.customer_id, + tan_medium=config.tan_medium, + **kwargs, + ) + + +def bootstrap_client(client: FinTS3PinTanClient, config: FinTSConfig) -> None: + if config.tan_mechanism and not client.get_current_tan_mechanism(): + client.set_tan_mechanism(config.tan_mechanism) + if config.interactive_bootstrap: + minimal_interactive_cli_bootstrap(client) + + +def persist_client(client: FinTS3PinTanClient, config: FinTSConfig) -> None: + if config.store_state: + FinTSProfileStore(config.state_dir, config.profile).save_client_data(client, config) + + +def with_fints_dialog(config: FinTSConfig, operation: Any) -> Any: + client = create_client(config) + bootstrap_client(client, config) + try: + with client: + while isinstance(client.init_tan_response, NeedTANResponse): + client.init_tan_response = answer_tan_challenge(client, client.init_tan_response, config) + return operation(client) + finally: + persist_client(client, config) + + +def list_accounts(config: FinTSConfig, *, include_balances: bool = False) -> list[dict[str, Any]]: + def operation(client: FinTS3PinTanClient) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for index, account in enumerate(client.get_sepa_accounts()): + row = account_to_dict(account) + row["index"] = index + if include_balances: + balance = resolve_tan_response(client, client.get_balance(account), config) + row["balance"] = balance_to_dict(balance) + rows.append(row) + return rows + + return with_fints_dialog(config, operation) + + +def fetch_transactions( + config: FinTSConfig, + *, + iban: str | None, + start_date: date, + end_date: date, + include_pending: bool = False, + household_id: str | None = None, + source_account_id: str | None = None, + owner_user_id: str | None = None, +) -> FinTSFetchResult: + def operation(client: FinTS3PinTanClient) -> FinTSFetchResult: + account = select_account(client.get_sepa_accounts(), iban) + response = client.get_transactions( + account, + start_date=start_date, + end_date=end_date, + include_pending=include_pending, + ) + transactions = resolve_tan_response(client, response, config) + observations = [ + transaction_to_observation( + transaction, + account=account, + household_id=household_id, + source_account_id=source_account_id, + owner_user_id=owner_user_id, + ) + for transaction in transactions + ] + return FinTSFetchResult(account=account, transactions=list(transactions), observations=observations) + + return with_fints_dialog(config, operation) + + +def resolve_tan_response( + client: FinTS3PinTanClient, + response: Any, + config: FinTSConfig, +) -> Any: + while isinstance(response, NeedTANResponse): + response = answer_tan_challenge(client, response, config) + return response + + +def answer_tan_challenge( + client: FinTS3PinTanClient, + response: NeedTANResponse, + config: FinTSConfig, +) -> Any: + print("\nTAN required by bank.") + challenge_text = challenge_to_text(response) + if challenge_text: + print(challenge_text) + + challenge_file = write_binary_challenge(response, config) + if challenge_file: + print(f"Challenge file: {challenge_file}") + + if getattr(response, "challenge_hhduc", None): + print("A chipTAN/flicker challenge is available.") + print("Use your bank reader/app if it can read the challenge shown by your terminal.") + + if getattr(response, "decoupled", False): + input("Confirm the login/order in your banking app, then press Enter here.") + return client.send_tan(response, "") + + tan = getpass.getpass("TAN: ") + return client.send_tan(response, tan) + + +def tan_challenge_to_dict(response: NeedTANResponse) -> dict[str, Any]: + matrix = getattr(response, "challenge_matrix", None) + matrix_payload = None + if matrix: + mime_type, payload = matrix + matrix_payload = { + "mime_type": str(mime_type), + "data_base64": base64.b64encode(payload).decode("ascii"), + } + return { + "challenge": challenge_to_text(response), + "challenge_html": str(getattr(response, "challenge_html", "") or ""), + "challenge_hhduc": getattr(response, "challenge_hhduc", None), + "challenge_matrix": matrix_payload, + "decoupled": bool(getattr(response, "decoupled", False)), + } + + +def challenge_to_text(response: NeedTANResponse) -> str: + if getattr(response, "challenge", None): + return str(response.challenge) + if getattr(response, "challenge_html", None): + parser = _HTMLTextExtractor() + parser.feed(str(response.challenge_html)) + return parser.text() + if getattr(response, "challenge_raw", None): + return str(response.challenge_raw) + return "" + + +def write_binary_challenge(response: NeedTANResponse, config: FinTSConfig) -> Path | None: + matrix = getattr(response, "challenge_matrix", None) + if not matrix: + return None + + mime_type, payload = matrix + suffix = ".bin" + if "png" in str(mime_type).lower(): + suffix = ".png" + elif "jpeg" in str(mime_type).lower() or "jpg" in str(mime_type).lower(): + suffix = ".jpg" + + output_dir = config.state_dir / "challenges" + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / f"{config.profile}-{datetime.now().strftime('%Y%m%d%H%M%S')}{suffix}" + path.write_bytes(payload) + os.chmod(path, 0o600) + return path + + +def select_account(accounts: list[SEPAAccount], iban: str | None) -> SEPAAccount: + if not accounts: + raise RuntimeError("No SEPA accounts returned by the bank.") + if not iban: + if len(accounts) > 1: + print("Multiple accounts returned; using the first one. Pass --iban to select explicitly.") + return accounts[0] + normalized = iban.replace(" ", "").upper() + for account in accounts: + if (account.iban or "").replace(" ", "").upper() == normalized: + return account + available = ", ".join(account.iban or "" for account in accounts) + raise RuntimeError(f"IBAN not found. Available accounts: {available}") + + +def date_range_from_args(args: Any) -> tuple[date, date]: + end_date = _parse_date_arg(getattr(args, "end_date", None)) or date.today() + start_date = _parse_date_arg(getattr(args, "start_date", None)) + if not start_date: + start_date = end_date - timedelta(days=getattr(args, "days", 30)) + return start_date, end_date + + +def transaction_to_observation( + transaction: Any, + *, + account: SEPAAccount, + household_id: str | None, + source_account_id: str | None, + owner_user_id: str | None, +) -> dict[str, Any]: + raw_data = transaction_data(transaction) + normalized_data = jsonable(raw_data) + amount, currency = extract_amount(raw_data) + booking_date = extract_date(raw_data, "date", "booking_date", "entry_date") + value_date = extract_date(raw_data, "entry_date", "valuta_date", "date") + counterparty = pick_text( + raw_data, + "applicant_name", + "applicant_iban", + "applicant_bin", + "counterparty", + "name", + ) + description = pick_text( + raw_data, + "purpose", + "transaction_details", + "extra_details", + "posting_text", + "prima_nota", + "additional_position_reference", + ) + reference = pick_text( + raw_data, + "id", + "transaction_reference", + "customer_reference", + "bank_reference", + "end_to_end_reference", + "eref", + "mref", + "svwz", + ) + fallback_reference = stable_transaction_reference(account, normalized_data) + signed_amount = amount + + return { + "household_id": household_id, + "source_type": "fints", + "source_account_id": source_account_id, + "owner_user_id": owner_user_id, + "source_reference": reference or fallback_reference, + "provider_transaction_id": reference or fallback_reference, + "raw_payload_json": { + "account": account_to_dict(account), + "transaction": normalized_data, + "source": "fints", + }, + "amount": str(abs(signed_amount)), + "currency": currency or "EUR", + "event_date": (booking_date or value_date or date.today()).isoformat(), + "booking_date": booking_date.isoformat() if booking_date else None, + "merchant": counterparty, + "counterparty": counterparty, + "description": description, + "event_type": infer_event_type(signed_amount, counterparty, description), + "status": "confirmed", + } + + +def transaction_data(transaction: Any) -> dict[str, Any]: + data = getattr(transaction, "data", transaction) + if isinstance(data, dict): + return data + if hasattr(data, "_asdict"): + return data._asdict() + return {"value": data} + + +def extract_amount(data: dict[str, Any]) -> tuple[Decimal, str | None]: + value = data.get("amount") or data.get("transaction_amount") + currency = data.get("currency") + if hasattr(value, "amount"): + currency = getattr(value, "currency", currency) + return Decimal(str(value.amount)), currency + if isinstance(value, dict): + amount = value.get("amount") or value.get("value") + currency = value.get("currency", currency) + return Decimal(str(amount)), currency + if value is None: + raise RuntimeError(f"Transaction has no amount: {data}") + return Decimal(str(value).replace(",", ".")), currency + + +def extract_date(data: dict[str, Any], *keys: str) -> date | None: + for key in keys: + value = data.get(key) + if isinstance(value, date): + return value + if isinstance(value, datetime): + return value.date() + if isinstance(value, str) and value: + for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%Y%m%d"): + try: + return datetime.strptime(value, fmt).date() + except ValueError: + continue + return None + + +def pick_text(data: dict[str, Any], *keys: str) -> str | None: + values: list[str] = [] + for key in keys: + value = data.get(key) + if value is None or value == "": + continue + if isinstance(value, list | tuple): + values.extend(str(item).strip() for item in value if str(item).strip()) + else: + values.append(str(value).strip()) + if not values: + return None + return " ".join(dict.fromkeys(values))[:500] + + +def infer_event_type(amount: Decimal, counterparty: str | None, description: str | None) -> str: + if amount >= 0: + return "income" + combined = f"{counterparty or ''} {description or ''}".lower() + liability_markers = ( + "paypal", + "klarna", + "kreditkarte", + "credit card", + "visa", + "mastercard", + "american express", + "amex", + ) + if any(marker in combined for marker in liability_markers): + return "liability_payment" + return "expense" + + +def account_to_dict(account: SEPAAccount) -> dict[str, Any]: + return { + "iban": account.iban, + "bic": account.bic, + "accountnumber": account.accountnumber, + "subaccount": account.subaccount, + "blz": account.blz, + } + + +def balance_to_dict(balance: Any) -> dict[str, Any]: + amount = getattr(balance, "amount", None) + return { + "amount": str(getattr(amount, "amount", amount)), + "currency": getattr(amount, "currency", None), + "date": getattr(balance, "date", None).isoformat() + if isinstance(getattr(balance, "date", None), date) + else str(getattr(balance, "date", "")), + "status": getattr(balance, "status", None), + } + + +def stable_transaction_reference(account: SEPAAccount, payload: dict[str, Any]) -> str: + digest = hashlib.sha256( + json.dumps( + {"account": account_to_dict(account), "transaction": payload}, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + return f"fints:{account.iban or account.accountnumber}:{digest}" + + +def jsonable(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): jsonable(item) for key, item in value.items()} + if isinstance(value, list | tuple | set): + return [jsonable(item) for item in value] + if isinstance(value, datetime | date): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if hasattr(value, "amount"): + return { + "amount": str(value.amount), + "currency": getattr(value, "currency", None), + } + if hasattr(value, "_asdict"): + return jsonable(value._asdict()) + return value if isinstance(value, str | int | float | bool | type(None)) else str(value) + + +def display_accounts(accounts: list[dict[str, Any]]) -> None: + rows = [] + for account in accounts: + balance = account.get("balance") + balance_text = "" + if isinstance(balance, dict) and balance.get("amount"): + balance_text = f"{balance['amount']} {balance.get('currency') or ''}".strip() + rows.append( + [ + str(account.get("index", "")), + account.get("iban") or "", + account.get("bic") or "", + account.get("accountnumber") or "", + balance_text, + ] + ) + print_table(["#", "IBAN", "BIC", "Account", "Balance"], rows) + + +def display_transactions(observations: list[dict[str, Any]]) -> None: + rows = [ + [ + observation.get("event_date") or "", + observation.get("event_type") or "", + observation.get("amount") or "", + observation.get("currency") or "", + observation.get("counterparty") or "", + observation.get("description") or "", + ] + for observation in observations + ] + print_table(["Date", "Type", "Amount", "Cur", "Counterparty", "Description"], rows) + + +def print_table(headers: list[str], rows: list[list[str]]) -> None: + widths = [ + max(len(headers[index]), *(len(row[index]) for row in rows)) if rows else len(headers[index]) + for index in range(len(headers)) + ] + print(" ".join(header.ljust(widths[index]) for index, header in enumerate(headers))) + print(" ".join("-" * width for width in widths)) + for row in rows: + print(" ".join(row[index].ljust(widths[index]) for index in range(len(headers)))) + + +def _parse_date_arg(value: str | None) -> date | None: + if not value: + return None + return date.fromisoformat(value) + + +def _required_arg(args: Any, attribute: str, env_name: str, message: str | None = None) -> str: + value = getattr(args, attribute, None) or os.environ.get(env_name) + if not value: + raise RuntimeError(message or f"{attribute.replace('_', '-')} or {env_name} is required") + return value diff --git a/apps/local-agent/mousehold_agent/http.py b/apps/local-agent/mousehold_agent/http.py new file mode 100644 index 0000000..f328abe --- /dev/null +++ b/apps/local-agent/mousehold_agent/http.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import json +from typing import Any +from urllib import request + + +def post_json(url: str, payload: dict[str, Any]) -> dict[str, Any]: + body = json.dumps(payload).encode("utf-8") + req = request.Request( + url, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with request.urlopen(req, timeout=30) as response: + return json.loads(response.read().decode("utf-8")) diff --git a/apps/local-agent/mousehold_agent/imap_importer.py b/apps/local-agent/mousehold_agent/imap_importer.py new file mode 100644 index 0000000..499ace9 --- /dev/null +++ b/apps/local-agent/mousehold_agent/imap_importer.py @@ -0,0 +1,676 @@ +from __future__ import annotations + +import email +import imaplib +import os +import re +import shlex +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal, InvalidOperation +from email import policy +from email.header import decode_header +from email.message import EmailMessage, Message +from email.utils import getaddresses, parsedate_to_datetime +from html.parser import HTMLParser +from typing import Any + +DEFAULT_SENDER_FILTERS = ( + "klarna", + "paypal", + "rechnung", + "invoice", + "receipt", + "order", + "bestellung", + "quittung", +) + +PARSER_NAME = "mousehold-imap-email" +PARSER_VERSION = "0.2.0" +MAX_RAW_TEXT_LENGTH = 20_000 + +_AMOUNT_VALUE = r"[+-]?(?:\d{1,3}(?:[.\s']\d{3})+|\d+)(?:[,.]\d{2})" +_AMOUNT_PATTERNS = ( + re.compile(rf"(?P€|EUR)\s*(?P{_AMOUNT_VALUE})", re.IGNORECASE), + re.compile(rf"(?P{_AMOUNT_VALUE})\s*(?P€|EUR)", re.IGNORECASE), +) +_DATE_VALUE = r"(?P\d{4}-\d{2}-\d{2}|\d{1,2}[./-]\d{1,2}[./-]\d{2,4})" +_DUE_DATE_PATTERNS = ( + re.compile(rf"\b(?:fällig(?:\s+am|\s+bis)?|zahlbar\s+bis|bezahlen\s+bis)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE), + re.compile(rf"\b(?:due(?:\s+on|\s+by)?|pay(?:ment)?\s+due|pay\s+by)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE), + re.compile(rf"\b(?:abbuchung\s+am|wird\s+am|debit(?:ed)?\s+on)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE), +) +_REFERENCE_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { + "order_number": ( + re.compile( + r"\b(?:bestell(?:nummer|nr\.?)|order\s+(?:number|no\.?))\s*[:#-]?\s*" + r"([A-Z0-9][A-Z0-9._/-]{2,})", + re.IGNORECASE, + ), + re.compile(r"\b(?:bestellung|order)\s*[:#-]\s*([A-Z0-9][A-Z0-9._/-]{2,})", re.IGNORECASE), + ), + "invoice_number": ( + re.compile( + r"\b(?:rechnungs(?:nummer|nr\.?)|invoice\s+(?:number|no\.?))\s*[:#-]?\s*" + r"([A-Z0-9][A-Z0-9._/-]{2,})", + re.IGNORECASE, + ), + re.compile(r"\b(?:rechnung|invoice)\s*[:#-]\s*([A-Z0-9][A-Z0-9._/-]{2,})", re.IGNORECASE), + ), + "paypal_transaction_id": ( + re.compile( + r"\b(?:paypal\s+)?(?:transaction|transaktion)(?:\s+id|\s+nr\.?|\s+number)?\s*[:#-]?\s*" + r"([A-Z0-9][A-Z0-9._-]{6,})", + re.IGNORECASE, + ), + ), + "klarna_reference": ( + re.compile( + r"\b(?:klarna\s+)?(?:referenz|reference|zahlung(?:s)?referenz)\s*[:#-]?\s*" + r"([A-Z0-9][A-Z0-9._/-]{4,})", + re.IGNORECASE, + ), + ), +} + + +@dataclass(frozen=True) +class IMAPConfig: + host: str + username: str + password: str + port: int = 993 + folder: str = "INBOX" + use_ssl: bool = True + + +@dataclass(frozen=True) +class AmountMatch: + amount: Decimal + currency: str + context: str + score: int + + +class _HTMLTextExtractor(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.parts: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + del attrs + if tag.lower() in {"br", "p", "div", "li", "tr"}: + self.parts.append("\n") + + def handle_data(self, data: str) -> None: + self.parts.append(data) + + def text(self) -> str: + return _clean_text(" ".join(self.parts)) + + +def config_from_env() -> IMAPConfig: + return IMAPConfig( + host=_required_env("MOUSEHOLD_IMAP_HOST"), + port=int(os.environ.get("MOUSEHOLD_IMAP_PORT", "993")), + username=_required_env("MOUSEHOLD_IMAP_USERNAME"), + password=_required_env("MOUSEHOLD_IMAP_PASSWORD"), + folder=os.environ.get("MOUSEHOLD_IMAP_FOLDER", "INBOX"), + use_ssl=_env_bool("MOUSEHOLD_IMAP_USE_SSL", default=True), + ) + + +def fetch_imap_observations( + *, + household_id: str, + owner_user_id: str | None, + source_account_id: str | None, + limit: int = 25, +) -> list[dict[str, Any]]: + sender_filters = tuple( + value.strip().lower() + for value in os.environ.get( + "MOUSEHOLD_IMAP_SENDER_FILTERS", ",".join(DEFAULT_SENDER_FILTERS) + ).split(",") + if value.strip() + ) + result = scan_imap_mailbox( + config_from_env(), + household_id=household_id, + owner_user_id=owner_user_id, + source_account_id=source_account_id, + limit=limit, + sender_filters=sender_filters, + ) + return result["observations"] + + +def list_imap_folders(config: IMAPConfig) -> list[dict[str, Any]]: + with _connect(config) as mailbox: + _login(mailbox, config) + status, data = mailbox.list() + _ensure_ok(status, data, "list folders") + return [_parse_folder_response(item) for item in data if item] + + +def scan_imap_mailbox( + config: IMAPConfig, + *, + household_id: str, + owner_user_id: str | None = None, + source_account_id: str | None = None, + limit: int = 25, + folder: str | None = None, + search_query: str = "ALL", + sender_filters: tuple[str, ...] | list[str] | None = DEFAULT_SENDER_FILTERS, +) -> dict[str, Any]: + selected_folder = folder or config.folder + filters = _normalize_filters(sender_filters) + messages: list[dict[str, Any]] = [] + observations: list[dict[str, Any]] = [] + scanned_count = 0 + filtered_count = 0 + + with _connect(config) as mailbox: + _login(mailbox, config) + status, data = mailbox.select(_mailbox_name(selected_folder), readonly=True) + _ensure_ok(status, data, f"select folder {selected_folder}") + status, data = mailbox.search(None, *_search_criteria(search_query)) + _ensure_ok(status, data, f"search {search_query}") + ids = data[0].split()[-limit:] if data and data[0] else [] + scanned_count = len(ids) + + for message_id in reversed(ids): + status, fetched = mailbox.fetch(message_id, "(RFC822)") + if _status_text(status) != "OK" or not fetched or not isinstance(fetched[0], tuple): + continue + message = email.message_from_bytes(fetched[0][1], policy=policy.default) + headers_text = _headers_text(message) + body = _message_text(message) + if filters and not _matches_filters(filters, headers_text, body): + filtered_count += 1 + continue + + parsed = parse_email_message( + message, + household_id=household_id, + owner_user_id=owner_user_id, + source_account_id=source_account_id, + folder=selected_folder, + body=body, + ) + messages.append(parsed["message"]) + if parsed["observation"]: + observations.append(parsed["observation"]) + + return { + "folder": selected_folder, + "scanned_count": scanned_count, + "filtered_count": filtered_count, + "matched_count": len(messages), + "importable_count": len(observations), + "messages": messages, + "observations": observations, + } + + +def parse_email_message( + message: Message, + *, + household_id: str, + owner_user_id: str | None, + source_account_id: str | None, + folder: str = "INBOX", + body: str | None = None, +) -> dict[str, Any]: + subject = _decode_mime_header(message.get("Subject")) + from_header = _decode_mime_header(message.get("From")) + to_header = _decode_mime_header(message.get("To")) + message_id = _compact_header_value(message.get("Message-ID")) + received_date = _message_date(message) or date.today() + text = body if body is not None else _message_text(message) + search_text = f"{subject}\n{text}" + source_type = _source_type(from_header, subject, text) + amount_match = _extract_amount_match(search_text) + due_date = _extract_due_date(search_text) + references = _extract_references(subject, text) + provider_transaction_id = _provider_transaction_id(references) + merchant = _merchant(from_header, subject, text, source_type) + event_type = _event_type(source_type, subject, text, due_date) + event_status = _event_status(source_type, event_type, due_date, subject, text) + source_reference = ( + message_id + or provider_transaction_id + or _fallback_source_reference(from_header, subject, received_date) + ) + if subject: + description = subject + elif merchant: + description = f"Email from {merchant}" + else: + description = "Email import" + + preview = { + "message_id": message_id, + "source_reference": source_reference, + "from": from_header, + "to": to_header, + "subject": subject, + "date": received_date.isoformat(), + "folder": folder, + "source_type": source_type, + "event_type": event_type, + "status": event_status, + "amount": str(amount_match.amount) if amount_match else None, + "currency": amount_match.currency if amount_match else None, + "merchant": merchant, + "due_date": due_date.isoformat() if due_date else None, + "references": references, + "importable": amount_match is not None, + "skip_reason": None if amount_match else "No EUR amount found", + } + + if not amount_match: + return {"message": preview, "observation": None} + + observation = { + "household_id": household_id, + "source_type": source_type, + "source_account_id": source_account_id, + "owner_user_id": owner_user_id, + "message_id": message_id, + "source_reference": source_reference, + "provider_transaction_id": provider_transaction_id, + "raw_text": text[:MAX_RAW_TEXT_LENGTH], + "raw_payload_json": { + "from": from_header, + "to": to_header, + "subject": subject, + "date": _compact_header_value(message.get("Date")), + "folder": folder, + "message_id": message_id, + "references": references, + "amount_context": amount_match.context, + "parser_name": PARSER_NAME, + "parser_version": PARSER_VERSION, + }, + "amount": str(amount_match.amount), + "currency": amount_match.currency, + "event_date": received_date.isoformat(), + "due_date": due_date.isoformat() if due_date else None, + "merchant": merchant, + "description": description[:240], + "event_type": event_type, + "status": event_status, + "payer_user_id": owner_user_id, + "payment_account_id": source_account_id, + } + return {"message": preview, "observation": observation} + + +def _connect(config: IMAPConfig) -> imaplib.IMAP4: + if config.use_ssl: + return imaplib.IMAP4_SSL(config.host, config.port) + return imaplib.IMAP4(config.host, config.port) + + +def _login(mailbox: imaplib.IMAP4, config: IMAPConfig) -> None: + status, data = mailbox.login(config.username, config.password) + _ensure_ok(status, data, "login") + + +def _required_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value + + +def _env_bool(name: str, *, default: bool) -> bool: + value = os.environ.get(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _status_text(status: str | bytes) -> str: + return status.decode("ascii", errors="replace").upper() if isinstance(status, bytes) else status.upper() + + +def _ensure_ok(status: str | bytes, data: object, action: str) -> None: + if _status_text(status) != "OK": + raise RuntimeError(f"IMAP {action} failed: {data!r}") + + +def _mailbox_name(folder: str) -> str: + if folder.startswith('"') and folder.endswith('"'): + return folder + if any(char in folder for char in ' "()[\\]'): + return '"' + folder.replace("\\", "\\\\").replace('"', '\\"') + '"' + return folder + + +def _search_criteria(search_query: str) -> list[str]: + query = search_query.strip() or "ALL" + try: + parts = shlex.split(query) + except ValueError: + parts = query.split() + return parts or ["ALL"] + + +def _normalize_filters(sender_filters: tuple[str, ...] | list[str] | None) -> tuple[str, ...]: + if sender_filters is None: + return () + return tuple(token.strip().lower() for token in sender_filters if token.strip()) + + +def _matches_filters(filters: tuple[str, ...], headers_text: str, body: str) -> bool: + haystack = f"{headers_text}\n{body[:5000]}".lower() + return any(token in haystack for token in filters) + + +def _headers_text(message: Message) -> str: + return "\n".join( + [ + _decode_mime_header(message.get("From")), + _decode_mime_header(message.get("To")), + _decode_mime_header(message.get("Subject")), + ] + ) + + +def _decode_mime_header(value: str | None) -> str: + if not value: + return "" + parts: list[str] = [] + for content, charset in decode_header(value): + if isinstance(content, bytes): + parts.append(content.decode(charset or "utf-8", errors="replace")) + else: + parts.append(content) + return _clean_text("".join(parts)) + + +def _compact_header_value(value: str | None) -> str | None: + cleaned = _decode_mime_header(value) + return cleaned or None + + +def _message_text(message: Message) -> str: + if isinstance(message, EmailMessage): + body = message.get_body(preferencelist=("plain", "html")) + if body: + content = body.get_content() + if body.get_content_type() == "text/html": + return _html_to_text(content) + return _clean_text(content) + + if message.is_multipart(): + plain_parts: list[str] = [] + html_parts: list[str] = [] + for part in message.walk(): + if part.get_content_maintype() != "text": + continue + payload = part.get_payload(decode=True) + if not payload: + continue + text = payload.decode(part.get_content_charset() or "utf-8", errors="replace") + if part.get_content_type() == "text/plain": + plain_parts.append(text) + elif part.get_content_type() == "text/html": + html_parts.append(_html_to_text(text)) + return _clean_text("\n".join(plain_parts or html_parts)) + + payload = message.get_payload(decode=True) + if payload: + text = payload.decode(message.get_content_charset() or "utf-8", errors="replace") + else: + text = str(message.get_payload()) + if message.get_content_type() == "text/html": + return _html_to_text(text) + return _clean_text(text) + + +def _html_to_text(html: str) -> str: + extractor = _HTMLTextExtractor() + extractor.feed(html) + return extractor.text() + + +def _clean_text(text: str) -> str: + text = text.replace("\xa0", " ") + text = re.sub(r"[ \t\r\f\v]+", " ", text) + text = re.sub(r"\n\s+", "\n", text) + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() + + +def _message_date(message: Message) -> date | None: + header = message.get("Date") + if not header: + return None + try: + parsed = parsedate_to_datetime(header) + except (TypeError, ValueError): + return None + return parsed.date() + + +def _extract_amount_match(text: str) -> AmountMatch | None: + candidates: list[AmountMatch] = [] + for pattern in _AMOUNT_PATTERNS: + for match in pattern.finditer(text): + amount = _parse_amount(match.group("amount")) + if amount is None: + continue + start = max(match.start() - 90, 0) + end = min(match.end() + 90, len(text)) + context = _clean_text(text[start:end]) + candidates.append( + AmountMatch( + amount=amount, + currency="EUR", + context=context, + score=_amount_context_score(context), + ) + ) + if not candidates: + return None + return sorted(candidates, key=lambda item: (item.score, item.amount), reverse=True)[0] + + +def _parse_amount(value: str) -> Decimal | None: + cleaned = value.replace("\xa0", "").replace(" ", "").replace("'", "") + if "," in cleaned and "." in cleaned: + if cleaned.rfind(",") > cleaned.rfind("."): + cleaned = cleaned.replace(".", "").replace(",", ".") + else: + cleaned = cleaned.replace(",", "") + elif "," in cleaned: + cleaned = cleaned.replace(".", "").replace(",", ".") + elif "." in cleaned and len(cleaned.rsplit(".", maxsplit=1)[-1]) != 2: + cleaned = cleaned.replace(".", "") + try: + return abs(Decimal(cleaned)).quantize(Decimal("0.01")) + except InvalidOperation: + return None + + +def _amount_context_score(context: str) -> int: + lowered = context.lower() + score = 0 + positive = ( + "gesamt", + "gesamtbetrag", + "summe", + "total", + "amount", + "betrag", + "zahlbetrag", + "payment", + "zahlung", + "fällig", + "due", + ) + negative = ("mwst", "vat", "tax", "steuer", "versand", "shipping", "rabatt", "discount", "gutschein") + if any(token in lowered for token in positive): + score += 20 + if any(token in lowered for token in ("fällig", "due", "zahlbar", "pay by")): + score += 8 + if any(token in lowered for token in negative): + score -= 10 + return score + + +def _extract_due_date(text: str) -> date | None: + for pattern in _DUE_DATE_PATTERNS: + match = pattern.search(text) + if match: + parsed = _parse_date_value(match.group("date")) + if parsed: + return parsed + return None + + +def _parse_date_value(value: str) -> date | None: + if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): + try: + return datetime.strptime(value, "%Y-%m-%d").date() + except ValueError: + return None + parts = re.split(r"[./-]", value) + if len(parts) != 3: + return None + day, month, year = parts + try: + numeric_year = int(year) + if numeric_year < 100: + numeric_year += 2000 + return date(numeric_year, int(month), int(day)) + except ValueError: + return None + + +def _extract_references(subject: str, text: str) -> dict[str, str]: + haystack = f"{subject}\n{text}" + references: dict[str, str] = {} + for key, patterns in _REFERENCE_PATTERNS.items(): + for pattern in patterns: + match = pattern.search(haystack) + if match: + references[key] = match.group(1).strip(" .,:;") + break + return references + + +def _provider_transaction_id(references: dict[str, str]) -> str | None: + for key in ("paypal_transaction_id", "klarna_reference", "order_number", "invoice_number"): + if references.get(key): + return references[key] + return None + + +def _source_type(sender: str, subject: str, text: str) -> str: + combined = f"{sender}\n{subject}\n{text[:3000]}".lower() + if "klarna" in combined: + return "klarna_email" + if "paypal" in combined or "pay pal" in combined: + return "paypal_email" + if any(token in combined for token in ("bestellung", "order confirmation", "your order", "deine bestellung")): + return "order_confirmation_email" + return "imap_email" + + +def _event_type(source_type: str, subject: str, text: str, due_date: date | None) -> str: + combined = f"{subject}\n{text[:3000]}".lower() + if any(token in combined for token in ("refund", "erstattung", "rückerstattung")): + return "refund" + if source_type == "klarna_email": + return "promised_payment" + if source_type == "paypal_email" and ( + due_date or any(token in combined for token in ("pay later", "später bezahlen", "fällig", "due")) + ): + return "promised_payment" + return "expense" + + +def _event_status( + source_type: str, + event_type: str, + due_date: date | None, + subject: str, + text: str, +) -> str: + del subject, text + if source_type == "klarna_email" or event_type == "promised_payment": + return "outstanding" if due_date else "announced" + return "announced" + + +def _merchant(sender: str, subject: str, text: str, source_type: str) -> str | None: + subject_merchant = _merchant_from_subject(subject) + if subject_merchant: + return subject_merchant + text_merchant = _merchant_from_text(text) + if text_merchant: + return text_merchant + addresses = getaddresses([sender]) + display_name = addresses[0][0] if addresses else "" + candidate = _clean_party_name(display_name or sender) + if source_type == "klarna_email" and not candidate: + return "Klarna" + if source_type == "paypal_email" and not candidate: + return "PayPal" + return candidate or None + + +def _merchant_from_subject(subject: str) -> str | None: + patterns = ( + r"\b(?:deine|ihre)\s+bestellung\s+(?:bei|von)\s+(.+)$", + r"\byour\s+order\s+(?:from|with|at)\s+(.+)$", + r"\b(?:zahlung|payment)\s+(?:an|to)\s+(.+)$", + r"\b(?:rechnung|invoice)\s+(?:von|from)\s+(.+)$", + ) + for pattern in patterns: + match = re.search(pattern, subject, re.IGNORECASE) + if match: + return _clean_party_name(match.group(1)) + return None + + +def _merchant_from_text(text: str) -> str | None: + patterns = ( + r"\b(?:händler|merchant|seller|shop)\s*:?\s*([^\n]{2,80})", + r"\b(?:zahlung\s+an|payment\s+to)\s*:?\s*([^\n]{2,80})", + ) + for pattern in patterns: + match = re.search(pattern, text, re.IGNORECASE) + if match: + return _clean_party_name(match.group(1)) + return None + + +def _clean_party_name(value: str) -> str: + cleaned = re.sub(r"<[^>]+>", "", value) + cleaned = re.split(r"\s[-|:]\s", cleaned, maxsplit=1)[0] + cleaned = re.sub(r"\s+", " ", cleaned).strip(" .,:;\"'") + return cleaned[:120] + + +def _fallback_source_reference(sender: str, subject: str, received_date: date) -> str: + return f"imap:{received_date.isoformat()}:{sender}:{subject}"[:240] + + +def _parse_folder_response(raw: bytes | str) -> dict[str, Any]: + text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw + match = re.match(r"\((?P.*?)\)\s+\"?(?P.*?)\"?\s+(?P.+)$", text) + if not match: + return {"name": text, "delimiter": None, "attributes": []} + name = match.group("name").strip() + if name.startswith('"') and name.endswith('"'): + name = name[1:-1].replace('\\"', '"').replace("\\\\", "\\") + attrs = [item.strip("\\") for item in match.group("attrs").split() if item.strip()] + delimiter = match.group("delimiter") or None + return {"name": name, "delimiter": delimiter, "attributes": attrs} diff --git a/apps/local-agent/mousehold_agent/paypal_connector.py b/apps/local-agent/mousehold_agent/paypal_connector.py new file mode 100644 index 0000000..f65f577 --- /dev/null +++ b/apps/local-agent/mousehold_agent/paypal_connector.py @@ -0,0 +1,459 @@ +from __future__ import annotations + +import base64 +import json +import re +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from datetime import UTC, date, datetime, time +from decimal import Decimal, InvalidOperation +from typing import Any, Literal + +PAYPAL_LIVE_BASE_URL = "https://api-m.paypal.com" +PAYPAL_SANDBOX_BASE_URL = "https://api-m.sandbox.paypal.com" +PAYPAL_REPORTING_MAX_DAYS = 31 +PAYPAL_TRANSACTION_SEARCH_SCOPE = "https://uri.paypal.com/services/reporting/search/read" + + +@dataclass(frozen=True) +class PayPalConfig: + client_id: str + client_secret: str + environment: Literal["live", "sandbox"] = "live" + base_url: str | None = None + + +@dataclass(frozen=True) +class PayPalFetchResult: + start_date: date + end_date: date + transactions: list[dict[str, Any]] + observations: list[dict[str, Any]] + raw_response_count: int + + +class PayPalAPIError(RuntimeError): + pass + + +def inspect_paypal_credentials(config: PayPalConfig) -> dict[str, Any]: + return _token_payload_to_auth_check(config, fetch_access_token_payload(config)) + + +def _token_payload_to_auth_check(config: PayPalConfig, payload: dict[str, Any]) -> dict[str, Any]: + scope = payload.get("scope") + scopes = [item for item in re.split(r"[\s,]+", scope.strip()) if item] if isinstance(scope, str) else [] + return { + "environment": config.environment, + "base_url": _base_url(config), + "app_id": _string_or_none(payload.get("app_id")), + "token_type": _string_or_none(payload.get("token_type")), + "expires_in": _int_or_none(payload.get("expires_in")), + "scope_text": scope if isinstance(scope, str) else None, + "scopes": scopes, + "required_scope": PAYPAL_TRANSACTION_SEARCH_SCOPE, + "has_transaction_search": PAYPAL_TRANSACTION_SEARCH_SCOPE in scopes, + } + + +def fetch_paypal_transactions( + config: PayPalConfig, + *, + start_date: date, + end_date: date, + household_id: str | None = None, + source_account_id: str | None = None, + owner_user_id: str | None = None, + transaction_currency: str | None = None, + page_size: int = 100, + max_pages: int = 10, +) -> PayPalFetchResult: + _validate_date_range(start_date, end_date) + token = fetch_access_token(config) + details = list_transaction_details( + config, + token=token, + start_date=start_date, + end_date=end_date, + transaction_currency=transaction_currency, + page_size=page_size, + max_pages=max_pages, + ) + observations = [ + paypal_transaction_to_observation( + transaction, + household_id=household_id, + source_account_id=source_account_id, + owner_user_id=owner_user_id, + ) + for transaction in details + if _transaction_amount(transaction) is not None + ] + return PayPalFetchResult( + start_date=start_date, + end_date=end_date, + transactions=[paypal_transaction_to_preview(transaction) for transaction in details], + observations=observations, + raw_response_count=len(details), + ) + + +def fetch_access_token(config: PayPalConfig) -> str: + payload = fetch_access_token_payload(config) + access_token = payload.get("access_token") + if not isinstance(access_token, str) or not access_token: + raise PayPalAPIError("PayPal token response did not include an access token.") + scope = payload.get("scope") + if isinstance(scope, str) and PAYPAL_TRANSACTION_SEARCH_SCOPE not in scope.split(): + raise PayPalAPIError( + "PayPal credentials do not include Transaction Search permission. " + f"Enable Transaction Search on the PayPal REST app and verify the OAuth token includes " + f"{PAYPAL_TRANSACTION_SEARCH_SCOPE}." + ) + return access_token + + +def fetch_access_token_payload(config: PayPalConfig) -> dict[str, Any]: + credentials = f"{config.client_id}:{config.client_secret}".encode() + body = urllib.parse.urlencode({"grant_type": "client_credentials"}).encode("ascii") + request = urllib.request.Request( + f"{_base_url(config)}/v1/oauth2/token", + data=body, + method="POST", + headers={ + "Authorization": f"Basic {base64.b64encode(credentials).decode('ascii')}", + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "application/json", + }, + ) + return _request_json(request) + + +def list_transaction_details( + config: PayPalConfig, + *, + token: str, + start_date: date, + end_date: date, + transaction_currency: str | None = None, + page_size: int = 100, + max_pages: int = 10, +) -> list[dict[str, Any]]: + page_size = max(1, min(page_size, 500)) + max_pages = max(1, max_pages) + transactions: list[dict[str, Any]] = [] + page = 1 + total_pages: int | None = None + + while page <= max_pages and (total_pages is None or page <= total_pages): + params = { + "start_date": _paypal_datetime(start_date, end_of_day=False), + "end_date": _paypal_datetime(end_date, end_of_day=True), + "fields": "all", + "page_size": str(page_size), + "page": str(page), + } + if transaction_currency: + params["transaction_currency"] = transaction_currency.upper() + + url = f"{_base_url(config)}/v1/reporting/transactions?{urllib.parse.urlencode(params)}" + request = urllib.request.Request( + url, + method="GET", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/json", + }, + ) + payload = _request_json(request) + page_transactions = payload.get("transaction_details") or [] + if not isinstance(page_transactions, list): + raise PayPalAPIError("PayPal transaction response has an unexpected shape.") + transactions.extend(item for item in page_transactions if isinstance(item, dict)) + + total_pages = _int_or_none(payload.get("total_pages")) + if total_pages is None and len(page_transactions) < page_size: + break + page += 1 + + return transactions + + +def paypal_transaction_to_preview(transaction: dict[str, Any]) -> dict[str, Any]: + info = _transaction_info(transaction) + amount = _transaction_amount(transaction) + currency = _currency_code(info) + event_date = _transaction_date(info) + return { + "transaction_id": _transaction_id(info), + "event_date": event_date.isoformat(), + "updated_date": _date_string(info.get("transaction_updated_date")), + "event_code": _string_or_none(info.get("transaction_event_code")), + "paypal_status": _string_or_none(info.get("transaction_status")), + "amount": str(abs(amount)) if amount is not None else None, + "currency": currency, + "counterparty": _counterparty(transaction), + "description": _description(transaction), + "event_type": _event_type(info, amount), + "status": _event_status(info), + } + + +def paypal_transaction_to_observation( + transaction: dict[str, Any], + *, + household_id: str | None, + source_account_id: str | None, + owner_user_id: str | None, +) -> dict[str, Any]: + info = _transaction_info(transaction) + amount = _transaction_amount(transaction) + if amount is None: + raise ValueError("PayPal transaction has no transaction_amount.value") + transaction_id = _transaction_id(info) + currency = _currency_code(info) + event_date = _transaction_date(info) + return { + "household_id": household_id, + "source_type": "paypal_api", + "source_account_id": source_account_id, + "owner_user_id": owner_user_id, + "source_reference": transaction_id, + "provider_transaction_id": transaction_id, + "raw_payload_json": transaction, + "amount": str(abs(amount)), + "currency": currency, + "event_date": event_date.isoformat(), + "booking_date": _date_string(info.get("transaction_updated_date")), + "merchant": _counterparty(transaction), + "counterparty": _counterparty(transaction), + "description": _description(transaction), + "event_type": _event_type(info, amount), + "status": _event_status(info), + "payer_user_id": owner_user_id, + "payment_account_id": source_account_id, + } + + +def _request_json(request: urllib.request.Request) -> dict[str, Any]: + try: + with urllib.request.urlopen(request, timeout=30) as response: + data = response.read() + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise PayPalAPIError(_paypal_error_message(exc.code, detail)) from exc + except urllib.error.URLError as exc: + raise PayPalAPIError(f"PayPal request failed: {exc.reason}") from exc + + try: + payload = json.loads(data.decode("utf-8")) + except json.JSONDecodeError as exc: + raise PayPalAPIError("PayPal returned invalid JSON.") from exc + if not isinstance(payload, dict): + raise PayPalAPIError("PayPal returned an unexpected JSON shape.") + return payload + + +def _paypal_error_message(status_code: int, detail: str) -> str: + try: + payload = json.loads(detail) + except json.JSONDecodeError: + return f"PayPal request failed with HTTP {status_code}: {detail[:500]}" + message = payload.get("message") or payload.get("error_description") or payload.get("error") + name = payload.get("name") + if status_code == 403 and (name == "NOT_AUTHORIZED" or "insufficient permissions" in str(message).lower()): + return ( + "PayPal Transaction Search is not authorized for these credentials. " + "Enable Transaction Search on the matching live/sandbox REST app, make sure you are using credentials " + "for the same environment selected in Mousehold, and verify the OAuth token includes " + f"{PAYPAL_TRANSACTION_SEARCH_SCOPE}. Third-party account access requires PayPal partner authorization." + ) + if name and message: + return f"PayPal request failed with HTTP {status_code}: {name}: {message}" + if message: + return f"PayPal request failed with HTTP {status_code}: {message}" + return f"PayPal request failed with HTTP {status_code}: {detail[:500]}" + + +def _base_url(config: PayPalConfig) -> str: + if config.base_url: + return config.base_url.rstrip("/") + return PAYPAL_SANDBOX_BASE_URL if config.environment == "sandbox" else PAYPAL_LIVE_BASE_URL + + +def _validate_date_range(start_date: date, end_date: date) -> None: + if end_date < start_date: + raise PayPalAPIError("PayPal end_date must be on or after start_date.") + if (end_date - start_date).days + 1 > PAYPAL_REPORTING_MAX_DAYS: + raise PayPalAPIError("PayPal Transaction Search supports a maximum date range of 31 days.") + + +def _paypal_datetime(value: date, *, end_of_day: bool) -> str: + wall_time = time(23, 59, 59) if end_of_day else time(0, 0, 0) + return datetime.combine(value, wall_time, tzinfo=UTC).isoformat().replace("+00:00", "Z") + + +def _transaction_info(transaction: dict[str, Any]) -> dict[str, Any]: + info = transaction.get("transaction_info") or {} + return info if isinstance(info, dict) else {} + + +def _transaction_amount(transaction: dict[str, Any]) -> Decimal | None: + info = _transaction_info(transaction) + amount = info.get("transaction_amount") or {} + if not isinstance(amount, dict): + return None + value = amount.get("value") + if value is None: + return None + try: + return Decimal(str(value)).quantize(Decimal("0.01")) + except InvalidOperation: + return None + + +def _currency_code(info: dict[str, Any]) -> str: + amount = info.get("transaction_amount") or {} + if isinstance(amount, dict): + currency = amount.get("currency_code") + if isinstance(currency, str) and currency: + return currency.upper() + return "EUR" + + +def _transaction_id(info: dict[str, Any]) -> str | None: + return _string_or_none(info.get("transaction_id")) or _string_or_none(info.get("paypal_reference_id")) + + +def _transaction_date(info: dict[str, Any]) -> date: + return _parse_paypal_date(info.get("transaction_initiation_date")) or date.today() + + +def _date_string(value: object) -> str | None: + parsed = _parse_paypal_date(value) + return parsed.isoformat() if parsed else None + + +def _parse_paypal_date(value: object) -> date | None: + if not isinstance(value, str) or not value: + return None + normalized = value.replace("Z", "+00:00") + if reworked := _parse_iso_date(normalized): + return reworked + for fmt in ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S.%f%z"): + try: + return datetime.strptime(value, fmt).date() + except ValueError: + continue + try: + return date.fromisoformat(value[:10]) + except ValueError: + return None + + +def _parse_iso_date(value: str) -> date | None: + try: + return datetime.fromisoformat(value).date() + except ValueError: + return None + + +def _counterparty(transaction: dict[str, Any]) -> str | None: + payer_info = transaction.get("payer_info") or {} + if isinstance(payer_info, dict): + payer_name = payer_info.get("payer_name") or {} + if isinstance(payer_name, dict): + full_name = _first_non_empty( + payer_name.get("alternate_full_name"), + " ".join( + str(part) + for part in (payer_name.get("given_name"), payer_name.get("surname")) + if part + ), + ) + if full_name: + return full_name + email = _string_or_none(payer_info.get("email_address")) + if email: + return email + + shipping_info = transaction.get("shipping_info") or {} + if isinstance(shipping_info, dict): + name = _string_or_none(shipping_info.get("name")) + if name: + return name + return None + + +def _description(transaction: dict[str, Any]) -> str | None: + info = _transaction_info(transaction) + cart_info = transaction.get("cart_info") or {} + if isinstance(cart_info, dict): + invoice_number = _string_or_none(cart_info.get("invoice_number")) + if invoice_number: + return f"PayPal invoice {invoice_number}" + item_details = cart_info.get("item_details") or [] + if isinstance(item_details, list) and item_details: + first_item = item_details[0] + if isinstance(first_item, dict): + item_name = _string_or_none(first_item.get("item_name")) + if item_name: + return item_name + return _first_non_empty( + info.get("transaction_subject"), + info.get("transaction_note"), + info.get("paypal_reference_id"), + info.get("transaction_id"), + ) + + +def _event_type(info: dict[str, Any], amount: Decimal | None) -> str: + description = " ".join( + str(value) + for value in ( + info.get("transaction_subject"), + info.get("transaction_note"), + info.get("transaction_event_code"), + ) + if value + ).lower() + if "fee" in description or "gebühr" in description: + return "fee" + if "refund" in description or "erstattung" in description or "rückerstattung" in description: + return "refund" + if amount is None: + return "expense" + return "expense" if amount < 0 else "income" + + +def _event_status(info: dict[str, Any]) -> str: + status = _string_or_none(info.get("transaction_status")) + if status == "P": + return "announced" + if status == "D": + return "cancelled" + return "confirmed" + + +def _string_or_none(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _first_non_empty(*values: object) -> str | None: + for value in values: + text = _string_or_none(value) + if text: + return text + return None + + +def _int_or_none(value: object) -> int | None: + try: + return int(str(value)) + except (TypeError, ValueError): + return None diff --git a/apps/local-agent/mousehold_agent/server.py b/apps/local-agent/mousehold_agent/server.py new file mode 100644 index 0000000..9a46ccd --- /dev/null +++ b/apps/local-agent/mousehold_agent/server.py @@ -0,0 +1,581 @@ +from __future__ import annotations + +import imaplib +import os +import threading +import uuid +from datetime import date, timedelta +from pathlib import Path +from typing import Any, Literal + +import uvicorn +from fastapi import APIRouter, FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from fints.client import FinTS3PinTanClient, NeedTANResponse +from pydantic import BaseModel, Field + +from .fints_connector import ( + PRODUCT_ID_HELP, + FinTSConfig, + FinTSConfigurationError, + account_to_dict, + balance_to_dict, + create_client, + persist_client, + select_account, + tan_challenge_to_dict, + transaction_to_observation, +) +from .imap_importer import DEFAULT_SENDER_FILTERS, IMAPConfig, list_imap_folders, scan_imap_mailbox +from .paypal_connector import PayPalAPIError, PayPalConfig, fetch_paypal_transactions, inspect_paypal_credentials +from .statement_importer import parse_klarna_statement, parse_paypal_statement + +LOCAL_AGENT_ORIGINS = [ + "http://127.0.0.1:5173", + "http://localhost:5173", +] + + +class FinTSRequest(BaseModel): + blz: str + server: str + bank_user_id: str + pin: str + customer_id: str | None = None + product_id: str | None = None + product_version: str | None = None + tan_mechanism: str | None = None + tan_medium: str | None = None + profile: str = "browser" + state_dir: str = "~/.config/mousehold/fints" + store_state: bool = True + + +class FinTSAccountsRequest(BaseModel): + credentials: FinTSRequest + with_balances: bool = False + + +class FinTSTransactionsRequest(BaseModel): + credentials: FinTSRequest + iban: str | None = None + days: int = Field(default=30, ge=1, le=730) + start_date: date | None = None + end_date: date | None = None + include_pending: bool = False + household_id: str | None = None + source_account_id: str | None = None + owner_user_id: str | None = None + + +class FinTSChoiceRequest(BaseModel): + value: str + + +class FinTSTanRequest(BaseModel): + tan: str = "" + + +class IMAPCredentials(BaseModel): + host: str + port: int = Field(default=993, ge=1, le=65535) + username: str + password: str + use_ssl: bool = True + + +class IMAPFoldersRequest(BaseModel): + credentials: IMAPCredentials + + +class IMAPScanRequest(BaseModel): + credentials: IMAPCredentials + folder: str = "INBOX" + search_query: str = "ALL" + sender_filters: list[str] = Field(default_factory=lambda: list(DEFAULT_SENDER_FILTERS)) + limit: int = Field(default=25, ge=1, le=500) + household_id: str + source_account_id: str | None = None + owner_user_id: str | None = None + + +class PayPalCredentials(BaseModel): + client_id: str + client_secret: str + environment: Literal["live", "sandbox"] = "live" + base_url: str | None = None + + +class PayPalTransactionsRequest(BaseModel): + credentials: PayPalCredentials + days: int = Field(default=30, ge=1, le=31) + start_date: date | None = None + end_date: date | None = None + transaction_currency: str | None = None + page_size: int = Field(default=100, ge=1, le=500) + max_pages: int = Field(default=10, ge=1, le=100) + household_id: str + source_account_id: str | None = None + owner_user_id: str | None = None + + +class PayPalAuthCheckRequest(BaseModel): + credentials: PayPalCredentials + + +class StatementParseRequest(BaseModel): + csv_text: str + filename: str | None = None + household_id: str + source_account_id: str | None = None + owner_user_id: str | None = None + + +JobStatus = Literal["running", "needs_tan", "needs_choice", "done", "error"] + + +class FinTSBrowserJob: + def __init__(self, kind: str) -> None: + self.id = str(uuid.uuid4()) + self.kind = kind + self.status: JobStatus = "running" + self.error: str | None = None + self.result: dict[str, Any] | None = None + self.challenge: dict[str, Any] | None = None + self.choice_kind: str | None = None + self.choice_options: list[dict[str, Any]] = [] + self._lock = threading.Lock() + self._event = threading.Event() + self._input_value: str | None = None + + def snapshot(self) -> dict[str, Any]: + with self._lock: + return { + "id": self.id, + "kind": self.kind, + "status": self.status, + "error": self.error, + "result": self.result, + "challenge": self.challenge, + "choice_kind": self.choice_kind, + "choice_options": self.choice_options, + } + + def set_done(self, result: dict[str, Any]) -> None: + with self._lock: + self.status = "done" + self.result = result + self.challenge = None + self.choice_kind = None + self.choice_options = [] + self._input_value = None + self._event.clear() + + def set_error(self, error: BaseException | str) -> None: + with self._lock: + self.status = "error" + self.error = str(error) + self.challenge = None + self.choice_kind = None + self.choice_options = [] + self._input_value = None + self._event.clear() + + def wait_for_tan(self, response: NeedTANResponse, timeout_seconds: int = 600) -> str: + with self._lock: + self.status = "needs_tan" + self.challenge = tan_challenge_to_dict(response) + self.choice_kind = None + self.choice_options = [] + self._input_value = None + self._event.clear() + if not self._event.wait(timeout_seconds): + raise TimeoutError("Timed out waiting for TAN input.") + with self._lock: + value = self._input_value or "" + self.status = "running" + self.challenge = None + self._input_value = None + self._event.clear() + return value + + def wait_for_choice( + self, + kind: str, + options: list[dict[str, Any]], + timeout_seconds: int = 600, + ) -> str: + with self._lock: + self.status = "needs_choice" + self.choice_kind = kind + self.choice_options = options + self.challenge = None + self._input_value = None + self._event.clear() + if not self._event.wait(timeout_seconds): + raise TimeoutError(f"Timed out waiting for {kind} selection.") + with self._lock: + value = self._input_value + self.status = "running" + self.choice_kind = None + self.choice_options = [] + self._input_value = None + self._event.clear() + if value is None: + raise RuntimeError(f"No {kind} selection received.") + return value + + def provide_value(self, value: str) -> None: + with self._lock: + if self.status not in {"needs_tan", "needs_choice"}: + raise RuntimeError("Job is not waiting for input.") + self._input_value = value + self._event.set() + + +jobs: dict[str, FinTSBrowserJob] = {} + + +def create_router() -> APIRouter: + router = APIRouter() + + @router.get("/health") + def health() -> dict[str, str]: + return {"status": "ok"} + + @router.post("/fints/accounts/start") + def start_accounts(payload: FinTSAccountsRequest) -> dict[str, Any]: + job = _start_job("accounts", lambda current_job: _run_accounts(current_job, payload)) + return job.snapshot() + + @router.post("/fints/transactions/start") + def start_transactions(payload: FinTSTransactionsRequest) -> dict[str, Any]: + job = _start_job("transactions", lambda current_job: _run_transactions(current_job, payload)) + return job.snapshot() + + @router.get("/fints/jobs/{job_id}") + def get_job(job_id: str) -> dict[str, Any]: + return _get_job(job_id).snapshot() + + @router.post("/fints/jobs/{job_id}/tan") + def provide_tan(job_id: str, payload: FinTSTanRequest) -> dict[str, Any]: + job = _get_job(job_id) + if job.snapshot()["status"] != "needs_tan": + raise HTTPException(status_code=409, detail="Job is not waiting for a TAN.") + try: + job.provide_value(payload.tan) + except RuntimeError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + return job.snapshot() + + @router.post("/fints/jobs/{job_id}/choice") + def provide_choice(job_id: str, payload: FinTSChoiceRequest) -> dict[str, Any]: + job = _get_job(job_id) + if job.snapshot()["status"] != "needs_choice": + raise HTTPException(status_code=409, detail="Job is not waiting for a choice.") + try: + job.provide_value(payload.value) + except RuntimeError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + return job.snapshot() + + @router.post("/imap/folders") + def imap_folders(payload: IMAPFoldersRequest) -> dict[str, Any]: + try: + return {"folders": list_imap_folders(_imap_config_from_payload(payload.credentials))} + except (OSError, RuntimeError, imaplib.IMAP4.error) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @router.post("/imap/scan") + def imap_scan(payload: IMAPScanRequest) -> dict[str, Any]: + try: + return scan_imap_mailbox( + _imap_config_from_payload(payload.credentials), + household_id=payload.household_id, + owner_user_id=payload.owner_user_id, + source_account_id=payload.source_account_id, + limit=payload.limit, + folder=payload.folder, + search_query=payload.search_query, + sender_filters=payload.sender_filters, + ) + except (OSError, RuntimeError, imaplib.IMAP4.error) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @router.post("/paypal/transactions") + def paypal_transactions(payload: PayPalTransactionsRequest) -> dict[str, Any]: + end_date = payload.end_date or date.today() + start_date = payload.start_date or (end_date - timedelta(days=payload.days - 1)) + try: + result = fetch_paypal_transactions( + _paypal_config_from_payload(payload.credentials), + start_date=start_date, + end_date=end_date, + household_id=payload.household_id, + owner_user_id=payload.owner_user_id, + source_account_id=payload.source_account_id, + transaction_currency=payload.transaction_currency, + page_size=payload.page_size, + max_pages=payload.max_pages, + ) + except PayPalAPIError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return { + "start_date": result.start_date.isoformat(), + "end_date": result.end_date.isoformat(), + "transactions": result.transactions, + "observations": result.observations, + "count": len(result.observations), + "raw_response_count": result.raw_response_count, + } + + @router.post("/paypal/auth-check") + def paypal_auth_check(payload: PayPalAuthCheckRequest) -> dict[str, Any]: + try: + return inspect_paypal_credentials(_paypal_config_from_payload(payload.credentials)) + except PayPalAPIError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @router.post("/paypal/statement/parse") + def paypal_statement_parse(payload: StatementParseRequest) -> dict[str, Any]: + return parse_paypal_statement( + payload.csv_text, + household_id=payload.household_id, + owner_user_id=payload.owner_user_id, + source_account_id=payload.source_account_id, + filename=payload.filename, + ) + + @router.post("/klarna/statement/parse") + def klarna_statement_parse(payload: StatementParseRequest) -> dict[str, Any]: + return parse_klarna_statement( + payload.csv_text, + household_id=payload.household_id, + owner_user_id=payload.owner_user_id, + source_account_id=payload.source_account_id, + filename=payload.filename, + ) + + return router + + +def create_app() -> FastAPI: + app = FastAPI(title="Mousehold Local Agent", version="0.1.0") + app.add_middleware( + CORSMiddleware, + allow_origins=_cors_origins(), + allow_credentials=False, + allow_methods=["*"], + allow_headers=["*"], + ) + app.include_router(create_router()) + return app + + +def run(host: str = "127.0.0.1", port: int = 8765) -> None: + uvicorn.run("mousehold_agent.server:app", host=host, port=port, reload=False) + + +def _start_job(kind: str, target: Any) -> FinTSBrowserJob: + job = FinTSBrowserJob(kind) + jobs[job.id] = job + + def wrapped() -> None: + try: + target(job) + except BaseException as exc: + job.set_error(exc) + + thread = threading.Thread(target=wrapped, name=f"fints-{kind}-{job.id}", daemon=True) + thread.start() + return job + + +def _get_job(job_id: str) -> FinTSBrowserJob: + job = jobs.get(job_id) + if not job: + raise HTTPException(status_code=404, detail="FinTS job not found") + return job + + +def _run_accounts(job: FinTSBrowserJob, payload: FinTSAccountsRequest) -> None: + config = _config_from_payload(payload.credentials) + + def operation(client: FinTS3PinTanClient) -> dict[str, Any]: + accounts: list[dict[str, Any]] = [] + for index, account in enumerate(client.get_sepa_accounts()): + row = account_to_dict(account) + row["index"] = index + if payload.with_balances: + balance = _resolve_tan_response(client, client.get_balance(account), job) + row["balance"] = balance_to_dict(balance) + accounts.append(row) + return {"accounts": accounts} + + job.set_done(_with_browser_dialog(config, job, operation)) + + +def _run_transactions(job: FinTSBrowserJob, payload: FinTSTransactionsRequest) -> None: + config = _config_from_payload(payload.credentials) + end_date = payload.end_date or date.today() + start_date = payload.start_date or (end_date - timedelta(days=payload.days)) + + def operation(client: FinTS3PinTanClient) -> dict[str, Any]: + account = select_account(client.get_sepa_accounts(), payload.iban) + transactions = _resolve_tan_response( + client, + client.get_transactions( + account, + start_date=start_date, + end_date=end_date, + include_pending=payload.include_pending, + ), + job, + ) + observations = [ + transaction_to_observation( + transaction, + account=account, + household_id=payload.household_id, + source_account_id=payload.source_account_id, + owner_user_id=payload.owner_user_id, + ) + for transaction in transactions + ] + return { + "account": account_to_dict(account), + "observations": observations, + "count": len(observations), + } + + job.set_done(_with_browser_dialog(config, job, operation)) + + +def _with_browser_dialog(config: FinTSConfig, job: FinTSBrowserJob, operation: Any) -> dict[str, Any]: + client = create_client(config) + _browser_bootstrap(client, config, job) + try: + with client: + while isinstance(client.init_tan_response, NeedTANResponse): + client.init_tan_response = _answer_tan(client, client.init_tan_response, job) + return operation(client) + finally: + persist_client(client, config) + + +def _browser_bootstrap(client: FinTS3PinTanClient, config: FinTSConfig, job: FinTSBrowserJob) -> None: + if not client.get_current_tan_mechanism(): + client.fetch_tan_mechanisms() + mechanisms = client.get_tan_mechanisms() + if config.tan_mechanism: + client.set_tan_mechanism(config.tan_mechanism) + elif len(mechanisms) == 1: + client.set_tan_mechanism(next(iter(mechanisms))) + elif len(mechanisms) > 1: + options = [ + { + "value": security_function, + "label": getattr(parameters, "name", security_function) or security_function, + "description": f"Function {security_function}", + } + for security_function, parameters in mechanisms.items() + ] + client.set_tan_mechanism(job.wait_for_choice("tan_mechanism", options)) + + if client.selected_tan_medium is None and client.is_tan_media_required(): + usage, media = client.get_tan_media() + del usage + if config.tan_medium: + client.selected_tan_medium = config.tan_medium + elif len(media) == 1: + client.set_tan_medium(media[0]) + elif len(media) == 0: + client.selected_tan_medium = "" + else: + options = [ + { + "value": str(index), + "label": getattr(item, "tan_medium_name", None) or f"Medium {index + 1}", + "description": _tan_medium_description(item), + } + for index, item in enumerate(media) + ] + client.set_tan_medium(media[int(job.wait_for_choice("tan_medium", options))]) + + +def _resolve_tan_response(client: FinTS3PinTanClient, response: Any, job: FinTSBrowserJob) -> Any: + while isinstance(response, NeedTANResponse): + response = _answer_tan(client, response, job) + return response + + +def _answer_tan( + client: FinTS3PinTanClient, + response: NeedTANResponse, + job: FinTSBrowserJob, +) -> Any: + if response.decoupled: + tan = job.wait_for_tan(response) + del tan + return client.send_tan(response, "") + return client.send_tan(response, job.wait_for_tan(response)) + + +def _tan_medium_description(item: Any) -> str: + parts = [] + mobile = getattr(item, "mobile_number_masked", None) + if mobile: + parts.append(str(mobile)) + last_use = getattr(item, "last_use", None) + if last_use: + parts.append(f"Last used {last_use}") + return " · ".join(parts) + + +def _config_from_payload(payload: FinTSRequest) -> FinTSConfig: + if not payload.product_id: + raise FinTSConfigurationError(PRODUCT_ID_HELP) + return FinTSConfig( + bank_identifier=payload.blz, + user_id=payload.bank_user_id, + pin=payload.pin, + server=payload.server, + customer_id=payload.customer_id or None, + product_id=payload.product_id, + product_version=payload.product_version or None, + tan_mechanism=payload.tan_mechanism or None, + tan_medium=payload.tan_medium or None, + profile=payload.profile, + state_dir=Path(payload.state_dir).expanduser(), + store_state=payload.store_state, + interactive_bootstrap=False, + ) + + +def _imap_config_from_payload(payload: IMAPCredentials) -> IMAPConfig: + return IMAPConfig( + host=payload.host, + port=payload.port, + username=payload.username, + password=payload.password, + use_ssl=payload.use_ssl, + ) + + +def _paypal_config_from_payload(payload: PayPalCredentials) -> PayPalConfig: + return PayPalConfig( + client_id=payload.client_id, + client_secret=payload.client_secret, + environment=payload.environment, + base_url=payload.base_url or None, + ) + + +def _cors_origins() -> list[str]: + configured = os.environ.get("MOUSEHOLD_LOCAL_AGENT_CORS_ORIGINS") + if not configured: + return LOCAL_AGENT_ORIGINS + return [origin.strip() for origin in configured.split(",") if origin.strip()] + + +app = create_app() diff --git a/apps/local-agent/mousehold_agent/statement_importer.py b/apps/local-agent/mousehold_agent/statement_importer.py new file mode 100644 index 0000000..0871ca6 --- /dev/null +++ b/apps/local-agent/mousehold_agent/statement_importer.py @@ -0,0 +1,487 @@ +from __future__ import annotations + +import csv +import hashlib +import io +import re +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal, InvalidOperation +from typing import Any, Literal + +StatementKind = Literal["paypal", "klarna"] + +PARSER_VERSION = "0.1.0" + +PAYPAL_FIELD_ALIASES = { + "date": ("date", "transaction date", "datum"), + "name": ("name", "payer name", "counterparty", "merchant", "händler", "haendler"), + "type": ("type", "transaction type", "typ"), + "status": ("status", "transaction status"), + "currency": ("currency", "currency code", "währung", "waehrung"), + "gross": ("gross", "amount", "transaction amount", "betrag", "brutto"), + "fee": ("fee", "fee amount", "gebühr", "gebuehr"), + "net": ("net", "net amount", "netto"), + "transaction_id": ("transaction id", "transaction_id", "transaktionscode", "transaktions-id"), + "reference_id": ("reference txn id", "reference transaction id", "referenztransaktionscode"), + "invoice": ("invoice number", "invoice id", "rechnungsnummer"), + "item": ("item title", "item name", "description", "beschreibung", "betreff"), + "from_email": ("from email address", "from email", "sender email"), + "to_email": ("to email address", "to email", "recipient email"), +} + +KLARNA_FIELD_ALIASES = { + "date": ("capture_date", "sale_date", "payout_date", "date", "datum", "transaction_date"), + "type": ("type", "transaction type", "typ"), + "detailed_type": ("detailed_type", "detail type", "details", "beschreibung"), + "amount": ("amount", "betrag", "transaction_amount", "original_capture_amount"), + "currency": ( + "posting_currency", + "settlement_currency", + "currency", + "währung", + "waehrung", + "original_capture_currency", + ), + "order_id": ("order_id", "order id", "bestellnummer", "bestellung", "short_order_id"), + "capture_id": ("capture_id", "capture id"), + "refund_id": ("refund_id", "refund id"), + "payment_reference": ("payment_reference", "payment reference", "zahlungsreferenz", "reference", "referenz"), + "merchant": ( + "merchant_reference1", + "merchant_reference2", + "terminal_name", + "merchant", + "händler", + "haendler", + "shop", + ), + "description": ("description", "details", "detailed_type", "type", "beschreibung", "verwendungszweck"), +} + + +@dataclass(frozen=True) +class ParsedStatement: + source: StatementKind + filename: str | None + row_count: int + importable_count: int + skipped_count: int + rows: list[dict[str, Any]] + observations: list[dict[str, Any]] + + def as_dict(self) -> dict[str, Any]: + return { + "source": self.source, + "filename": self.filename, + "row_count": self.row_count, + "importable_count": self.importable_count, + "skipped_count": self.skipped_count, + "rows": self.rows, + "observations": self.observations, + } + + +def parse_paypal_statement( + csv_text: str, + *, + household_id: str, + owner_user_id: str | None, + source_account_id: str | None, + filename: str | None = None, +) -> dict[str, Any]: + rows = _dict_rows(csv_text) + previews: list[dict[str, Any]] = [] + observations: list[dict[str, Any]] = [] + for index, row in enumerate(rows, start=1): + preview, observation = _parse_paypal_row( + row, + index=index, + household_id=household_id, + owner_user_id=owner_user_id, + source_account_id=source_account_id, + filename=filename, + ) + previews.append(preview) + if observation: + observations.append(observation) + return ParsedStatement( + source="paypal", + filename=filename, + row_count=len(rows), + importable_count=len(observations), + skipped_count=len(previews) - len(observations), + rows=previews, + observations=observations, + ).as_dict() + + +def parse_klarna_statement( + csv_text: str, + *, + household_id: str, + owner_user_id: str | None, + source_account_id: str | None, + filename: str | None = None, +) -> dict[str, Any]: + rows = _klarna_transaction_rows(csv_text) + previews: list[dict[str, Any]] = [] + observations: list[dict[str, Any]] = [] + for index, row in enumerate(rows, start=1): + preview, observation = _parse_klarna_row( + row, + index=index, + household_id=household_id, + owner_user_id=owner_user_id, + source_account_id=source_account_id, + filename=filename, + ) + previews.append(preview) + if observation: + observations.append(observation) + return ParsedStatement( + source="klarna", + filename=filename, + row_count=len(rows), + importable_count=len(observations), + skipped_count=len(previews) - len(observations), + rows=previews, + observations=observations, + ).as_dict() + + +def _parse_paypal_row( + row: dict[str, str], + *, + index: int, + household_id: str, + owner_user_id: str | None, + source_account_id: str | None, + filename: str | None, +) -> tuple[dict[str, Any], dict[str, Any] | None]: + amount = _parse_amount(_pick(row, PAYPAL_FIELD_ALIASES["net"]) or _pick(row, PAYPAL_FIELD_ALIASES["gross"])) + event_date = _parse_date(_pick(row, PAYPAL_FIELD_ALIASES["date"])) + currency = _pick(row, PAYPAL_FIELD_ALIASES["currency"]) or "EUR" + raw_type = _pick(row, PAYPAL_FIELD_ALIASES["type"]) or "" + status_text = _pick(row, PAYPAL_FIELD_ALIASES["status"]) or "" + transaction_id = _pick(row, PAYPAL_FIELD_ALIASES["transaction_id"]) + reference_id = _pick(row, PAYPAL_FIELD_ALIASES["reference_id"]) + source_reference = transaction_id or reference_id or _row_reference("paypal", row, index) + merchant = _pick(row, PAYPAL_FIELD_ALIASES["name"]) or _pick(row, PAYPAL_FIELD_ALIASES["from_email"]) + description = _first_non_empty( + _pick(row, PAYPAL_FIELD_ALIASES["item"]), + _pick(row, PAYPAL_FIELD_ALIASES["invoice"]), + raw_type, + merchant, + ) + + preview = _preview( + row=index, + source_type="paypal_csv", + source_reference=source_reference, + event_date=event_date, + amount=amount, + currency=currency, + merchant=merchant, + description=description, + raw_type=raw_type, + event_type=_paypal_event_type(raw_type, description, amount), + status=_paypal_status(status_text), + ) + if amount is None: + preview["skip_reason"] = "No amount found" + return preview, None + if event_date is None: + preview["skip_reason"] = "No date found" + return preview, None + + observation = { + "household_id": household_id, + "source_type": "paypal_csv", + "source_account_id": source_account_id, + "owner_user_id": owner_user_id, + "source_reference": source_reference, + "provider_transaction_id": transaction_id, + "raw_payload_json": { + "filename": filename, + "row": row, + "parser_name": "mousehold-paypal-statement", + "parser_version": PARSER_VERSION, + }, + "amount": str(abs(amount)), + "currency": currency.upper(), + "event_date": event_date.isoformat(), + "booking_date": event_date.isoformat(), + "merchant": merchant, + "counterparty": merchant, + "description": description, + "event_type": preview["event_type"], + "status": preview["status"], + "payer_user_id": owner_user_id, + "payment_account_id": source_account_id, + } + return preview, observation + + +def _parse_klarna_row( + row: dict[str, str], + *, + index: int, + household_id: str, + owner_user_id: str | None, + source_account_id: str | None, + filename: str | None, +) -> tuple[dict[str, Any], dict[str, Any] | None]: + amount = _parse_amount(_pick(row, KLARNA_FIELD_ALIASES["amount"])) + event_date = _parse_date(_pick(row, KLARNA_FIELD_ALIASES["date"])) + currency = _pick(row, KLARNA_FIELD_ALIASES["currency"]) or "EUR" + row_type = _pick(row, KLARNA_FIELD_ALIASES["type"]) or "" + detailed_type = _pick(row, KLARNA_FIELD_ALIASES["detailed_type"]) or "" + raw_type = " / ".join(part for part in (row_type, detailed_type) if part) + source_reference = _first_non_empty( + _pick(row, KLARNA_FIELD_ALIASES["capture_id"]), + _pick(row, KLARNA_FIELD_ALIASES["refund_id"]), + _pick(row, KLARNA_FIELD_ALIASES["order_id"]), + _pick(row, KLARNA_FIELD_ALIASES["payment_reference"]), + _row_reference("klarna", row, index), + ) + merchant = _pick(row, KLARNA_FIELD_ALIASES["merchant"]) or "Klarna" + description = _first_non_empty(_pick(row, KLARNA_FIELD_ALIASES["description"]), raw_type, merchant) + event_type = _klarna_event_type(row_type, detailed_type, description, amount) + status = ( + "outstanding" + if row_type.upper() == "CHARGE" and "PAYMENT_REMINDER" in detailed_type.upper() + else "confirmed" + ) + + preview = _preview( + row=index, + source_type="klarna_csv", + source_reference=source_reference, + event_date=event_date, + amount=amount, + currency=currency, + merchant=merchant, + description=description, + raw_type=raw_type, + event_type=event_type, + status=status, + ) + if amount is None: + preview["skip_reason"] = "No amount found" + return preview, None + if event_date is None: + preview["skip_reason"] = "No date found" + return preview, None + + observation = { + "household_id": household_id, + "source_type": "klarna_csv", + "source_account_id": source_account_id, + "owner_user_id": owner_user_id, + "source_reference": source_reference, + "provider_transaction_id": source_reference, + "raw_payload_json": { + "filename": filename, + "row": row, + "parser_name": "mousehold-klarna-statement", + "parser_version": PARSER_VERSION, + }, + "amount": str(abs(amount)), + "currency": currency.upper(), + "event_date": event_date.isoformat(), + "booking_date": event_date.isoformat(), + "merchant": merchant, + "counterparty": "Klarna", + "description": description, + "event_type": event_type, + "status": status, + "payer_user_id": owner_user_id, + "payment_account_id": source_account_id, + } + return preview, observation + + +def _dict_rows(csv_text: str) -> list[dict[str, str]]: + text = csv_text.lstrip("\ufeff") + dialect = _sniff_dialect(text) + reader = csv.DictReader(io.StringIO(text), dialect=dialect) + return [ + {_normalize_header(key): (value or "").strip() for key, value in row.items() if key} + for row in reader + if any((value or "").strip() for value in row.values()) + ] + + +def _klarna_transaction_rows(csv_text: str) -> list[dict[str, str]]: + text = csv_text.lstrip("\ufeff") + dialect = _sniff_dialect(text) + reader = csv.reader(io.StringIO(text), dialect=dialect) + header: list[str] | None = None + rows: list[dict[str, str]] = [] + for raw_row in reader: + cells = [cell.strip() for cell in raw_row] + normalized = [_normalize_header(cell) for cell in cells] + if "type" in normalized and "amount" in normalized: + header = normalized + continue + if header is None or not any(cells): + continue + row = { + key: cells[position].strip() + for position, key in enumerate(header) + if key and position < len(cells) + } + if _pick(row, KLARNA_FIELD_ALIASES["type"]) or _pick(row, KLARNA_FIELD_ALIASES["amount"]): + rows.append(row) + if rows: + return rows + return _dict_rows(text) + + +def _sniff_dialect(csv_text: str) -> csv.Dialect: + sample = csv_text[:4096] + try: + return csv.Sniffer().sniff(sample, delimiters=",;\t") + except csv.Error: + class Fallback(csv.excel): + delimiter = ";" + + return Fallback + + +def _pick(row: dict[str, str], aliases: tuple[str, ...]) -> str | None: + for alias in aliases: + value = row.get(_normalize_header(alias)) + if value: + return value.strip() + return None + + +def _parse_amount(value: str | None) -> Decimal | None: + if not value: + return None + cleaned = value.strip().replace("\xa0", " ") + cleaned = re.sub(r"[^0-9,.\-+() ]", "", cleaned) + negative = cleaned.startswith("(") and cleaned.endswith(")") + cleaned = cleaned.strip("() ").replace(" ", "") + if not cleaned: + return None + if "," in cleaned and "." in cleaned: + if cleaned.rfind(",") > cleaned.rfind("."): + cleaned = cleaned.replace(".", "").replace(",", ".") + else: + cleaned = cleaned.replace(",", "") + elif "," in cleaned: + cleaned = cleaned.replace(".", "").replace(",", ".") + try: + amount = Decimal(cleaned).quantize(Decimal("0.01")) + except InvalidOperation: + return None + return -amount if negative else amount + + +def _parse_date(value: str | None) -> date | None: + if not value: + return None + stripped = value.strip() + if "T" in stripped: + try: + return datetime.fromisoformat(stripped.replace("Z", "+00:00")).date() + except ValueError: + pass + for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%m-%d-%Y"): + try: + return datetime.strptime(stripped[:10], fmt).date() + except ValueError: + continue + return None + + +def _paypal_event_type(raw_type: str, description: str | None, amount: Decimal | None) -> str: + combined = f"{raw_type} {description or ''}".lower() + if "fee" in combined or "gebühr" in combined: + return "fee" + if "refund" in combined or "erstattung" in combined or "reversal" in combined: + return "refund" + if "transfer" in combined or "bank account" in combined: + return "transfer" + if amount is None: + return "expense" + return "expense" if amount < 0 else "income" + + +def _paypal_status(value: str) -> str: + normalized = value.strip().lower() + if normalized in {"pending", "processing", "created", "payment pending"}: + return "announced" + if normalized in {"denied", "canceled", "cancelled", "voided", "expired", "failed"}: + return "cancelled" + return "confirmed" + + +def _klarna_event_type( + row_type: str, + detailed_type: str, + description: str | None, + amount: Decimal | None, +) -> str: + combined = f"{row_type} {detailed_type} {description or ''}".lower() + if "fee" in combined or "commission" in combined: + return "fee" + if "return" in combined or "refund" in combined or "credit" in combined or "release" in combined: + return "refund" + if "payment_reminder" in combined or "liability" in combined or "payment" in combined or "zahlung" in combined: + return "liability_payment" + if amount is None: + return "expense" + return "expense" if amount < 0 else "income" + + +def _preview( + *, + row: int, + source_type: str, + source_reference: str | None, + event_date: date | None, + amount: Decimal | None, + currency: str, + merchant: str | None, + description: str | None, + raw_type: str, + event_type: str, + status: str, +) -> dict[str, Any]: + return { + "row": row, + "source_type": source_type, + "source_reference": source_reference, + "event_date": event_date.isoformat() if event_date else None, + "amount": str(abs(amount)) if amount is not None else None, + "currency": currency.upper(), + "merchant": merchant, + "description": description, + "raw_type": raw_type, + "event_type": event_type, + "status": status, + "importable": amount is not None and event_date is not None, + "skip_reason": None, + } + + +def _normalize_header(value: str) -> str: + lowered = value.strip().lower().replace("\ufeff", "") + lowered = lowered.replace("ä", "ae").replace("ö", "oe").replace("ü", "ue").replace("ß", "ss") + return re.sub(r"[^a-z0-9]+", "_", lowered).strip("_") + + +def _row_reference(source: str, row: dict[str, str], index: int) -> str: + digest = hashlib.sha256(repr(sorted(row.items())).encode()).hexdigest()[:16] + return f"{source}-statement-row-{index}-{digest}" + + +def _first_non_empty(*values: str | None) -> str | None: + for value in values: + if value: + return value + return None diff --git a/apps/web/.env.development b/apps/web/.env.development new file mode 100644 index 0000000..9ba6d2c --- /dev/null +++ b/apps/web/.env.development @@ -0,0 +1 @@ +VITE_API_URL=http://127.0.0.1:8001 diff --git a/apps/web/.env.example b/apps/web/.env.example new file mode 100644 index 0000000..145acbf --- /dev/null +++ b/apps/web/.env.example @@ -0,0 +1,5 @@ +VITE_API_URL=http://127.0.0.1:8001 + +# Optional. Leave unset to use the embedded API route at ${VITE_API_URL}/agent. +# Set this only if you run `mousehold-agent serve` as a separate process. +VITE_LOCAL_AGENT_URL=http://127.0.0.1:8765 diff --git a/apps/web/index.html b/apps/web/index.html new file mode 100644 index 0000000..76795d2 --- /dev/null +++ b/apps/web/index.html @@ -0,0 +1,12 @@ + + + + + + Mousehold + + +
+ + + diff --git a/apps/web/package.json b/apps/web/package.json new file mode 100644 index 0000000..181e935 --- /dev/null +++ b/apps/web/package.json @@ -0,0 +1,25 @@ +{ + "name": "@mousehold/web", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite --host 127.0.0.1", + "build": "tsc -b && vite build", + "lint": "tsc --noEmit", + "test:e2e": "playwright test" + }, + "dependencies": { + "@vitejs/plugin-react": "^5.0.0", + "lucide-react": "^0.468.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "typescript": "^5.8.0", + "vite": "^7.0.0" + }, + "devDependencies": { + "@playwright/test": "^1.61.1", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0" + } +} diff --git a/apps/web/playwright.config.ts b/apps/web/playwright.config.ts new file mode 100644 index 0000000..5da8b4b --- /dev/null +++ b/apps/web/playwright.config.ts @@ -0,0 +1,16 @@ +import { defineConfig, devices } from "@playwright/test"; + +export default defineConfig({ + testDir: "./tests", + timeout: 30_000, + use: { + baseURL: process.env.PLAYWRIGHT_WEB_URL ?? "http://127.0.0.1:5173", + trace: "retain-on-failure" + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] } + } + ] +}); diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx new file mode 100644 index 0000000..924c006 --- /dev/null +++ b/apps/web/src/App.tsx @@ -0,0 +1,4071 @@ +import { FormEvent, useEffect, useMemo, useRef, useState } from "react"; +import type { ReactNode } from "react"; +import { + BadgeCheck, + BookOpen, + Download, + FileText, + FolderKanban, + Inbox, + KeyRound, + Landmark, + LayoutDashboard, + Link as LinkIcon, + ListFilter, + Mail, + PencilLine, + Plus, + RefreshCw, + Save, + Settings, + Split, + Trash2, + UploadCloud, + WalletCards +} from "lucide-react"; +import type { LucideIcon } from "lucide-react"; +import { api, localAgentApi } from "./api"; +import type { + Account, + AuthSession, + BackupResult, + Category, + Connection, + Event, + EventDetail, + FinTSAccount, + FinTSJob, + Household, + HouseholdExport, + IMAPFolder, + IMAPScanResult, + ImportRun, + ImportRunDetail, + Observation, + ObservationBatchResponse, + ObservationReviewItem, + Project, + ProjectSummary, + ReconciliationAcceptResponse, + ReconciliationEventSummary, + ReconciliationIgnoreResponse, + ReconciliationSuggestion, + RecurringCommitment, + StatementParseResult, + SettlementSummary, + SyncObservationsResponse, + Tag, + User +} from "./types"; + +const today = new Date().toISOString().slice(0, 10); +const fintsBankPresets = [ + { + id: "comdirect", + label: "comdirect", + profile: "comdirect", + blz: "20041111", + server: "https://fints.comdirect.de/fints", + customer_id: "", + product_version: "0.1.0" + } +]; + +type PageId = "setup" | "dashboard" | "inbox" | "ledger" | "entry" | "bank" | "mail" | "statements" | "projects"; + +const pages: Array<{ id: PageId; label: string; icon: LucideIcon }> = [ + { id: "setup", label: "Setup", icon: Settings }, + { id: "dashboard", label: "Dashboard", icon: LayoutDashboard }, + { id: "inbox", label: "Inbox", icon: Inbox }, + { id: "ledger", label: "Ledger", icon: BookOpen }, + { id: "entry", label: "Entry", icon: PencilLine }, + { id: "bank", label: "Bank", icon: Landmark }, + { id: "mail", label: "Mail", icon: Mail }, + { id: "statements", label: "Statements", icon: FileText }, + { id: "projects", label: "Projects", icon: FolderKanban } +]; + +export function App() { + const [households, setHouseholds] = useState([]); + const [selectedHouseholdId, setSelectedHouseholdId] = useState(""); + const [users, setUsers] = useState([]); + const [accounts, setAccounts] = useState([]); + const [observations, setObservations] = useState([]); + const [events, setEvents] = useState([]); + const [settlement, setSettlement] = useState(null); + const [projects, setProjects] = useState([]); + const [selectedProjectId, setSelectedProjectId] = useState(""); + const [projectSummary, setProjectSummary] = useState(null); + const [connections, setConnections] = useState([]); + const [importRuns, setImportRuns] = useState([]); + const [recurring, setRecurring] = useState([]); + const [categories, setCategories] = useState([]); + const [tags, setTags] = useState([]); + const [authSession, setAuthSession] = useState(null); + const [currentPage, setCurrentPage] = useState("dashboard"); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + + const selectedHousehold = households.find((item) => item.id === selectedHouseholdId) ?? households[0]; + const userName = useMemo(() => new Map(users.map((user) => [user.id, user.display_name])), [users]); + const accountName = useMemo(() => new Map(accounts.map((account) => [account.id, account.name])), [accounts]); + const categoryName = useMemo(() => new Map(categories.map((category) => [category.id, category.path])), [categories]); + const tagName = useMemo(() => new Map(tags.map((tag) => [tag.id, tag.name])), [tags]); + + async function loadHouseholds() { + const data = await api("/households"); + setHouseholds(data); + if (!selectedHouseholdId && data[0]) { + setSelectedHouseholdId(data[0].id); + } + } + + async function loadHouseholdData(householdId = selectedHouseholdId) { + if (!householdId) return; + const [ + nextUsers, + nextAccounts, + nextObservations, + nextEvents, + nextSettlement, + nextProjects, + nextConnections, + nextImportRuns, + nextRecurring, + nextCategories, + nextTags + ] = + await Promise.all([ + api(`/users?household_id=${householdId}`), + api(`/accounts?household_id=${householdId}`), + api(`/observations/inbox?household_id=${householdId}`), + api(`/events?household_id=${householdId}&hide_related_confirmations=true`), + api(`/settlement/summary?household_id=${householdId}`), + api(`/projects?household_id=${householdId}`), + api(`/connections?household_id=${householdId}`), + api(`/import-runs?household_id=${householdId}`), + api(`/recurring-commitments?household_id=${householdId}`), + api(`/categories?household_id=${householdId}&include_inactive=true`), + api(`/tags?household_id=${householdId}`) + ]); + setUsers(nextUsers); + setAccounts(nextAccounts); + setObservations(nextObservations); + setEvents(nextEvents); + setSettlement(nextSettlement); + setProjects(nextProjects); + setConnections(nextConnections); + setImportRuns(nextImportRuns); + setRecurring(nextRecurring); + setCategories(nextCategories); + setTags(nextTags); + if (!selectedProjectId && nextProjects[0]) setSelectedProjectId(nextProjects[0].id); + } + + async function refresh() { + setBusy(true); + setError(null); + try { + await loadHouseholds(); + await loadHouseholdData(); + if (selectedProjectId) await loadProjectSummary(selectedProjectId); + } catch (caught) { + setError(caught instanceof Error ? caught.message : "Request failed"); + } finally { + setBusy(false); + } + } + + async function loadAuth() { + setAuthSession(await api("/auth/me")); + } + + async function loginAs(userId: string) { + if (!userId) return; + const nextSession = await api("/auth/login", { + method: "POST", + body: JSON.stringify({ user_id: userId }) + }); + setAuthSession(nextSession); + if (nextSession.household.id !== selectedHouseholdId) { + setSelectedHouseholdId(nextSession.household.id); + } + } + + async function logout() { + await api<{ logged_out: boolean }>("/auth/logout", { method: "POST" }); + setAuthSession(null); + } + + async function loadProjectSummary(projectId: string) { + if (!projectId) { + setProjectSummary(null); + return; + } + setProjectSummary(await api(`/projects/${projectId}/summary`)); + } + + useEffect(() => { + loadHouseholds().catch((caught: unknown) => setError(caught instanceof Error ? caught.message : "API unavailable")); + loadAuth().catch(() => setAuthSession(null)); + }, []); + + useEffect(() => { + loadHouseholdData(selectedHouseholdId).catch((caught: unknown) => + setError(caught instanceof Error ? caught.message : "API unavailable") + ); + }, [selectedHouseholdId]); + + useEffect(() => { + loadProjectSummary(selectedProjectId).catch(() => setProjectSummary(null)); + }, [selectedProjectId]); + + return ( +
+
+
+

Mousehold

+ {selectedHousehold?.name ?? "No household"} +
+
+ + + + + + + {authSession && ( + + )} + +
+
+ + + + {error &&
{error}
} + +
+ {currentPage === "setup" && ( + + )} + + {currentPage === "dashboard" && ( +
+
+ + +
+ + +
+ )} + + {currentPage === "inbox" && ( + + )} + + {currentPage === "ledger" && ( + + )} + + {currentPage === "entry" && ( +
+ + +
+ )} + + {currentPage === "bank" && ( +
+ + +
+ )} + + {currentPage === "mail" && ( +
+ + +
+ )} + + {currentPage === "statements" && ( +
+ + +
+ )} + + {currentPage === "projects" && ( + + )} +
+
+ ); +} + +function SetupPage({ + households, + selectedHouseholdId, + onSelectHousehold, + users, + accounts, + connections, + categories, + tags, + onChanged +}: { + households: Household[]; + selectedHouseholdId: string; + onSelectHousehold: (householdId: string) => void; + users: User[]; + accounts: Account[]; + connections: Connection[]; + categories: Category[]; + tags: Tag[]; + onChanged: () => Promise; +}) { + return ( +
+ + + + +
+ ); +} + +function SetupPanel({ + households, + selectedHouseholdId, + onSelectHousehold, + users, + accounts, + onChanged +}: { + households: Household[]; + selectedHouseholdId: string; + onSelectHousehold: (householdId: string) => void; + users: User[]; + accounts: Account[]; + onChanged: () => Promise; +}) { + async function createHousehold(event: FormEvent) { + event.preventDefault(); + const form = new FormData(event.currentTarget); + await api("/households", { + method: "POST", + body: JSON.stringify({ name: form.get("name"), default_currency: "EUR" }) + }); + event.currentTarget.reset(); + await onChanged(); + } + + async function createUser(event: FormEvent) { + event.preventDefault(); + const form = new FormData(event.currentTarget); + await api("/users", { + method: "POST", + body: JSON.stringify({ + household_id: selectedHouseholdId, + display_name: form.get("display_name"), + email: form.get("email") || null + }) + }); + event.currentTarget.reset(); + await onChanged(); + } + + async function createAccount(event: FormEvent) { + event.preventDefault(); + const form = new FormData(event.currentTarget); + await api("/accounts", { + method: "POST", + body: JSON.stringify({ + household_id: selectedHouseholdId, + owner_user_id: form.get("owner_user_id") || null, + name: form.get("name"), + account_type: form.get("account_type"), + institution_name: form.get("institution_name") || null, + currency: "EUR", + visibility_mode: form.get("visibility_mode"), + is_active: true + }) + }); + event.currentTarget.reset(); + await onChanged(); + } + + async function updateHousehold(event: FormEvent, householdId: string) { + event.preventDefault(); + const form = new FormData(event.currentTarget); + await api(`/households/${householdId}`, { + method: "PATCH", + body: JSON.stringify({ + name: form.get("name"), + default_currency: form.get("default_currency") || "EUR" + }) + }); + await onChanged(); + } + + async function deleteHousehold(householdId: string) { + if (!window.confirm("Delete this household and its local data?")) return; + await api(`/households/${householdId}`, { method: "DELETE" }); + if (selectedHouseholdId === householdId) onSelectHousehold(""); + await onChanged(); + } + + async function updateUser(event: FormEvent, userId: string) { + event.preventDefault(); + const form = new FormData(event.currentTarget); + await api(`/users/${userId}`, { + method: "PATCH", + body: JSON.stringify({ + display_name: form.get("display_name"), + email: form.get("email") || null + }) + }); + await onChanged(); + } + + async function deleteUser(userId: string) { + if (!window.confirm("Delete this user?")) return; + await api(`/users/${userId}`, { method: "DELETE" }); + await onChanged(); + } + + async function updateAccount(event: FormEvent, accountId: string) { + event.preventDefault(); + const form = new FormData(event.currentTarget); + await api(`/accounts/${accountId}`, { + method: "PATCH", + body: JSON.stringify({ + owner_user_id: form.get("owner_user_id") || null, + name: form.get("name"), + institution_name: form.get("institution_name") || null, + account_type: form.get("account_type"), + visibility_mode: form.get("visibility_mode"), + is_active: form.get("is_active") === "on" + }) + }); + await onChanged(); + } + + async function deleteAccount(accountId: string) { + if (!window.confirm("Delete this account? Existing events may still reference it.")) return; + await api(`/accounts/${accountId}`, { method: "DELETE" }); + await onChanged(); + } + + return ( +
+
+

Households

+
+ + + + +
+
+ {households.map((household) => ( +
updateHousehold(event, household.id)}> + + + + + + + + + +
+ ))} +
+
+ +
+

Users

+
+ + + + + + + +
+
+ {users.map((user) => ( +
updateUser(event, user.id)}> + + + + + + + + +
+ ))} +
+
+ +
+

Accounts

+
+ + + + + + + + + + + + + + + + +
+
+ {accounts.map((account) => ( +
updateAccount(event, account.id)}> + + + + + + + + + + + + + + + + + + +
+ ))} +
+
+
+ ); +} + +function AccountTypeOptions() { + return ( + <> + + + + + + + + + + + ); +} + +function VisibilityOptions() { + return ( + <> + + + + + + ); +} + +function ConnectionPanel({ + selectedHouseholdId, + users, + accounts, + connections, + onChanged +}: { + selectedHouseholdId: string; + users: User[]; + accounts: Account[]; + connections: Connection[]; + onChanged: () => Promise; +}) { + async function createConnection(event: FormEvent) { + event.preventDefault(); + const form = new FormData(event.currentTarget); + const connectorConfig = connectorConfigFromForm(form); + if (!connectorConfig) return; + await api("/connections", { + method: "POST", + body: JSON.stringify({ + household_id: selectedHouseholdId, + owner_user_id: form.get("owner_user_id"), + connection_type: form.get("connection_type"), + display_name: form.get("display_name"), + status: "needs_auth", + runs_locally: form.get("runs_locally") === "on", + secret_storage_ref: form.get("secret_storage_ref") || null, + connector_config_json: { ...connectorConfig, source_account_id: form.get("source_account_id") || null } + }) + }); + event.currentTarget.reset(); + await onChanged(); + } + + async function updateConnection(event: FormEvent, connectionId: string) { + event.preventDefault(); + const form = new FormData(event.currentTarget); + const connectorConfig = connectorConfigFromForm(form); + if (!connectorConfig) return; + await api(`/connections/${connectionId}`, { + method: "PATCH", + body: JSON.stringify({ + owner_user_id: form.get("owner_user_id"), + connection_type: form.get("connection_type"), + display_name: form.get("display_name"), + status: form.get("status"), + runs_locally: form.get("runs_locally") === "on", + secret_storage_ref: form.get("secret_storage_ref") || null, + connector_config_json: { ...connectorConfig, source_account_id: form.get("source_account_id") || null } + }) + }); + await onChanged(); + } + + async function deleteConnection(connectionId: string) { + if (!window.confirm("Delete this connector profile?")) return; + await api(`/connections/${connectionId}`, { method: "DELETE" }); + await onChanged(); + } + + async function importDemo() { + const owner = users[0]?.id; + const account = accounts[0]?.id; + if (!selectedHouseholdId || !owner) return; + await api("/observations/sync", { + method: "POST", + body: JSON.stringify({ + connection_id: connections[0]?.id ?? "web-demo", + observations: [ + demoObservation(selectedHouseholdId, owner, account, "bank_csv", "demo-bank-web", "42.50", "REWE", "confirmed"), + demoObservation(selectedHouseholdId, owner, account, "paypal_email", "demo-paypal-web", "18.99", "PayPal", "outstanding"), + demoObservation(selectedHouseholdId, owner, account, "klarna_email", "demo-klarna-web", "84.00", "Klarna", "outstanding") + ] + }) + }); + await onChanged(); + } + + return ( +
+

Connectors

+
+ + + + + + + + + + + + + + + + +