first ruggedy draft

apps/local-agent/mousehold_agent/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

apps/local-agent/mousehold_agent/cli.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+from .csv_importer import read_transaction_csv
+from .fints_connector import (
+    account_to_dict,
+    config_from_args,
+    date_range_from_args,
+    display_accounts,
+    display_transactions,
+    fetch_transactions,
+    list_accounts,
+)
+from .http import post_json
+from .imap_importer import fetch_imap_observations
+from .server import run as run_local_agent_server
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(prog="mousehold-agent")
+    parser.add_argument("--api-url", default="http://127.0.0.1:8000")
+    subcommands = parser.add_subparsers(dest="command", required=True)
+
+    register = subcommands.add_parser("register")
+    register.add_argument("--household-id", required=True)
+    register.add_argument("--owner-user-id", required=True)
+    register.add_argument("--display-name", default="Local connector agent")
+
+    csv_sync = subcommands.add_parser("sync-csv")
+    csv_sync.add_argument("--connection-id", required=True)
+    csv_sync.add_argument("--household-id", required=True)
+    csv_sync.add_argument("--source-type", choices=["bank_csv", "paypal_csv"], required=True)
+    csv_sync.add_argument("--path", type=Path, required=True)
+    csv_sync.add_argument("--source-account-id")
+    csv_sync.add_argument("--owner-user-id")
+
+    imap_sync = subcommands.add_parser("sync-imap")
+    imap_sync.add_argument("--connection-id", required=True)
+    imap_sync.add_argument("--household-id", required=True)
+    imap_sync.add_argument("--owner-user-id")
+    imap_sync.add_argument("--source-account-id")
+    imap_sync.add_argument("--limit", type=int, default=25)
+
+    demo = subcommands.add_parser("sync-demo")
+    demo.add_argument("--connection-id", required=True)
+    demo.add_argument("--household-id", required=True)
+    demo.add_argument("--owner-user-id")
+    demo.add_argument("--source-account-id")
+
+    serve = subcommands.add_parser("serve")
+    serve.add_argument("--host", default="127.0.0.1")
+    serve.add_argument("--port", type=int, default=8765)
+
+    fints_accounts = subcommands.add_parser("fints-accounts")
+    _add_fints_common_args(fints_accounts)
+    fints_accounts.add_argument("--with-balances", action="store_true")
+    fints_accounts.add_argument("--json", action="store_true")
+
+    fints_transactions = subcommands.add_parser("fints-transactions")
+    _add_fints_common_args(fints_transactions)
+    _add_fints_transaction_args(fints_transactions)
+    fints_transactions.add_argument("--json", action="store_true")
+
+    fints_sync = subcommands.add_parser("sync-fints")
+    _add_fints_common_args(fints_sync)
+    _add_fints_transaction_args(fints_sync)
+    fints_sync.add_argument("--connection-id", required=True)
+    fints_sync.add_argument("--household-id", required=True)
+    fints_sync.add_argument("--owner-user-id")
+    fints_sync.add_argument("--source-account-id")
+    fints_sync.add_argument("--display", action="store_true")
+
+    args = parser.parse_args()
+    if args.command == "register":
+        result = post_json(
+            f"{args.api_url}/local-agent/register",
+            {
+                "household_id": args.household_id,
+                "owner_user_id": args.owner_user_id,
+                "display_name": args.display_name,
+            },
+        )
+    elif args.command == "sync-csv":
+        observations = read_transaction_csv(
+            args.path,
+            household_id=args.household_id,
+            source_type=args.source_type,
+            source_account_id=args.source_account_id,
+            owner_user_id=args.owner_user_id,
+        )
+        result = _sync(args.api_url, args.connection_id, observations)
+    elif args.command == "sync-imap":
+        observations = fetch_imap_observations(
+            household_id=args.household_id,
+            owner_user_id=args.owner_user_id,
+            source_account_id=args.source_account_id,
+            limit=args.limit,
+        )
+        result = _sync(args.api_url, args.connection_id, observations)
+    elif args.command == "sync-demo":
+        result = _sync(
+            args.api_url,
+            args.connection_id,
+            _demo_observations(args.household_id, args.owner_user_id, args.source_account_id),
+        )
+    elif args.command == "serve":
+        run_local_agent_server(host=args.host, port=args.port)
+        return
+    elif args.command == "fints-accounts":
+        accounts = list_accounts(config_from_args(args), include_balances=args.with_balances)
+        if args.json:
+            result = accounts
+        else:
+            display_accounts(accounts)
+            return
+    elif args.command == "fints-transactions":
+        start_date, end_date = date_range_from_args(args)
+        fetched = fetch_transactions(
+            config_from_args(args),
+            iban=args.iban,
+            start_date=start_date,
+            end_date=end_date,
+            include_pending=args.include_pending,
+        )
+        if args.json:
+            result = {
+                "account": account_to_dict(fetched.account),
+                "observations": fetched.observations,
+            }
+        else:
+            display_transactions(fetched.observations)
+            return
+    elif args.command == "sync-fints":
+        start_date, end_date = date_range_from_args(args)
+        fetched = fetch_transactions(
+            config_from_args(args),
+            iban=args.iban,
+            start_date=start_date,
+            end_date=end_date,
+            include_pending=args.include_pending,
+            household_id=args.household_id,
+            source_account_id=args.source_account_id,
+            owner_user_id=args.owner_user_id,
+        )
+        if args.display:
+            display_transactions(fetched.observations)
+        result = _sync(args.api_url, args.connection_id, fetched.observations)
+    else:
+        raise AssertionError(args.command)
+    print(json.dumps(result, indent=2, sort_keys=True))
+
+
+def _sync(api_url: str, connection_id: str, observations: list[dict[str, Any]]) -> dict[str, Any]:
+    return post_json(
+        f"{api_url}/observations/sync",
+        {
+            "connection_id": connection_id,
+            "observations": observations,
+        },
+    )
+
+
+def _add_fints_common_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--profile", default="default")
+    parser.add_argument("--state-dir", default="~/.config/mousehold/fints")
+    parser.add_argument("--no-state", action="store_true")
+    parser.add_argument("--no-bootstrap", action="store_true")
+    parser.add_argument("--blz", help="Bank identifier/BLZ. Env fallback: MOUSEHOLD_FINTS_BLZ")
+    parser.add_argument("--server", help="FinTS endpoint URL. Env fallback: MOUSEHOLD_FINTS_SERVER")
+    parser.add_argument("--bank-user-id", help="FinTS login/user ID. Env fallback: MOUSEHOLD_FINTS_USER_ID")
+    parser.add_argument("--customer-id", help="Optional customer ID. Env fallback: MOUSEHOLD_FINTS_CUSTOMER_ID")
+    parser.add_argument("--pin", help="FinTS PIN. Prefer prompt or MOUSEHOLD_FINTS_PIN over shell history.")
+    parser.add_argument("--pin-env", default="MOUSEHOLD_FINTS_PIN")
+    parser.add_argument("--tan-mechanism", help="Optional TAN mechanism. Env fallback: MOUSEHOLD_FINTS_TAN_MECHANISM")
+    parser.add_argument("--tan-medium", help="Optional TAN medium name. Env fallback: MOUSEHOLD_FINTS_TAN_MEDIUM")
+    parser.add_argument("--product-id", help="Registered FinTS product ID. Env fallback: MOUSEHOLD_FINTS_PRODUCT_ID")
+    parser.add_argument(
+        "--product-version",
+        help="Optional product version. Env fallback: MOUSEHOLD_FINTS_PRODUCT_VERSION",
+    )
+
+
+def _add_fints_transaction_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--iban", help="IBAN to fetch. Defaults to the first account returned by the bank.")
+    parser.add_argument("--days", type=int, default=30)
+    parser.add_argument("--start-date", help="YYYY-MM-DD")
+    parser.add_argument("--end-date", help="YYYY-MM-DD")
+    parser.add_argument("--include-pending", action="store_true")
+
+
+def _demo_observations(
+    household_id: str,
+    owner_user_id: str | None,
+    source_account_id: str | None,
+) -> list[dict[str, Any]]:
+    today = date.today().isoformat()
+    return [
+        {
+            "household_id": household_id,
+            "source_type": "bank_csv",
+            "source_account_id": source_account_id,
+            "owner_user_id": owner_user_id,
+            "source_reference": "demo-bank-001",
+            "provider_transaction_id": "demo-bank-001",
+            "amount": "42.50",
+            "currency": "EUR",
+            "event_date": today,
+            "merchant": "REWE",
+            "description": "Groceries",
+            "event_type": "expense",
+            "status": "confirmed",
+        },
+        {
+            "household_id": household_id,
+            "source_type": "paypal_email",
+            "source_account_id": source_account_id,
+            "owner_user_id": owner_user_id,
+            "source_reference": "demo-paypal-email-001",
+            "message_id": "demo-paypal-email-001",
+            "amount": "18.99",
+            "currency": "EUR",
+            "event_date": today,
+            "due_date": today,
+            "merchant": "PayPal Merchant",
+            "description": "PayPal purchase announced by email",
+            "event_type": "promised_payment",
+            "status": "outstanding",
+        },
+        {
+            "household_id": household_id,
+            "source_type": "klarna_email",
+            "source_account_id": source_account_id,
+            "owner_user_id": owner_user_id,
+            "source_reference": "demo-klarna-email-001",
+            "message_id": "demo-klarna-email-001",
+            "amount": "84.00",
+            "currency": "EUR",
+            "event_date": today,
+            "due_date": today,
+            "merchant": "Klarna Shop",
+            "description": "Klarna pay-later obligation",
+            "event_type": "promised_payment",
+            "status": "outstanding",
+        },
+    ]

apps/local-agent/mousehold_agent/csv_importer.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import csv
+from datetime import date
+from decimal import Decimal
+from pathlib import Path
+from typing import Any
+
+
+def read_transaction_csv(
+    path: Path,
+    *,
+    household_id: str,
+    source_type: str,
+    source_account_id: str | None,
+    owner_user_id: str | None,
+) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    with path.open(newline="", encoding="utf-8-sig") as handle:
+        reader = csv.DictReader(handle)
+        for row in reader:
+            event_date = _pick(
+                row, "event_date", "date", "booking_date", "Buchungstag", "Valutadatum"
+            )
+            amount = _pick(row, "amount", "Betrag", "value")
+            if not event_date or not amount:
+                continue
+            merchant = _pick(
+                row,
+                "merchant",
+                "counterparty",
+                "Auftraggeber/Empfänger",
+                "Name Zahlungsbeteiligter",
+            )
+            description = _pick(row, "description", "purpose", "Verwendungszweck", "Buchungstext")
+            currency = _pick(row, "currency", "Waehrung", "Währung") or "EUR"
+            provider_transaction_id = _pick(
+                row, "provider_transaction_id", "transaction_id", "Umsatz ID"
+            )
+            normalized = {
+                "household_id": household_id,
+                "source_type": source_type,
+                "source_account_id": source_account_id,
+                "owner_user_id": owner_user_id,
+                "source_reference": provider_transaction_id,
+                "provider_transaction_id": provider_transaction_id,
+                "raw_payload_json": row,
+                "amount": str(_parse_amount(amount)),
+                "currency": currency,
+                "event_date": _parse_date(event_date).isoformat(),
+                "merchant": merchant,
+                "description": description,
+                "event_type": "expense" if _parse_amount(amount) < 0 else "income",
+                "status": "confirmed",
+            }
+            normalized["amount"] = str(abs(_parse_amount(amount)))
+            rows.append(normalized)
+    return rows
+
+
+def _pick(row: dict[str, str], *keys: str) -> str | None:
+    lowered = {key.lower(): value for key, value in row.items()}
+    for key in keys:
+        value = row.get(key) or lowered.get(key.lower())
+        if value:
+            return value.strip()
+    return None
+
+
+def _parse_amount(value: str) -> Decimal:
+    normalized = value.strip().replace(".", "").replace(",", ".")
+    return Decimal(normalized)
+
+
+def _parse_date(value: str) -> date:
+    stripped = value.strip()
+    for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%d/%m/%Y"):
+        try:
+            return (
+                date.fromisoformat(stripped)
+                if fmt == "%Y-%m-%d"
+                else __import__("datetime").datetime.strptime(stripped, fmt).date()
+            )
+        except ValueError:
+            continue
+    raise ValueError(f"Unsupported date format: {value}")

apps/local-agent/mousehold_agent/fints_connector.py

@@ -0,0 +1,582 @@
+from __future__ import annotations
+
+import base64
+import getpass
+import hashlib
+import json
+import os
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta
+from decimal import Decimal
+from html.parser import HTMLParser
+from pathlib import Path
+from typing import Any
+
+from fints.client import FinTS3PinTanClient, NeedTANResponse
+from fints.models import SEPAAccount
+from fints.utils import minimal_interactive_cli_bootstrap
+
+DEFAULT_PROFILE = "default"
+DEFAULT_STATE_DIR = Path.home() / ".config" / "mousehold" / "fints"
+PRODUCT_ID_HELP = (
+    "FinTS product_id is required by python-fints >=4. Register one with Deutsche Kreditwirtschaft/ZKA "
+    "and provide it as MOUSEHOLD_FINTS_PRODUCT_ID, --product-id, or in the FinTS web form."
+)
+
+
+class FinTSConfigurationError(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True)
+class FinTSConfig:
+    bank_identifier: str
+    user_id: str
+    pin: str
+    server: str
+    product_id: str
+    customer_id: str | None = None
+    product_version: str | None = None
+    tan_mechanism: str | None = None
+    tan_medium: str | None = None
+    profile: str = DEFAULT_PROFILE
+    state_dir: Path = DEFAULT_STATE_DIR
+    store_state: bool = True
+    interactive_bootstrap: bool = True
+
+
+@dataclass(frozen=True)
+class FinTSFetchResult:
+    account: SEPAAccount
+    transactions: list[Any]
+    observations: list[dict[str, Any]]
+
+
+class _HTMLTextExtractor(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.parts: list[str] = []
+
+    def handle_data(self, data: str) -> None:
+        stripped = data.strip()
+        if stripped:
+            self.parts.append(stripped)
+
+    def text(self) -> str:
+        return " ".join(self.parts)
+
+
+class FinTSProfileStore:
+    def __init__(self, state_dir: Path, profile: str) -> None:
+        self.state_dir = state_dir
+        self.profile = profile
+        self.path = state_dir / f"{profile}.json"
+
+    def load_client_data(self) -> bytes | None:
+        if not self.path.exists():
+            return None
+        data = json.loads(self.path.read_text(encoding="utf-8"))
+        client_data = data.get("client_data")
+        if not client_data:
+            return None
+        return base64.b64decode(client_data.encode("ascii"))
+
+    def save_client_data(self, client: FinTS3PinTanClient, config: FinTSConfig) -> None:
+        self.state_dir.mkdir(parents=True, exist_ok=True)
+        payload = {
+            "profile": self.profile,
+            "bank_identifier": config.bank_identifier,
+            "server": config.server,
+            "user_id": config.user_id,
+            "customer_id": config.customer_id,
+            "updated_at": datetime.now().isoformat(timespec="seconds"),
+            "client_data": base64.b64encode(client.deconstruct()).decode("ascii"),
+        }
+        tmp_path = self.path.with_suffix(".tmp")
+        tmp_path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
+        os.chmod(tmp_path, 0o600)
+        tmp_path.replace(self.path)
+        os.chmod(self.path, 0o600)
+
+
+def config_from_args(args: Any) -> FinTSConfig:
+    pin = getattr(args, "pin", None) or os.environ.get(getattr(args, "pin_env", "MOUSEHOLD_FINTS_PIN"))
+    if not pin:
+        pin = getpass.getpass("FinTS PIN: ")
+
+    product_id = _required_arg(args, "product_id", "MOUSEHOLD_FINTS_PRODUCT_ID", PRODUCT_ID_HELP)
+    product_version = getattr(args, "product_version", None) or os.environ.get("MOUSEHOLD_FINTS_PRODUCT_VERSION")
+
+    return FinTSConfig(
+        bank_identifier=_required_arg(args, "blz", "MOUSEHOLD_FINTS_BLZ"),
+        user_id=_required_arg(args, "bank_user_id", "MOUSEHOLD_FINTS_USER_ID"),
+        pin=pin,
+        server=_required_arg(args, "server", "MOUSEHOLD_FINTS_SERVER"),
+        customer_id=getattr(args, "customer_id", None) or os.environ.get("MOUSEHOLD_FINTS_CUSTOMER_ID"),
+        product_id=product_id,
+        product_version=product_version,
+        tan_mechanism=getattr(args, "tan_mechanism", None) or os.environ.get("MOUSEHOLD_FINTS_TAN_MECHANISM"),
+        tan_medium=getattr(args, "tan_medium", None) or os.environ.get("MOUSEHOLD_FINTS_TAN_MEDIUM"),
+        profile=getattr(args, "profile", DEFAULT_PROFILE),
+        state_dir=Path(getattr(args, "state_dir", DEFAULT_STATE_DIR)).expanduser(),
+        store_state=not getattr(args, "no_state", False),
+        interactive_bootstrap=not getattr(args, "no_bootstrap", False),
+    )
+
+
+def create_client(config: FinTSConfig) -> FinTS3PinTanClient:
+    if not config.product_id:
+        raise FinTSConfigurationError(PRODUCT_ID_HELP)
+
+    kwargs: dict[str, Any] = {"product_id": config.product_id}
+    if config.product_version:
+        kwargs["product_version"] = config.product_version
+
+    store = FinTSProfileStore(config.state_dir, config.profile)
+    if config.store_state and (client_data := store.load_client_data()):
+        kwargs["from_data"] = client_data
+
+    return FinTS3PinTanClient(
+        config.bank_identifier,
+        config.user_id,
+        config.pin,
+        config.server,
+        customer_id=config.customer_id,
+        tan_medium=config.tan_medium,
+        **kwargs,
+    )
+
+
+def bootstrap_client(client: FinTS3PinTanClient, config: FinTSConfig) -> None:
+    if config.tan_mechanism and not client.get_current_tan_mechanism():
+        client.set_tan_mechanism(config.tan_mechanism)
+    if config.interactive_bootstrap:
+        minimal_interactive_cli_bootstrap(client)
+
+
+def persist_client(client: FinTS3PinTanClient, config: FinTSConfig) -> None:
+    if config.store_state:
+        FinTSProfileStore(config.state_dir, config.profile).save_client_data(client, config)
+
+
+def with_fints_dialog(config: FinTSConfig, operation: Any) -> Any:
+    client = create_client(config)
+    bootstrap_client(client, config)
+    try:
+        with client:
+            while isinstance(client.init_tan_response, NeedTANResponse):
+                client.init_tan_response = answer_tan_challenge(client, client.init_tan_response, config)
+            return operation(client)
+    finally:
+        persist_client(client, config)
+
+
+def list_accounts(config: FinTSConfig, *, include_balances: bool = False) -> list[dict[str, Any]]:
+    def operation(client: FinTS3PinTanClient) -> list[dict[str, Any]]:
+        rows: list[dict[str, Any]] = []
+        for index, account in enumerate(client.get_sepa_accounts()):
+            row = account_to_dict(account)
+            row["index"] = index
+            if include_balances:
+                balance = resolve_tan_response(client, client.get_balance(account), config)
+                row["balance"] = balance_to_dict(balance)
+            rows.append(row)
+        return rows
+
+    return with_fints_dialog(config, operation)
+
+
+def fetch_transactions(
+    config: FinTSConfig,
+    *,
+    iban: str | None,
+    start_date: date,
+    end_date: date,
+    include_pending: bool = False,
+    household_id: str | None = None,
+    source_account_id: str | None = None,
+    owner_user_id: str | None = None,
+) -> FinTSFetchResult:
+    def operation(client: FinTS3PinTanClient) -> FinTSFetchResult:
+        account = select_account(client.get_sepa_accounts(), iban)
+        response = client.get_transactions(
+            account,
+            start_date=start_date,
+            end_date=end_date,
+            include_pending=include_pending,
+        )
+        transactions = resolve_tan_response(client, response, config)
+        observations = [
+            transaction_to_observation(
+                transaction,
+                account=account,
+                household_id=household_id,
+                source_account_id=source_account_id,
+                owner_user_id=owner_user_id,
+            )
+            for transaction in transactions
+        ]
+        return FinTSFetchResult(account=account, transactions=list(transactions), observations=observations)
+
+    return with_fints_dialog(config, operation)
+
+
+def resolve_tan_response(
+    client: FinTS3PinTanClient,
+    response: Any,
+    config: FinTSConfig,
+) -> Any:
+    while isinstance(response, NeedTANResponse):
+        response = answer_tan_challenge(client, response, config)
+    return response
+
+
+def answer_tan_challenge(
+    client: FinTS3PinTanClient,
+    response: NeedTANResponse,
+    config: FinTSConfig,
+) -> Any:
+    print("\nTAN required by bank.")
+    challenge_text = challenge_to_text(response)
+    if challenge_text:
+        print(challenge_text)
+
+    challenge_file = write_binary_challenge(response, config)
+    if challenge_file:
+        print(f"Challenge file: {challenge_file}")
+
+    if getattr(response, "challenge_hhduc", None):
+        print("A chipTAN/flicker challenge is available.")
+        print("Use your bank reader/app if it can read the challenge shown by your terminal.")
+
+    if getattr(response, "decoupled", False):
+        input("Confirm the login/order in your banking app, then press Enter here.")
+        return client.send_tan(response, "")
+
+    tan = getpass.getpass("TAN: ")
+    return client.send_tan(response, tan)
+
+
+def tan_challenge_to_dict(response: NeedTANResponse) -> dict[str, Any]:
+    matrix = getattr(response, "challenge_matrix", None)
+    matrix_payload = None
+    if matrix:
+        mime_type, payload = matrix
+        matrix_payload = {
+            "mime_type": str(mime_type),
+            "data_base64": base64.b64encode(payload).decode("ascii"),
+        }
+    return {
+        "challenge": challenge_to_text(response),
+        "challenge_html": str(getattr(response, "challenge_html", "") or ""),
+        "challenge_hhduc": getattr(response, "challenge_hhduc", None),
+        "challenge_matrix": matrix_payload,
+        "decoupled": bool(getattr(response, "decoupled", False)),
+    }
+
+
+def challenge_to_text(response: NeedTANResponse) -> str:
+    if getattr(response, "challenge", None):
+        return str(response.challenge)
+    if getattr(response, "challenge_html", None):
+        parser = _HTMLTextExtractor()
+        parser.feed(str(response.challenge_html))
+        return parser.text()
+    if getattr(response, "challenge_raw", None):
+        return str(response.challenge_raw)
+    return ""
+
+
+def write_binary_challenge(response: NeedTANResponse, config: FinTSConfig) -> Path | None:
+    matrix = getattr(response, "challenge_matrix", None)
+    if not matrix:
+        return None
+
+    mime_type, payload = matrix
+    suffix = ".bin"
+    if "png" in str(mime_type).lower():
+        suffix = ".png"
+    elif "jpeg" in str(mime_type).lower() or "jpg" in str(mime_type).lower():
+        suffix = ".jpg"
+
+    output_dir = config.state_dir / "challenges"
+    output_dir.mkdir(parents=True, exist_ok=True)
+    path = output_dir / f"{config.profile}-{datetime.now().strftime('%Y%m%d%H%M%S')}{suffix}"
+    path.write_bytes(payload)
+    os.chmod(path, 0o600)
+    return path
+
+
+def select_account(accounts: list[SEPAAccount], iban: str | None) -> SEPAAccount:
+    if not accounts:
+        raise RuntimeError("No SEPA accounts returned by the bank.")
+    if not iban:
+        if len(accounts) > 1:
+            print("Multiple accounts returned; using the first one. Pass --iban to select explicitly.")
+        return accounts[0]
+    normalized = iban.replace(" ", "").upper()
+    for account in accounts:
+        if (account.iban or "").replace(" ", "").upper() == normalized:
+            return account
+    available = ", ".join(account.iban or "<no iban>" for account in accounts)
+    raise RuntimeError(f"IBAN not found. Available accounts: {available}")
+
+
+def date_range_from_args(args: Any) -> tuple[date, date]:
+    end_date = _parse_date_arg(getattr(args, "end_date", None)) or date.today()
+    start_date = _parse_date_arg(getattr(args, "start_date", None))
+    if not start_date:
+        start_date = end_date - timedelta(days=getattr(args, "days", 30))
+    return start_date, end_date
+
+
+def transaction_to_observation(
+    transaction: Any,
+    *,
+    account: SEPAAccount,
+    household_id: str | None,
+    source_account_id: str | None,
+    owner_user_id: str | None,
+) -> dict[str, Any]:
+    raw_data = transaction_data(transaction)
+    normalized_data = jsonable(raw_data)
+    amount, currency = extract_amount(raw_data)
+    booking_date = extract_date(raw_data, "date", "booking_date", "entry_date")
+    value_date = extract_date(raw_data, "entry_date", "valuta_date", "date")
+    counterparty = pick_text(
+        raw_data,
+        "applicant_name",
+        "applicant_iban",
+        "applicant_bin",
+        "counterparty",
+        "name",
+    )
+    description = pick_text(
+        raw_data,
+        "purpose",
+        "transaction_details",
+        "extra_details",
+        "posting_text",
+        "prima_nota",
+        "additional_position_reference",
+    )
+    reference = pick_text(
+        raw_data,
+        "id",
+        "transaction_reference",
+        "customer_reference",
+        "bank_reference",
+        "end_to_end_reference",
+        "eref",
+        "mref",
+        "svwz",
+    )
+    fallback_reference = stable_transaction_reference(account, normalized_data)
+    signed_amount = amount
+
+    return {
+        "household_id": household_id,
+        "source_type": "fints",
+        "source_account_id": source_account_id,
+        "owner_user_id": owner_user_id,
+        "source_reference": reference or fallback_reference,
+        "provider_transaction_id": reference or fallback_reference,
+        "raw_payload_json": {
+            "account": account_to_dict(account),
+            "transaction": normalized_data,
+            "source": "fints",
+        },
+        "amount": str(abs(signed_amount)),
+        "currency": currency or "EUR",
+        "event_date": (booking_date or value_date or date.today()).isoformat(),
+        "booking_date": booking_date.isoformat() if booking_date else None,
+        "merchant": counterparty,
+        "counterparty": counterparty,
+        "description": description,
+        "event_type": infer_event_type(signed_amount, counterparty, description),
+        "status": "confirmed",
+    }
+
+
+def transaction_data(transaction: Any) -> dict[str, Any]:
+    data = getattr(transaction, "data", transaction)
+    if isinstance(data, dict):
+        return data
+    if hasattr(data, "_asdict"):
+        return data._asdict()
+    return {"value": data}
+
+
+def extract_amount(data: dict[str, Any]) -> tuple[Decimal, str | None]:
+    value = data.get("amount") or data.get("transaction_amount")
+    currency = data.get("currency")
+    if hasattr(value, "amount"):
+        currency = getattr(value, "currency", currency)
+        return Decimal(str(value.amount)), currency
+    if isinstance(value, dict):
+        amount = value.get("amount") or value.get("value")
+        currency = value.get("currency", currency)
+        return Decimal(str(amount)), currency
+    if value is None:
+        raise RuntimeError(f"Transaction has no amount: {data}")
+    return Decimal(str(value).replace(",", ".")), currency
+
+
+def extract_date(data: dict[str, Any], *keys: str) -> date | None:
+    for key in keys:
+        value = data.get(key)
+        if isinstance(value, date):
+            return value
+        if isinstance(value, datetime):
+            return value.date()
+        if isinstance(value, str) and value:
+            for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%Y%m%d"):
+                try:
+                    return datetime.strptime(value, fmt).date()
+                except ValueError:
+                    continue
+    return None
+
+
+def pick_text(data: dict[str, Any], *keys: str) -> str | None:
+    values: list[str] = []
+    for key in keys:
+        value = data.get(key)
+        if value is None or value == "":
+            continue
+        if isinstance(value, list | tuple):
+            values.extend(str(item).strip() for item in value if str(item).strip())
+        else:
+            values.append(str(value).strip())
+    if not values:
+        return None
+    return " ".join(dict.fromkeys(values))[:500]
+
+
+def infer_event_type(amount: Decimal, counterparty: str | None, description: str | None) -> str:
+    if amount >= 0:
+        return "income"
+    combined = f"{counterparty or ''} {description or ''}".lower()
+    liability_markers = (
+        "paypal",
+        "klarna",
+        "kreditkarte",
+        "credit card",
+        "visa",
+        "mastercard",
+        "american express",
+        "amex",
+    )
+    if any(marker in combined for marker in liability_markers):
+        return "liability_payment"
+    return "expense"
+
+
+def account_to_dict(account: SEPAAccount) -> dict[str, Any]:
+    return {
+        "iban": account.iban,
+        "bic": account.bic,
+        "accountnumber": account.accountnumber,
+        "subaccount": account.subaccount,
+        "blz": account.blz,
+    }
+
+
+def balance_to_dict(balance: Any) -> dict[str, Any]:
+    amount = getattr(balance, "amount", None)
+    return {
+        "amount": str(getattr(amount, "amount", amount)),
+        "currency": getattr(amount, "currency", None),
+        "date": getattr(balance, "date", None).isoformat()
+        if isinstance(getattr(balance, "date", None), date)
+        else str(getattr(balance, "date", "")),
+        "status": getattr(balance, "status", None),
+    }
+
+
+def stable_transaction_reference(account: SEPAAccount, payload: dict[str, Any]) -> str:
+    digest = hashlib.sha256(
+        json.dumps(
+            {"account": account_to_dict(account), "transaction": payload},
+            sort_keys=True,
+            separators=(",", ":"),
+        ).encode("utf-8")
+    ).hexdigest()
+    return f"fints:{account.iban or account.accountnumber}:{digest}"
+
+
+def jsonable(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {str(key): jsonable(item) for key, item in value.items()}
+    if isinstance(value, list | tuple | set):
+        return [jsonable(item) for item in value]
+    if isinstance(value, datetime | date):
+        return value.isoformat()
+    if isinstance(value, Decimal):
+        return str(value)
+    if hasattr(value, "amount"):
+        return {
+            "amount": str(value.amount),
+            "currency": getattr(value, "currency", None),
+        }
+    if hasattr(value, "_asdict"):
+        return jsonable(value._asdict())
+    return value if isinstance(value, str | int | float | bool | type(None)) else str(value)
+
+
+def display_accounts(accounts: list[dict[str, Any]]) -> None:
+    rows = []
+    for account in accounts:
+        balance = account.get("balance")
+        balance_text = ""
+        if isinstance(balance, dict) and balance.get("amount"):
+            balance_text = f"{balance['amount']} {balance.get('currency') or ''}".strip()
+        rows.append(
+            [
+                str(account.get("index", "")),
+                account.get("iban") or "",
+                account.get("bic") or "",
+                account.get("accountnumber") or "",
+                balance_text,
+            ]
+        )
+    print_table(["#", "IBAN", "BIC", "Account", "Balance"], rows)
+
+
+def display_transactions(observations: list[dict[str, Any]]) -> None:
+    rows = [
+        [
+            observation.get("event_date") or "",
+            observation.get("event_type") or "",
+            observation.get("amount") or "",
+            observation.get("currency") or "",
+            observation.get("counterparty") or "",
+            observation.get("description") or "",
+        ]
+        for observation in observations
+    ]
+    print_table(["Date", "Type", "Amount", "Cur", "Counterparty", "Description"], rows)
+
+
+def print_table(headers: list[str], rows: list[list[str]]) -> None:
+    widths = [
+        max(len(headers[index]), *(len(row[index]) for row in rows)) if rows else len(headers[index])
+        for index in range(len(headers))
+    ]
+    print("  ".join(header.ljust(widths[index]) for index, header in enumerate(headers)))
+    print("  ".join("-" * width for width in widths))
+    for row in rows:
+        print("  ".join(row[index].ljust(widths[index]) for index in range(len(headers))))
+
+
+def _parse_date_arg(value: str | None) -> date | None:
+    if not value:
+        return None
+    return date.fromisoformat(value)
+
+
+def _required_arg(args: Any, attribute: str, env_name: str, message: str | None = None) -> str:
+    value = getattr(args, attribute, None) or os.environ.get(env_name)
+    if not value:
+        raise RuntimeError(message or f"{attribute.replace('_', '-')} or {env_name} is required")
+    return value

apps/local-agent/mousehold_agent/http.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+from urllib import request
+
+
+def post_json(url: str, payload: dict[str, Any]) -> dict[str, Any]:
+    body = json.dumps(payload).encode("utf-8")
+    req = request.Request(
+        url,
+        data=body,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=30) as response:
+        return json.loads(response.read().decode("utf-8"))

apps/local-agent/mousehold_agent/imap_importer.py

@@ -0,0 +1,676 @@
+from __future__ import annotations
+
+import email
+import imaplib
+import os
+import re
+import shlex
+from dataclasses import dataclass
+from datetime import date, datetime
+from decimal import Decimal, InvalidOperation
+from email import policy
+from email.header import decode_header
+from email.message import EmailMessage, Message
+from email.utils import getaddresses, parsedate_to_datetime
+from html.parser import HTMLParser
+from typing import Any
+
+DEFAULT_SENDER_FILTERS = (
+    "klarna",
+    "paypal",
+    "rechnung",
+    "invoice",
+    "receipt",
+    "order",
+    "bestellung",
+    "quittung",
+)
+
+PARSER_NAME = "mousehold-imap-email"
+PARSER_VERSION = "0.2.0"
+MAX_RAW_TEXT_LENGTH = 20_000
+
+_AMOUNT_VALUE = r"[+-]?(?:\d{1,3}(?:[.\s']\d{3})+|\d+)(?:[,.]\d{2})"
+_AMOUNT_PATTERNS = (
+    re.compile(rf"(?P<currency>€|EUR)\s*(?P<amount>{_AMOUNT_VALUE})", re.IGNORECASE),
+    re.compile(rf"(?P<amount>{_AMOUNT_VALUE})\s*(?P<currency>€|EUR)", re.IGNORECASE),
+)
+_DATE_VALUE = r"(?P<date>\d{4}-\d{2}-\d{2}|\d{1,2}[./-]\d{1,2}[./-]\d{2,4})"
+_DUE_DATE_PATTERNS = (
+    re.compile(rf"\b(?:fällig(?:\s+am|\s+bis)?|zahlbar\s+bis|bezahlen\s+bis)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE),
+    re.compile(rf"\b(?:due(?:\s+on|\s+by)?|pay(?:ment)?\s+due|pay\s+by)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE),
+    re.compile(rf"\b(?:abbuchung\s+am|wird\s+am|debit(?:ed)?\s+on)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE),
+)
+_REFERENCE_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
+    "order_number": (
+        re.compile(
+            r"\b(?:bestell(?:nummer|nr\.?)|order\s+(?:number|no\.?))\s*[:#-]?\s*"
+            r"([A-Z0-9][A-Z0-9._/-]{2,})",
+            re.IGNORECASE,
+        ),
+        re.compile(r"\b(?:bestellung|order)\s*[:#-]\s*([A-Z0-9][A-Z0-9._/-]{2,})", re.IGNORECASE),
+    ),
+    "invoice_number": (
+        re.compile(
+            r"\b(?:rechnungs(?:nummer|nr\.?)|invoice\s+(?:number|no\.?))\s*[:#-]?\s*"
+            r"([A-Z0-9][A-Z0-9._/-]{2,})",
+            re.IGNORECASE,
+        ),
+        re.compile(r"\b(?:rechnung|invoice)\s*[:#-]\s*([A-Z0-9][A-Z0-9._/-]{2,})", re.IGNORECASE),
+    ),
+    "paypal_transaction_id": (
+        re.compile(
+            r"\b(?:paypal\s+)?(?:transaction|transaktion)(?:\s+id|\s+nr\.?|\s+number)?\s*[:#-]?\s*"
+            r"([A-Z0-9][A-Z0-9._-]{6,})",
+            re.IGNORECASE,
+        ),
+    ),
+    "klarna_reference": (
+        re.compile(
+            r"\b(?:klarna\s+)?(?:referenz|reference|zahlung(?:s)?referenz)\s*[:#-]?\s*"
+            r"([A-Z0-9][A-Z0-9._/-]{4,})",
+            re.IGNORECASE,
+        ),
+    ),
+}
+
+
+@dataclass(frozen=True)
+class IMAPConfig:
+    host: str
+    username: str
+    password: str
+    port: int = 993
+    folder: str = "INBOX"
+    use_ssl: bool = True
+
+
+@dataclass(frozen=True)
+class AmountMatch:
+    amount: Decimal
+    currency: str
+    context: str
+    score: int
+
+
+class _HTMLTextExtractor(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.parts: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        del attrs
+        if tag.lower() in {"br", "p", "div", "li", "tr"}:
+            self.parts.append("\n")
+
+    def handle_data(self, data: str) -> None:
+        self.parts.append(data)
+
+    def text(self) -> str:
+        return _clean_text(" ".join(self.parts))
+
+
+def config_from_env() -> IMAPConfig:
+    return IMAPConfig(
+        host=_required_env("MOUSEHOLD_IMAP_HOST"),
+        port=int(os.environ.get("MOUSEHOLD_IMAP_PORT", "993")),
+        username=_required_env("MOUSEHOLD_IMAP_USERNAME"),
+        password=_required_env("MOUSEHOLD_IMAP_PASSWORD"),
+        folder=os.environ.get("MOUSEHOLD_IMAP_FOLDER", "INBOX"),
+        use_ssl=_env_bool("MOUSEHOLD_IMAP_USE_SSL", default=True),
+    )
+
+
+def fetch_imap_observations(
+    *,
+    household_id: str,
+    owner_user_id: str | None,
+    source_account_id: str | None,
+    limit: int = 25,
+) -> list[dict[str, Any]]:
+    sender_filters = tuple(
+        value.strip().lower()
+        for value in os.environ.get(
+            "MOUSEHOLD_IMAP_SENDER_FILTERS", ",".join(DEFAULT_SENDER_FILTERS)
+        ).split(",")
+        if value.strip()
+    )
+    result = scan_imap_mailbox(
+        config_from_env(),
+        household_id=household_id,
+        owner_user_id=owner_user_id,
+        source_account_id=source_account_id,
+        limit=limit,
+        sender_filters=sender_filters,
+    )
+    return result["observations"]
+
+
+def list_imap_folders(config: IMAPConfig) -> list[dict[str, Any]]:
+    with _connect(config) as mailbox:
+        _login(mailbox, config)
+        status, data = mailbox.list()
+        _ensure_ok(status, data, "list folders")
+        return [_parse_folder_response(item) for item in data if item]
+
+
+def scan_imap_mailbox(
+    config: IMAPConfig,
+    *,
+    household_id: str,
+    owner_user_id: str | None = None,
+    source_account_id: str | None = None,
+    limit: int = 25,
+    folder: str | None = None,
+    search_query: str = "ALL",
+    sender_filters: tuple[str, ...] | list[str] | None = DEFAULT_SENDER_FILTERS,
+) -> dict[str, Any]:
+    selected_folder = folder or config.folder
+    filters = _normalize_filters(sender_filters)
+    messages: list[dict[str, Any]] = []
+    observations: list[dict[str, Any]] = []
+    scanned_count = 0
+    filtered_count = 0
+
+    with _connect(config) as mailbox:
+        _login(mailbox, config)
+        status, data = mailbox.select(_mailbox_name(selected_folder), readonly=True)
+        _ensure_ok(status, data, f"select folder {selected_folder}")
+        status, data = mailbox.search(None, *_search_criteria(search_query))
+        _ensure_ok(status, data, f"search {search_query}")
+        ids = data[0].split()[-limit:] if data and data[0] else []
+        scanned_count = len(ids)
+
+        for message_id in reversed(ids):
+            status, fetched = mailbox.fetch(message_id, "(RFC822)")
+            if _status_text(status) != "OK" or not fetched or not isinstance(fetched[0], tuple):
+                continue
+            message = email.message_from_bytes(fetched[0][1], policy=policy.default)
+            headers_text = _headers_text(message)
+            body = _message_text(message)
+            if filters and not _matches_filters(filters, headers_text, body):
+                filtered_count += 1
+                continue
+
+            parsed = parse_email_message(
+                message,
+                household_id=household_id,
+                owner_user_id=owner_user_id,
+                source_account_id=source_account_id,
+                folder=selected_folder,
+                body=body,
+            )
+            messages.append(parsed["message"])
+            if parsed["observation"]:
+                observations.append(parsed["observation"])
+
+    return {
+        "folder": selected_folder,
+        "scanned_count": scanned_count,
+        "filtered_count": filtered_count,
+        "matched_count": len(messages),
+        "importable_count": len(observations),
+        "messages": messages,
+        "observations": observations,
+    }
+
+
+def parse_email_message(
+    message: Message,
+    *,
+    household_id: str,
+    owner_user_id: str | None,
+    source_account_id: str | None,
+    folder: str = "INBOX",
+    body: str | None = None,
+) -> dict[str, Any]:
+    subject = _decode_mime_header(message.get("Subject"))
+    from_header = _decode_mime_header(message.get("From"))
+    to_header = _decode_mime_header(message.get("To"))
+    message_id = _compact_header_value(message.get("Message-ID"))
+    received_date = _message_date(message) or date.today()
+    text = body if body is not None else _message_text(message)
+    search_text = f"{subject}\n{text}"
+    source_type = _source_type(from_header, subject, text)
+    amount_match = _extract_amount_match(search_text)
+    due_date = _extract_due_date(search_text)
+    references = _extract_references(subject, text)
+    provider_transaction_id = _provider_transaction_id(references)
+    merchant = _merchant(from_header, subject, text, source_type)
+    event_type = _event_type(source_type, subject, text, due_date)
+    event_status = _event_status(source_type, event_type, due_date, subject, text)
+    source_reference = (
+        message_id
+        or provider_transaction_id
+        or _fallback_source_reference(from_header, subject, received_date)
+    )
+    if subject:
+        description = subject
+    elif merchant:
+        description = f"Email from {merchant}"
+    else:
+        description = "Email import"
+
+    preview = {
+        "message_id": message_id,
+        "source_reference": source_reference,
+        "from": from_header,
+        "to": to_header,
+        "subject": subject,
+        "date": received_date.isoformat(),
+        "folder": folder,
+        "source_type": source_type,
+        "event_type": event_type,
+        "status": event_status,
+        "amount": str(amount_match.amount) if amount_match else None,
+        "currency": amount_match.currency if amount_match else None,
+        "merchant": merchant,
+        "due_date": due_date.isoformat() if due_date else None,
+        "references": references,
+        "importable": amount_match is not None,
+        "skip_reason": None if amount_match else "No EUR amount found",
+    }
+
+    if not amount_match:
+        return {"message": preview, "observation": None}
+
+    observation = {
+        "household_id": household_id,
+        "source_type": source_type,
+        "source_account_id": source_account_id,
+        "owner_user_id": owner_user_id,
+        "message_id": message_id,
+        "source_reference": source_reference,
+        "provider_transaction_id": provider_transaction_id,
+        "raw_text": text[:MAX_RAW_TEXT_LENGTH],
+        "raw_payload_json": {
+            "from": from_header,
+            "to": to_header,
+            "subject": subject,
+            "date": _compact_header_value(message.get("Date")),
+            "folder": folder,
+            "message_id": message_id,
+            "references": references,
+            "amount_context": amount_match.context,
+            "parser_name": PARSER_NAME,
+            "parser_version": PARSER_VERSION,
+        },
+        "amount": str(amount_match.amount),
+        "currency": amount_match.currency,
+        "event_date": received_date.isoformat(),
+        "due_date": due_date.isoformat() if due_date else None,
+        "merchant": merchant,
+        "description": description[:240],
+        "event_type": event_type,
+        "status": event_status,
+        "payer_user_id": owner_user_id,
+        "payment_account_id": source_account_id,
+    }
+    return {"message": preview, "observation": observation}
+
+
+def _connect(config: IMAPConfig) -> imaplib.IMAP4:
+    if config.use_ssl:
+        return imaplib.IMAP4_SSL(config.host, config.port)
+    return imaplib.IMAP4(config.host, config.port)
+
+
+def _login(mailbox: imaplib.IMAP4, config: IMAPConfig) -> None:
+    status, data = mailbox.login(config.username, config.password)
+    _ensure_ok(status, data, "login")
+
+
+def _required_env(name: str) -> str:
+    value = os.environ.get(name)
+    if not value:
+        raise RuntimeError(f"{name} is required")
+    return value
+
+
+def _env_bool(name: str, *, default: bool) -> bool:
+    value = os.environ.get(name)
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _status_text(status: str | bytes) -> str:
+    return status.decode("ascii", errors="replace").upper() if isinstance(status, bytes) else status.upper()
+
+
+def _ensure_ok(status: str | bytes, data: object, action: str) -> None:
+    if _status_text(status) != "OK":
+        raise RuntimeError(f"IMAP {action} failed: {data!r}")
+
+
+def _mailbox_name(folder: str) -> str:
+    if folder.startswith('"') and folder.endswith('"'):
+        return folder
+    if any(char in folder for char in ' "()[\\]'):
+        return '"' + folder.replace("\\", "\\\\").replace('"', '\\"') + '"'
+    return folder
+
+
+def _search_criteria(search_query: str) -> list[str]:
+    query = search_query.strip() or "ALL"
+    try:
+        parts = shlex.split(query)
+    except ValueError:
+        parts = query.split()
+    return parts or ["ALL"]
+
+
+def _normalize_filters(sender_filters: tuple[str, ...] | list[str] | None) -> tuple[str, ...]:
+    if sender_filters is None:
+        return ()
+    return tuple(token.strip().lower() for token in sender_filters if token.strip())
+
+
+def _matches_filters(filters: tuple[str, ...], headers_text: str, body: str) -> bool:
+    haystack = f"{headers_text}\n{body[:5000]}".lower()
+    return any(token in haystack for token in filters)
+
+
+def _headers_text(message: Message) -> str:
+    return "\n".join(
+        [
+            _decode_mime_header(message.get("From")),
+            _decode_mime_header(message.get("To")),
+            _decode_mime_header(message.get("Subject")),
+        ]
+    )
+
+
+def _decode_mime_header(value: str | None) -> str:
+    if not value:
+        return ""
+    parts: list[str] = []
+    for content, charset in decode_header(value):
+        if isinstance(content, bytes):
+            parts.append(content.decode(charset or "utf-8", errors="replace"))
+        else:
+            parts.append(content)
+    return _clean_text("".join(parts))
+
+
+def _compact_header_value(value: str | None) -> str | None:
+    cleaned = _decode_mime_header(value)
+    return cleaned or None
+
+
+def _message_text(message: Message) -> str:
+    if isinstance(message, EmailMessage):
+        body = message.get_body(preferencelist=("plain", "html"))
+        if body:
+            content = body.get_content()
+            if body.get_content_type() == "text/html":
+                return _html_to_text(content)
+            return _clean_text(content)
+
+    if message.is_multipart():
+        plain_parts: list[str] = []
+        html_parts: list[str] = []
+        for part in message.walk():
+            if part.get_content_maintype() != "text":
+                continue
+            payload = part.get_payload(decode=True)
+            if not payload:
+                continue
+            text = payload.decode(part.get_content_charset() or "utf-8", errors="replace")
+            if part.get_content_type() == "text/plain":
+                plain_parts.append(text)
+            elif part.get_content_type() == "text/html":
+                html_parts.append(_html_to_text(text))
+        return _clean_text("\n".join(plain_parts or html_parts))
+
+    payload = message.get_payload(decode=True)
+    if payload:
+        text = payload.decode(message.get_content_charset() or "utf-8", errors="replace")
+    else:
+        text = str(message.get_payload())
+    if message.get_content_type() == "text/html":
+        return _html_to_text(text)
+    return _clean_text(text)
+
+
+def _html_to_text(html: str) -> str:
+    extractor = _HTMLTextExtractor()
+    extractor.feed(html)
+    return extractor.text()
+
+
+def _clean_text(text: str) -> str:
+    text = text.replace("\xa0", " ")
+    text = re.sub(r"[ \t\r\f\v]+", " ", text)
+    text = re.sub(r"\n\s+", "\n", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
+def _message_date(message: Message) -> date | None:
+    header = message.get("Date")
+    if not header:
+        return None
+    try:
+        parsed = parsedate_to_datetime(header)
+    except (TypeError, ValueError):
+        return None
+    return parsed.date()
+
+
+def _extract_amount_match(text: str) -> AmountMatch | None:
+    candidates: list[AmountMatch] = []
+    for pattern in _AMOUNT_PATTERNS:
+        for match in pattern.finditer(text):
+            amount = _parse_amount(match.group("amount"))
+            if amount is None:
+                continue
+            start = max(match.start() - 90, 0)
+            end = min(match.end() + 90, len(text))
+            context = _clean_text(text[start:end])
+            candidates.append(
+                AmountMatch(
+                    amount=amount,
+                    currency="EUR",
+                    context=context,
+                    score=_amount_context_score(context),
+                )
+            )
+    if not candidates:
+        return None
+    return sorted(candidates, key=lambda item: (item.score, item.amount), reverse=True)[0]
+
+
+def _parse_amount(value: str) -> Decimal | None:
+    cleaned = value.replace("\xa0", "").replace(" ", "").replace("'", "")
+    if "," in cleaned and "." in cleaned:
+        if cleaned.rfind(",") > cleaned.rfind("."):
+            cleaned = cleaned.replace(".", "").replace(",", ".")
+        else:
+            cleaned = cleaned.replace(",", "")
+    elif "," in cleaned:
+        cleaned = cleaned.replace(".", "").replace(",", ".")
+    elif "." in cleaned and len(cleaned.rsplit(".", maxsplit=1)[-1]) != 2:
+        cleaned = cleaned.replace(".", "")
+    try:
+        return abs(Decimal(cleaned)).quantize(Decimal("0.01"))
+    except InvalidOperation:
+        return None
+
+
+def _amount_context_score(context: str) -> int:
+    lowered = context.lower()
+    score = 0
+    positive = (
+        "gesamt",
+        "gesamtbetrag",
+        "summe",
+        "total",
+        "amount",
+        "betrag",
+        "zahlbetrag",
+        "payment",
+        "zahlung",
+        "fällig",
+        "due",
+    )
+    negative = ("mwst", "vat", "tax", "steuer", "versand", "shipping", "rabatt", "discount", "gutschein")
+    if any(token in lowered for token in positive):
+        score += 20
+    if any(token in lowered for token in ("fällig", "due", "zahlbar", "pay by")):
+        score += 8
+    if any(token in lowered for token in negative):
+        score -= 10
+    return score
+
+
+def _extract_due_date(text: str) -> date | None:
+    for pattern in _DUE_DATE_PATTERNS:
+        match = pattern.search(text)
+        if match:
+            parsed = _parse_date_value(match.group("date"))
+            if parsed:
+                return parsed
+    return None
+
+
+def _parse_date_value(value: str) -> date | None:
+    if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
+        try:
+            return datetime.strptime(value, "%Y-%m-%d").date()
+        except ValueError:
+            return None
+    parts = re.split(r"[./-]", value)
+    if len(parts) != 3:
+        return None
+    day, month, year = parts
+    try:
+        numeric_year = int(year)
+        if numeric_year < 100:
+            numeric_year += 2000
+        return date(numeric_year, int(month), int(day))
+    except ValueError:
+        return None
+
+
+def _extract_references(subject: str, text: str) -> dict[str, str]:
+    haystack = f"{subject}\n{text}"
+    references: dict[str, str] = {}
+    for key, patterns in _REFERENCE_PATTERNS.items():
+        for pattern in patterns:
+            match = pattern.search(haystack)
+            if match:
+                references[key] = match.group(1).strip(" .,:;")
+                break
+    return references
+
+
+def _provider_transaction_id(references: dict[str, str]) -> str | None:
+    for key in ("paypal_transaction_id", "klarna_reference", "order_number", "invoice_number"):
+        if references.get(key):
+            return references[key]
+    return None
+
+
+def _source_type(sender: str, subject: str, text: str) -> str:
+    combined = f"{sender}\n{subject}\n{text[:3000]}".lower()
+    if "klarna" in combined:
+        return "klarna_email"
+    if "paypal" in combined or "pay pal" in combined:
+        return "paypal_email"
+    if any(token in combined for token in ("bestellung", "order confirmation", "your order", "deine bestellung")):
+        return "order_confirmation_email"
+    return "imap_email"
+
+
+def _event_type(source_type: str, subject: str, text: str, due_date: date | None) -> str:
+    combined = f"{subject}\n{text[:3000]}".lower()
+    if any(token in combined for token in ("refund", "erstattung", "rückerstattung")):
+        return "refund"
+    if source_type == "klarna_email":
+        return "promised_payment"
+    if source_type == "paypal_email" and (
+        due_date or any(token in combined for token in ("pay later", "später bezahlen", "fällig", "due"))
+    ):
+        return "promised_payment"
+    return "expense"
+
+
+def _event_status(
+    source_type: str,
+    event_type: str,
+    due_date: date | None,
+    subject: str,
+    text: str,
+) -> str:
+    del subject, text
+    if source_type == "klarna_email" or event_type == "promised_payment":
+        return "outstanding" if due_date else "announced"
+    return "announced"
+
+
+def _merchant(sender: str, subject: str, text: str, source_type: str) -> str | None:
+    subject_merchant = _merchant_from_subject(subject)
+    if subject_merchant:
+        return subject_merchant
+    text_merchant = _merchant_from_text(text)
+    if text_merchant:
+        return text_merchant
+    addresses = getaddresses([sender])
+    display_name = addresses[0][0] if addresses else ""
+    candidate = _clean_party_name(display_name or sender)
+    if source_type == "klarna_email" and not candidate:
+        return "Klarna"
+    if source_type == "paypal_email" and not candidate:
+        return "PayPal"
+    return candidate or None
+
+
+def _merchant_from_subject(subject: str) -> str | None:
+    patterns = (
+        r"\b(?:deine|ihre)\s+bestellung\s+(?:bei|von)\s+(.+)$",
+        r"\byour\s+order\s+(?:from|with|at)\s+(.+)$",
+        r"\b(?:zahlung|payment)\s+(?:an|to)\s+(.+)$",
+        r"\b(?:rechnung|invoice)\s+(?:von|from)\s+(.+)$",
+    )
+    for pattern in patterns:
+        match = re.search(pattern, subject, re.IGNORECASE)
+        if match:
+            return _clean_party_name(match.group(1))
+    return None
+
+
+def _merchant_from_text(text: str) -> str | None:
+    patterns = (
+        r"\b(?:händler|merchant|seller|shop)\s*:?\s*([^\n]{2,80})",
+        r"\b(?:zahlung\s+an|payment\s+to)\s*:?\s*([^\n]{2,80})",
+    )
+    for pattern in patterns:
+        match = re.search(pattern, text, re.IGNORECASE)
+        if match:
+            return _clean_party_name(match.group(1))
+    return None
+
+
+def _clean_party_name(value: str) -> str:
+    cleaned = re.sub(r"<[^>]+>", "", value)
+    cleaned = re.split(r"\s[-|:]\s", cleaned, maxsplit=1)[0]
+    cleaned = re.sub(r"\s+", " ", cleaned).strip(" .,:;\"'")
+    return cleaned[:120]
+
+
+def _fallback_source_reference(sender: str, subject: str, received_date: date) -> str:
+    return f"imap:{received_date.isoformat()}:{sender}:{subject}"[:240]
+
+
+def _parse_folder_response(raw: bytes | str) -> dict[str, Any]:
+    text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
+    match = re.match(r"\((?P<attrs>.*?)\)\s+\"?(?P<delimiter>.*?)\"?\s+(?P<name>.+)$", text)
+    if not match:
+        return {"name": text, "delimiter": None, "attributes": []}
+    name = match.group("name").strip()
+    if name.startswith('"') and name.endswith('"'):
+        name = name[1:-1].replace('\\"', '"').replace("\\\\", "\\")
+    attrs = [item.strip("\\") for item in match.group("attrs").split() if item.strip()]
+    delimiter = match.group("delimiter") or None
+    return {"name": name, "delimiter": delimiter, "attributes": attrs}

apps/local-agent/mousehold_agent/paypal_connector.py

@@ -0,0 +1,459 @@
+from __future__ import annotations
+
+import base64
+import json
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from datetime import UTC, date, datetime, time
+from decimal import Decimal, InvalidOperation
+from typing import Any, Literal
+
+PAYPAL_LIVE_BASE_URL = "https://api-m.paypal.com"
+PAYPAL_SANDBOX_BASE_URL = "https://api-m.sandbox.paypal.com"
+PAYPAL_REPORTING_MAX_DAYS = 31
+PAYPAL_TRANSACTION_SEARCH_SCOPE = "https://uri.paypal.com/services/reporting/search/read"
+
+
+@dataclass(frozen=True)
+class PayPalConfig:
+    client_id: str
+    client_secret: str
+    environment: Literal["live", "sandbox"] = "live"
+    base_url: str | None = None
+
+
+@dataclass(frozen=True)
+class PayPalFetchResult:
+    start_date: date
+    end_date: date
+    transactions: list[dict[str, Any]]
+    observations: list[dict[str, Any]]
+    raw_response_count: int
+
+
+class PayPalAPIError(RuntimeError):
+    pass
+
+
+def inspect_paypal_credentials(config: PayPalConfig) -> dict[str, Any]:
+    return _token_payload_to_auth_check(config, fetch_access_token_payload(config))
+
+
+def _token_payload_to_auth_check(config: PayPalConfig, payload: dict[str, Any]) -> dict[str, Any]:
+    scope = payload.get("scope")
+    scopes = [item for item in re.split(r"[\s,]+", scope.strip()) if item] if isinstance(scope, str) else []
+    return {
+        "environment": config.environment,
+        "base_url": _base_url(config),
+        "app_id": _string_or_none(payload.get("app_id")),
+        "token_type": _string_or_none(payload.get("token_type")),
+        "expires_in": _int_or_none(payload.get("expires_in")),
+        "scope_text": scope if isinstance(scope, str) else None,
+        "scopes": scopes,
+        "required_scope": PAYPAL_TRANSACTION_SEARCH_SCOPE,
+        "has_transaction_search": PAYPAL_TRANSACTION_SEARCH_SCOPE in scopes,
+    }
+
+
+def fetch_paypal_transactions(
+    config: PayPalConfig,
+    *,
+    start_date: date,
+    end_date: date,
+    household_id: str | None = None,
+    source_account_id: str | None = None,
+    owner_user_id: str | None = None,
+    transaction_currency: str | None = None,
+    page_size: int = 100,
+    max_pages: int = 10,
+) -> PayPalFetchResult:
+    _validate_date_range(start_date, end_date)
+    token = fetch_access_token(config)
+    details = list_transaction_details(
+        config,
+        token=token,
+        start_date=start_date,
+        end_date=end_date,
+        transaction_currency=transaction_currency,
+        page_size=page_size,
+        max_pages=max_pages,
+    )
+    observations = [
+        paypal_transaction_to_observation(
+            transaction,
+            household_id=household_id,
+            source_account_id=source_account_id,
+            owner_user_id=owner_user_id,
+        )
+        for transaction in details
+        if _transaction_amount(transaction) is not None
+    ]
+    return PayPalFetchResult(
+        start_date=start_date,
+        end_date=end_date,
+        transactions=[paypal_transaction_to_preview(transaction) for transaction in details],
+        observations=observations,
+        raw_response_count=len(details),
+    )
+
+
+def fetch_access_token(config: PayPalConfig) -> str:
+    payload = fetch_access_token_payload(config)
+    access_token = payload.get("access_token")
+    if not isinstance(access_token, str) or not access_token:
+        raise PayPalAPIError("PayPal token response did not include an access token.")
+    scope = payload.get("scope")
+    if isinstance(scope, str) and PAYPAL_TRANSACTION_SEARCH_SCOPE not in scope.split():
+        raise PayPalAPIError(
+            "PayPal credentials do not include Transaction Search permission. "
+            f"Enable Transaction Search on the PayPal REST app and verify the OAuth token includes "
+            f"{PAYPAL_TRANSACTION_SEARCH_SCOPE}."
+        )
+    return access_token
+
+
+def fetch_access_token_payload(config: PayPalConfig) -> dict[str, Any]:
+    credentials = f"{config.client_id}:{config.client_secret}".encode()
+    body = urllib.parse.urlencode({"grant_type": "client_credentials"}).encode("ascii")
+    request = urllib.request.Request(
+        f"{_base_url(config)}/v1/oauth2/token",
+        data=body,
+        method="POST",
+        headers={
+            "Authorization": f"Basic {base64.b64encode(credentials).decode('ascii')}",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Accept": "application/json",
+        },
+    )
+    return _request_json(request)
+
+
+def list_transaction_details(
+    config: PayPalConfig,
+    *,
+    token: str,
+    start_date: date,
+    end_date: date,
+    transaction_currency: str | None = None,
+    page_size: int = 100,
+    max_pages: int = 10,
+) -> list[dict[str, Any]]:
+    page_size = max(1, min(page_size, 500))
+    max_pages = max(1, max_pages)
+    transactions: list[dict[str, Any]] = []
+    page = 1
+    total_pages: int | None = None
+
+    while page <= max_pages and (total_pages is None or page <= total_pages):
+        params = {
+            "start_date": _paypal_datetime(start_date, end_of_day=False),
+            "end_date": _paypal_datetime(end_date, end_of_day=True),
+            "fields": "all",
+            "page_size": str(page_size),
+            "page": str(page),
+        }
+        if transaction_currency:
+            params["transaction_currency"] = transaction_currency.upper()
+
+        url = f"{_base_url(config)}/v1/reporting/transactions?{urllib.parse.urlencode(params)}"
+        request = urllib.request.Request(
+            url,
+            method="GET",
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Accept": "application/json",
+            },
+        )
+        payload = _request_json(request)
+        page_transactions = payload.get("transaction_details") or []
+        if not isinstance(page_transactions, list):
+            raise PayPalAPIError("PayPal transaction response has an unexpected shape.")
+        transactions.extend(item for item in page_transactions if isinstance(item, dict))
+
+        total_pages = _int_or_none(payload.get("total_pages"))
+        if total_pages is None and len(page_transactions) < page_size:
+            break
+        page += 1
+
+    return transactions
+
+
+def paypal_transaction_to_preview(transaction: dict[str, Any]) -> dict[str, Any]:
+    info = _transaction_info(transaction)
+    amount = _transaction_amount(transaction)
+    currency = _currency_code(info)
+    event_date = _transaction_date(info)
+    return {
+        "transaction_id": _transaction_id(info),
+        "event_date": event_date.isoformat(),
+        "updated_date": _date_string(info.get("transaction_updated_date")),
+        "event_code": _string_or_none(info.get("transaction_event_code")),
+        "paypal_status": _string_or_none(info.get("transaction_status")),
+        "amount": str(abs(amount)) if amount is not None else None,
+        "currency": currency,
+        "counterparty": _counterparty(transaction),
+        "description": _description(transaction),
+        "event_type": _event_type(info, amount),
+        "status": _event_status(info),
+    }
+
+
+def paypal_transaction_to_observation(
+    transaction: dict[str, Any],
+    *,
+    household_id: str | None,
+    source_account_id: str | None,
+    owner_user_id: str | None,
+) -> dict[str, Any]:
+    info = _transaction_info(transaction)
+    amount = _transaction_amount(transaction)
+    if amount is None:
+        raise ValueError("PayPal transaction has no transaction_amount.value")
+    transaction_id = _transaction_id(info)
+    currency = _currency_code(info)
+    event_date = _transaction_date(info)
+    return {
+        "household_id": household_id,
+        "source_type": "paypal_api",
+        "source_account_id": source_account_id,
+        "owner_user_id": owner_user_id,
+        "source_reference": transaction_id,
+        "provider_transaction_id": transaction_id,
+        "raw_payload_json": transaction,
+        "amount": str(abs(amount)),
+        "currency": currency,
+        "event_date": event_date.isoformat(),
+        "booking_date": _date_string(info.get("transaction_updated_date")),
+        "merchant": _counterparty(transaction),
+        "counterparty": _counterparty(transaction),
+        "description": _description(transaction),
+        "event_type": _event_type(info, amount),
+        "status": _event_status(info),
+        "payer_user_id": owner_user_id,
+        "payment_account_id": source_account_id,
+    }
+
+
+def _request_json(request: urllib.request.Request) -> dict[str, Any]:
+    try:
+        with urllib.request.urlopen(request, timeout=30) as response:
+            data = response.read()
+    except urllib.error.HTTPError as exc:
+        detail = exc.read().decode("utf-8", errors="replace")
+        raise PayPalAPIError(_paypal_error_message(exc.code, detail)) from exc
+    except urllib.error.URLError as exc:
+        raise PayPalAPIError(f"PayPal request failed: {exc.reason}") from exc
+
+    try:
+        payload = json.loads(data.decode("utf-8"))
+    except json.JSONDecodeError as exc:
+        raise PayPalAPIError("PayPal returned invalid JSON.") from exc
+    if not isinstance(payload, dict):
+        raise PayPalAPIError("PayPal returned an unexpected JSON shape.")
+    return payload
+
+
+def _paypal_error_message(status_code: int, detail: str) -> str:
+    try:
+        payload = json.loads(detail)
+    except json.JSONDecodeError:
+        return f"PayPal request failed with HTTP {status_code}: {detail[:500]}"
+    message = payload.get("message") or payload.get("error_description") or payload.get("error")
+    name = payload.get("name")
+    if status_code == 403 and (name == "NOT_AUTHORIZED" or "insufficient permissions" in str(message).lower()):
+        return (
+            "PayPal Transaction Search is not authorized for these credentials. "
+            "Enable Transaction Search on the matching live/sandbox REST app, make sure you are using credentials "
+            "for the same environment selected in Mousehold, and verify the OAuth token includes "
+            f"{PAYPAL_TRANSACTION_SEARCH_SCOPE}. Third-party account access requires PayPal partner authorization."
+        )
+    if name and message:
+        return f"PayPal request failed with HTTP {status_code}: {name}: {message}"
+    if message:
+        return f"PayPal request failed with HTTP {status_code}: {message}"
+    return f"PayPal request failed with HTTP {status_code}: {detail[:500]}"
+
+
+def _base_url(config: PayPalConfig) -> str:
+    if config.base_url:
+        return config.base_url.rstrip("/")
+    return PAYPAL_SANDBOX_BASE_URL if config.environment == "sandbox" else PAYPAL_LIVE_BASE_URL
+
+
+def _validate_date_range(start_date: date, end_date: date) -> None:
+    if end_date < start_date:
+        raise PayPalAPIError("PayPal end_date must be on or after start_date.")
+    if (end_date - start_date).days + 1 > PAYPAL_REPORTING_MAX_DAYS:
+        raise PayPalAPIError("PayPal Transaction Search supports a maximum date range of 31 days.")
+
+
+def _paypal_datetime(value: date, *, end_of_day: bool) -> str:
+    wall_time = time(23, 59, 59) if end_of_day else time(0, 0, 0)
+    return datetime.combine(value, wall_time, tzinfo=UTC).isoformat().replace("+00:00", "Z")
+
+
+def _transaction_info(transaction: dict[str, Any]) -> dict[str, Any]:
+    info = transaction.get("transaction_info") or {}
+    return info if isinstance(info, dict) else {}
+
+
+def _transaction_amount(transaction: dict[str, Any]) -> Decimal | None:
+    info = _transaction_info(transaction)
+    amount = info.get("transaction_amount") or {}
+    if not isinstance(amount, dict):
+        return None
+    value = amount.get("value")
+    if value is None:
+        return None
+    try:
+        return Decimal(str(value)).quantize(Decimal("0.01"))
+    except InvalidOperation:
+        return None
+
+
+def _currency_code(info: dict[str, Any]) -> str:
+    amount = info.get("transaction_amount") or {}
+    if isinstance(amount, dict):
+        currency = amount.get("currency_code")
+        if isinstance(currency, str) and currency:
+            return currency.upper()
+    return "EUR"
+
+
+def _transaction_id(info: dict[str, Any]) -> str | None:
+    return _string_or_none(info.get("transaction_id")) or _string_or_none(info.get("paypal_reference_id"))
+
+
+def _transaction_date(info: dict[str, Any]) -> date:
+    return _parse_paypal_date(info.get("transaction_initiation_date")) or date.today()
+
+
+def _date_string(value: object) -> str | None:
+    parsed = _parse_paypal_date(value)
+    return parsed.isoformat() if parsed else None
+
+
+def _parse_paypal_date(value: object) -> date | None:
+    if not isinstance(value, str) or not value:
+        return None
+    normalized = value.replace("Z", "+00:00")
+    if reworked := _parse_iso_date(normalized):
+        return reworked
+    for fmt in ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S.%f%z"):
+        try:
+            return datetime.strptime(value, fmt).date()
+        except ValueError:
+            continue
+    try:
+        return date.fromisoformat(value[:10])
+    except ValueError:
+        return None
+
+
+def _parse_iso_date(value: str) -> date | None:
+    try:
+        return datetime.fromisoformat(value).date()
+    except ValueError:
+        return None
+
+
+def _counterparty(transaction: dict[str, Any]) -> str | None:
+    payer_info = transaction.get("payer_info") or {}
+    if isinstance(payer_info, dict):
+        payer_name = payer_info.get("payer_name") or {}
+        if isinstance(payer_name, dict):
+            full_name = _first_non_empty(
+                payer_name.get("alternate_full_name"),
+                " ".join(
+                    str(part)
+                    for part in (payer_name.get("given_name"), payer_name.get("surname"))
+                    if part
+                ),
+            )
+            if full_name:
+                return full_name
+        email = _string_or_none(payer_info.get("email_address"))
+        if email:
+            return email
+
+    shipping_info = transaction.get("shipping_info") or {}
+    if isinstance(shipping_info, dict):
+        name = _string_or_none(shipping_info.get("name"))
+        if name:
+            return name
+    return None
+
+
+def _description(transaction: dict[str, Any]) -> str | None:
+    info = _transaction_info(transaction)
+    cart_info = transaction.get("cart_info") or {}
+    if isinstance(cart_info, dict):
+        invoice_number = _string_or_none(cart_info.get("invoice_number"))
+        if invoice_number:
+            return f"PayPal invoice {invoice_number}"
+        item_details = cart_info.get("item_details") or []
+        if isinstance(item_details, list) and item_details:
+            first_item = item_details[0]
+            if isinstance(first_item, dict):
+                item_name = _string_or_none(first_item.get("item_name"))
+                if item_name:
+                    return item_name
+    return _first_non_empty(
+        info.get("transaction_subject"),
+        info.get("transaction_note"),
+        info.get("paypal_reference_id"),
+        info.get("transaction_id"),
+    )
+
+
+def _event_type(info: dict[str, Any], amount: Decimal | None) -> str:
+    description = " ".join(
+        str(value)
+        for value in (
+            info.get("transaction_subject"),
+            info.get("transaction_note"),
+            info.get("transaction_event_code"),
+        )
+        if value
+    ).lower()
+    if "fee" in description or "gebühr" in description:
+        return "fee"
+    if "refund" in description or "erstattung" in description or "rückerstattung" in description:
+        return "refund"
+    if amount is None:
+        return "expense"
+    return "expense" if amount < 0 else "income"
+
+
+def _event_status(info: dict[str, Any]) -> str:
+    status = _string_or_none(info.get("transaction_status"))
+    if status == "P":
+        return "announced"
+    if status == "D":
+        return "cancelled"
+    return "confirmed"
+
+
+def _string_or_none(value: object) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+def _first_non_empty(*values: object) -> str | None:
+    for value in values:
+        text = _string_or_none(value)
+        if text:
+            return text
+    return None
+
+
+def _int_or_none(value: object) -> int | None:
+    try:
+        return int(str(value))
+    except (TypeError, ValueError):
+        return None

apps/local-agent/mousehold_agent/server.py

@@ -0,0 +1,581 @@
+from __future__ import annotations
+
+import imaplib
+import os
+import threading
+import uuid
+from datetime import date, timedelta
+from pathlib import Path
+from typing import Any, Literal
+
+import uvicorn
+from fastapi import APIRouter, FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fints.client import FinTS3PinTanClient, NeedTANResponse
+from pydantic import BaseModel, Field
+
+from .fints_connector import (
+    PRODUCT_ID_HELP,
+    FinTSConfig,
+    FinTSConfigurationError,
+    account_to_dict,
+    balance_to_dict,
+    create_client,
+    persist_client,
+    select_account,
+    tan_challenge_to_dict,
+    transaction_to_observation,
+)
+from .imap_importer import DEFAULT_SENDER_FILTERS, IMAPConfig, list_imap_folders, scan_imap_mailbox
+from .paypal_connector import PayPalAPIError, PayPalConfig, fetch_paypal_transactions, inspect_paypal_credentials
+from .statement_importer import parse_klarna_statement, parse_paypal_statement
+
+LOCAL_AGENT_ORIGINS = [
+    "http://127.0.0.1:5173",
+    "http://localhost:5173",
+]
+
+
+class FinTSRequest(BaseModel):
+    blz: str
+    server: str
+    bank_user_id: str
+    pin: str
+    customer_id: str | None = None
+    product_id: str | None = None
+    product_version: str | None = None
+    tan_mechanism: str | None = None
+    tan_medium: str | None = None
+    profile: str = "browser"
+    state_dir: str = "~/.config/mousehold/fints"
+    store_state: bool = True
+
+
+class FinTSAccountsRequest(BaseModel):
+    credentials: FinTSRequest
+    with_balances: bool = False
+
+
+class FinTSTransactionsRequest(BaseModel):
+    credentials: FinTSRequest
+    iban: str | None = None
+    days: int = Field(default=30, ge=1, le=730)
+    start_date: date | None = None
+    end_date: date | None = None
+    include_pending: bool = False
+    household_id: str | None = None
+    source_account_id: str | None = None
+    owner_user_id: str | None = None
+
+
+class FinTSChoiceRequest(BaseModel):
+    value: str
+
+
+class FinTSTanRequest(BaseModel):
+    tan: str = ""
+
+
+class IMAPCredentials(BaseModel):
+    host: str
+    port: int = Field(default=993, ge=1, le=65535)
+    username: str
+    password: str
+    use_ssl: bool = True
+
+
+class IMAPFoldersRequest(BaseModel):
+    credentials: IMAPCredentials
+
+
+class IMAPScanRequest(BaseModel):
+    credentials: IMAPCredentials
+    folder: str = "INBOX"
+    search_query: str = "ALL"
+    sender_filters: list[str] = Field(default_factory=lambda: list(DEFAULT_SENDER_FILTERS))
+    limit: int = Field(default=25, ge=1, le=500)
+    household_id: str
+    source_account_id: str | None = None
+    owner_user_id: str | None = None
+
+
+class PayPalCredentials(BaseModel):
+    client_id: str
+    client_secret: str
+    environment: Literal["live", "sandbox"] = "live"
+    base_url: str | None = None
+
+
+class PayPalTransactionsRequest(BaseModel):
+    credentials: PayPalCredentials
+    days: int = Field(default=30, ge=1, le=31)
+    start_date: date | None = None
+    end_date: date | None = None
+    transaction_currency: str | None = None
+    page_size: int = Field(default=100, ge=1, le=500)
+    max_pages: int = Field(default=10, ge=1, le=100)
+    household_id: str
+    source_account_id: str | None = None
+    owner_user_id: str | None = None
+
+
+class PayPalAuthCheckRequest(BaseModel):
+    credentials: PayPalCredentials
+
+
+class StatementParseRequest(BaseModel):
+    csv_text: str
+    filename: str | None = None
+    household_id: str
+    source_account_id: str | None = None
+    owner_user_id: str | None = None
+
+
+JobStatus = Literal["running", "needs_tan", "needs_choice", "done", "error"]
+
+
+class FinTSBrowserJob:
+    def __init__(self, kind: str) -> None:
+        self.id = str(uuid.uuid4())
+        self.kind = kind
+        self.status: JobStatus = "running"
+        self.error: str | None = None
+        self.result: dict[str, Any] | None = None
+        self.challenge: dict[str, Any] | None = None
+        self.choice_kind: str | None = None
+        self.choice_options: list[dict[str, Any]] = []
+        self._lock = threading.Lock()
+        self._event = threading.Event()
+        self._input_value: str | None = None
+
+    def snapshot(self) -> dict[str, Any]:
+        with self._lock:
+            return {
+                "id": self.id,
+                "kind": self.kind,
+                "status": self.status,
+                "error": self.error,
+                "result": self.result,
+                "challenge": self.challenge,
+                "choice_kind": self.choice_kind,
+                "choice_options": self.choice_options,
+            }
+
+    def set_done(self, result: dict[str, Any]) -> None:
+        with self._lock:
+            self.status = "done"
+            self.result = result
+            self.challenge = None
+            self.choice_kind = None
+            self.choice_options = []
+            self._input_value = None
+            self._event.clear()
+
+    def set_error(self, error: BaseException | str) -> None:
+        with self._lock:
+            self.status = "error"
+            self.error = str(error)
+            self.challenge = None
+            self.choice_kind = None
+            self.choice_options = []
+            self._input_value = None
+            self._event.clear()
+
+    def wait_for_tan(self, response: NeedTANResponse, timeout_seconds: int = 600) -> str:
+        with self._lock:
+            self.status = "needs_tan"
+            self.challenge = tan_challenge_to_dict(response)
+            self.choice_kind = None
+            self.choice_options = []
+            self._input_value = None
+            self._event.clear()
+        if not self._event.wait(timeout_seconds):
+            raise TimeoutError("Timed out waiting for TAN input.")
+        with self._lock:
+            value = self._input_value or ""
+            self.status = "running"
+            self.challenge = None
+            self._input_value = None
+            self._event.clear()
+            return value
+
+    def wait_for_choice(
+        self,
+        kind: str,
+        options: list[dict[str, Any]],
+        timeout_seconds: int = 600,
+    ) -> str:
+        with self._lock:
+            self.status = "needs_choice"
+            self.choice_kind = kind
+            self.choice_options = options
+            self.challenge = None
+            self._input_value = None
+            self._event.clear()
+        if not self._event.wait(timeout_seconds):
+            raise TimeoutError(f"Timed out waiting for {kind} selection.")
+        with self._lock:
+            value = self._input_value
+            self.status = "running"
+            self.choice_kind = None
+            self.choice_options = []
+            self._input_value = None
+            self._event.clear()
+        if value is None:
+            raise RuntimeError(f"No {kind} selection received.")
+        return value
+
+    def provide_value(self, value: str) -> None:
+        with self._lock:
+            if self.status not in {"needs_tan", "needs_choice"}:
+                raise RuntimeError("Job is not waiting for input.")
+            self._input_value = value
+            self._event.set()
+
+
+jobs: dict[str, FinTSBrowserJob] = {}
+
+
+def create_router() -> APIRouter:
+    router = APIRouter()
+
+    @router.get("/health")
+    def health() -> dict[str, str]:
+        return {"status": "ok"}
+
+    @router.post("/fints/accounts/start")
+    def start_accounts(payload: FinTSAccountsRequest) -> dict[str, Any]:
+        job = _start_job("accounts", lambda current_job: _run_accounts(current_job, payload))
+        return job.snapshot()
+
+    @router.post("/fints/transactions/start")
+    def start_transactions(payload: FinTSTransactionsRequest) -> dict[str, Any]:
+        job = _start_job("transactions", lambda current_job: _run_transactions(current_job, payload))
+        return job.snapshot()
+
+    @router.get("/fints/jobs/{job_id}")
+    def get_job(job_id: str) -> dict[str, Any]:
+        return _get_job(job_id).snapshot()
+
+    @router.post("/fints/jobs/{job_id}/tan")
+    def provide_tan(job_id: str, payload: FinTSTanRequest) -> dict[str, Any]:
+        job = _get_job(job_id)
+        if job.snapshot()["status"] != "needs_tan":
+            raise HTTPException(status_code=409, detail="Job is not waiting for a TAN.")
+        try:
+            job.provide_value(payload.tan)
+        except RuntimeError as exc:
+            raise HTTPException(status_code=409, detail=str(exc)) from exc
+        return job.snapshot()
+
+    @router.post("/fints/jobs/{job_id}/choice")
+    def provide_choice(job_id: str, payload: FinTSChoiceRequest) -> dict[str, Any]:
+        job = _get_job(job_id)
+        if job.snapshot()["status"] != "needs_choice":
+            raise HTTPException(status_code=409, detail="Job is not waiting for a choice.")
+        try:
+            job.provide_value(payload.value)
+        except RuntimeError as exc:
+            raise HTTPException(status_code=409, detail=str(exc)) from exc
+        return job.snapshot()
+
+    @router.post("/imap/folders")
+    def imap_folders(payload: IMAPFoldersRequest) -> dict[str, Any]:
+        try:
+            return {"folders": list_imap_folders(_imap_config_from_payload(payload.credentials))}
+        except (OSError, RuntimeError, imaplib.IMAP4.error) as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @router.post("/imap/scan")
+    def imap_scan(payload: IMAPScanRequest) -> dict[str, Any]:
+        try:
+            return scan_imap_mailbox(
+                _imap_config_from_payload(payload.credentials),
+                household_id=payload.household_id,
+                owner_user_id=payload.owner_user_id,
+                source_account_id=payload.source_account_id,
+                limit=payload.limit,
+                folder=payload.folder,
+                search_query=payload.search_query,
+                sender_filters=payload.sender_filters,
+            )
+        except (OSError, RuntimeError, imaplib.IMAP4.error) as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @router.post("/paypal/transactions")
+    def paypal_transactions(payload: PayPalTransactionsRequest) -> dict[str, Any]:
+        end_date = payload.end_date or date.today()
+        start_date = payload.start_date or (end_date - timedelta(days=payload.days - 1))
+        try:
+            result = fetch_paypal_transactions(
+                _paypal_config_from_payload(payload.credentials),
+                start_date=start_date,
+                end_date=end_date,
+                household_id=payload.household_id,
+                owner_user_id=payload.owner_user_id,
+                source_account_id=payload.source_account_id,
+                transaction_currency=payload.transaction_currency,
+                page_size=payload.page_size,
+                max_pages=payload.max_pages,
+            )
+        except PayPalAPIError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+        return {
+            "start_date": result.start_date.isoformat(),
+            "end_date": result.end_date.isoformat(),
+            "transactions": result.transactions,
+            "observations": result.observations,
+            "count": len(result.observations),
+            "raw_response_count": result.raw_response_count,
+        }
+
+    @router.post("/paypal/auth-check")
+    def paypal_auth_check(payload: PayPalAuthCheckRequest) -> dict[str, Any]:
+        try:
+            return inspect_paypal_credentials(_paypal_config_from_payload(payload.credentials))
+        except PayPalAPIError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @router.post("/paypal/statement/parse")
+    def paypal_statement_parse(payload: StatementParseRequest) -> dict[str, Any]:
+        return parse_paypal_statement(
+            payload.csv_text,
+            household_id=payload.household_id,
+            owner_user_id=payload.owner_user_id,
+            source_account_id=payload.source_account_id,
+            filename=payload.filename,
+        )
+
+    @router.post("/klarna/statement/parse")
+    def klarna_statement_parse(payload: StatementParseRequest) -> dict[str, Any]:
+        return parse_klarna_statement(
+            payload.csv_text,
+            household_id=payload.household_id,
+            owner_user_id=payload.owner_user_id,
+            source_account_id=payload.source_account_id,
+            filename=payload.filename,
+        )
+
+    return router
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(title="Mousehold Local Agent", version="0.1.0")
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=_cors_origins(),
+        allow_credentials=False,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+    app.include_router(create_router())
+    return app
+
+
+def run(host: str = "127.0.0.1", port: int = 8765) -> None:
+    uvicorn.run("mousehold_agent.server:app", host=host, port=port, reload=False)
+
+
+def _start_job(kind: str, target: Any) -> FinTSBrowserJob:
+    job = FinTSBrowserJob(kind)
+    jobs[job.id] = job
+
+    def wrapped() -> None:
+        try:
+            target(job)
+        except BaseException as exc:
+            job.set_error(exc)
+
+    thread = threading.Thread(target=wrapped, name=f"fints-{kind}-{job.id}", daemon=True)
+    thread.start()
+    return job
+
+
+def _get_job(job_id: str) -> FinTSBrowserJob:
+    job = jobs.get(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="FinTS job not found")
+    return job
+
+
+def _run_accounts(job: FinTSBrowserJob, payload: FinTSAccountsRequest) -> None:
+    config = _config_from_payload(payload.credentials)
+
+    def operation(client: FinTS3PinTanClient) -> dict[str, Any]:
+        accounts: list[dict[str, Any]] = []
+        for index, account in enumerate(client.get_sepa_accounts()):
+            row = account_to_dict(account)
+            row["index"] = index
+            if payload.with_balances:
+                balance = _resolve_tan_response(client, client.get_balance(account), job)
+                row["balance"] = balance_to_dict(balance)
+            accounts.append(row)
+        return {"accounts": accounts}
+
+    job.set_done(_with_browser_dialog(config, job, operation))
+
+
+def _run_transactions(job: FinTSBrowserJob, payload: FinTSTransactionsRequest) -> None:
+    config = _config_from_payload(payload.credentials)
+    end_date = payload.end_date or date.today()
+    start_date = payload.start_date or (end_date - timedelta(days=payload.days))
+
+    def operation(client: FinTS3PinTanClient) -> dict[str, Any]:
+        account = select_account(client.get_sepa_accounts(), payload.iban)
+        transactions = _resolve_tan_response(
+            client,
+            client.get_transactions(
+                account,
+                start_date=start_date,
+                end_date=end_date,
+                include_pending=payload.include_pending,
+            ),
+            job,
+        )
+        observations = [
+            transaction_to_observation(
+                transaction,
+                account=account,
+                household_id=payload.household_id,
+                source_account_id=payload.source_account_id,
+                owner_user_id=payload.owner_user_id,
+            )
+            for transaction in transactions
+        ]
+        return {
+            "account": account_to_dict(account),
+            "observations": observations,
+            "count": len(observations),
+        }
+
+    job.set_done(_with_browser_dialog(config, job, operation))
+
+
+def _with_browser_dialog(config: FinTSConfig, job: FinTSBrowserJob, operation: Any) -> dict[str, Any]:
+    client = create_client(config)
+    _browser_bootstrap(client, config, job)
+    try:
+        with client:
+            while isinstance(client.init_tan_response, NeedTANResponse):
+                client.init_tan_response = _answer_tan(client, client.init_tan_response, job)
+            return operation(client)
+    finally:
+        persist_client(client, config)
+
+
+def _browser_bootstrap(client: FinTS3PinTanClient, config: FinTSConfig, job: FinTSBrowserJob) -> None:
+    if not client.get_current_tan_mechanism():
+        client.fetch_tan_mechanisms()
+        mechanisms = client.get_tan_mechanisms()
+        if config.tan_mechanism:
+            client.set_tan_mechanism(config.tan_mechanism)
+        elif len(mechanisms) == 1:
+            client.set_tan_mechanism(next(iter(mechanisms)))
+        elif len(mechanisms) > 1:
+            options = [
+                {
+                    "value": security_function,
+                    "label": getattr(parameters, "name", security_function) or security_function,
+                    "description": f"Function {security_function}",
+                }
+                for security_function, parameters in mechanisms.items()
+            ]
+            client.set_tan_mechanism(job.wait_for_choice("tan_mechanism", options))
+
+    if client.selected_tan_medium is None and client.is_tan_media_required():
+        usage, media = client.get_tan_media()
+        del usage
+        if config.tan_medium:
+            client.selected_tan_medium = config.tan_medium
+        elif len(media) == 1:
+            client.set_tan_medium(media[0])
+        elif len(media) == 0:
+            client.selected_tan_medium = ""
+        else:
+            options = [
+                {
+                    "value": str(index),
+                    "label": getattr(item, "tan_medium_name", None) or f"Medium {index + 1}",
+                    "description": _tan_medium_description(item),
+                }
+                for index, item in enumerate(media)
+            ]
+            client.set_tan_medium(media[int(job.wait_for_choice("tan_medium", options))])
+
+
+def _resolve_tan_response(client: FinTS3PinTanClient, response: Any, job: FinTSBrowserJob) -> Any:
+    while isinstance(response, NeedTANResponse):
+        response = _answer_tan(client, response, job)
+    return response
+
+
+def _answer_tan(
+    client: FinTS3PinTanClient,
+    response: NeedTANResponse,
+    job: FinTSBrowserJob,
+) -> Any:
+    if response.decoupled:
+        tan = job.wait_for_tan(response)
+        del tan
+        return client.send_tan(response, "")
+    return client.send_tan(response, job.wait_for_tan(response))
+
+
+def _tan_medium_description(item: Any) -> str:
+    parts = []
+    mobile = getattr(item, "mobile_number_masked", None)
+    if mobile:
+        parts.append(str(mobile))
+    last_use = getattr(item, "last_use", None)
+    if last_use:
+        parts.append(f"Last used {last_use}")
+    return " · ".join(parts)
+
+
+def _config_from_payload(payload: FinTSRequest) -> FinTSConfig:
+    if not payload.product_id:
+        raise FinTSConfigurationError(PRODUCT_ID_HELP)
+    return FinTSConfig(
+        bank_identifier=payload.blz,
+        user_id=payload.bank_user_id,
+        pin=payload.pin,
+        server=payload.server,
+        customer_id=payload.customer_id or None,
+        product_id=payload.product_id,
+        product_version=payload.product_version or None,
+        tan_mechanism=payload.tan_mechanism or None,
+        tan_medium=payload.tan_medium or None,
+        profile=payload.profile,
+        state_dir=Path(payload.state_dir).expanduser(),
+        store_state=payload.store_state,
+        interactive_bootstrap=False,
+    )
+
+
+def _imap_config_from_payload(payload: IMAPCredentials) -> IMAPConfig:
+    return IMAPConfig(
+        host=payload.host,
+        port=payload.port,
+        username=payload.username,
+        password=payload.password,
+        use_ssl=payload.use_ssl,
+    )
+
+
+def _paypal_config_from_payload(payload: PayPalCredentials) -> PayPalConfig:
+    return PayPalConfig(
+        client_id=payload.client_id,
+        client_secret=payload.client_secret,
+        environment=payload.environment,
+        base_url=payload.base_url or None,
+    )
+
+
+def _cors_origins() -> list[str]:
+    configured = os.environ.get("MOUSEHOLD_LOCAL_AGENT_CORS_ORIGINS")
+    if not configured:
+        return LOCAL_AGENT_ORIGINS
+    return [origin.strip() for origin in configured.split(",") if origin.strip()]
+
+
+app = create_app()

apps/local-agent/mousehold_agent/statement_importer.py

@@ -0,0 +1,487 @@
+from __future__ import annotations
+
+import csv
+import hashlib
+import io
+import re
+from dataclasses import dataclass
+from datetime import date, datetime
+from decimal import Decimal, InvalidOperation
+from typing import Any, Literal
+
+StatementKind = Literal["paypal", "klarna"]
+
+PARSER_VERSION = "0.1.0"
+
+PAYPAL_FIELD_ALIASES = {
+    "date": ("date", "transaction date", "datum"),
+    "name": ("name", "payer name", "counterparty", "merchant", "händler", "haendler"),
+    "type": ("type", "transaction type", "typ"),
+    "status": ("status", "transaction status"),
+    "currency": ("currency", "currency code", "währung", "waehrung"),
+    "gross": ("gross", "amount", "transaction amount", "betrag", "brutto"),
+    "fee": ("fee", "fee amount", "gebühr", "gebuehr"),
+    "net": ("net", "net amount", "netto"),
+    "transaction_id": ("transaction id", "transaction_id", "transaktionscode", "transaktions-id"),
+    "reference_id": ("reference txn id", "reference transaction id", "referenztransaktionscode"),
+    "invoice": ("invoice number", "invoice id", "rechnungsnummer"),
+    "item": ("item title", "item name", "description", "beschreibung", "betreff"),
+    "from_email": ("from email address", "from email", "sender email"),
+    "to_email": ("to email address", "to email", "recipient email"),
+}
+
+KLARNA_FIELD_ALIASES = {
+    "date": ("capture_date", "sale_date", "payout_date", "date", "datum", "transaction_date"),
+    "type": ("type", "transaction type", "typ"),
+    "detailed_type": ("detailed_type", "detail type", "details", "beschreibung"),
+    "amount": ("amount", "betrag", "transaction_amount", "original_capture_amount"),
+    "currency": (
+        "posting_currency",
+        "settlement_currency",
+        "currency",
+        "währung",
+        "waehrung",
+        "original_capture_currency",
+    ),
+    "order_id": ("order_id", "order id", "bestellnummer", "bestellung", "short_order_id"),
+    "capture_id": ("capture_id", "capture id"),
+    "refund_id": ("refund_id", "refund id"),
+    "payment_reference": ("payment_reference", "payment reference", "zahlungsreferenz", "reference", "referenz"),
+    "merchant": (
+        "merchant_reference1",
+        "merchant_reference2",
+        "terminal_name",
+        "merchant",
+        "händler",
+        "haendler",
+        "shop",
+    ),
+    "description": ("description", "details", "detailed_type", "type", "beschreibung", "verwendungszweck"),
+}
+
+
+@dataclass(frozen=True)
+class ParsedStatement:
+    source: StatementKind
+    filename: str | None
+    row_count: int
+    importable_count: int
+    skipped_count: int
+    rows: list[dict[str, Any]]
+    observations: list[dict[str, Any]]
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "source": self.source,
+            "filename": self.filename,
+            "row_count": self.row_count,
+            "importable_count": self.importable_count,
+            "skipped_count": self.skipped_count,
+            "rows": self.rows,
+            "observations": self.observations,
+        }
+
+
+def parse_paypal_statement(
+    csv_text: str,
+    *,
+    household_id: str,
+    owner_user_id: str | None,
+    source_account_id: str | None,
+    filename: str | None = None,
+) -> dict[str, Any]:
+    rows = _dict_rows(csv_text)
+    previews: list[dict[str, Any]] = []
+    observations: list[dict[str, Any]] = []
+    for index, row in enumerate(rows, start=1):
+        preview, observation = _parse_paypal_row(
+            row,
+            index=index,
+            household_id=household_id,
+            owner_user_id=owner_user_id,
+            source_account_id=source_account_id,
+            filename=filename,
+        )
+        previews.append(preview)
+        if observation:
+            observations.append(observation)
+    return ParsedStatement(
+        source="paypal",
+        filename=filename,
+        row_count=len(rows),
+        importable_count=len(observations),
+        skipped_count=len(previews) - len(observations),
+        rows=previews,
+        observations=observations,
+    ).as_dict()
+
+
+def parse_klarna_statement(
+    csv_text: str,
+    *,
+    household_id: str,
+    owner_user_id: str | None,
+    source_account_id: str | None,
+    filename: str | None = None,
+) -> dict[str, Any]:
+    rows = _klarna_transaction_rows(csv_text)
+    previews: list[dict[str, Any]] = []
+    observations: list[dict[str, Any]] = []
+    for index, row in enumerate(rows, start=1):
+        preview, observation = _parse_klarna_row(
+            row,
+            index=index,
+            household_id=household_id,
+            owner_user_id=owner_user_id,
+            source_account_id=source_account_id,
+            filename=filename,
+        )
+        previews.append(preview)
+        if observation:
+            observations.append(observation)
+    return ParsedStatement(
+        source="klarna",
+        filename=filename,
+        row_count=len(rows),
+        importable_count=len(observations),
+        skipped_count=len(previews) - len(observations),
+        rows=previews,
+        observations=observations,
+    ).as_dict()
+
+
+def _parse_paypal_row(
+    row: dict[str, str],
+    *,
+    index: int,
+    household_id: str,
+    owner_user_id: str | None,
+    source_account_id: str | None,
+    filename: str | None,
+) -> tuple[dict[str, Any], dict[str, Any] | None]:
+    amount = _parse_amount(_pick(row, PAYPAL_FIELD_ALIASES["net"]) or _pick(row, PAYPAL_FIELD_ALIASES["gross"]))
+    event_date = _parse_date(_pick(row, PAYPAL_FIELD_ALIASES["date"]))
+    currency = _pick(row, PAYPAL_FIELD_ALIASES["currency"]) or "EUR"
+    raw_type = _pick(row, PAYPAL_FIELD_ALIASES["type"]) or ""
+    status_text = _pick(row, PAYPAL_FIELD_ALIASES["status"]) or ""
+    transaction_id = _pick(row, PAYPAL_FIELD_ALIASES["transaction_id"])
+    reference_id = _pick(row, PAYPAL_FIELD_ALIASES["reference_id"])
+    source_reference = transaction_id or reference_id or _row_reference("paypal", row, index)
+    merchant = _pick(row, PAYPAL_FIELD_ALIASES["name"]) or _pick(row, PAYPAL_FIELD_ALIASES["from_email"])
+    description = _first_non_empty(
+        _pick(row, PAYPAL_FIELD_ALIASES["item"]),
+        _pick(row, PAYPAL_FIELD_ALIASES["invoice"]),
+        raw_type,
+        merchant,
+    )
+
+    preview = _preview(
+        row=index,
+        source_type="paypal_csv",
+        source_reference=source_reference,
+        event_date=event_date,
+        amount=amount,
+        currency=currency,
+        merchant=merchant,
+        description=description,
+        raw_type=raw_type,
+        event_type=_paypal_event_type(raw_type, description, amount),
+        status=_paypal_status(status_text),
+    )
+    if amount is None:
+        preview["skip_reason"] = "No amount found"
+        return preview, None
+    if event_date is None:
+        preview["skip_reason"] = "No date found"
+        return preview, None
+
+    observation = {
+        "household_id": household_id,
+        "source_type": "paypal_csv",
+        "source_account_id": source_account_id,
+        "owner_user_id": owner_user_id,
+        "source_reference": source_reference,
+        "provider_transaction_id": transaction_id,
+        "raw_payload_json": {
+            "filename": filename,
+            "row": row,
+            "parser_name": "mousehold-paypal-statement",
+            "parser_version": PARSER_VERSION,
+        },
+        "amount": str(abs(amount)),
+        "currency": currency.upper(),
+        "event_date": event_date.isoformat(),
+        "booking_date": event_date.isoformat(),
+        "merchant": merchant,
+        "counterparty": merchant,
+        "description": description,
+        "event_type": preview["event_type"],
+        "status": preview["status"],
+        "payer_user_id": owner_user_id,
+        "payment_account_id": source_account_id,
+    }
+    return preview, observation
+
+
+def _parse_klarna_row(
+    row: dict[str, str],
+    *,
+    index: int,
+    household_id: str,
+    owner_user_id: str | None,
+    source_account_id: str | None,
+    filename: str | None,
+) -> tuple[dict[str, Any], dict[str, Any] | None]:
+    amount = _parse_amount(_pick(row, KLARNA_FIELD_ALIASES["amount"]))
+    event_date = _parse_date(_pick(row, KLARNA_FIELD_ALIASES["date"]))
+    currency = _pick(row, KLARNA_FIELD_ALIASES["currency"]) or "EUR"
+    row_type = _pick(row, KLARNA_FIELD_ALIASES["type"]) or ""
+    detailed_type = _pick(row, KLARNA_FIELD_ALIASES["detailed_type"]) or ""
+    raw_type = " / ".join(part for part in (row_type, detailed_type) if part)
+    source_reference = _first_non_empty(
+        _pick(row, KLARNA_FIELD_ALIASES["capture_id"]),
+        _pick(row, KLARNA_FIELD_ALIASES["refund_id"]),
+        _pick(row, KLARNA_FIELD_ALIASES["order_id"]),
+        _pick(row, KLARNA_FIELD_ALIASES["payment_reference"]),
+        _row_reference("klarna", row, index),
+    )
+    merchant = _pick(row, KLARNA_FIELD_ALIASES["merchant"]) or "Klarna"
+    description = _first_non_empty(_pick(row, KLARNA_FIELD_ALIASES["description"]), raw_type, merchant)
+    event_type = _klarna_event_type(row_type, detailed_type, description, amount)
+    status = (
+        "outstanding"
+        if row_type.upper() == "CHARGE" and "PAYMENT_REMINDER" in detailed_type.upper()
+        else "confirmed"
+    )
+
+    preview = _preview(
+        row=index,
+        source_type="klarna_csv",
+        source_reference=source_reference,
+        event_date=event_date,
+        amount=amount,
+        currency=currency,
+        merchant=merchant,
+        description=description,
+        raw_type=raw_type,
+        event_type=event_type,
+        status=status,
+    )
+    if amount is None:
+        preview["skip_reason"] = "No amount found"
+        return preview, None
+    if event_date is None:
+        preview["skip_reason"] = "No date found"
+        return preview, None
+
+    observation = {
+        "household_id": household_id,
+        "source_type": "klarna_csv",
+        "source_account_id": source_account_id,
+        "owner_user_id": owner_user_id,
+        "source_reference": source_reference,
+        "provider_transaction_id": source_reference,
+        "raw_payload_json": {
+            "filename": filename,
+            "row": row,
+            "parser_name": "mousehold-klarna-statement",
+            "parser_version": PARSER_VERSION,
+        },
+        "amount": str(abs(amount)),
+        "currency": currency.upper(),
+        "event_date": event_date.isoformat(),
+        "booking_date": event_date.isoformat(),
+        "merchant": merchant,
+        "counterparty": "Klarna",
+        "description": description,
+        "event_type": event_type,
+        "status": status,
+        "payer_user_id": owner_user_id,
+        "payment_account_id": source_account_id,
+    }
+    return preview, observation
+
+
+def _dict_rows(csv_text: str) -> list[dict[str, str]]:
+    text = csv_text.lstrip("\ufeff")
+    dialect = _sniff_dialect(text)
+    reader = csv.DictReader(io.StringIO(text), dialect=dialect)
+    return [
+        {_normalize_header(key): (value or "").strip() for key, value in row.items() if key}
+        for row in reader
+        if any((value or "").strip() for value in row.values())
+    ]
+
+
+def _klarna_transaction_rows(csv_text: str) -> list[dict[str, str]]:
+    text = csv_text.lstrip("\ufeff")
+    dialect = _sniff_dialect(text)
+    reader = csv.reader(io.StringIO(text), dialect=dialect)
+    header: list[str] | None = None
+    rows: list[dict[str, str]] = []
+    for raw_row in reader:
+        cells = [cell.strip() for cell in raw_row]
+        normalized = [_normalize_header(cell) for cell in cells]
+        if "type" in normalized and "amount" in normalized:
+            header = normalized
+            continue
+        if header is None or not any(cells):
+            continue
+        row = {
+            key: cells[position].strip()
+            for position, key in enumerate(header)
+            if key and position < len(cells)
+        }
+        if _pick(row, KLARNA_FIELD_ALIASES["type"]) or _pick(row, KLARNA_FIELD_ALIASES["amount"]):
+            rows.append(row)
+    if rows:
+        return rows
+    return _dict_rows(text)
+
+
+def _sniff_dialect(csv_text: str) -> csv.Dialect:
+    sample = csv_text[:4096]
+    try:
+        return csv.Sniffer().sniff(sample, delimiters=",;\t")
+    except csv.Error:
+        class Fallback(csv.excel):
+            delimiter = ";"
+
+        return Fallback
+
+
+def _pick(row: dict[str, str], aliases: tuple[str, ...]) -> str | None:
+    for alias in aliases:
+        value = row.get(_normalize_header(alias))
+        if value:
+            return value.strip()
+    return None
+
+
+def _parse_amount(value: str | None) -> Decimal | None:
+    if not value:
+        return None
+    cleaned = value.strip().replace("\xa0", " ")
+    cleaned = re.sub(r"[^0-9,.\-+() ]", "", cleaned)
+    negative = cleaned.startswith("(") and cleaned.endswith(")")
+    cleaned = cleaned.strip("() ").replace(" ", "")
+    if not cleaned:
+        return None
+    if "," in cleaned and "." in cleaned:
+        if cleaned.rfind(",") > cleaned.rfind("."):
+            cleaned = cleaned.replace(".", "").replace(",", ".")
+        else:
+            cleaned = cleaned.replace(",", "")
+    elif "," in cleaned:
+        cleaned = cleaned.replace(".", "").replace(",", ".")
+    try:
+        amount = Decimal(cleaned).quantize(Decimal("0.01"))
+    except InvalidOperation:
+        return None
+    return -amount if negative else amount
+
+
+def _parse_date(value: str | None) -> date | None:
+    if not value:
+        return None
+    stripped = value.strip()
+    if "T" in stripped:
+        try:
+            return datetime.fromisoformat(stripped.replace("Z", "+00:00")).date()
+        except ValueError:
+            pass
+    for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%m-%d-%Y"):
+        try:
+            return datetime.strptime(stripped[:10], fmt).date()
+        except ValueError:
+            continue
+    return None
+
+
+def _paypal_event_type(raw_type: str, description: str | None, amount: Decimal | None) -> str:
+    combined = f"{raw_type} {description or ''}".lower()
+    if "fee" in combined or "gebühr" in combined:
+        return "fee"
+    if "refund" in combined or "erstattung" in combined or "reversal" in combined:
+        return "refund"
+    if "transfer" in combined or "bank account" in combined:
+        return "transfer"
+    if amount is None:
+        return "expense"
+    return "expense" if amount < 0 else "income"
+
+
+def _paypal_status(value: str) -> str:
+    normalized = value.strip().lower()
+    if normalized in {"pending", "processing", "created", "payment pending"}:
+        return "announced"
+    if normalized in {"denied", "canceled", "cancelled", "voided", "expired", "failed"}:
+        return "cancelled"
+    return "confirmed"
+
+
+def _klarna_event_type(
+    row_type: str,
+    detailed_type: str,
+    description: str | None,
+    amount: Decimal | None,
+) -> str:
+    combined = f"{row_type} {detailed_type} {description or ''}".lower()
+    if "fee" in combined or "commission" in combined:
+        return "fee"
+    if "return" in combined or "refund" in combined or "credit" in combined or "release" in combined:
+        return "refund"
+    if "payment_reminder" in combined or "liability" in combined or "payment" in combined or "zahlung" in combined:
+        return "liability_payment"
+    if amount is None:
+        return "expense"
+    return "expense" if amount < 0 else "income"
+
+
+def _preview(
+    *,
+    row: int,
+    source_type: str,
+    source_reference: str | None,
+    event_date: date | None,
+    amount: Decimal | None,
+    currency: str,
+    merchant: str | None,
+    description: str | None,
+    raw_type: str,
+    event_type: str,
+    status: str,
+) -> dict[str, Any]:
+    return {
+        "row": row,
+        "source_type": source_type,
+        "source_reference": source_reference,
+        "event_date": event_date.isoformat() if event_date else None,
+        "amount": str(abs(amount)) if amount is not None else None,
+        "currency": currency.upper(),
+        "merchant": merchant,
+        "description": description,
+        "raw_type": raw_type,
+        "event_type": event_type,
+        "status": status,
+        "importable": amount is not None and event_date is not None,
+        "skip_reason": None,
+    }
+
+
+def _normalize_header(value: str) -> str:
+    lowered = value.strip().lower().replace("\ufeff", "")
+    lowered = lowered.replace("ä", "ae").replace("ö", "oe").replace("ü", "ue").replace("ß", "ss")
+    return re.sub(r"[^a-z0-9]+", "_", lowered).strip("_")
+
+
+def _row_reference(source: str, row: dict[str, str], index: int) -> str:
+    digest = hashlib.sha256(repr(sorted(row.items())).encode()).hexdigest()[:16]
+    return f"{source}-statement-row-{index}-{digest}"
+
+
+def _first_non_empty(*values: str | None) -> str | None:
+    for value in values:
+        if value:
+            return value
+    return None