from __future__ import annotations import csv import hashlib import io import re from dataclasses import dataclass from datetime import date, datetime from decimal import Decimal, InvalidOperation from typing import Any, Literal StatementKind = Literal["paypal", "klarna"] PARSER_VERSION = "0.1.0" PAYPAL_FIELD_ALIASES = { "date": ("date", "transaction date", "datum"), "name": ("name", "payer name", "counterparty", "merchant", "händler", "haendler"), "type": ("type", "transaction type", "typ"), "status": ("status", "transaction status"), "currency": ("currency", "currency code", "währung", "waehrung"), "gross": ("gross", "amount", "transaction amount", "betrag", "brutto"), "fee": ("fee", "fee amount", "gebühr", "gebuehr"), "net": ("net", "net amount", "netto"), "transaction_id": ("transaction id", "transaction_id", "transaktionscode", "transaktions-id"), "reference_id": ("reference txn id", "reference transaction id", "referenztransaktionscode"), "invoice": ("invoice number", "invoice id", "rechnungsnummer"), "item": ("item title", "item name", "description", "beschreibung", "betreff"), "from_email": ("from email address", "from email", "sender email"), "to_email": ("to email address", "to email", "recipient email"), } KLARNA_FIELD_ALIASES = { "date": ("capture_date", "sale_date", "payout_date", "date", "datum", "transaction_date"), "type": ("type", "transaction type", "typ"), "detailed_type": ("detailed_type", "detail type", "details", "beschreibung"), "amount": ("amount", "betrag", "transaction_amount", "original_capture_amount"), "currency": ( "posting_currency", "settlement_currency", "currency", "währung", "waehrung", "original_capture_currency", ), "order_id": ("order_id", "order id", "bestellnummer", "bestellung", "short_order_id"), "capture_id": ("capture_id", "capture id"), "refund_id": ("refund_id", "refund id"), "payment_reference": ("payment_reference", "payment reference", "zahlungsreferenz", "reference", "referenz"), "merchant": ( "merchant_reference1", "merchant_reference2", "terminal_name", "merchant", "händler", "haendler", "shop", ), "description": ("description", "details", "detailed_type", "type", "beschreibung", "verwendungszweck"), } @dataclass(frozen=True) class ParsedStatement: source: StatementKind filename: str | None row_count: int importable_count: int skipped_count: int rows: list[dict[str, Any]] observations: list[dict[str, Any]] def as_dict(self) -> dict[str, Any]: return { "source": self.source, "filename": self.filename, "row_count": self.row_count, "importable_count": self.importable_count, "skipped_count": self.skipped_count, "rows": self.rows, "observations": self.observations, } def parse_paypal_statement( csv_text: str, *, household_id: str, owner_user_id: str | None, source_account_id: str | None, filename: str | None = None, ) -> dict[str, Any]: rows = _dict_rows(csv_text) previews: list[dict[str, Any]] = [] observations: list[dict[str, Any]] = [] for index, row in enumerate(rows, start=1): preview, observation = _parse_paypal_row( row, index=index, household_id=household_id, owner_user_id=owner_user_id, source_account_id=source_account_id, filename=filename, ) previews.append(preview) if observation: observations.append(observation) return ParsedStatement( source="paypal", filename=filename, row_count=len(rows), importable_count=len(observations), skipped_count=len(previews) - len(observations), rows=previews, observations=observations, ).as_dict() def parse_klarna_statement( csv_text: str, *, household_id: str, owner_user_id: str | None, source_account_id: str | None, filename: str | None = None, ) -> dict[str, Any]: rows = _klarna_transaction_rows(csv_text) previews: list[dict[str, Any]] = [] observations: list[dict[str, Any]] = [] for index, row in enumerate(rows, start=1): preview, observation = _parse_klarna_row( row, index=index, household_id=household_id, owner_user_id=owner_user_id, source_account_id=source_account_id, filename=filename, ) previews.append(preview) if observation: observations.append(observation) return ParsedStatement( source="klarna", filename=filename, row_count=len(rows), importable_count=len(observations), skipped_count=len(previews) - len(observations), rows=previews, observations=observations, ).as_dict() def _parse_paypal_row( row: dict[str, str], *, index: int, household_id: str, owner_user_id: str | None, source_account_id: str | None, filename: str | None, ) -> tuple[dict[str, Any], dict[str, Any] | None]: amount = _parse_amount(_pick(row, PAYPAL_FIELD_ALIASES["net"]) or _pick(row, PAYPAL_FIELD_ALIASES["gross"])) event_date = _parse_date(_pick(row, PAYPAL_FIELD_ALIASES["date"])) currency = _pick(row, PAYPAL_FIELD_ALIASES["currency"]) or "EUR" raw_type = _pick(row, PAYPAL_FIELD_ALIASES["type"]) or "" status_text = _pick(row, PAYPAL_FIELD_ALIASES["status"]) or "" transaction_id = _pick(row, PAYPAL_FIELD_ALIASES["transaction_id"]) reference_id = _pick(row, PAYPAL_FIELD_ALIASES["reference_id"]) source_reference = transaction_id or reference_id or _row_reference("paypal", row, index) merchant = _pick(row, PAYPAL_FIELD_ALIASES["name"]) or _pick(row, PAYPAL_FIELD_ALIASES["from_email"]) description = _first_non_empty( _pick(row, PAYPAL_FIELD_ALIASES["item"]), _pick(row, PAYPAL_FIELD_ALIASES["invoice"]), raw_type, merchant, ) preview = _preview( row=index, source_type="paypal_csv", source_reference=source_reference, event_date=event_date, amount=amount, currency=currency, merchant=merchant, description=description, raw_type=raw_type, event_type=_paypal_event_type(raw_type, description, amount), status=_paypal_status(status_text), ) if amount is None: preview["skip_reason"] = "No amount found" return preview, None if event_date is None: preview["skip_reason"] = "No date found" return preview, None observation = { "household_id": household_id, "source_type": "paypal_csv", "source_account_id": source_account_id, "owner_user_id": owner_user_id, "source_reference": source_reference, "provider_transaction_id": transaction_id, "raw_payload_json": { "filename": filename, "row": row, "parser_name": "mousehold-paypal-statement", "parser_version": PARSER_VERSION, }, "amount": str(abs(amount)), "currency": currency.upper(), "event_date": event_date.isoformat(), "booking_date": event_date.isoformat(), "merchant": merchant, "counterparty": merchant, "description": description, "event_type": preview["event_type"], "status": preview["status"], "payer_user_id": owner_user_id, "payment_account_id": source_account_id, } return preview, observation def _parse_klarna_row( row: dict[str, str], *, index: int, household_id: str, owner_user_id: str | None, source_account_id: str | None, filename: str | None, ) -> tuple[dict[str, Any], dict[str, Any] | None]: amount = _parse_amount(_pick(row, KLARNA_FIELD_ALIASES["amount"])) event_date = _parse_date(_pick(row, KLARNA_FIELD_ALIASES["date"])) currency = _pick(row, KLARNA_FIELD_ALIASES["currency"]) or "EUR" row_type = _pick(row, KLARNA_FIELD_ALIASES["type"]) or "" detailed_type = _pick(row, KLARNA_FIELD_ALIASES["detailed_type"]) or "" raw_type = " / ".join(part for part in (row_type, detailed_type) if part) source_reference = _first_non_empty( _pick(row, KLARNA_FIELD_ALIASES["capture_id"]), _pick(row, KLARNA_FIELD_ALIASES["refund_id"]), _pick(row, KLARNA_FIELD_ALIASES["order_id"]), _pick(row, KLARNA_FIELD_ALIASES["payment_reference"]), _row_reference("klarna", row, index), ) merchant = _pick(row, KLARNA_FIELD_ALIASES["merchant"]) or "Klarna" description = _first_non_empty(_pick(row, KLARNA_FIELD_ALIASES["description"]), raw_type, merchant) event_type = _klarna_event_type(row_type, detailed_type, description, amount) status = ( "outstanding" if row_type.upper() == "CHARGE" and "PAYMENT_REMINDER" in detailed_type.upper() else "confirmed" ) preview = _preview( row=index, source_type="klarna_csv", source_reference=source_reference, event_date=event_date, amount=amount, currency=currency, merchant=merchant, description=description, raw_type=raw_type, event_type=event_type, status=status, ) if amount is None: preview["skip_reason"] = "No amount found" return preview, None if event_date is None: preview["skip_reason"] = "No date found" return preview, None observation = { "household_id": household_id, "source_type": "klarna_csv", "source_account_id": source_account_id, "owner_user_id": owner_user_id, "source_reference": source_reference, "provider_transaction_id": source_reference, "raw_payload_json": { "filename": filename, "row": row, "parser_name": "mousehold-klarna-statement", "parser_version": PARSER_VERSION, }, "amount": str(abs(amount)), "currency": currency.upper(), "event_date": event_date.isoformat(), "booking_date": event_date.isoformat(), "merchant": merchant, "counterparty": "Klarna", "description": description, "event_type": event_type, "status": status, "payer_user_id": owner_user_id, "payment_account_id": source_account_id, } return preview, observation def _dict_rows(csv_text: str) -> list[dict[str, str]]: text = csv_text.lstrip("\ufeff") dialect = _sniff_dialect(text) reader = csv.DictReader(io.StringIO(text), dialect=dialect) return [ {_normalize_header(key): (value or "").strip() for key, value in row.items() if key} for row in reader if any((value or "").strip() for value in row.values()) ] def _klarna_transaction_rows(csv_text: str) -> list[dict[str, str]]: text = csv_text.lstrip("\ufeff") dialect = _sniff_dialect(text) reader = csv.reader(io.StringIO(text), dialect=dialect) header: list[str] | None = None rows: list[dict[str, str]] = [] for raw_row in reader: cells = [cell.strip() for cell in raw_row] normalized = [_normalize_header(cell) for cell in cells] if "type" in normalized and "amount" in normalized: header = normalized continue if header is None or not any(cells): continue row = { key: cells[position].strip() for position, key in enumerate(header) if key and position < len(cells) } if _pick(row, KLARNA_FIELD_ALIASES["type"]) or _pick(row, KLARNA_FIELD_ALIASES["amount"]): rows.append(row) if rows: return rows return _dict_rows(text) def _sniff_dialect(csv_text: str) -> csv.Dialect: sample = csv_text[:4096] try: return csv.Sniffer().sniff(sample, delimiters=",;\t") except csv.Error: class Fallback(csv.excel): delimiter = ";" return Fallback def _pick(row: dict[str, str], aliases: tuple[str, ...]) -> str | None: for alias in aliases: value = row.get(_normalize_header(alias)) if value: return value.strip() return None def _parse_amount(value: str | None) -> Decimal | None: if not value: return None cleaned = value.strip().replace("\xa0", " ") cleaned = re.sub(r"[^0-9,.\-+() ]", "", cleaned) negative = cleaned.startswith("(") and cleaned.endswith(")") cleaned = cleaned.strip("() ").replace(" ", "") if not cleaned: return None if "," in cleaned and "." in cleaned: if cleaned.rfind(",") > cleaned.rfind("."): cleaned = cleaned.replace(".", "").replace(",", ".") else: cleaned = cleaned.replace(",", "") elif "," in cleaned: cleaned = cleaned.replace(".", "").replace(",", ".") try: amount = Decimal(cleaned).quantize(Decimal("0.01")) except InvalidOperation: return None return -amount if negative else amount def _parse_date(value: str | None) -> date | None: if not value: return None stripped = value.strip() if "T" in stripped: try: return datetime.fromisoformat(stripped.replace("Z", "+00:00")).date() except ValueError: pass for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%m-%d-%Y"): try: return datetime.strptime(stripped[:10], fmt).date() except ValueError: continue return None def _paypal_event_type(raw_type: str, description: str | None, amount: Decimal | None) -> str: combined = f"{raw_type} {description or ''}".lower() if "fee" in combined or "gebühr" in combined: return "fee" if "refund" in combined or "erstattung" in combined or "reversal" in combined: return "refund" if "transfer" in combined or "bank account" in combined: return "transfer" if amount is None: return "expense" return "expense" if amount < 0 else "income" def _paypal_status(value: str) -> str: normalized = value.strip().lower() if normalized in {"pending", "processing", "created", "payment pending"}: return "announced" if normalized in {"denied", "canceled", "cancelled", "voided", "expired", "failed"}: return "cancelled" return "confirmed" def _klarna_event_type( row_type: str, detailed_type: str, description: str | None, amount: Decimal | None, ) -> str: combined = f"{row_type} {detailed_type} {description or ''}".lower() if "fee" in combined or "commission" in combined: return "fee" if "return" in combined or "refund" in combined or "credit" in combined or "release" in combined: return "refund" if "payment_reminder" in combined or "liability" in combined or "payment" in combined or "zahlung" in combined: return "liability_payment" if amount is None: return "expense" return "expense" if amount < 0 else "income" def _preview( *, row: int, source_type: str, source_reference: str | None, event_date: date | None, amount: Decimal | None, currency: str, merchant: str | None, description: str | None, raw_type: str, event_type: str, status: str, ) -> dict[str, Any]: return { "row": row, "source_type": source_type, "source_reference": source_reference, "event_date": event_date.isoformat() if event_date else None, "amount": str(abs(amount)) if amount is not None else None, "currency": currency.upper(), "merchant": merchant, "description": description, "raw_type": raw_type, "event_type": event_type, "status": status, "importable": amount is not None and event_date is not None, "skip_reason": None, } def _normalize_header(value: str) -> str: lowered = value.strip().lower().replace("\ufeff", "") lowered = lowered.replace("ä", "ae").replace("ö", "oe").replace("ü", "ue").replace("ß", "ss") return re.sub(r"[^a-z0-9]+", "_", lowered).strip("_") def _row_reference(source: str, row: dict[str, str], index: int) -> str: digest = hashlib.sha256(repr(sorted(row.items())).encode()).hexdigest()[:16] return f"{source}-statement-row-{index}-{digest}" def _first_non_empty(*values: str | None) -> str | None: for value in values: if value: return value return None