488 lines
16 KiB
Python
488 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import hashlib
|
|
import io
|
|
import re
|
|
from dataclasses import dataclass
|
|
from datetime import date, datetime
|
|
from decimal import Decimal, InvalidOperation
|
|
from typing import Any, Literal
|
|
|
|
StatementKind = Literal["paypal", "klarna"]
|
|
|
|
PARSER_VERSION = "0.1.0"
|
|
|
|
PAYPAL_FIELD_ALIASES = {
|
|
"date": ("date", "transaction date", "datum"),
|
|
"name": ("name", "payer name", "counterparty", "merchant", "händler", "haendler"),
|
|
"type": ("type", "transaction type", "typ"),
|
|
"status": ("status", "transaction status"),
|
|
"currency": ("currency", "currency code", "währung", "waehrung"),
|
|
"gross": ("gross", "amount", "transaction amount", "betrag", "brutto"),
|
|
"fee": ("fee", "fee amount", "gebühr", "gebuehr"),
|
|
"net": ("net", "net amount", "netto"),
|
|
"transaction_id": ("transaction id", "transaction_id", "transaktionscode", "transaktions-id"),
|
|
"reference_id": ("reference txn id", "reference transaction id", "referenztransaktionscode"),
|
|
"invoice": ("invoice number", "invoice id", "rechnungsnummer"),
|
|
"item": ("item title", "item name", "description", "beschreibung", "betreff"),
|
|
"from_email": ("from email address", "from email", "sender email"),
|
|
"to_email": ("to email address", "to email", "recipient email"),
|
|
}
|
|
|
|
KLARNA_FIELD_ALIASES = {
|
|
"date": ("capture_date", "sale_date", "payout_date", "date", "datum", "transaction_date"),
|
|
"type": ("type", "transaction type", "typ"),
|
|
"detailed_type": ("detailed_type", "detail type", "details", "beschreibung"),
|
|
"amount": ("amount", "betrag", "transaction_amount", "original_capture_amount"),
|
|
"currency": (
|
|
"posting_currency",
|
|
"settlement_currency",
|
|
"currency",
|
|
"währung",
|
|
"waehrung",
|
|
"original_capture_currency",
|
|
),
|
|
"order_id": ("order_id", "order id", "bestellnummer", "bestellung", "short_order_id"),
|
|
"capture_id": ("capture_id", "capture id"),
|
|
"refund_id": ("refund_id", "refund id"),
|
|
"payment_reference": ("payment_reference", "payment reference", "zahlungsreferenz", "reference", "referenz"),
|
|
"merchant": (
|
|
"merchant_reference1",
|
|
"merchant_reference2",
|
|
"terminal_name",
|
|
"merchant",
|
|
"händler",
|
|
"haendler",
|
|
"shop",
|
|
),
|
|
"description": ("description", "details", "detailed_type", "type", "beschreibung", "verwendungszweck"),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ParsedStatement:
|
|
source: StatementKind
|
|
filename: str | None
|
|
row_count: int
|
|
importable_count: int
|
|
skipped_count: int
|
|
rows: list[dict[str, Any]]
|
|
observations: list[dict[str, Any]]
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"source": self.source,
|
|
"filename": self.filename,
|
|
"row_count": self.row_count,
|
|
"importable_count": self.importable_count,
|
|
"skipped_count": self.skipped_count,
|
|
"rows": self.rows,
|
|
"observations": self.observations,
|
|
}
|
|
|
|
|
|
def parse_paypal_statement(
|
|
csv_text: str,
|
|
*,
|
|
household_id: str,
|
|
owner_user_id: str | None,
|
|
source_account_id: str | None,
|
|
filename: str | None = None,
|
|
) -> dict[str, Any]:
|
|
rows = _dict_rows(csv_text)
|
|
previews: list[dict[str, Any]] = []
|
|
observations: list[dict[str, Any]] = []
|
|
for index, row in enumerate(rows, start=1):
|
|
preview, observation = _parse_paypal_row(
|
|
row,
|
|
index=index,
|
|
household_id=household_id,
|
|
owner_user_id=owner_user_id,
|
|
source_account_id=source_account_id,
|
|
filename=filename,
|
|
)
|
|
previews.append(preview)
|
|
if observation:
|
|
observations.append(observation)
|
|
return ParsedStatement(
|
|
source="paypal",
|
|
filename=filename,
|
|
row_count=len(rows),
|
|
importable_count=len(observations),
|
|
skipped_count=len(previews) - len(observations),
|
|
rows=previews,
|
|
observations=observations,
|
|
).as_dict()
|
|
|
|
|
|
def parse_klarna_statement(
|
|
csv_text: str,
|
|
*,
|
|
household_id: str,
|
|
owner_user_id: str | None,
|
|
source_account_id: str | None,
|
|
filename: str | None = None,
|
|
) -> dict[str, Any]:
|
|
rows = _klarna_transaction_rows(csv_text)
|
|
previews: list[dict[str, Any]] = []
|
|
observations: list[dict[str, Any]] = []
|
|
for index, row in enumerate(rows, start=1):
|
|
preview, observation = _parse_klarna_row(
|
|
row,
|
|
index=index,
|
|
household_id=household_id,
|
|
owner_user_id=owner_user_id,
|
|
source_account_id=source_account_id,
|
|
filename=filename,
|
|
)
|
|
previews.append(preview)
|
|
if observation:
|
|
observations.append(observation)
|
|
return ParsedStatement(
|
|
source="klarna",
|
|
filename=filename,
|
|
row_count=len(rows),
|
|
importable_count=len(observations),
|
|
skipped_count=len(previews) - len(observations),
|
|
rows=previews,
|
|
observations=observations,
|
|
).as_dict()
|
|
|
|
|
|
def _parse_paypal_row(
|
|
row: dict[str, str],
|
|
*,
|
|
index: int,
|
|
household_id: str,
|
|
owner_user_id: str | None,
|
|
source_account_id: str | None,
|
|
filename: str | None,
|
|
) -> tuple[dict[str, Any], dict[str, Any] | None]:
|
|
amount = _parse_amount(_pick(row, PAYPAL_FIELD_ALIASES["net"]) or _pick(row, PAYPAL_FIELD_ALIASES["gross"]))
|
|
event_date = _parse_date(_pick(row, PAYPAL_FIELD_ALIASES["date"]))
|
|
currency = _pick(row, PAYPAL_FIELD_ALIASES["currency"]) or "EUR"
|
|
raw_type = _pick(row, PAYPAL_FIELD_ALIASES["type"]) or ""
|
|
status_text = _pick(row, PAYPAL_FIELD_ALIASES["status"]) or ""
|
|
transaction_id = _pick(row, PAYPAL_FIELD_ALIASES["transaction_id"])
|
|
reference_id = _pick(row, PAYPAL_FIELD_ALIASES["reference_id"])
|
|
source_reference = transaction_id or reference_id or _row_reference("paypal", row, index)
|
|
merchant = _pick(row, PAYPAL_FIELD_ALIASES["name"]) or _pick(row, PAYPAL_FIELD_ALIASES["from_email"])
|
|
description = _first_non_empty(
|
|
_pick(row, PAYPAL_FIELD_ALIASES["item"]),
|
|
_pick(row, PAYPAL_FIELD_ALIASES["invoice"]),
|
|
raw_type,
|
|
merchant,
|
|
)
|
|
|
|
preview = _preview(
|
|
row=index,
|
|
source_type="paypal_csv",
|
|
source_reference=source_reference,
|
|
event_date=event_date,
|
|
amount=amount,
|
|
currency=currency,
|
|
merchant=merchant,
|
|
description=description,
|
|
raw_type=raw_type,
|
|
event_type=_paypal_event_type(raw_type, description, amount),
|
|
status=_paypal_status(status_text),
|
|
)
|
|
if amount is None:
|
|
preview["skip_reason"] = "No amount found"
|
|
return preview, None
|
|
if event_date is None:
|
|
preview["skip_reason"] = "No date found"
|
|
return preview, None
|
|
|
|
observation = {
|
|
"household_id": household_id,
|
|
"source_type": "paypal_csv",
|
|
"source_account_id": source_account_id,
|
|
"owner_user_id": owner_user_id,
|
|
"source_reference": source_reference,
|
|
"provider_transaction_id": transaction_id,
|
|
"raw_payload_json": {
|
|
"filename": filename,
|
|
"row": row,
|
|
"parser_name": "mousehold-paypal-statement",
|
|
"parser_version": PARSER_VERSION,
|
|
},
|
|
"amount": str(abs(amount)),
|
|
"currency": currency.upper(),
|
|
"event_date": event_date.isoformat(),
|
|
"booking_date": event_date.isoformat(),
|
|
"merchant": merchant,
|
|
"counterparty": merchant,
|
|
"description": description,
|
|
"event_type": preview["event_type"],
|
|
"status": preview["status"],
|
|
"payer_user_id": owner_user_id,
|
|
"payment_account_id": source_account_id,
|
|
}
|
|
return preview, observation
|
|
|
|
|
|
def _parse_klarna_row(
|
|
row: dict[str, str],
|
|
*,
|
|
index: int,
|
|
household_id: str,
|
|
owner_user_id: str | None,
|
|
source_account_id: str | None,
|
|
filename: str | None,
|
|
) -> tuple[dict[str, Any], dict[str, Any] | None]:
|
|
amount = _parse_amount(_pick(row, KLARNA_FIELD_ALIASES["amount"]))
|
|
event_date = _parse_date(_pick(row, KLARNA_FIELD_ALIASES["date"]))
|
|
currency = _pick(row, KLARNA_FIELD_ALIASES["currency"]) or "EUR"
|
|
row_type = _pick(row, KLARNA_FIELD_ALIASES["type"]) or ""
|
|
detailed_type = _pick(row, KLARNA_FIELD_ALIASES["detailed_type"]) or ""
|
|
raw_type = " / ".join(part for part in (row_type, detailed_type) if part)
|
|
source_reference = _first_non_empty(
|
|
_pick(row, KLARNA_FIELD_ALIASES["capture_id"]),
|
|
_pick(row, KLARNA_FIELD_ALIASES["refund_id"]),
|
|
_pick(row, KLARNA_FIELD_ALIASES["order_id"]),
|
|
_pick(row, KLARNA_FIELD_ALIASES["payment_reference"]),
|
|
_row_reference("klarna", row, index),
|
|
)
|
|
merchant = _pick(row, KLARNA_FIELD_ALIASES["merchant"]) or "Klarna"
|
|
description = _first_non_empty(_pick(row, KLARNA_FIELD_ALIASES["description"]), raw_type, merchant)
|
|
event_type = _klarna_event_type(row_type, detailed_type, description, amount)
|
|
status = (
|
|
"outstanding"
|
|
if row_type.upper() == "CHARGE" and "PAYMENT_REMINDER" in detailed_type.upper()
|
|
else "confirmed"
|
|
)
|
|
|
|
preview = _preview(
|
|
row=index,
|
|
source_type="klarna_csv",
|
|
source_reference=source_reference,
|
|
event_date=event_date,
|
|
amount=amount,
|
|
currency=currency,
|
|
merchant=merchant,
|
|
description=description,
|
|
raw_type=raw_type,
|
|
event_type=event_type,
|
|
status=status,
|
|
)
|
|
if amount is None:
|
|
preview["skip_reason"] = "No amount found"
|
|
return preview, None
|
|
if event_date is None:
|
|
preview["skip_reason"] = "No date found"
|
|
return preview, None
|
|
|
|
observation = {
|
|
"household_id": household_id,
|
|
"source_type": "klarna_csv",
|
|
"source_account_id": source_account_id,
|
|
"owner_user_id": owner_user_id,
|
|
"source_reference": source_reference,
|
|
"provider_transaction_id": source_reference,
|
|
"raw_payload_json": {
|
|
"filename": filename,
|
|
"row": row,
|
|
"parser_name": "mousehold-klarna-statement",
|
|
"parser_version": PARSER_VERSION,
|
|
},
|
|
"amount": str(abs(amount)),
|
|
"currency": currency.upper(),
|
|
"event_date": event_date.isoformat(),
|
|
"booking_date": event_date.isoformat(),
|
|
"merchant": merchant,
|
|
"counterparty": "Klarna",
|
|
"description": description,
|
|
"event_type": event_type,
|
|
"status": status,
|
|
"payer_user_id": owner_user_id,
|
|
"payment_account_id": source_account_id,
|
|
}
|
|
return preview, observation
|
|
|
|
|
|
def _dict_rows(csv_text: str) -> list[dict[str, str]]:
|
|
text = csv_text.lstrip("\ufeff")
|
|
dialect = _sniff_dialect(text)
|
|
reader = csv.DictReader(io.StringIO(text), dialect=dialect)
|
|
return [
|
|
{_normalize_header(key): (value or "").strip() for key, value in row.items() if key}
|
|
for row in reader
|
|
if any((value or "").strip() for value in row.values())
|
|
]
|
|
|
|
|
|
def _klarna_transaction_rows(csv_text: str) -> list[dict[str, str]]:
|
|
text = csv_text.lstrip("\ufeff")
|
|
dialect = _sniff_dialect(text)
|
|
reader = csv.reader(io.StringIO(text), dialect=dialect)
|
|
header: list[str] | None = None
|
|
rows: list[dict[str, str]] = []
|
|
for raw_row in reader:
|
|
cells = [cell.strip() for cell in raw_row]
|
|
normalized = [_normalize_header(cell) for cell in cells]
|
|
if "type" in normalized and "amount" in normalized:
|
|
header = normalized
|
|
continue
|
|
if header is None or not any(cells):
|
|
continue
|
|
row = {
|
|
key: cells[position].strip()
|
|
for position, key in enumerate(header)
|
|
if key and position < len(cells)
|
|
}
|
|
if _pick(row, KLARNA_FIELD_ALIASES["type"]) or _pick(row, KLARNA_FIELD_ALIASES["amount"]):
|
|
rows.append(row)
|
|
if rows:
|
|
return rows
|
|
return _dict_rows(text)
|
|
|
|
|
|
def _sniff_dialect(csv_text: str) -> csv.Dialect:
|
|
sample = csv_text[:4096]
|
|
try:
|
|
return csv.Sniffer().sniff(sample, delimiters=",;\t")
|
|
except csv.Error:
|
|
class Fallback(csv.excel):
|
|
delimiter = ";"
|
|
|
|
return Fallback
|
|
|
|
|
|
def _pick(row: dict[str, str], aliases: tuple[str, ...]) -> str | None:
|
|
for alias in aliases:
|
|
value = row.get(_normalize_header(alias))
|
|
if value:
|
|
return value.strip()
|
|
return None
|
|
|
|
|
|
def _parse_amount(value: str | None) -> Decimal | None:
|
|
if not value:
|
|
return None
|
|
cleaned = value.strip().replace("\xa0", " ")
|
|
cleaned = re.sub(r"[^0-9,.\-+() ]", "", cleaned)
|
|
negative = cleaned.startswith("(") and cleaned.endswith(")")
|
|
cleaned = cleaned.strip("() ").replace(" ", "")
|
|
if not cleaned:
|
|
return None
|
|
if "," in cleaned and "." in cleaned:
|
|
if cleaned.rfind(",") > cleaned.rfind("."):
|
|
cleaned = cleaned.replace(".", "").replace(",", ".")
|
|
else:
|
|
cleaned = cleaned.replace(",", "")
|
|
elif "," in cleaned:
|
|
cleaned = cleaned.replace(".", "").replace(",", ".")
|
|
try:
|
|
amount = Decimal(cleaned).quantize(Decimal("0.01"))
|
|
except InvalidOperation:
|
|
return None
|
|
return -amount if negative else amount
|
|
|
|
|
|
def _parse_date(value: str | None) -> date | None:
|
|
if not value:
|
|
return None
|
|
stripped = value.strip()
|
|
if "T" in stripped:
|
|
try:
|
|
return datetime.fromisoformat(stripped.replace("Z", "+00:00")).date()
|
|
except ValueError:
|
|
pass
|
|
for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%m-%d-%Y"):
|
|
try:
|
|
return datetime.strptime(stripped[:10], fmt).date()
|
|
except ValueError:
|
|
continue
|
|
return None
|
|
|
|
|
|
def _paypal_event_type(raw_type: str, description: str | None, amount: Decimal | None) -> str:
|
|
combined = f"{raw_type} {description or ''}".lower()
|
|
if "fee" in combined or "gebühr" in combined:
|
|
return "fee"
|
|
if "refund" in combined or "erstattung" in combined or "reversal" in combined:
|
|
return "refund"
|
|
if "transfer" in combined or "bank account" in combined:
|
|
return "transfer"
|
|
if amount is None:
|
|
return "expense"
|
|
return "expense" if amount < 0 else "income"
|
|
|
|
|
|
def _paypal_status(value: str) -> str:
|
|
normalized = value.strip().lower()
|
|
if normalized in {"pending", "processing", "created", "payment pending"}:
|
|
return "announced"
|
|
if normalized in {"denied", "canceled", "cancelled", "voided", "expired", "failed"}:
|
|
return "cancelled"
|
|
return "confirmed"
|
|
|
|
|
|
def _klarna_event_type(
|
|
row_type: str,
|
|
detailed_type: str,
|
|
description: str | None,
|
|
amount: Decimal | None,
|
|
) -> str:
|
|
combined = f"{row_type} {detailed_type} {description or ''}".lower()
|
|
if "fee" in combined or "commission" in combined:
|
|
return "fee"
|
|
if "return" in combined or "refund" in combined or "credit" in combined or "release" in combined:
|
|
return "refund"
|
|
if "payment_reminder" in combined or "liability" in combined or "payment" in combined or "zahlung" in combined:
|
|
return "liability_payment"
|
|
if amount is None:
|
|
return "expense"
|
|
return "expense" if amount < 0 else "income"
|
|
|
|
|
|
def _preview(
|
|
*,
|
|
row: int,
|
|
source_type: str,
|
|
source_reference: str | None,
|
|
event_date: date | None,
|
|
amount: Decimal | None,
|
|
currency: str,
|
|
merchant: str | None,
|
|
description: str | None,
|
|
raw_type: str,
|
|
event_type: str,
|
|
status: str,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"row": row,
|
|
"source_type": source_type,
|
|
"source_reference": source_reference,
|
|
"event_date": event_date.isoformat() if event_date else None,
|
|
"amount": str(abs(amount)) if amount is not None else None,
|
|
"currency": currency.upper(),
|
|
"merchant": merchant,
|
|
"description": description,
|
|
"raw_type": raw_type,
|
|
"event_type": event_type,
|
|
"status": status,
|
|
"importable": amount is not None and event_date is not None,
|
|
"skip_reason": None,
|
|
}
|
|
|
|
|
|
def _normalize_header(value: str) -> str:
|
|
lowered = value.strip().lower().replace("\ufeff", "")
|
|
lowered = lowered.replace("ä", "ae").replace("ö", "oe").replace("ü", "ue").replace("ß", "ss")
|
|
return re.sub(r"[^a-z0-9]+", "_", lowered).strip("_")
|
|
|
|
|
|
def _row_reference(source: str, row: dict[str, str], index: int) -> str:
|
|
digest = hashlib.sha256(repr(sorted(row.items())).encode()).hexdigest()[:16]
|
|
return f"{source}-statement-row-{index}-{digest}"
|
|
|
|
|
|
def _first_non_empty(*values: str | None) -> str | None:
|
|
for value in values:
|
|
if value:
|
|
return value
|
|
return None
|