677 lines
22 KiB
Python
677 lines
22 KiB
Python
from __future__ import annotations
|
|
|
|
import email
|
|
import imaplib
|
|
import os
|
|
import re
|
|
import shlex
|
|
from dataclasses import dataclass
|
|
from datetime import date, datetime
|
|
from decimal import Decimal, InvalidOperation
|
|
from email import policy
|
|
from email.header import decode_header
|
|
from email.message import EmailMessage, Message
|
|
from email.utils import getaddresses, parsedate_to_datetime
|
|
from html.parser import HTMLParser
|
|
from typing import Any
|
|
|
|
DEFAULT_SENDER_FILTERS = (
|
|
"klarna",
|
|
"paypal",
|
|
"rechnung",
|
|
"invoice",
|
|
"receipt",
|
|
"order",
|
|
"bestellung",
|
|
"quittung",
|
|
)
|
|
|
|
PARSER_NAME = "mousehold-imap-email"
|
|
PARSER_VERSION = "0.2.0"
|
|
MAX_RAW_TEXT_LENGTH = 20_000
|
|
|
|
_AMOUNT_VALUE = r"[+-]?(?:\d{1,3}(?:[.\s']\d{3})+|\d+)(?:[,.]\d{2})"
|
|
_AMOUNT_PATTERNS = (
|
|
re.compile(rf"(?P<currency>€|EUR)\s*(?P<amount>{_AMOUNT_VALUE})", re.IGNORECASE),
|
|
re.compile(rf"(?P<amount>{_AMOUNT_VALUE})\s*(?P<currency>€|EUR)", re.IGNORECASE),
|
|
)
|
|
_DATE_VALUE = r"(?P<date>\d{4}-\d{2}-\d{2}|\d{1,2}[./-]\d{1,2}[./-]\d{2,4})"
|
|
_DUE_DATE_PATTERNS = (
|
|
re.compile(rf"\b(?:fällig(?:\s+am|\s+bis)?|zahlbar\s+bis|bezahlen\s+bis)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE),
|
|
re.compile(rf"\b(?:due(?:\s+on|\s+by)?|pay(?:ment)?\s+due|pay\s+by)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE),
|
|
re.compile(rf"\b(?:abbuchung\s+am|wird\s+am|debit(?:ed)?\s+on)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE),
|
|
)
|
|
_REFERENCE_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|
"order_number": (
|
|
re.compile(
|
|
r"\b(?:bestell(?:nummer|nr\.?)|order\s+(?:number|no\.?))\s*[:#-]?\s*"
|
|
r"([A-Z0-9][A-Z0-9._/-]{2,})",
|
|
re.IGNORECASE,
|
|
),
|
|
re.compile(r"\b(?:bestellung|order)\s*[:#-]\s*([A-Z0-9][A-Z0-9._/-]{2,})", re.IGNORECASE),
|
|
),
|
|
"invoice_number": (
|
|
re.compile(
|
|
r"\b(?:rechnungs(?:nummer|nr\.?)|invoice\s+(?:number|no\.?))\s*[:#-]?\s*"
|
|
r"([A-Z0-9][A-Z0-9._/-]{2,})",
|
|
re.IGNORECASE,
|
|
),
|
|
re.compile(r"\b(?:rechnung|invoice)\s*[:#-]\s*([A-Z0-9][A-Z0-9._/-]{2,})", re.IGNORECASE),
|
|
),
|
|
"paypal_transaction_id": (
|
|
re.compile(
|
|
r"\b(?:paypal\s+)?(?:transaction|transaktion)(?:\s+id|\s+nr\.?|\s+number)?\s*[:#-]?\s*"
|
|
r"([A-Z0-9][A-Z0-9._-]{6,})",
|
|
re.IGNORECASE,
|
|
),
|
|
),
|
|
"klarna_reference": (
|
|
re.compile(
|
|
r"\b(?:klarna\s+)?(?:referenz|reference|zahlung(?:s)?referenz)\s*[:#-]?\s*"
|
|
r"([A-Z0-9][A-Z0-9._/-]{4,})",
|
|
re.IGNORECASE,
|
|
),
|
|
),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class IMAPConfig:
|
|
host: str
|
|
username: str
|
|
password: str
|
|
port: int = 993
|
|
folder: str = "INBOX"
|
|
use_ssl: bool = True
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AmountMatch:
|
|
amount: Decimal
|
|
currency: str
|
|
context: str
|
|
score: int
|
|
|
|
|
|
class _HTMLTextExtractor(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.parts: list[str] = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
del attrs
|
|
if tag.lower() in {"br", "p", "div", "li", "tr"}:
|
|
self.parts.append("\n")
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
self.parts.append(data)
|
|
|
|
def text(self) -> str:
|
|
return _clean_text(" ".join(self.parts))
|
|
|
|
|
|
def config_from_env() -> IMAPConfig:
|
|
return IMAPConfig(
|
|
host=_required_env("MOUSEHOLD_IMAP_HOST"),
|
|
port=int(os.environ.get("MOUSEHOLD_IMAP_PORT", "993")),
|
|
username=_required_env("MOUSEHOLD_IMAP_USERNAME"),
|
|
password=_required_env("MOUSEHOLD_IMAP_PASSWORD"),
|
|
folder=os.environ.get("MOUSEHOLD_IMAP_FOLDER", "INBOX"),
|
|
use_ssl=_env_bool("MOUSEHOLD_IMAP_USE_SSL", default=True),
|
|
)
|
|
|
|
|
|
def fetch_imap_observations(
|
|
*,
|
|
household_id: str,
|
|
owner_user_id: str | None,
|
|
source_account_id: str | None,
|
|
limit: int = 25,
|
|
) -> list[dict[str, Any]]:
|
|
sender_filters = tuple(
|
|
value.strip().lower()
|
|
for value in os.environ.get(
|
|
"MOUSEHOLD_IMAP_SENDER_FILTERS", ",".join(DEFAULT_SENDER_FILTERS)
|
|
).split(",")
|
|
if value.strip()
|
|
)
|
|
result = scan_imap_mailbox(
|
|
config_from_env(),
|
|
household_id=household_id,
|
|
owner_user_id=owner_user_id,
|
|
source_account_id=source_account_id,
|
|
limit=limit,
|
|
sender_filters=sender_filters,
|
|
)
|
|
return result["observations"]
|
|
|
|
|
|
def list_imap_folders(config: IMAPConfig) -> list[dict[str, Any]]:
|
|
with _connect(config) as mailbox:
|
|
_login(mailbox, config)
|
|
status, data = mailbox.list()
|
|
_ensure_ok(status, data, "list folders")
|
|
return [_parse_folder_response(item) for item in data if item]
|
|
|
|
|
|
def scan_imap_mailbox(
|
|
config: IMAPConfig,
|
|
*,
|
|
household_id: str,
|
|
owner_user_id: str | None = None,
|
|
source_account_id: str | None = None,
|
|
limit: int = 25,
|
|
folder: str | None = None,
|
|
search_query: str = "ALL",
|
|
sender_filters: tuple[str, ...] | list[str] | None = DEFAULT_SENDER_FILTERS,
|
|
) -> dict[str, Any]:
|
|
selected_folder = folder or config.folder
|
|
filters = _normalize_filters(sender_filters)
|
|
messages: list[dict[str, Any]] = []
|
|
observations: list[dict[str, Any]] = []
|
|
scanned_count = 0
|
|
filtered_count = 0
|
|
|
|
with _connect(config) as mailbox:
|
|
_login(mailbox, config)
|
|
status, data = mailbox.select(_mailbox_name(selected_folder), readonly=True)
|
|
_ensure_ok(status, data, f"select folder {selected_folder}")
|
|
status, data = mailbox.search(None, *_search_criteria(search_query))
|
|
_ensure_ok(status, data, f"search {search_query}")
|
|
ids = data[0].split()[-limit:] if data and data[0] else []
|
|
scanned_count = len(ids)
|
|
|
|
for message_id in reversed(ids):
|
|
status, fetched = mailbox.fetch(message_id, "(RFC822)")
|
|
if _status_text(status) != "OK" or not fetched or not isinstance(fetched[0], tuple):
|
|
continue
|
|
message = email.message_from_bytes(fetched[0][1], policy=policy.default)
|
|
headers_text = _headers_text(message)
|
|
body = _message_text(message)
|
|
if filters and not _matches_filters(filters, headers_text, body):
|
|
filtered_count += 1
|
|
continue
|
|
|
|
parsed = parse_email_message(
|
|
message,
|
|
household_id=household_id,
|
|
owner_user_id=owner_user_id,
|
|
source_account_id=source_account_id,
|
|
folder=selected_folder,
|
|
body=body,
|
|
)
|
|
messages.append(parsed["message"])
|
|
if parsed["observation"]:
|
|
observations.append(parsed["observation"])
|
|
|
|
return {
|
|
"folder": selected_folder,
|
|
"scanned_count": scanned_count,
|
|
"filtered_count": filtered_count,
|
|
"matched_count": len(messages),
|
|
"importable_count": len(observations),
|
|
"messages": messages,
|
|
"observations": observations,
|
|
}
|
|
|
|
|
|
def parse_email_message(
|
|
message: Message,
|
|
*,
|
|
household_id: str,
|
|
owner_user_id: str | None,
|
|
source_account_id: str | None,
|
|
folder: str = "INBOX",
|
|
body: str | None = None,
|
|
) -> dict[str, Any]:
|
|
subject = _decode_mime_header(message.get("Subject"))
|
|
from_header = _decode_mime_header(message.get("From"))
|
|
to_header = _decode_mime_header(message.get("To"))
|
|
message_id = _compact_header_value(message.get("Message-ID"))
|
|
received_date = _message_date(message) or date.today()
|
|
text = body if body is not None else _message_text(message)
|
|
search_text = f"{subject}\n{text}"
|
|
source_type = _source_type(from_header, subject, text)
|
|
amount_match = _extract_amount_match(search_text)
|
|
due_date = _extract_due_date(search_text)
|
|
references = _extract_references(subject, text)
|
|
provider_transaction_id = _provider_transaction_id(references)
|
|
merchant = _merchant(from_header, subject, text, source_type)
|
|
event_type = _event_type(source_type, subject, text, due_date)
|
|
event_status = _event_status(source_type, event_type, due_date, subject, text)
|
|
source_reference = (
|
|
message_id
|
|
or provider_transaction_id
|
|
or _fallback_source_reference(from_header, subject, received_date)
|
|
)
|
|
if subject:
|
|
description = subject
|
|
elif merchant:
|
|
description = f"Email from {merchant}"
|
|
else:
|
|
description = "Email import"
|
|
|
|
preview = {
|
|
"message_id": message_id,
|
|
"source_reference": source_reference,
|
|
"from": from_header,
|
|
"to": to_header,
|
|
"subject": subject,
|
|
"date": received_date.isoformat(),
|
|
"folder": folder,
|
|
"source_type": source_type,
|
|
"event_type": event_type,
|
|
"status": event_status,
|
|
"amount": str(amount_match.amount) if amount_match else None,
|
|
"currency": amount_match.currency if amount_match else None,
|
|
"merchant": merchant,
|
|
"due_date": due_date.isoformat() if due_date else None,
|
|
"references": references,
|
|
"importable": amount_match is not None,
|
|
"skip_reason": None if amount_match else "No EUR amount found",
|
|
}
|
|
|
|
if not amount_match:
|
|
return {"message": preview, "observation": None}
|
|
|
|
observation = {
|
|
"household_id": household_id,
|
|
"source_type": source_type,
|
|
"source_account_id": source_account_id,
|
|
"owner_user_id": owner_user_id,
|
|
"message_id": message_id,
|
|
"source_reference": source_reference,
|
|
"provider_transaction_id": provider_transaction_id,
|
|
"raw_text": text[:MAX_RAW_TEXT_LENGTH],
|
|
"raw_payload_json": {
|
|
"from": from_header,
|
|
"to": to_header,
|
|
"subject": subject,
|
|
"date": _compact_header_value(message.get("Date")),
|
|
"folder": folder,
|
|
"message_id": message_id,
|
|
"references": references,
|
|
"amount_context": amount_match.context,
|
|
"parser_name": PARSER_NAME,
|
|
"parser_version": PARSER_VERSION,
|
|
},
|
|
"amount": str(amount_match.amount),
|
|
"currency": amount_match.currency,
|
|
"event_date": received_date.isoformat(),
|
|
"due_date": due_date.isoformat() if due_date else None,
|
|
"merchant": merchant,
|
|
"description": description[:240],
|
|
"event_type": event_type,
|
|
"status": event_status,
|
|
"payer_user_id": owner_user_id,
|
|
"payment_account_id": source_account_id,
|
|
}
|
|
return {"message": preview, "observation": observation}
|
|
|
|
|
|
def _connect(config: IMAPConfig) -> imaplib.IMAP4:
|
|
if config.use_ssl:
|
|
return imaplib.IMAP4_SSL(config.host, config.port)
|
|
return imaplib.IMAP4(config.host, config.port)
|
|
|
|
|
|
def _login(mailbox: imaplib.IMAP4, config: IMAPConfig) -> None:
|
|
status, data = mailbox.login(config.username, config.password)
|
|
_ensure_ok(status, data, "login")
|
|
|
|
|
|
def _required_env(name: str) -> str:
|
|
value = os.environ.get(name)
|
|
if not value:
|
|
raise RuntimeError(f"{name} is required")
|
|
return value
|
|
|
|
|
|
def _env_bool(name: str, *, default: bool) -> bool:
|
|
value = os.environ.get(name)
|
|
if value is None:
|
|
return default
|
|
return value.strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
def _status_text(status: str | bytes) -> str:
|
|
return status.decode("ascii", errors="replace").upper() if isinstance(status, bytes) else status.upper()
|
|
|
|
|
|
def _ensure_ok(status: str | bytes, data: object, action: str) -> None:
|
|
if _status_text(status) != "OK":
|
|
raise RuntimeError(f"IMAP {action} failed: {data!r}")
|
|
|
|
|
|
def _mailbox_name(folder: str) -> str:
|
|
if folder.startswith('"') and folder.endswith('"'):
|
|
return folder
|
|
if any(char in folder for char in ' "()[\\]'):
|
|
return '"' + folder.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
return folder
|
|
|
|
|
|
def _search_criteria(search_query: str) -> list[str]:
|
|
query = search_query.strip() or "ALL"
|
|
try:
|
|
parts = shlex.split(query)
|
|
except ValueError:
|
|
parts = query.split()
|
|
return parts or ["ALL"]
|
|
|
|
|
|
def _normalize_filters(sender_filters: tuple[str, ...] | list[str] | None) -> tuple[str, ...]:
|
|
if sender_filters is None:
|
|
return ()
|
|
return tuple(token.strip().lower() for token in sender_filters if token.strip())
|
|
|
|
|
|
def _matches_filters(filters: tuple[str, ...], headers_text: str, body: str) -> bool:
|
|
haystack = f"{headers_text}\n{body[:5000]}".lower()
|
|
return any(token in haystack for token in filters)
|
|
|
|
|
|
def _headers_text(message: Message) -> str:
|
|
return "\n".join(
|
|
[
|
|
_decode_mime_header(message.get("From")),
|
|
_decode_mime_header(message.get("To")),
|
|
_decode_mime_header(message.get("Subject")),
|
|
]
|
|
)
|
|
|
|
|
|
def _decode_mime_header(value: str | None) -> str:
|
|
if not value:
|
|
return ""
|
|
parts: list[str] = []
|
|
for content, charset in decode_header(value):
|
|
if isinstance(content, bytes):
|
|
parts.append(content.decode(charset or "utf-8", errors="replace"))
|
|
else:
|
|
parts.append(content)
|
|
return _clean_text("".join(parts))
|
|
|
|
|
|
def _compact_header_value(value: str | None) -> str | None:
|
|
cleaned = _decode_mime_header(value)
|
|
return cleaned or None
|
|
|
|
|
|
def _message_text(message: Message) -> str:
|
|
if isinstance(message, EmailMessage):
|
|
body = message.get_body(preferencelist=("plain", "html"))
|
|
if body:
|
|
content = body.get_content()
|
|
if body.get_content_type() == "text/html":
|
|
return _html_to_text(content)
|
|
return _clean_text(content)
|
|
|
|
if message.is_multipart():
|
|
plain_parts: list[str] = []
|
|
html_parts: list[str] = []
|
|
for part in message.walk():
|
|
if part.get_content_maintype() != "text":
|
|
continue
|
|
payload = part.get_payload(decode=True)
|
|
if not payload:
|
|
continue
|
|
text = payload.decode(part.get_content_charset() or "utf-8", errors="replace")
|
|
if part.get_content_type() == "text/plain":
|
|
plain_parts.append(text)
|
|
elif part.get_content_type() == "text/html":
|
|
html_parts.append(_html_to_text(text))
|
|
return _clean_text("\n".join(plain_parts or html_parts))
|
|
|
|
payload = message.get_payload(decode=True)
|
|
if payload:
|
|
text = payload.decode(message.get_content_charset() or "utf-8", errors="replace")
|
|
else:
|
|
text = str(message.get_payload())
|
|
if message.get_content_type() == "text/html":
|
|
return _html_to_text(text)
|
|
return _clean_text(text)
|
|
|
|
|
|
def _html_to_text(html: str) -> str:
|
|
extractor = _HTMLTextExtractor()
|
|
extractor.feed(html)
|
|
return extractor.text()
|
|
|
|
|
|
def _clean_text(text: str) -> str:
|
|
text = text.replace("\xa0", " ")
|
|
text = re.sub(r"[ \t\r\f\v]+", " ", text)
|
|
text = re.sub(r"\n\s+", "\n", text)
|
|
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
return text.strip()
|
|
|
|
|
|
def _message_date(message: Message) -> date | None:
|
|
header = message.get("Date")
|
|
if not header:
|
|
return None
|
|
try:
|
|
parsed = parsedate_to_datetime(header)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return parsed.date()
|
|
|
|
|
|
def _extract_amount_match(text: str) -> AmountMatch | None:
|
|
candidates: list[AmountMatch] = []
|
|
for pattern in _AMOUNT_PATTERNS:
|
|
for match in pattern.finditer(text):
|
|
amount = _parse_amount(match.group("amount"))
|
|
if amount is None:
|
|
continue
|
|
start = max(match.start() - 90, 0)
|
|
end = min(match.end() + 90, len(text))
|
|
context = _clean_text(text[start:end])
|
|
candidates.append(
|
|
AmountMatch(
|
|
amount=amount,
|
|
currency="EUR",
|
|
context=context,
|
|
score=_amount_context_score(context),
|
|
)
|
|
)
|
|
if not candidates:
|
|
return None
|
|
return sorted(candidates, key=lambda item: (item.score, item.amount), reverse=True)[0]
|
|
|
|
|
|
def _parse_amount(value: str) -> Decimal | None:
|
|
cleaned = value.replace("\xa0", "").replace(" ", "").replace("'", "")
|
|
if "," in cleaned and "." in cleaned:
|
|
if cleaned.rfind(",") > cleaned.rfind("."):
|
|
cleaned = cleaned.replace(".", "").replace(",", ".")
|
|
else:
|
|
cleaned = cleaned.replace(",", "")
|
|
elif "," in cleaned:
|
|
cleaned = cleaned.replace(".", "").replace(",", ".")
|
|
elif "." in cleaned and len(cleaned.rsplit(".", maxsplit=1)[-1]) != 2:
|
|
cleaned = cleaned.replace(".", "")
|
|
try:
|
|
return abs(Decimal(cleaned)).quantize(Decimal("0.01"))
|
|
except InvalidOperation:
|
|
return None
|
|
|
|
|
|
def _amount_context_score(context: str) -> int:
|
|
lowered = context.lower()
|
|
score = 0
|
|
positive = (
|
|
"gesamt",
|
|
"gesamtbetrag",
|
|
"summe",
|
|
"total",
|
|
"amount",
|
|
"betrag",
|
|
"zahlbetrag",
|
|
"payment",
|
|
"zahlung",
|
|
"fällig",
|
|
"due",
|
|
)
|
|
negative = ("mwst", "vat", "tax", "steuer", "versand", "shipping", "rabatt", "discount", "gutschein")
|
|
if any(token in lowered for token in positive):
|
|
score += 20
|
|
if any(token in lowered for token in ("fällig", "due", "zahlbar", "pay by")):
|
|
score += 8
|
|
if any(token in lowered for token in negative):
|
|
score -= 10
|
|
return score
|
|
|
|
|
|
def _extract_due_date(text: str) -> date | None:
|
|
for pattern in _DUE_DATE_PATTERNS:
|
|
match = pattern.search(text)
|
|
if match:
|
|
parsed = _parse_date_value(match.group("date"))
|
|
if parsed:
|
|
return parsed
|
|
return None
|
|
|
|
|
|
def _parse_date_value(value: str) -> date | None:
|
|
if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
|
try:
|
|
return datetime.strptime(value, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
return None
|
|
parts = re.split(r"[./-]", value)
|
|
if len(parts) != 3:
|
|
return None
|
|
day, month, year = parts
|
|
try:
|
|
numeric_year = int(year)
|
|
if numeric_year < 100:
|
|
numeric_year += 2000
|
|
return date(numeric_year, int(month), int(day))
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _extract_references(subject: str, text: str) -> dict[str, str]:
|
|
haystack = f"{subject}\n{text}"
|
|
references: dict[str, str] = {}
|
|
for key, patterns in _REFERENCE_PATTERNS.items():
|
|
for pattern in patterns:
|
|
match = pattern.search(haystack)
|
|
if match:
|
|
references[key] = match.group(1).strip(" .,:;")
|
|
break
|
|
return references
|
|
|
|
|
|
def _provider_transaction_id(references: dict[str, str]) -> str | None:
|
|
for key in ("paypal_transaction_id", "klarna_reference", "order_number", "invoice_number"):
|
|
if references.get(key):
|
|
return references[key]
|
|
return None
|
|
|
|
|
|
def _source_type(sender: str, subject: str, text: str) -> str:
|
|
combined = f"{sender}\n{subject}\n{text[:3000]}".lower()
|
|
if "klarna" in combined:
|
|
return "klarna_email"
|
|
if "paypal" in combined or "pay pal" in combined:
|
|
return "paypal_email"
|
|
if any(token in combined for token in ("bestellung", "order confirmation", "your order", "deine bestellung")):
|
|
return "order_confirmation_email"
|
|
return "imap_email"
|
|
|
|
|
|
def _event_type(source_type: str, subject: str, text: str, due_date: date | None) -> str:
|
|
combined = f"{subject}\n{text[:3000]}".lower()
|
|
if any(token in combined for token in ("refund", "erstattung", "rückerstattung")):
|
|
return "refund"
|
|
if source_type == "klarna_email":
|
|
return "promised_payment"
|
|
if source_type == "paypal_email" and (
|
|
due_date or any(token in combined for token in ("pay later", "später bezahlen", "fällig", "due"))
|
|
):
|
|
return "promised_payment"
|
|
return "expense"
|
|
|
|
|
|
def _event_status(
|
|
source_type: str,
|
|
event_type: str,
|
|
due_date: date | None,
|
|
subject: str,
|
|
text: str,
|
|
) -> str:
|
|
del subject, text
|
|
if source_type == "klarna_email" or event_type == "promised_payment":
|
|
return "outstanding" if due_date else "announced"
|
|
return "announced"
|
|
|
|
|
|
def _merchant(sender: str, subject: str, text: str, source_type: str) -> str | None:
|
|
subject_merchant = _merchant_from_subject(subject)
|
|
if subject_merchant:
|
|
return subject_merchant
|
|
text_merchant = _merchant_from_text(text)
|
|
if text_merchant:
|
|
return text_merchant
|
|
addresses = getaddresses([sender])
|
|
display_name = addresses[0][0] if addresses else ""
|
|
candidate = _clean_party_name(display_name or sender)
|
|
if source_type == "klarna_email" and not candidate:
|
|
return "Klarna"
|
|
if source_type == "paypal_email" and not candidate:
|
|
return "PayPal"
|
|
return candidate or None
|
|
|
|
|
|
def _merchant_from_subject(subject: str) -> str | None:
|
|
patterns = (
|
|
r"\b(?:deine|ihre)\s+bestellung\s+(?:bei|von)\s+(.+)$",
|
|
r"\byour\s+order\s+(?:from|with|at)\s+(.+)$",
|
|
r"\b(?:zahlung|payment)\s+(?:an|to)\s+(.+)$",
|
|
r"\b(?:rechnung|invoice)\s+(?:von|from)\s+(.+)$",
|
|
)
|
|
for pattern in patterns:
|
|
match = re.search(pattern, subject, re.IGNORECASE)
|
|
if match:
|
|
return _clean_party_name(match.group(1))
|
|
return None
|
|
|
|
|
|
def _merchant_from_text(text: str) -> str | None:
|
|
patterns = (
|
|
r"\b(?:händler|merchant|seller|shop)\s*:?\s*([^\n]{2,80})",
|
|
r"\b(?:zahlung\s+an|payment\s+to)\s*:?\s*([^\n]{2,80})",
|
|
)
|
|
for pattern in patterns:
|
|
match = re.search(pattern, text, re.IGNORECASE)
|
|
if match:
|
|
return _clean_party_name(match.group(1))
|
|
return None
|
|
|
|
|
|
def _clean_party_name(value: str) -> str:
|
|
cleaned = re.sub(r"<[^>]+>", "", value)
|
|
cleaned = re.split(r"\s[-|:]\s", cleaned, maxsplit=1)[0]
|
|
cleaned = re.sub(r"\s+", " ", cleaned).strip(" .,:;\"'")
|
|
return cleaned[:120]
|
|
|
|
|
|
def _fallback_source_reference(sender: str, subject: str, received_date: date) -> str:
|
|
return f"imap:{received_date.isoformat()}:{sender}:{subject}"[:240]
|
|
|
|
|
|
def _parse_folder_response(raw: bytes | str) -> dict[str, Any]:
|
|
text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
|
|
match = re.match(r"\((?P<attrs>.*?)\)\s+\"?(?P<delimiter>.*?)\"?\s+(?P<name>.+)$", text)
|
|
if not match:
|
|
return {"name": text, "delimiter": None, "attributes": []}
|
|
name = match.group("name").strip()
|
|
if name.startswith('"') and name.endswith('"'):
|
|
name = name[1:-1].replace('\\"', '"').replace("\\\\", "\\")
|
|
attrs = [item.strip("\\") for item in match.group("attrs").split() if item.strip()]
|
|
delimiter = match.group("delimiter") or None
|
|
return {"name": name, "delimiter": delimiter, "attributes": attrs}
|