first ruggedy draft
This commit is contained in:
676
apps/local-agent/mousehold_agent/imap_importer.py
Normal file
676
apps/local-agent/mousehold_agent/imap_importer.py
Normal file
@@ -0,0 +1,676 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import email
|
||||
import imaplib
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from email import policy
|
||||
from email.header import decode_header
|
||||
from email.message import EmailMessage, Message
|
||||
from email.utils import getaddresses, parsedate_to_datetime
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_SENDER_FILTERS = (
|
||||
"klarna",
|
||||
"paypal",
|
||||
"rechnung",
|
||||
"invoice",
|
||||
"receipt",
|
||||
"order",
|
||||
"bestellung",
|
||||
"quittung",
|
||||
)
|
||||
|
||||
PARSER_NAME = "mousehold-imap-email"
|
||||
PARSER_VERSION = "0.2.0"
|
||||
MAX_RAW_TEXT_LENGTH = 20_000
|
||||
|
||||
_AMOUNT_VALUE = r"[+-]?(?:\d{1,3}(?:[.\s']\d{3})+|\d+)(?:[,.]\d{2})"
|
||||
_AMOUNT_PATTERNS = (
|
||||
re.compile(rf"(?P<currency>€|EUR)\s*(?P<amount>{_AMOUNT_VALUE})", re.IGNORECASE),
|
||||
re.compile(rf"(?P<amount>{_AMOUNT_VALUE})\s*(?P<currency>€|EUR)", re.IGNORECASE),
|
||||
)
|
||||
_DATE_VALUE = r"(?P<date>\d{4}-\d{2}-\d{2}|\d{1,2}[./-]\d{1,2}[./-]\d{2,4})"
|
||||
_DUE_DATE_PATTERNS = (
|
||||
re.compile(rf"\b(?:fällig(?:\s+am|\s+bis)?|zahlbar\s+bis|bezahlen\s+bis)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE),
|
||||
re.compile(rf"\b(?:due(?:\s+on|\s+by)?|pay(?:ment)?\s+due|pay\s+by)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE),
|
||||
re.compile(rf"\b(?:abbuchung\s+am|wird\s+am|debit(?:ed)?\s+on)\s*:?\s*{_DATE_VALUE}", re.IGNORECASE),
|
||||
)
|
||||
_REFERENCE_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
||||
"order_number": (
|
||||
re.compile(
|
||||
r"\b(?:bestell(?:nummer|nr\.?)|order\s+(?:number|no\.?))\s*[:#-]?\s*"
|
||||
r"([A-Z0-9][A-Z0-9._/-]{2,})",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
re.compile(r"\b(?:bestellung|order)\s*[:#-]\s*([A-Z0-9][A-Z0-9._/-]{2,})", re.IGNORECASE),
|
||||
),
|
||||
"invoice_number": (
|
||||
re.compile(
|
||||
r"\b(?:rechnungs(?:nummer|nr\.?)|invoice\s+(?:number|no\.?))\s*[:#-]?\s*"
|
||||
r"([A-Z0-9][A-Z0-9._/-]{2,})",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
re.compile(r"\b(?:rechnung|invoice)\s*[:#-]\s*([A-Z0-9][A-Z0-9._/-]{2,})", re.IGNORECASE),
|
||||
),
|
||||
"paypal_transaction_id": (
|
||||
re.compile(
|
||||
r"\b(?:paypal\s+)?(?:transaction|transaktion)(?:\s+id|\s+nr\.?|\s+number)?\s*[:#-]?\s*"
|
||||
r"([A-Z0-9][A-Z0-9._-]{6,})",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
),
|
||||
"klarna_reference": (
|
||||
re.compile(
|
||||
r"\b(?:klarna\s+)?(?:referenz|reference|zahlung(?:s)?referenz)\s*[:#-]?\s*"
|
||||
r"([A-Z0-9][A-Z0-9._/-]{4,})",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IMAPConfig:
|
||||
host: str
|
||||
username: str
|
||||
password: str
|
||||
port: int = 993
|
||||
folder: str = "INBOX"
|
||||
use_ssl: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AmountMatch:
|
||||
amount: Decimal
|
||||
currency: str
|
||||
context: str
|
||||
score: int
|
||||
|
||||
|
||||
class _HTMLTextExtractor(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.parts: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
del attrs
|
||||
if tag.lower() in {"br", "p", "div", "li", "tr"}:
|
||||
self.parts.append("\n")
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
self.parts.append(data)
|
||||
|
||||
def text(self) -> str:
|
||||
return _clean_text(" ".join(self.parts))
|
||||
|
||||
|
||||
def config_from_env() -> IMAPConfig:
|
||||
return IMAPConfig(
|
||||
host=_required_env("MOUSEHOLD_IMAP_HOST"),
|
||||
port=int(os.environ.get("MOUSEHOLD_IMAP_PORT", "993")),
|
||||
username=_required_env("MOUSEHOLD_IMAP_USERNAME"),
|
||||
password=_required_env("MOUSEHOLD_IMAP_PASSWORD"),
|
||||
folder=os.environ.get("MOUSEHOLD_IMAP_FOLDER", "INBOX"),
|
||||
use_ssl=_env_bool("MOUSEHOLD_IMAP_USE_SSL", default=True),
|
||||
)
|
||||
|
||||
|
||||
def fetch_imap_observations(
|
||||
*,
|
||||
household_id: str,
|
||||
owner_user_id: str | None,
|
||||
source_account_id: str | None,
|
||||
limit: int = 25,
|
||||
) -> list[dict[str, Any]]:
|
||||
sender_filters = tuple(
|
||||
value.strip().lower()
|
||||
for value in os.environ.get(
|
||||
"MOUSEHOLD_IMAP_SENDER_FILTERS", ",".join(DEFAULT_SENDER_FILTERS)
|
||||
).split(",")
|
||||
if value.strip()
|
||||
)
|
||||
result = scan_imap_mailbox(
|
||||
config_from_env(),
|
||||
household_id=household_id,
|
||||
owner_user_id=owner_user_id,
|
||||
source_account_id=source_account_id,
|
||||
limit=limit,
|
||||
sender_filters=sender_filters,
|
||||
)
|
||||
return result["observations"]
|
||||
|
||||
|
||||
def list_imap_folders(config: IMAPConfig) -> list[dict[str, Any]]:
|
||||
with _connect(config) as mailbox:
|
||||
_login(mailbox, config)
|
||||
status, data = mailbox.list()
|
||||
_ensure_ok(status, data, "list folders")
|
||||
return [_parse_folder_response(item) for item in data if item]
|
||||
|
||||
|
||||
def scan_imap_mailbox(
|
||||
config: IMAPConfig,
|
||||
*,
|
||||
household_id: str,
|
||||
owner_user_id: str | None = None,
|
||||
source_account_id: str | None = None,
|
||||
limit: int = 25,
|
||||
folder: str | None = None,
|
||||
search_query: str = "ALL",
|
||||
sender_filters: tuple[str, ...] | list[str] | None = DEFAULT_SENDER_FILTERS,
|
||||
) -> dict[str, Any]:
|
||||
selected_folder = folder or config.folder
|
||||
filters = _normalize_filters(sender_filters)
|
||||
messages: list[dict[str, Any]] = []
|
||||
observations: list[dict[str, Any]] = []
|
||||
scanned_count = 0
|
||||
filtered_count = 0
|
||||
|
||||
with _connect(config) as mailbox:
|
||||
_login(mailbox, config)
|
||||
status, data = mailbox.select(_mailbox_name(selected_folder), readonly=True)
|
||||
_ensure_ok(status, data, f"select folder {selected_folder}")
|
||||
status, data = mailbox.search(None, *_search_criteria(search_query))
|
||||
_ensure_ok(status, data, f"search {search_query}")
|
||||
ids = data[0].split()[-limit:] if data and data[0] else []
|
||||
scanned_count = len(ids)
|
||||
|
||||
for message_id in reversed(ids):
|
||||
status, fetched = mailbox.fetch(message_id, "(RFC822)")
|
||||
if _status_text(status) != "OK" or not fetched or not isinstance(fetched[0], tuple):
|
||||
continue
|
||||
message = email.message_from_bytes(fetched[0][1], policy=policy.default)
|
||||
headers_text = _headers_text(message)
|
||||
body = _message_text(message)
|
||||
if filters and not _matches_filters(filters, headers_text, body):
|
||||
filtered_count += 1
|
||||
continue
|
||||
|
||||
parsed = parse_email_message(
|
||||
message,
|
||||
household_id=household_id,
|
||||
owner_user_id=owner_user_id,
|
||||
source_account_id=source_account_id,
|
||||
folder=selected_folder,
|
||||
body=body,
|
||||
)
|
||||
messages.append(parsed["message"])
|
||||
if parsed["observation"]:
|
||||
observations.append(parsed["observation"])
|
||||
|
||||
return {
|
||||
"folder": selected_folder,
|
||||
"scanned_count": scanned_count,
|
||||
"filtered_count": filtered_count,
|
||||
"matched_count": len(messages),
|
||||
"importable_count": len(observations),
|
||||
"messages": messages,
|
||||
"observations": observations,
|
||||
}
|
||||
|
||||
|
||||
def parse_email_message(
|
||||
message: Message,
|
||||
*,
|
||||
household_id: str,
|
||||
owner_user_id: str | None,
|
||||
source_account_id: str | None,
|
||||
folder: str = "INBOX",
|
||||
body: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
subject = _decode_mime_header(message.get("Subject"))
|
||||
from_header = _decode_mime_header(message.get("From"))
|
||||
to_header = _decode_mime_header(message.get("To"))
|
||||
message_id = _compact_header_value(message.get("Message-ID"))
|
||||
received_date = _message_date(message) or date.today()
|
||||
text = body if body is not None else _message_text(message)
|
||||
search_text = f"{subject}\n{text}"
|
||||
source_type = _source_type(from_header, subject, text)
|
||||
amount_match = _extract_amount_match(search_text)
|
||||
due_date = _extract_due_date(search_text)
|
||||
references = _extract_references(subject, text)
|
||||
provider_transaction_id = _provider_transaction_id(references)
|
||||
merchant = _merchant(from_header, subject, text, source_type)
|
||||
event_type = _event_type(source_type, subject, text, due_date)
|
||||
event_status = _event_status(source_type, event_type, due_date, subject, text)
|
||||
source_reference = (
|
||||
message_id
|
||||
or provider_transaction_id
|
||||
or _fallback_source_reference(from_header, subject, received_date)
|
||||
)
|
||||
if subject:
|
||||
description = subject
|
||||
elif merchant:
|
||||
description = f"Email from {merchant}"
|
||||
else:
|
||||
description = "Email import"
|
||||
|
||||
preview = {
|
||||
"message_id": message_id,
|
||||
"source_reference": source_reference,
|
||||
"from": from_header,
|
||||
"to": to_header,
|
||||
"subject": subject,
|
||||
"date": received_date.isoformat(),
|
||||
"folder": folder,
|
||||
"source_type": source_type,
|
||||
"event_type": event_type,
|
||||
"status": event_status,
|
||||
"amount": str(amount_match.amount) if amount_match else None,
|
||||
"currency": amount_match.currency if amount_match else None,
|
||||
"merchant": merchant,
|
||||
"due_date": due_date.isoformat() if due_date else None,
|
||||
"references": references,
|
||||
"importable": amount_match is not None,
|
||||
"skip_reason": None if amount_match else "No EUR amount found",
|
||||
}
|
||||
|
||||
if not amount_match:
|
||||
return {"message": preview, "observation": None}
|
||||
|
||||
observation = {
|
||||
"household_id": household_id,
|
||||
"source_type": source_type,
|
||||
"source_account_id": source_account_id,
|
||||
"owner_user_id": owner_user_id,
|
||||
"message_id": message_id,
|
||||
"source_reference": source_reference,
|
||||
"provider_transaction_id": provider_transaction_id,
|
||||
"raw_text": text[:MAX_RAW_TEXT_LENGTH],
|
||||
"raw_payload_json": {
|
||||
"from": from_header,
|
||||
"to": to_header,
|
||||
"subject": subject,
|
||||
"date": _compact_header_value(message.get("Date")),
|
||||
"folder": folder,
|
||||
"message_id": message_id,
|
||||
"references": references,
|
||||
"amount_context": amount_match.context,
|
||||
"parser_name": PARSER_NAME,
|
||||
"parser_version": PARSER_VERSION,
|
||||
},
|
||||
"amount": str(amount_match.amount),
|
||||
"currency": amount_match.currency,
|
||||
"event_date": received_date.isoformat(),
|
||||
"due_date": due_date.isoformat() if due_date else None,
|
||||
"merchant": merchant,
|
||||
"description": description[:240],
|
||||
"event_type": event_type,
|
||||
"status": event_status,
|
||||
"payer_user_id": owner_user_id,
|
||||
"payment_account_id": source_account_id,
|
||||
}
|
||||
return {"message": preview, "observation": observation}
|
||||
|
||||
|
||||
def _connect(config: IMAPConfig) -> imaplib.IMAP4:
|
||||
if config.use_ssl:
|
||||
return imaplib.IMAP4_SSL(config.host, config.port)
|
||||
return imaplib.IMAP4(config.host, config.port)
|
||||
|
||||
|
||||
def _login(mailbox: imaplib.IMAP4, config: IMAPConfig) -> None:
|
||||
status, data = mailbox.login(config.username, config.password)
|
||||
_ensure_ok(status, data, "login")
|
||||
|
||||
|
||||
def _required_env(name: str) -> str:
|
||||
value = os.environ.get(name)
|
||||
if not value:
|
||||
raise RuntimeError(f"{name} is required")
|
||||
return value
|
||||
|
||||
|
||||
def _env_bool(name: str, *, default: bool) -> bool:
|
||||
value = os.environ.get(name)
|
||||
if value is None:
|
||||
return default
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _status_text(status: str | bytes) -> str:
|
||||
return status.decode("ascii", errors="replace").upper() if isinstance(status, bytes) else status.upper()
|
||||
|
||||
|
||||
def _ensure_ok(status: str | bytes, data: object, action: str) -> None:
|
||||
if _status_text(status) != "OK":
|
||||
raise RuntimeError(f"IMAP {action} failed: {data!r}")
|
||||
|
||||
|
||||
def _mailbox_name(folder: str) -> str:
|
||||
if folder.startswith('"') and folder.endswith('"'):
|
||||
return folder
|
||||
if any(char in folder for char in ' "()[\\]'):
|
||||
return '"' + folder.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
return folder
|
||||
|
||||
|
||||
def _search_criteria(search_query: str) -> list[str]:
|
||||
query = search_query.strip() or "ALL"
|
||||
try:
|
||||
parts = shlex.split(query)
|
||||
except ValueError:
|
||||
parts = query.split()
|
||||
return parts or ["ALL"]
|
||||
|
||||
|
||||
def _normalize_filters(sender_filters: tuple[str, ...] | list[str] | None) -> tuple[str, ...]:
|
||||
if sender_filters is None:
|
||||
return ()
|
||||
return tuple(token.strip().lower() for token in sender_filters if token.strip())
|
||||
|
||||
|
||||
def _matches_filters(filters: tuple[str, ...], headers_text: str, body: str) -> bool:
|
||||
haystack = f"{headers_text}\n{body[:5000]}".lower()
|
||||
return any(token in haystack for token in filters)
|
||||
|
||||
|
||||
def _headers_text(message: Message) -> str:
|
||||
return "\n".join(
|
||||
[
|
||||
_decode_mime_header(message.get("From")),
|
||||
_decode_mime_header(message.get("To")),
|
||||
_decode_mime_header(message.get("Subject")),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _decode_mime_header(value: str | None) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
parts: list[str] = []
|
||||
for content, charset in decode_header(value):
|
||||
if isinstance(content, bytes):
|
||||
parts.append(content.decode(charset or "utf-8", errors="replace"))
|
||||
else:
|
||||
parts.append(content)
|
||||
return _clean_text("".join(parts))
|
||||
|
||||
|
||||
def _compact_header_value(value: str | None) -> str | None:
|
||||
cleaned = _decode_mime_header(value)
|
||||
return cleaned or None
|
||||
|
||||
|
||||
def _message_text(message: Message) -> str:
|
||||
if isinstance(message, EmailMessage):
|
||||
body = message.get_body(preferencelist=("plain", "html"))
|
||||
if body:
|
||||
content = body.get_content()
|
||||
if body.get_content_type() == "text/html":
|
||||
return _html_to_text(content)
|
||||
return _clean_text(content)
|
||||
|
||||
if message.is_multipart():
|
||||
plain_parts: list[str] = []
|
||||
html_parts: list[str] = []
|
||||
for part in message.walk():
|
||||
if part.get_content_maintype() != "text":
|
||||
continue
|
||||
payload = part.get_payload(decode=True)
|
||||
if not payload:
|
||||
continue
|
||||
text = payload.decode(part.get_content_charset() or "utf-8", errors="replace")
|
||||
if part.get_content_type() == "text/plain":
|
||||
plain_parts.append(text)
|
||||
elif part.get_content_type() == "text/html":
|
||||
html_parts.append(_html_to_text(text))
|
||||
return _clean_text("\n".join(plain_parts or html_parts))
|
||||
|
||||
payload = message.get_payload(decode=True)
|
||||
if payload:
|
||||
text = payload.decode(message.get_content_charset() or "utf-8", errors="replace")
|
||||
else:
|
||||
text = str(message.get_payload())
|
||||
if message.get_content_type() == "text/html":
|
||||
return _html_to_text(text)
|
||||
return _clean_text(text)
|
||||
|
||||
|
||||
def _html_to_text(html: str) -> str:
|
||||
extractor = _HTMLTextExtractor()
|
||||
extractor.feed(html)
|
||||
return extractor.text()
|
||||
|
||||
|
||||
def _clean_text(text: str) -> str:
|
||||
text = text.replace("\xa0", " ")
|
||||
text = re.sub(r"[ \t\r\f\v]+", " ", text)
|
||||
text = re.sub(r"\n\s+", "\n", text)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _message_date(message: Message) -> date | None:
|
||||
header = message.get("Date")
|
||||
if not header:
|
||||
return None
|
||||
try:
|
||||
parsed = parsedate_to_datetime(header)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed.date()
|
||||
|
||||
|
||||
def _extract_amount_match(text: str) -> AmountMatch | None:
|
||||
candidates: list[AmountMatch] = []
|
||||
for pattern in _AMOUNT_PATTERNS:
|
||||
for match in pattern.finditer(text):
|
||||
amount = _parse_amount(match.group("amount"))
|
||||
if amount is None:
|
||||
continue
|
||||
start = max(match.start() - 90, 0)
|
||||
end = min(match.end() + 90, len(text))
|
||||
context = _clean_text(text[start:end])
|
||||
candidates.append(
|
||||
AmountMatch(
|
||||
amount=amount,
|
||||
currency="EUR",
|
||||
context=context,
|
||||
score=_amount_context_score(context),
|
||||
)
|
||||
)
|
||||
if not candidates:
|
||||
return None
|
||||
return sorted(candidates, key=lambda item: (item.score, item.amount), reverse=True)[0]
|
||||
|
||||
|
||||
def _parse_amount(value: str) -> Decimal | None:
|
||||
cleaned = value.replace("\xa0", "").replace(" ", "").replace("'", "")
|
||||
if "," in cleaned and "." in cleaned:
|
||||
if cleaned.rfind(",") > cleaned.rfind("."):
|
||||
cleaned = cleaned.replace(".", "").replace(",", ".")
|
||||
else:
|
||||
cleaned = cleaned.replace(",", "")
|
||||
elif "," in cleaned:
|
||||
cleaned = cleaned.replace(".", "").replace(",", ".")
|
||||
elif "." in cleaned and len(cleaned.rsplit(".", maxsplit=1)[-1]) != 2:
|
||||
cleaned = cleaned.replace(".", "")
|
||||
try:
|
||||
return abs(Decimal(cleaned)).quantize(Decimal("0.01"))
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
|
||||
def _amount_context_score(context: str) -> int:
|
||||
lowered = context.lower()
|
||||
score = 0
|
||||
positive = (
|
||||
"gesamt",
|
||||
"gesamtbetrag",
|
||||
"summe",
|
||||
"total",
|
||||
"amount",
|
||||
"betrag",
|
||||
"zahlbetrag",
|
||||
"payment",
|
||||
"zahlung",
|
||||
"fällig",
|
||||
"due",
|
||||
)
|
||||
negative = ("mwst", "vat", "tax", "steuer", "versand", "shipping", "rabatt", "discount", "gutschein")
|
||||
if any(token in lowered for token in positive):
|
||||
score += 20
|
||||
if any(token in lowered for token in ("fällig", "due", "zahlbar", "pay by")):
|
||||
score += 8
|
||||
if any(token in lowered for token in negative):
|
||||
score -= 10
|
||||
return score
|
||||
|
||||
|
||||
def _extract_due_date(text: str) -> date | None:
|
||||
for pattern in _DUE_DATE_PATTERNS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
parsed = _parse_date_value(match.group("date"))
|
||||
if parsed:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
|
||||
def _parse_date_value(value: str) -> date | None:
|
||||
if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
||||
try:
|
||||
return datetime.strptime(value, "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
return None
|
||||
parts = re.split(r"[./-]", value)
|
||||
if len(parts) != 3:
|
||||
return None
|
||||
day, month, year = parts
|
||||
try:
|
||||
numeric_year = int(year)
|
||||
if numeric_year < 100:
|
||||
numeric_year += 2000
|
||||
return date(numeric_year, int(month), int(day))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _extract_references(subject: str, text: str) -> dict[str, str]:
|
||||
haystack = f"{subject}\n{text}"
|
||||
references: dict[str, str] = {}
|
||||
for key, patterns in _REFERENCE_PATTERNS.items():
|
||||
for pattern in patterns:
|
||||
match = pattern.search(haystack)
|
||||
if match:
|
||||
references[key] = match.group(1).strip(" .,:;")
|
||||
break
|
||||
return references
|
||||
|
||||
|
||||
def _provider_transaction_id(references: dict[str, str]) -> str | None:
|
||||
for key in ("paypal_transaction_id", "klarna_reference", "order_number", "invoice_number"):
|
||||
if references.get(key):
|
||||
return references[key]
|
||||
return None
|
||||
|
||||
|
||||
def _source_type(sender: str, subject: str, text: str) -> str:
|
||||
combined = f"{sender}\n{subject}\n{text[:3000]}".lower()
|
||||
if "klarna" in combined:
|
||||
return "klarna_email"
|
||||
if "paypal" in combined or "pay pal" in combined:
|
||||
return "paypal_email"
|
||||
if any(token in combined for token in ("bestellung", "order confirmation", "your order", "deine bestellung")):
|
||||
return "order_confirmation_email"
|
||||
return "imap_email"
|
||||
|
||||
|
||||
def _event_type(source_type: str, subject: str, text: str, due_date: date | None) -> str:
|
||||
combined = f"{subject}\n{text[:3000]}".lower()
|
||||
if any(token in combined for token in ("refund", "erstattung", "rückerstattung")):
|
||||
return "refund"
|
||||
if source_type == "klarna_email":
|
||||
return "promised_payment"
|
||||
if source_type == "paypal_email" and (
|
||||
due_date or any(token in combined for token in ("pay later", "später bezahlen", "fällig", "due"))
|
||||
):
|
||||
return "promised_payment"
|
||||
return "expense"
|
||||
|
||||
|
||||
def _event_status(
|
||||
source_type: str,
|
||||
event_type: str,
|
||||
due_date: date | None,
|
||||
subject: str,
|
||||
text: str,
|
||||
) -> str:
|
||||
del subject, text
|
||||
if source_type == "klarna_email" or event_type == "promised_payment":
|
||||
return "outstanding" if due_date else "announced"
|
||||
return "announced"
|
||||
|
||||
|
||||
def _merchant(sender: str, subject: str, text: str, source_type: str) -> str | None:
|
||||
subject_merchant = _merchant_from_subject(subject)
|
||||
if subject_merchant:
|
||||
return subject_merchant
|
||||
text_merchant = _merchant_from_text(text)
|
||||
if text_merchant:
|
||||
return text_merchant
|
||||
addresses = getaddresses([sender])
|
||||
display_name = addresses[0][0] if addresses else ""
|
||||
candidate = _clean_party_name(display_name or sender)
|
||||
if source_type == "klarna_email" and not candidate:
|
||||
return "Klarna"
|
||||
if source_type == "paypal_email" and not candidate:
|
||||
return "PayPal"
|
||||
return candidate or None
|
||||
|
||||
|
||||
def _merchant_from_subject(subject: str) -> str | None:
|
||||
patterns = (
|
||||
r"\b(?:deine|ihre)\s+bestellung\s+(?:bei|von)\s+(.+)$",
|
||||
r"\byour\s+order\s+(?:from|with|at)\s+(.+)$",
|
||||
r"\b(?:zahlung|payment)\s+(?:an|to)\s+(.+)$",
|
||||
r"\b(?:rechnung|invoice)\s+(?:von|from)\s+(.+)$",
|
||||
)
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, subject, re.IGNORECASE)
|
||||
if match:
|
||||
return _clean_party_name(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def _merchant_from_text(text: str) -> str | None:
|
||||
patterns = (
|
||||
r"\b(?:händler|merchant|seller|shop)\s*:?\s*([^\n]{2,80})",
|
||||
r"\b(?:zahlung\s+an|payment\s+to)\s*:?\s*([^\n]{2,80})",
|
||||
)
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, text, re.IGNORECASE)
|
||||
if match:
|
||||
return _clean_party_name(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def _clean_party_name(value: str) -> str:
|
||||
cleaned = re.sub(r"<[^>]+>", "", value)
|
||||
cleaned = re.split(r"\s[-|:]\s", cleaned, maxsplit=1)[0]
|
||||
cleaned = re.sub(r"\s+", " ", cleaned).strip(" .,:;\"'")
|
||||
return cleaned[:120]
|
||||
|
||||
|
||||
def _fallback_source_reference(sender: str, subject: str, received_date: date) -> str:
|
||||
return f"imap:{received_date.isoformat()}:{sender}:{subject}"[:240]
|
||||
|
||||
|
||||
def _parse_folder_response(raw: bytes | str) -> dict[str, Any]:
|
||||
text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
|
||||
match = re.match(r"\((?P<attrs>.*?)\)\s+\"?(?P<delimiter>.*?)\"?\s+(?P<name>.+)$", text)
|
||||
if not match:
|
||||
return {"name": text, "delimiter": None, "attributes": []}
|
||||
name = match.group("name").strip()
|
||||
if name.startswith('"') and name.endswith('"'):
|
||||
name = name[1:-1].replace('\\"', '"').replace("\\\\", "\\")
|
||||
attrs = [item.strip("\\") for item in match.group("attrs").split() if item.strip()]
|
||||
delimiter = match.group("delimiter") or None
|
||||
return {"name": name, "delimiter": delimiter, "attributes": attrs}
|
||||
Reference in New Issue
Block a user