from __future__ import annotations import base64 import binascii from collections import Counter, defaultdict import hashlib import json import os from typing import Any from sqlalchemy.orm import Session, selectinload from govoplan_addresses.backend.db.models import ( AddressImportRun, AddressListEntry, Contact, ) from govoplan_addresses.backend.schemas import ( ContactCreateRequest, ContactUpdateRequest, ) from govoplan_addresses.backend.service import ( AddressBookError, create_contact, get_visible_address_book, get_visible_address_list, get_visible_contact, update_contact, ) from govoplan_addresses.backend.vcard import ( MAX_VCARD_CARDS, VCARD_PARSER_VERSION, contacts_to_vcard, parse_vcards_with_issues, ) from govoplan_addresses.backend.vcard_batch_schemas import ( VCardBatchCancelRequest, VCardBatchCommitRequest, VCardBatchPreviewRequest, VCardExportRequest, ) from govoplan_core.auth import ApiPrincipal from govoplan_core.db.base import utcnow MAX_VCARD_BATCH_BYTES = 10_000_000 DEFAULT_PERSISTED_BATCH_THRESHOLD = 500 def preview_vcard_batch( session: Session, principal: ApiPrincipal, address_book_id: str, payload: VCardBatchPreviewRequest, ) -> AddressImportRun: book = get_visible_address_book(session, principal, address_book_id) if book.read_only: raise AddressBookError("Static vCard imports require a writable address book.") decoded = _decode_files(payload) input_hash = _hash_json( { "files": [ { "filename": filename, "sha256": hashlib.sha256(raw).hexdigest(), "size": len(raw), } for filename, raw in decoded ] } ) parsed_cards, diagnostics = _parse_files(decoded) plan = _plan_cards( session, book.id, parsed_cards, duplicate_card_policy=payload.duplicate_card_policy, existing_contact_policy=payload.existing_contact_policy, ) statistics: dict[str, int | str] = dict( Counter(str(item["action"]) for item in plan) ) statistics.update( { "files": len(decoded), "cards": len(parsed_cards), "errors": sum(item["severity"] == "error" for item in diagnostics), "warnings": sum(item["severity"] == "warning" for item in diagnostics), "parser_version": VCARD_PARSER_VERSION, "execution_mode": _execution_mode(len(parsed_cards)), } ) plan_hash = _hash_json( { "address_book_id": book.id, "input_hash": input_hash, "parser_version": VCARD_PARSER_VERSION, "duplicate_card_policy": payload.duplicate_card_policy, "existing_contact_policy": payload.existing_contact_policy, "plan": plan, } ) source_filename = decoded[0][0] if len(decoded) > 1: source_filename = f"{source_filename} (+{len(decoded) - 1} files)" run = AddressImportRun( tenant_id=book.tenant_id, address_book_id=book.id, profile_id=None, source_filename=source_filename[:500], source_format="vcard", input_hash=input_hash, plan_hash=plan_hash, status="previewed", row_count=len(parsed_cards), statistics=statistics, diagnostics=diagnostics, plan_data=plan, result_evidence={ "parser_version": VCARD_PARSER_VERSION, "file_manifest": [ { "filename": filename, "sha256": hashlib.sha256(raw).hexdigest(), "size": len(raw), } for filename, raw in decoded ], "progress": _progress(len(plan)), }, created_by_account_id=principal.account_id, ) session.add(run) session.flush() return run def get_vcard_batch_run( session: Session, principal: ApiPrincipal, run_id: str, ) -> AddressImportRun: book_ids = [book.id for book in _visible_books(session, principal)] if not book_ids: raise AddressBookError("vCard batch run not found.") item = ( session.query(AddressImportRun) .filter( AddressImportRun.id == run_id, AddressImportRun.address_book_id.in_(book_ids), AddressImportRun.source_format == "vcard", ) .one_or_none() ) if item is None: raise AddressBookError("vCard batch run not found.") return item def apply_vcard_batch( session: Session, principal: ApiPrincipal, run_id: str, payload: VCardBatchCommitRequest, ) -> AddressImportRun: run = get_vcard_batch_run(session, principal, run_id) if run.plan_hash != payload.expected_plan_hash: raise AddressBookError("The reviewed vCard plan changed; create a new preview.") selections = _selection_map(payload) commit_hash = _hash_json( { "plan_hash": run.plan_hash, "selections": [ {"source_key": key, "action": selections[key]} for key in sorted(selections) ], } ) evidence = dict(run.result_evidence or {}) if run.status == "applied": if evidence.get("commit_hash") != commit_hash: raise AddressBookError( "This vCard batch was already applied with a different selection." ) return run if run.status != "previewed": raise AddressBookError( f"vCard batch cannot be applied from status {run.status!r}." ) if not selections: raise AddressBookError( "Select at least one vCard action before applying the batch." ) plan_by_key = {str(item["source_key"]): item for item in run.plan_data or []} unknown = sorted(set(selections).difference(plan_by_key)) if unknown: raise AddressBookError( "The selection contains cards that are not part of the reviewed plan." ) created_ids: list[str] = [] updated_ids: list[str] = [] ignored = 0 for source_key in sorted(plan_by_key): item = plan_by_key[source_key] action = selections.get(source_key, "ignore") allowed = set(item.get("allowed_actions") or []) if action not in allowed: raise AddressBookError( f'Action {action!r} is not allowed for vCard "{item.get("display_name") or source_key}".' ) if action == "ignore": ignored += 1 continue contact = _apply_plan_item( session, principal, run=run, item=item, action=action, ) if action == "create": created_ids.append(contact.id) else: updated_ids.append(contact.id) run.status = "applied" run.applied_at = utcnow() run.result_evidence = { **evidence, "commit_hash": commit_hash, "selection_count": len(selections), "created_contact_ids": created_ids, "updated_contact_ids": updated_ids, "ignored_count": ignored, "applied_by_account_id": principal.account_id, "applied_at": run.applied_at.isoformat(), "progress": { "total": len(plan_by_key), "completed": len(plan_by_key), "created": len(created_ids), "updated": len(updated_ids), "ignored": ignored, "failed": 0, }, } run.statistics = { **dict(run.statistics or {}), "applied_create": len(created_ids), "applied_update": len(updated_ids), "applied_ignore": ignored, } return run def cancel_vcard_batch( session: Session, principal: ApiPrincipal, run_id: str, payload: VCardBatchCancelRequest, ) -> AddressImportRun: run = get_vcard_batch_run(session, principal, run_id) if run.plan_hash != payload.expected_plan_hash: raise AddressBookError("The reviewed vCard plan changed; reload the batch.") if run.status == "cancelled": return run if run.status != "previewed": raise AddressBookError("Only a previewed vCard batch can be cancelled.") run.status = "cancelled" run.result_evidence = { **dict(run.result_evidence or {}), "cancel_reason": payload.reason.strip(), "cancelled_by_account_id": principal.account_id, "cancelled_at": utcnow().isoformat(), } return run def vcard_batch_payload(run: AddressImportRun) -> dict[str, Any]: evidence = dict(run.result_evidence or {}) progress = dict(evidence.get("progress") or _progress(run.row_count)) return { "id": run.id, "address_book_id": run.address_book_id, "status": run.status, "input_hash": run.input_hash, "plan_hash": run.plan_hash, "parser_version": str(evidence.get("parser_version") or VCARD_PARSER_VERSION), "execution_mode": str( (run.statistics or {}).get("execution_mode") or "bounded_sync" ), "file_count": int((run.statistics or {}).get("files") or 0), "card_count": run.row_count, "statistics": dict(run.statistics or {}), "diagnostics": list(run.diagnostics or []), "plan": [_public_plan_item(item) for item in run.plan_data or []], "progress": progress, "can_apply": run.status == "previewed" and bool(run.plan_data), "can_cancel": run.status == "previewed", "commit_hash": evidence.get("commit_hash"), "created_at": run.created_at, "updated_at": run.updated_at, "applied_at": run.applied_at, } def vcard_diagnostics_payload(run: AddressImportRun) -> dict[str, Any]: return { "schema_version": "1.0", "run_id": run.id, "status": run.status, "input_hash": run.input_hash, "plan_hash": run.plan_hash, "parser_version": (run.result_evidence or {}).get("parser_version"), "statistics": dict(run.statistics or {}), "diagnostics": list(run.diagnostics or []), "effects": [_public_plan_item(item) for item in run.plan_data or []], } def export_vcards( session: Session, principal: ApiPrincipal, address_book_id: str, payload: VCardExportRequest, ) -> dict[str, Any]: book = get_visible_address_book(session, principal, address_book_id) contacts = _export_contacts(session, principal, book.id, payload) contacts.sort(key=lambda item: (item.display_name.casefold(), item.id)) content = contacts_to_vcard(contacts, version=payload.version) scope_label = { "address_book": book.name, "address_list": "address-list", "contacts": "selected-contacts", }[payload.scope] return { "filename": f"{_safe_filename(scope_label)}-{payload.version.replace('.', '')}.vcf", "media_type": "text/vcard", "scope": payload.scope, "version": payload.version, "ordering": "display_name_casefold_then_contact_id", "contact_count": len(contacts), "content_hash": hashlib.sha256(content.encode("utf-8")).hexdigest(), "content": content, } def _decode_files(payload: VCardBatchPreviewRequest) -> list[tuple[str, bytes]]: decoded: list[tuple[str, bytes]] = [] total = 0 for item in payload.files: filename = item.filename.strip() if not filename.casefold().endswith(".vcf"): raise AddressBookError("vCard batch uploads accept only .vcf files.") try: raw = base64.b64decode(item.content_base64, validate=True) except (binascii.Error, ValueError) as exc: raise AddressBookError( f'vCard file "{filename}" is not valid base64.' ) from exc if not raw: raise AddressBookError(f'vCard file "{filename}" is empty.') total += len(raw) if total > MAX_VCARD_BATCH_BYTES: raise AddressBookError( f"Combined vCard uploads are limited to {MAX_VCARD_BATCH_BYTES} bytes." ) decoded.append((filename, raw)) return decoded def _parse_files( decoded: list[tuple[str, bytes]], ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: cards: list[dict[str, Any]] = [] diagnostics: list[dict[str, Any]] = [] for file_index, (filename, raw) in enumerate(decoded): try: content = raw.decode("utf-8-sig") except UnicodeDecodeError as exc: raise AddressBookError( f'vCard file "{filename}" is not valid UTF-8: {exc}.' ) from exc remaining = MAX_VCARD_CARDS - len(cards) if remaining < 1: raise AddressBookError( f"vCard batches are limited to {MAX_VCARD_CARDS} cards." ) result = parse_vcards_with_issues(content, max_cards=remaining) for issue in result.issues: diagnostics.append( { "severity": issue.severity, "code": "vcard_parse_error" if issue.severity == "error" else "vcard_parse_warning", "message": issue.message, "source_filename": filename, "card_index": issue.index or None, "field": issue.field, "details": {"line": issue.line} if issue.line is not None else {}, } ) for card_index, parsed in enumerate(result.cards, start=1): raw_hash = hashlib.sha256(parsed.raw.encode("utf-8")).hexdigest() identity = ( f"uid:{parsed.source_ref.strip()}" if parsed.source_ref and parsed.source_ref.strip() else f"sha256:{raw_hash}" ) source_key = hashlib.sha256( f"{file_index}:{filename}:{card_index}:{raw_hash}".encode("utf-8") ).hexdigest() cards.append( { "source_key": source_key, "source_identity": identity, "source_filename": filename, "card_index": card_index, "raw": parsed.raw, "source_ref": parsed.source_ref.strip() if parsed.source_ref else None, "source_revision": parsed.source_revision.strip() if parsed.source_revision else None, "payload": parsed.payload.model_dump(mode="json"), } ) return cards, diagnostics def _plan_cards( session: Session, address_book_id: str, cards: list[dict[str, Any]], *, duplicate_card_policy: str, existing_contact_policy: str, ) -> list[dict[str, Any]]: contacts = ( session.query(Contact) .options( selectinload(Contact.emails), selectinload(Contact.phones), selectinload(Contact.postal_addresses), ) .filter( Contact.address_book_id == address_book_id, Contact.deleted_at.is_(None) ) .all() ) by_source: dict[str, list[Contact]] = defaultdict(list) by_email: dict[str, list[Contact]] = defaultdict(list) for contact in contacts: if contact.source_ref: by_source[contact.source_ref.strip()].append(contact) for email in contact.emails: normalized = (email.normalized_email or email.email).strip().casefold() if normalized: by_email[normalized].append(contact) identity_positions: dict[str, list[int]] = defaultdict(list) for index, card in enumerate(cards): identity_positions[str(card["source_identity"])].append(index) result: list[dict[str, Any]] = [] for index, card in enumerate(cards): positions = identity_positions[str(card["source_identity"])] if len(positions) > 1: chosen = positions[0] if duplicate_card_policy == "first" else positions[-1] if duplicate_card_policy == "reject": result.append( _planned_card( card, action="conflict", allowed=["ignore"], message="Duplicate UID or identical card appears in this batch.", ) ) continue if index != chosen: result.append( _planned_card( card, action="ignore", allowed=["ignore"], message=f"Duplicate card ignored by {duplicate_card_policy} policy.", ) ) continue suggestions = _duplicate_candidates( card, by_source=by_source, by_email=by_email ) exact_source = [item for item in suggestions if "source_uid" in item["reasons"]] candidates = exact_source or suggestions if len(candidates) > 1: result.append( _planned_card( card, action="conflict", allowed=["create", "ignore"], suggestions=suggestions, message="Multiple existing contacts match this card; create explicitly or ignore it.", ) ) continue existing = next( ( contact for contact in contacts if candidates and contact.id == candidates[0]["contact_id"] ), None, ) if existing is None: result.append( _planned_card( card, action="create", allowed=["create", "ignore"], suggestions=suggestions, ) ) continue changed = _changed_fields(existing, card["payload"]) if not changed: result.append( _planned_card( card, action="unchanged", allowed=["ignore"], contact=existing, suggestions=suggestions, message="Existing contact already matches the parsed card.", ) ) elif existing_contact_policy == "reject": result.append( _planned_card( card, action="conflict", allowed=["ignore"], contact=existing, suggestions=suggestions, changed=changed, message="An existing contact matches and the preview policy rejects updates.", ) ) elif existing_contact_policy == "ignore": result.append( _planned_card( card, action="ignore", allowed=["update", "ignore"], contact=existing, suggestions=suggestions, changed=changed, message="Existing contact is ignored by preview policy.", ) ) else: result.append( _planned_card( card, action="update", allowed=["update", "ignore"], contact=existing, suggestions=suggestions, changed=changed, ) ) return result def _planned_card( card: dict[str, Any], *, action: str, allowed: list[str], contact: Contact | None = None, suggestions: list[dict[str, Any]] | None = None, changed: list[str] | None = None, message: str | None = None, ) -> dict[str, Any]: return { **card, "row_number": int(card["card_index"]), "action": action, "allowed_actions": allowed, "contact_id": contact.id if contact is not None else None, "expected_contact_hash": _contact_hash(contact) if contact is not None else None, "display_name": card["payload"].get("display_name"), "changed_fields": changed or [], "duplicate_suggestions": suggestions or [], "message": message, } def _duplicate_candidates( card: dict[str, Any], *, by_source: dict[str, list[Contact]], by_email: dict[str, list[Contact]], ) -> list[dict[str, Any]]: reasons: dict[str, set[str]] = defaultdict(set) contacts: dict[str, Contact] = {} source_ref = card.get("source_ref") if source_ref: for contact in by_source.get(str(source_ref), []): contacts[contact.id] = contact reasons[contact.id].add("source_uid") for item in card["payload"].get("emails") or []: normalized = str(item.get("email") or "").strip().casefold() for contact in by_email.get(normalized, []): contacts[contact.id] = contact reasons[contact.id].add("email") return [ { "contact_id": contact_id, "display_name": contacts[contact_id].display_name, "reasons": sorted(reasons[contact_id]), } for contact_id in sorted( contacts, key=lambda item: (contacts[item].display_name.casefold(), item) )[:5] ] def _apply_plan_item( session: Session, principal: ApiPrincipal, *, run: AddressImportRun, item: dict[str, Any], action: str, ) -> Contact: contact_payload = ContactCreateRequest.model_validate(item["payload"]) if action == "create": if item.get("source_ref"): appeared = ( session.query(Contact) .filter( Contact.address_book_id == run.address_book_id, Contact.source_ref == item["source_ref"], Contact.deleted_at.is_(None), ) .first() ) if appeared is not None: raise AddressBookError( "A matching vCard UID appeared after preview; preview the batch again." ) contact = create_contact( session, principal, run.address_book_id, contact_payload ) else: contact_id = str(item.get("contact_id") or "") if not contact_id: raise AddressBookError( "The reviewed vCard update has no stable target contact." ) current = get_visible_contact(session, principal, contact_id) if _contact_hash(current) != item.get("expected_contact_hash"): raise AddressBookError( f'Contact "{current.display_name}" changed after preview; preview the batch again.' ) contact = update_contact( session, principal, current.id, ContactUpdateRequest.model_validate(item["payload"]), ) contact.source_kind = "vcard" contact.source_ref = ( item.get("source_ref") or f"vcard-sha256:{str(item['source_identity']).split(':', 1)[-1]}" ) contact.source_payload_kind = "vcard" contact.source_payload_raw = item["raw"] contact.source_revision = item.get("source_revision") provenance = dict(contact.provenance or {}) provenance["vcard_batch"] = { "run_id": run.id, "input_hash": run.input_hash, "plan_hash": run.plan_hash, "parser_version": VCARD_PARSER_VERSION, "source_filename": item["source_filename"], "card_index": item["card_index"], } contact.provenance = provenance session.flush() return contact def _changed_fields(contact: Contact, payload: dict[str, Any]) -> list[str]: current = _contact_projection(contact) incoming = _payload_projection(payload) return sorted(key for key in incoming if current.get(key) != incoming.get(key)) def _contact_hash(contact: Contact) -> str: return _hash_json(_contact_projection(contact)) def _contact_projection(contact: Contact) -> dict[str, Any]: return { "display_name": contact.display_name, "given_name": contact.given_name, "family_name": contact.family_name, "organization": contact.organization, "role_title": contact.role_title, "note": contact.note, "tags": list(contact.tags or []), "emails": [ {"label": item.label, "email": item.email, "is_primary": item.is_primary} for item in contact.emails ], "phones": [ {"label": item.label, "phone": item.phone, "is_primary": item.is_primary} for item in contact.phones ], "postal_addresses": [ { "label": item.label, "street": item.street, "postal_code": item.postal_code, "locality": item.locality, "region": item.region, "country": item.country, "is_primary": item.is_primary, } for item in contact.postal_addresses ], } def _payload_projection(payload: dict[str, Any]) -> dict[str, Any]: return { key: payload.get(key) for key in ( "display_name", "given_name", "family_name", "organization", "role_title", "note", "tags", "emails", "phones", "postal_addresses", ) } def _selection_map(payload: VCardBatchCommitRequest) -> dict[str, str]: result: dict[str, str] = {} for selection in payload.selections: if selection.source_key in result: raise AddressBookError("Each vCard may be selected only once.") result[selection.source_key] = selection.action return result def _public_plan_item(item: dict[str, Any]) -> dict[str, Any]: return { key: item.get(key) for key in ( "source_key", "source_filename", "card_index", "action", "allowed_actions", "contact_id", "display_name", "changed_fields", "duplicate_suggestions", "message", ) } def _export_contacts( session: Session, principal: ApiPrincipal, address_book_id: str, payload: VCardExportRequest, ) -> list[Contact]: if payload.scope == "address_book": return _loaded_contacts(session, address_book_id=address_book_id) if payload.scope == "contacts": contacts = [ get_visible_contact(session, principal, contact_id) for contact_id in payload.contact_ids ] if any(contact.address_book_id != address_book_id for contact in contacts): raise AddressBookError( "Every selected contact must belong to the exported address book." ) return contacts address_list = get_visible_address_list( session, principal, str(payload.address_list_id) ) if address_list.address_book_id != address_book_id: raise AddressBookError( "The selected address list does not belong to the exported address book." ) contact_ids = [ item.contact_id for item in ( session.query(AddressListEntry) .filter(AddressListEntry.address_list_id == address_list.id) .order_by(AddressListEntry.order_index.asc(), AddressListEntry.id.asc()) .all() ) ] if not contact_ids: return [] return _loaded_contacts( session, address_book_id=address_book_id, contact_ids=set(contact_ids) ) def _loaded_contacts( session: Session, *, address_book_id: str, contact_ids: set[str] | None = None, ) -> list[Contact]: query = ( session.query(Contact) .options( selectinload(Contact.emails), selectinload(Contact.phones), selectinload(Contact.postal_addresses), ) .filter( Contact.address_book_id == address_book_id, Contact.deleted_at.is_(None) ) ) if contact_ids is not None: query = query.filter(Contact.id.in_(contact_ids)) return query.all() def _visible_books(session: Session, principal: ApiPrincipal): from govoplan_addresses.backend.service import list_address_books return list_address_books(session, principal) def _execution_mode(card_count: int) -> str: raw = os.getenv( "GOVOPLAN_ADDRESSES_VCARD_JOB_THRESHOLD", str(DEFAULT_PERSISTED_BATCH_THRESHOLD) ) try: threshold = max(1, min(MAX_VCARD_CARDS, int(raw))) except ValueError: threshold = DEFAULT_PERSISTED_BATCH_THRESHOLD return "persisted_batch" if card_count >= threshold else "bounded_sync" def _progress(total: int) -> dict[str, int]: return { "total": total, "completed": 0, "created": 0, "updated": 0, "ignored": 0, "failed": 0, } def _hash_json(value: Any) -> str: return hashlib.sha256( json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=False ).encode("utf-8") ).hexdigest() def _safe_filename(value: str) -> str: safe = "".join( character if character.isalnum() or character in {"-", "_"} else "-" for character in value.strip() ) return safe.strip("-")[:120] or "contacts" __all__ = [ "apply_vcard_batch", "cancel_vcard_batch", "export_vcards", "get_vcard_batch_run", "preview_vcard_batch", "vcard_batch_payload", "vcard_diagnostics_payload", ]