From 42fe262376fe8c1ea885b7e0a6040c32a5b68723 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Thu, 20 Aug 2026 17:24:42 +0200 Subject: [PATCH] feat: add selective vCard batch workflows --- README.md | 10 +- docs/INTERFACE_PATTERN_MIGRATION.md | 5 + src/govoplan_addresses/backend/db/models.py | 6 +- .../backend/import_schemas.py | 2 +- src/govoplan_addresses/backend/manifest.py | 29 +- .../versions/d6e8f9a0b1c2_vcard_batch_runs.py | 34 + src/govoplan_addresses/backend/router.py | 427 +++++++-- src/govoplan_addresses/backend/vcard.py | 119 ++- .../backend/vcard_batch_schemas.py | 144 +++ .../backend/vcard_batches.py | 901 ++++++++++++++++++ tests/test_migrations.py | 2 +- tests/test_vcard_batches.py | 270 ++++++ webui/src/api/addresses.ts | 113 ++- .../features/addressbook/AddressBookPage.tsx | 212 ++++- 14 files changed, 2116 insertions(+), 158 deletions(-) create mode 100644 src/govoplan_addresses/backend/migrations/versions/d6e8f9a0b1c2_vcard_batch_runs.py create mode 100644 src/govoplan_addresses/backend/vcard_batch_schemas.py create mode 100644 src/govoplan_addresses/backend/vcard_batches.py create mode 100644 tests/test_vcard_batches.py diff --git a/README.md b/README.md index 1e82745..f39145e 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,14 @@ tenant summaries, and uninstall guards. The first UI supports user, group, tenant, and system-scoped address books, multi-value contact methods, soft deletion, restore, read-only lookup/search, -and vCard import/export for common contact fields. Imported vCards preserve -source payload and revision metadata for later sync/conflict work. +and vCard import/export for common contact fields. Multi-file vCard imports now +create a persisted preview before mutation, expose duplicate suggestions and +per-card create/update/ignore choices, reject stale plans, and make identical +commit retries idempotent. Pending batches can be reloaded or cancelled. +Address-book, address-list, and selected-contact exports explicitly support +vCard 3.0 or 4.0 with deterministic ordering and a recorded content hash. +Imported vCards preserve source payload and revision metadata for later +sync/conflict work, while batch diagnostics expose only bounded metadata. The backend and WebUI also support classical address lists: reusable groupings of contacts or specific contact methods within one address book. Campaigns can diff --git a/docs/INTERFACE_PATTERN_MIGRATION.md b/docs/INTERFACE_PATTERN_MIGRATION.md index 8b78977..75eb72a 100644 --- a/docs/INTERFACE_PATTERN_MIGRATION.md +++ b/docs/INTERFACE_PATTERN_MIGRATION.md @@ -26,6 +26,11 @@ imports, synchronization, quality review, and reversible merge operations. source diagnostics and conflict state as explicitly stated. - Imports and synchronization separate preview from apply; incomplete external reads never infer deletions. +- vCard batch upload accepts multiple files, persists a non-mutating preview, + and requires an explicit create/update/ignore choice for every reviewed card. + Reload and cancellation preserve the pending plan; only applying the matching + plan hash mutates contacts. Scoped exports name the selected vCard version and + use deterministic ordering. - Merge and communication-governance operations append auditable evidence and never silently erase prior state. - Request feedback is rendered as a compact shared alert over the full-height diff --git a/src/govoplan_addresses/backend/db/models.py b/src/govoplan_addresses/backend/db/models.py index b20b7ea..b37e79b 100644 --- a/src/govoplan_addresses/backend/db/models.py +++ b/src/govoplan_addresses/backend/db/models.py @@ -587,9 +587,9 @@ class AddressImportRun(Base, TimestampMixin): nullable=False, index=True, ) - profile_id: Mapped[str] = mapped_column( + profile_id: Mapped[str | None] = mapped_column( ForeignKey("addresses_import_profiles.id", ondelete="RESTRICT"), - nullable=False, + nullable=True, index=True, ) source_filename: Mapped[str] = mapped_column(String(500), nullable=False) @@ -607,7 +607,7 @@ class AddressImportRun(Base, TimestampMixin): rolled_back_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) address_book: Mapped[AddressBook] = relationship() - profile: Mapped[AddressImportProfile] = relationship() + profile: Mapped[AddressImportProfile | None] = relationship() __all__ = [ diff --git a/src/govoplan_addresses/backend/import_schemas.py b/src/govoplan_addresses/backend/import_schemas.py index 9c77c45..b1b4c66 100644 --- a/src/govoplan_addresses/backend/import_schemas.py +++ b/src/govoplan_addresses/backend/import_schemas.py @@ -129,7 +129,7 @@ class AddressImportDiagnosticResponse(BaseModel): class AddressImportRunResponse(BaseModel): id: str address_book_id: str - profile_id: str + profile_id: str | None source_filename: str source_format: str input_hash: str diff --git a/src/govoplan_addresses/backend/manifest.py b/src/govoplan_addresses/backend/manifest.py index 667e070..8046e7a 100644 --- a/src/govoplan_addresses/backend/manifest.py +++ b/src/govoplan_addresses/backend/manifest.py @@ -460,6 +460,31 @@ manifest = ModuleManifest( related_modules=("connectors", "datasources", "dataflow", "files", "audit"), order=33, ), + DocumentationTopic( + id="addresses.vcard-batches", + title="Selective vCard batch import and export", + summary="Preview multiple vCard files, choose each card's effect, and export deterministic scoped files.", + body=( + "One or more UTF-8 .vcf files are parsed into a persisted, non-mutating preview with bounded diagnostics, " + "duplicate suggestions, an input hash, a parser version, and a deterministic plan hash. Operators choose " + "create, update, or ignore only where the reviewed plan permits it. Apply rejects stale contact targets and " + "is idempotent for the same selection; a different retry is rejected. Pending runs can be reloaded or cancelled " + "without changing contacts. Upload size, file count, card count, line count, and unfolded-line length are bounded. " + "Exports can target a complete address book, one address list, or explicit contacts; vCard 3.0 or 4.0 is selected " + "explicitly and contacts use deterministic display-name and stable-ID ordering. Export and import evidence records " + "hashes and counts, while diagnostics never disclose raw contact payloads. Large previews remain persisted and expose " + "their batch execution mode so a runtime job capability can execute them asynchronously when available." + ), + layer="configured", + documentation_types=("admin", "user"), + audience=("tenant_admin", "operator", "module_admin", "power_user"), + related_modules=("files", "audit", "connectors"), + order=34, + metadata={ + "seed": True, + "help_contexts": ["addresses.action.import", "addresses.contacts", "addresses.sources"], + }, + ), DocumentationTopic( id="addresses.ldap-directory", title="LDAP and Active Directory address sources", @@ -475,7 +500,7 @@ manifest = ModuleManifest( documentation_types=("admin", "user"), audience=("tenant_admin", "operator", "module_admin"), related_modules=("connectors", "idm", "access", "policy", "audit"), - order=34, + order=35, ), DocumentationTopic( id="addresses.quality-and-merge", @@ -513,7 +538,7 @@ manifest = ModuleManifest( documentation_types=("admin", "user"), audience=("tenant_admin", "operator", "module_admin", "power_user"), related_modules=("dist_lists", "connectors", "datasources", "campaigns", "policy", "audit"), - order=35, + order=36, metadata={ "seed": True, "help_contexts": [ diff --git a/src/govoplan_addresses/backend/migrations/versions/d6e8f9a0b1c2_vcard_batch_runs.py b/src/govoplan_addresses/backend/migrations/versions/d6e8f9a0b1c2_vcard_batch_runs.py new file mode 100644 index 0000000..f8bf490 --- /dev/null +++ b/src/govoplan_addresses/backend/migrations/versions/d6e8f9a0b1c2_vcard_batch_runs.py @@ -0,0 +1,34 @@ +"""Allow profile-free persisted vCard batch runs. + +Revision ID: d6e8f9a0b1c2 +Revises: c5d7e8f9a0b1 +""" + +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "d6e8f9a0b1c2" +down_revision = "c5d7e8f9a0b1" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("addresses_import_runs") as batch: + batch.alter_column( + "profile_id", + existing_type=sa.String(length=36), + nullable=True, + ) + + +def downgrade() -> None: + with op.batch_alter_table("addresses_import_runs") as batch: + batch.alter_column( + "profile_id", + existing_type=sa.String(length=36), + nullable=False, + ) diff --git a/src/govoplan_addresses/backend/router.py b/src/govoplan_addresses/backend/router.py index c54f33b..d3a0a4d 100644 --- a/src/govoplan_addresses/backend/router.py +++ b/src/govoplan_addresses/backend/router.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import asdict +import json import re from fastapi import APIRouter, Depends, HTTPException, Query, Response, status @@ -56,6 +57,23 @@ from govoplan_addresses.backend.imports import ( rollback_address_import, update_import_profile, ) +from govoplan_addresses.backend.vcard_batch_schemas import ( + VCardBatchCancelRequest, + VCardBatchCommitRequest, + VCardBatchPreviewRequest, + VCardBatchRunResponse, + VCardExportRequest, + VCardExportResponse, +) +from govoplan_addresses.backend.vcard_batches import ( + apply_vcard_batch, + cancel_vcard_batch, + export_vcards, + get_vcard_batch_run, + preview_vcard_batch, + vcard_batch_payload, + vcard_diagnostics_payload, +) from govoplan_addresses.backend.capabilities import ( AddressesContactPointResolutionCapability, AddressesContactWriterCapability, @@ -238,9 +256,7 @@ def _contact_response( decision = quality.get((channel, point_id)) or quality.get((channel, None)) return { "quality_state": decision.state if decision is not None else "valid", - "quality_reason_code": ( - decision.reason_code if decision is not None else None - ), + "quality_reason_code": (decision.reason_code if decision is not None else None), } return ContactResponse.model_validate( @@ -323,9 +339,7 @@ def _contact_point_audit_details( "postal": [item.id for item in contact.postal_addresses], } return { - f"{key_prefix}contact_point_counts": { - channel: len(ids) for channel, ids in point_ids.items() - }, + f"{key_prefix}contact_point_counts": {channel: len(ids) for channel, ids in point_ids.items()}, f"{key_prefix}contact_point_ids": point_ids, } @@ -454,7 +468,9 @@ def _sync_source_response(sync_source: AddressSyncSource) -> AddressSyncSourceRe ) -def _sync_diagnostic_response(diagnostic: AddressSyncDiagnostic) -> AddressSyncDiagnosticResponse: +def _sync_diagnostic_response( + diagnostic: AddressSyncDiagnostic, +) -> AddressSyncDiagnosticResponse: return AddressSyncDiagnosticResponse.model_validate( { "id": diagnostic.id, @@ -470,7 +486,9 @@ def _sync_diagnostic_response(diagnostic: AddressSyncDiagnostic) -> AddressSyncD ) -def _sync_tombstone_response(tombstone: AddressSyncTombstone) -> AddressSyncTombstoneResponse: +def _sync_tombstone_response( + tombstone: AddressSyncTombstone, +) -> AddressSyncTombstoneResponse: return AddressSyncTombstoneResponse.model_validate( { "id": tombstone.id, @@ -490,7 +508,9 @@ def _sync_tombstone_response(tombstone: AddressSyncTombstone) -> AddressSyncTomb ) -def _sync_conflict_response(conflict: AddressSyncConflict) -> AddressSyncConflictResponse: +def _sync_conflict_response( + conflict: AddressSyncConflict, +) -> AddressSyncConflictResponse: return AddressSyncConflictResponse.model_validate( { "id": conflict.id, @@ -583,7 +603,11 @@ def api_list_address_books( return AddressBookListResponse(address_books=[_book_response(book, contact_count=counts.get(book.id, 0)) for book in books]) -@router.post("/address-books", response_model=AddressBookResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/address-books", + response_model=AddressBookResponse, + status_code=status.HTTP_201_CREATED, +) def api_create_address_book( payload: AddressBookCreateRequest, principal: ApiPrincipal = Depends(get_api_principal), @@ -591,7 +615,12 @@ def api_create_address_book( ): _require_scope(principal, "addresses:address_book:write") try: - book = create_address_book(session, principal, payload, allow_system=has_scope(principal, "addresses:address_book:admin")) + book = create_address_book( + session, + principal, + payload, + allow_system=has_scope(principal, "addresses:address_book:admin"), + ) session.commit() session.refresh(book) return _book_response(book) @@ -738,10 +767,7 @@ def api_suggest_duplicate_contacts( right=_contact_response(item.right), score=item.score, confidence=item.confidence, - features=[ - ContactDuplicateFeatureResponse(**asdict(feature)) - for feature in item.features - ], + features=[ContactDuplicateFeatureResponse(**asdict(feature)) for feature in item.features], ) for item in scan.suggestions ], @@ -778,10 +804,7 @@ def api_address_quality_summary( quality_counts=summary.quality_counts, duplicate_suggestion_count=summary.duplicate_suggestion_count, correction_count=summary.correction_count, - corrections=[ - AddressQualityCorrectionResponse(**asdict(item)) - for item in summary.corrections - ], + corrections=[AddressQualityCorrectionResponse(**asdict(item)) for item in summary.corrections], truncated=summary.truncated, ) except AddressBookError as exc: @@ -895,9 +918,7 @@ def api_resolve_contact_redirect( ): _require_scope(principal, "addresses:contact:read") try: - return ContactRedirectResponse.model_validate( - asdict(resolve_contact_redirect(session, principal, contact_id)) - ) + return ContactRedirectResponse.model_validate(asdict(resolve_contact_redirect(session, principal, contact_id))) except AddressBookError as exc: raise _error(exc) from exc @@ -1167,16 +1188,23 @@ def api_list_address_lists( ): _require_scope(principal, "addresses:address_list:read") try: - address_lists = list_address_lists(session, principal, address_book_id=address_book_id, include_deleted=include_deleted) - counts = address_list_entry_counts(session, [address_list.id for address_list in address_lists]) - return AddressListListResponse( - address_lists=[_address_list_response(address_list, entry_count=counts.get(address_list.id, 0)) for address_list in address_lists] + address_lists = list_address_lists( + session, + principal, + address_book_id=address_book_id, + include_deleted=include_deleted, ) + counts = address_list_entry_counts(session, [address_list.id for address_list in address_lists]) + return AddressListListResponse(address_lists=[_address_list_response(address_list, entry_count=counts.get(address_list.id, 0)) for address_list in address_lists]) except AddressBookError as exc: raise _error(exc) from exc -@router.post("/address-books/{book_id}/address-lists", response_model=AddressListResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/address-books/{book_id}/address-lists", + response_model=AddressListResponse, + status_code=status.HTTP_201_CREATED, +) def api_create_address_list( book_id: str, payload: AddressListCreateRequest, @@ -1247,7 +1275,10 @@ def api_restore_address_list( raise _error(exc) from exc -@router.get("/address-lists/{address_list_id}/entries", response_model=AddressListEntryListResponse) +@router.get( + "/address-lists/{address_list_id}/entries", + response_model=AddressListEntryListResponse, +) def api_list_address_list_entries( address_list_id: str, principal: ApiPrincipal = Depends(get_api_principal), @@ -1261,7 +1292,11 @@ def api_list_address_list_entries( raise _error(exc) from exc -@router.post("/address-lists/{address_list_id}/entries", response_model=AddressListEntryResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/address-lists/{address_list_id}/entries", + response_model=AddressListEntryResponse, + status_code=status.HTTP_201_CREATED, +) def api_create_address_list_entry( address_list_id: str, payload: AddressListEntryCreateRequest, @@ -1332,7 +1367,10 @@ def api_list_write_targets( return AddressBookWriteTargetsResponse(targets=[_write_decision_response(decision) for decision in decisions]) -@router.get("/address-books/{book_id}/write-decision", response_model=AddressBookWriteDecisionResponse) +@router.get( + "/address-books/{book_id}/write-decision", + response_model=AddressBookWriteDecisionResponse, +) def api_get_address_book_write_decision( book_id: str, operation: str = Query(default="create_contact"), @@ -1378,9 +1416,7 @@ def api_discover_ldap_base_dns( ): _require_scope(principal, "addresses:sync:write") try: - return AddressLdapDiscoveryResponse( - base_dns=list(discover_ldap_base_dns(session, principal, payload)) - ) + return AddressLdapDiscoveryResponse(base_dns=list(discover_ldap_base_dns(session, principal, payload))) except (AddressBookError, AddressLdapError) as exc: raise _error(AddressBookError(str(exc))) from exc @@ -1438,7 +1474,11 @@ def api_list_address_credentials( ) -@router.post("/address-books/{book_id}/carddav/sources", response_model=AddressSyncSourceResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/address-books/{book_id}/carddav/sources", + response_model=AddressSyncSourceResponse, + status_code=status.HTTP_201_CREATED, +) def api_create_carddav_sync_source( book_id: str, payload: AddressCardDavSourceCreateRequest, @@ -1454,7 +1494,11 @@ def api_create_carddav_sync_source( action="addresses.sync_source_created", object_type="address_sync_source", object_id=sync_source.id, - details={"address_book_id": book_id, "connector_type": "carddav", "sync_direction": sync_source.sync_direction}, + details={ + "address_book_id": book_id, + "connector_type": "carddav", + "sync_direction": sync_source.sync_direction, + }, ) session.commit() session.refresh(sync_source) @@ -1473,7 +1517,12 @@ def api_list_sync_sources( ): _require_scope(principal, "addresses:sync:read") try: - sync_sources = list_sync_sources(session, principal, address_book_id=address_book_id, include_disabled=include_disabled) + sync_sources = list_sync_sources( + session, + principal, + address_book_id=address_book_id, + include_disabled=include_disabled, + ) return AddressSyncSourceListResponse(sync_sources=[_sync_source_response(sync_source) for sync_source in sync_sources]) except AddressBookError as exc: raise _error(exc) from exc @@ -1575,7 +1624,11 @@ def api_run_sync_source( raise _error(AddressBookError(message)) from exc -@router.post("/address-books/{book_id}/sync-sources", response_model=AddressSyncSourceResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/address-books/{book_id}/sync-sources", + response_model=AddressSyncSourceResponse, + status_code=status.HTTP_201_CREATED, +) def api_create_sync_source( book_id: str, payload: AddressSyncSourceCreateRequest, @@ -1591,7 +1644,11 @@ def api_create_sync_source( action="addresses.sync_source_created", object_type="address_sync_source", object_id=sync_source.id, - details={"address_book_id": book_id, "connector_type": sync_source.connector_type, "sync_direction": sync_source.sync_direction}, + details={ + "address_book_id": book_id, + "connector_type": sync_source.connector_type, + "sync_direction": sync_source.sync_direction, + }, ) session.commit() session.refresh(sync_source) @@ -1617,7 +1674,11 @@ def api_update_sync_source( action="addresses.sync_source_updated", object_type="address_sync_source", object_id=sync_source.id, - details={"connector_type": sync_source.connector_type, "sync_direction": sync_source.sync_direction, "enabled": sync_source.enabled}, + details={ + "connector_type": sync_source.connector_type, + "sync_direction": sync_source.sync_direction, + "enabled": sync_source.enabled, + }, ) session.commit() session.refresh(sync_source) @@ -1651,7 +1712,10 @@ def api_delete_sync_source( raise _error(exc) from exc -@router.post("/sync-sources/{sync_source_id}/attempts/start", response_model=AddressSyncSourceResponse) +@router.post( + "/sync-sources/{sync_source_id}/attempts/start", + response_model=AddressSyncSourceResponse, +) def api_start_sync_attempt( sync_source_id: str, principal: ApiPrincipal = Depends(get_api_principal), @@ -1676,7 +1740,10 @@ def api_start_sync_attempt( raise _error(exc) from exc -@router.post("/sync-sources/{sync_source_id}/attempts/finish", response_model=AddressSyncSourceResponse) +@router.post( + "/sync-sources/{sync_source_id}/attempts/finish", + response_model=AddressSyncSourceResponse, +) def api_finish_sync_attempt( sync_source_id: str, payload: AddressSyncAttemptFinishRequest, @@ -1692,7 +1759,11 @@ def api_finish_sync_attempt( action="addresses.sync_finished", object_type="address_sync_source", object_id=sync_source.id, - details={"connector_type": sync_source.connector_type, "status": sync_source.status, "error": sync_source.last_error}, + details={ + "connector_type": sync_source.connector_type, + "status": sync_source.status, + "error": sync_source.last_error, + }, ) session.commit() session.refresh(sync_source) @@ -1702,7 +1773,10 @@ def api_finish_sync_attempt( raise _error(exc) from exc -@router.get("/sync-sources/{sync_source_id}/diagnostics", response_model=AddressSyncDiagnosticListResponse) +@router.get( + "/sync-sources/{sync_source_id}/diagnostics", + response_model=AddressSyncDiagnosticListResponse, +) def api_list_sync_diagnostics( sync_source_id: str, limit: int = Query(default=100, ge=1, le=500), @@ -1717,7 +1791,11 @@ def api_list_sync_diagnostics( raise _error(exc) from exc -@router.post("/sync-sources/{sync_source_id}/diagnostics", response_model=AddressSyncDiagnosticResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/sync-sources/{sync_source_id}/diagnostics", + response_model=AddressSyncDiagnosticResponse, + status_code=status.HTTP_201_CREATED, +) def api_record_sync_diagnostic( sync_source_id: str, payload: AddressSyncDiagnosticCreateRequest, @@ -1735,7 +1813,10 @@ def api_record_sync_diagnostic( raise _error(exc) from exc -@router.get("/sync-sources/{sync_source_id}/tombstones", response_model=AddressSyncTombstoneListResponse) +@router.get( + "/sync-sources/{sync_source_id}/tombstones", + response_model=AddressSyncTombstoneListResponse, +) def api_list_sync_tombstones( sync_source_id: str, limit: int = Query(default=200, ge=1, le=1000), @@ -1750,7 +1831,11 @@ def api_list_sync_tombstones( raise _error(exc) from exc -@router.post("/sync-sources/{sync_source_id}/tombstones", response_model=AddressSyncTombstoneResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/sync-sources/{sync_source_id}/tombstones", + response_model=AddressSyncTombstoneResponse, + status_code=status.HTTP_201_CREATED, +) def api_record_sync_tombstone( sync_source_id: str, payload: AddressSyncTombstoneCreateRequest, @@ -1768,7 +1853,10 @@ def api_record_sync_tombstone( raise _error(exc) from exc -@router.get("/sync-sources/{sync_source_id}/conflicts", response_model=AddressSyncConflictListResponse) +@router.get( + "/sync-sources/{sync_source_id}/conflicts", + response_model=AddressSyncConflictListResponse, +) def api_list_sync_conflicts( sync_source_id: str, status_filter: str | None = Query(default="open", alias="status"), @@ -1784,7 +1872,11 @@ def api_list_sync_conflicts( raise _error(exc) from exc -@router.post("/sync-sources/{sync_source_id}/conflicts", response_model=AddressSyncConflictResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/sync-sources/{sync_source_id}/conflicts", + response_model=AddressSyncConflictResponse, + status_code=status.HTTP_201_CREATED, +) def api_record_sync_conflict( sync_source_id: str, payload: AddressSyncConflictCreateRequest, @@ -1818,7 +1910,11 @@ def api_resolve_sync_conflict( action="addresses.sync_conflict_resolved", object_type="address_sync_conflict", object_id=conflict.id, - details={"sync_source_id": conflict.sync_source_id, "status": conflict.status, "resolution": conflict.resolution}, + details={ + "sync_source_id": conflict.sync_source_id, + "status": conflict.status, + "resolution": conflict.resolution, + }, ) session.commit() session.refresh(conflict) @@ -1828,7 +1924,11 @@ def api_resolve_sync_conflict( raise _error(exc) from exc -@router.post("/address-books/{book_id}/contacts", response_model=ContactResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/address-books/{book_id}/contacts", + response_model=ContactResponse, + status_code=status.HTTP_201_CREATED, +) def api_create_contact( book_id: str, payload: ContactCreateRequest, @@ -1869,11 +1969,7 @@ def api_update_contact( _require_scope(principal, "addresses:contact:write") try: previous_contact = session.get(Contact, contact_id) - previous_point_details = ( - _contact_point_audit_details(previous_contact, prefix="previous") - if previous_contact is not None - else {} - ) + previous_point_details = _contact_point_audit_details(previous_contact, prefix="previous") if previous_contact is not None else {} contact = update_contact(session, principal, contact_id, payload) audit_from_principal( session, @@ -1897,7 +1993,10 @@ def api_update_contact( raise _error(exc) from exc -@router.get("/contacts/{contact_id}/channel-rules", response_model=ContactChannelRuleListResponse) +@router.get( + "/contacts/{contact_id}/channel-rules", + response_model=ContactChannelRuleListResponse, +) def api_list_contact_channel_rules( contact_id: str, principal: ApiPrincipal = Depends(get_api_principal), @@ -1905,12 +2004,7 @@ def api_list_contact_channel_rules( ): _require_scope(principal, "addresses:governance:read") try: - return ContactChannelRuleListResponse( - rules=[ - _channel_rule_response(rule) - for rule in list_contact_channel_rules(session, principal, contact_id) - ] - ) + return ContactChannelRuleListResponse(rules=[_channel_rule_response(rule) for rule in list_contact_channel_rules(session, principal, contact_id)]) except AddressBookError as exc: raise _error(exc) from exc @@ -2044,7 +2138,11 @@ def api_restore_contact( raise _error(exc) from exc -@router.post("/address-books/{book_id}/vcards/import", response_model=VCardImportResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/address-books/{book_id}/vcards/import", + response_model=VCardImportResponse, + status_code=status.HTTP_201_CREATED, +) def api_import_address_book_vcards( book_id: str, payload: VCardImportRequest, @@ -2091,6 +2189,136 @@ def api_import_address_book_vcards( raise _error(AddressBookError(str(exc))) from exc +@router.post( + "/address-books/{book_id}/vcard-batches/preview", + response_model=VCardBatchRunResponse, + status_code=status.HTTP_201_CREATED, +) +def api_preview_vcard_batch( + book_id: str, + payload: VCardBatchPreviewRequest, + principal: ApiPrincipal = Depends(get_api_principal), + session: Session = Depends(get_session), +): + _require_scope(principal, "addresses:contact:write") + try: + run = preview_vcard_batch(session, principal, book_id, payload) + audit_from_principal( + session, + principal, + action="addresses.vcard_batch_previewed", + object_type="address_import_run", + object_id=run.id, + details={ + "address_book_id": book_id, + "input_hash": run.input_hash, + "plan_hash": run.plan_hash, + "statistics": run.statistics, + }, + ) + session.commit() + session.refresh(run) + return VCardBatchRunResponse.model_validate(vcard_batch_payload(run)) + except AddressBookError as exc: + session.rollback() + raise _error(exc) from exc + + +@router.get("/vcard-batches/{run_id}", response_model=VCardBatchRunResponse) +def api_get_vcard_batch( + run_id: str, + principal: ApiPrincipal = Depends(get_api_principal), + session: Session = Depends(get_session), +): + _require_scope(principal, "addresses:contact:read") + try: + return VCardBatchRunResponse.model_validate(vcard_batch_payload(get_vcard_batch_run(session, principal, run_id))) + except AddressBookError as exc: + raise _error(exc) from exc + + +@router.post("/vcard-batches/{run_id}/apply", response_model=VCardBatchRunResponse) +def api_apply_vcard_batch( + run_id: str, + payload: VCardBatchCommitRequest, + principal: ApiPrincipal = Depends(get_api_principal), + session: Session = Depends(get_session), +): + _require_scope(principal, "addresses:contact:write") + try: + run = apply_vcard_batch(session, principal, run_id, payload) + session.flush() + audit_from_principal( + session, + principal, + action="addresses.vcard_batch_applied", + object_type="address_import_run", + object_id=run.id, + details={ + "address_book_id": run.address_book_id, + "input_hash": run.input_hash, + "plan_hash": run.plan_hash, + "statistics": run.statistics, + }, + ) + session.commit() + session.refresh(run) + return VCardBatchRunResponse.model_validate(vcard_batch_payload(run)) + except AddressBookError as exc: + session.rollback() + raise _error(exc) from exc + + +@router.post("/vcard-batches/{run_id}/cancel", response_model=VCardBatchRunResponse) +def api_cancel_vcard_batch( + run_id: str, + payload: VCardBatchCancelRequest, + principal: ApiPrincipal = Depends(get_api_principal), + session: Session = Depends(get_session), +): + _require_scope(principal, "addresses:contact:write") + try: + run = cancel_vcard_batch(session, principal, run_id, payload) + audit_from_principal( + session, + principal, + action="addresses.vcard_batch_cancelled", + object_type="address_import_run", + object_id=run.id, + details={"address_book_id": run.address_book_id, "reason": payload.reason}, + ) + session.commit() + session.refresh(run) + return VCardBatchRunResponse.model_validate(vcard_batch_payload(run)) + except AddressBookError as exc: + session.rollback() + raise _error(exc) from exc + + +@router.get("/vcard-batches/{run_id}/diagnostics") +def api_export_vcard_batch_diagnostics( + run_id: str, + principal: ApiPrincipal = Depends(get_api_principal), + session: Session = Depends(get_session), +): + _require_scope(principal, "addresses:contact:read") + try: + run = get_vcard_batch_run(session, principal, run_id) + content = json.dumps( + vcard_diagnostics_payload(run), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return Response( + content=content, + media_type="application/json", + headers={"Content-Disposition": f'attachment; filename="vcard-batch-{run.id}.json"'}, + ) + except AddressBookError as exc: + raise _error(exc) from exc + + @router.get("/import-profiles", response_model=AddressImportProfileListResponse) def api_list_address_import_profiles( include_history: bool = Query(default=False), @@ -2098,15 +2326,14 @@ def api_list_address_import_profiles( session: Session = Depends(get_session), ): _require_scope(principal, "addresses:contact:read") - return AddressImportProfileListResponse( - profiles=[ - AddressImportProfileResponse.model_validate(item) - for item in list_import_profiles(session, principal, include_history=include_history) - ] - ) + return AddressImportProfileListResponse(profiles=[AddressImportProfileResponse.model_validate(item) for item in list_import_profiles(session, principal, include_history=include_history)]) -@router.post("/import-profiles", response_model=AddressImportProfileResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/import-profiles", + response_model=AddressImportProfileResponse, + status_code=status.HTTP_201_CREATED, +) def api_create_address_import_profile( payload: AddressImportProfileCreateRequest, principal: ApiPrincipal = Depends(get_api_principal), @@ -2122,7 +2349,11 @@ def api_create_address_import_profile( action="addresses.import_profile_created", object_type="address_import_profile", object_id=profile.profile_key, - details={"version": profile.version, "source_format": profile.source_format, "scope_type": profile.scope_type}, + details={ + "version": profile.version, + "source_format": profile.source_format, + "scope_type": profile.scope_type, + }, ) session.commit() session.refresh(profile) @@ -2149,7 +2380,10 @@ def api_update_address_import_profile( action="addresses.import_profile_versioned", object_type="address_import_profile", object_id=profile.profile_key, - details={"version": profile.version, "source_format": profile.source_format}, + details={ + "version": profile.version, + "source_format": profile.source_format, + }, ) session.commit() session.refresh(profile) @@ -2202,7 +2436,11 @@ def api_preview_address_import( action="addresses.import_previewed", object_type="address_import_run", object_id=run.id, - details={"input_hash": run.input_hash, "plan_hash": run.plan_hash, "statistics": run.statistics}, + details={ + "input_hash": run.input_hash, + "plan_hash": run.plan_hash, + "statistics": run.statistics, + }, ) session.commit() session.refresh(run) @@ -2220,9 +2458,7 @@ def api_get_address_import_run( ): _require_scope(principal, "addresses:contact:read") try: - return AddressImportRunResponse.model_validate( - import_run_payload(get_import_run(session, principal, run_id)) - ) + return AddressImportRunResponse.model_validate(import_run_payload(get_import_run(session, principal, run_id))) except AddressBookError as exc: raise _error(exc) from exc @@ -2244,7 +2480,11 @@ def api_apply_address_import( action="addresses.import_applied", object_type="address_import_run", object_id=run.id, - details={"input_hash": run.input_hash, "plan_hash": run.plan_hash, "statistics": run.statistics}, + details={ + "input_hash": run.input_hash, + "plan_hash": run.plan_hash, + "statistics": run.statistics, + }, ) session.commit() session.refresh(run) @@ -2300,6 +2540,39 @@ def api_export_address_book_vcards( raise _error(exc) from exc +@router.post( + "/address-books/{book_id}/vcards/export", + response_model=VCardExportResponse, +) +def api_export_selected_vcards( + book_id: str, + payload: VCardExportRequest, + principal: ApiPrincipal = Depends(get_api_principal), + session: Session = Depends(get_session), +): + _require_scope(principal, "addresses:contact:read") + try: + result = export_vcards(session, principal, book_id, payload) + audit_from_principal( + session, + principal, + action="addresses.vcards_exported", + object_type="address_book", + object_id=book_id, + details={ + "scope": result["scope"], + "version": result["version"], + "contact_count": result["contact_count"], + "content_hash": result["content_hash"], + }, + ) + session.commit() + return VCardExportResponse.model_validate(result) + except AddressBookError as exc: + session.rollback() + raise _error(exc) from exc + + @router.get("/contacts/{contact_id}/vcard") def api_export_contact_vcard( contact_id: str, diff --git a/src/govoplan_addresses/backend/vcard.py b/src/govoplan_addresses/backend/vcard.py index 7ed4ea1..c87b752 100644 --- a/src/govoplan_addresses/backend/vcard.py +++ b/src/govoplan_addresses/backend/vcard.py @@ -16,6 +16,12 @@ class VCardError(ValueError): pass +VCARD_PARSER_VERSION = "govoplan-vcard/2" +MAX_VCARD_CARDS = 10_000 +MAX_VCARD_LINES = 200_000 +MAX_VCARD_UNFOLDED_LINE_CHARS = 16_384 + + @dataclass(frozen=True, slots=True) class ParsedVCard: payload: ContactCreateRequest @@ -60,12 +66,16 @@ class _VCardDraft: def _normalize_lines(content: str) -> list[str]: raw_lines = content.replace("\r\n", "\n").replace("\r", "\n").split("\n") + if len(raw_lines) > MAX_VCARD_LINES: + raise VCardError(f"vCard input exceeds the {MAX_VCARD_LINES}-line parser limit.") lines: list[str] = [] for line in raw_lines: if line.startswith((" ", "\t")) and lines: lines[-1] += line[1:] elif line: lines.append(line) + if lines and len(lines[-1]) > MAX_VCARD_UNFOLDED_LINE_CHARS: + raise VCardError(f"vCard unfolded lines are limited to {MAX_VCARD_UNFOLDED_LINE_CHARS} characters.") return lines @@ -90,27 +100,13 @@ def _split_unescaped(value: str, separator: str) -> list[str]: def _unescape_text(value: str) -> str: - return ( - value.replace("\\n", "\n") - .replace("\\N", "\n") - .replace("\\,", ",") - .replace("\\;", ";") - .replace("\\\\", "\\") - .strip() - ) + return value.replace("\\n", "\n").replace("\\N", "\n").replace("\\,", ",").replace("\\;", ";").replace("\\\\", "\\").strip() def _escape_text(value: str | None) -> str: if value is None: return "" - return ( - value.replace("\\", "\\\\") - .replace("\r\n", "\n") - .replace("\r", "\n") - .replace("\n", "\\n") - .replace(";", "\\;") - .replace(",", "\\,") - ) + return value.replace("\\", "\\\\").replace("\r\n", "\n").replace("\r", "\n").replace("\n", "\\n").replace(";", "\\;").replace(",", "\\,") def _parse_head(head: str) -> tuple[str, dict[str, list[str]]]: @@ -154,7 +150,7 @@ def _is_pref(params: dict[str, list[str]]) -> bool: def _card_blocks(content: str) -> list[list[str]]: - result = _card_blocks_with_issues(content) + result = _card_blocks_with_issues(content, max_cards=MAX_VCARD_CARDS) if result.issues: raise VCardError(result.issues[0].message) return result.cards @@ -166,8 +162,14 @@ class _CardBlockResult: issues: list[ParsedVCardIssue] -def _card_blocks_with_issues(content: str) -> _CardBlockResult: - lines = _normalize_lines(content) +def _card_blocks_with_issues(content: str, *, max_cards: int) -> _CardBlockResult: + try: + lines = _normalize_lines(content) + except VCardError as exc: + return _CardBlockResult( + cards=[], + issues=[ParsedVCardIssue(index=0, message=str(exc))], + ) blocks: list[list[str]] = [] issues: list[ParsedVCardIssue] = [] current: list[str] | None = None @@ -189,6 +191,15 @@ def _card_blocks_with_issues(content: str) -> _CardBlockResult: issues.append(ParsedVCardIssue(index=card_index, message="vCard END appears before BEGIN.")) continue current.append(line) + if len(blocks) >= max_cards: + issues.append( + ParsedVCardIssue( + index=len(blocks) + 1, + message=f"vCard input exceeds the configured {max_cards}-card limit.", + ) + ) + current = None + break blocks.append(current) current = None elif current is not None: @@ -258,7 +269,14 @@ def _apply_card_metadata( if name == "VERSION": draft.version = value.strip() if draft.version and draft.version not in {"3.0", "4.0"}: - issues.append(ParsedVCardIssue(index=index, severity="warning", field="VERSION", message=f"vCard version {draft.version} is not fully supported.")) + issues.append( + ParsedVCardIssue( + index=index, + severity="warning", + field="VERSION", + message=f"vCard version {draft.version} is not fully supported.", + ) + ) return True if name == "UID": draft.uid = _unescape_text(value) or draft.uid @@ -325,15 +343,34 @@ def _append_card_email( if not email: return if "@" not in email: - issues.append(ParsedVCardIssue(index=index, severity="warning", field="EMAIL", message=f"Skipped invalid email address: {email}")) + issues.append( + ParsedVCardIssue( + index=index, + severity="warning", + field="EMAIL", + message=f"Skipped invalid email address: {email}", + ) + ) return - draft.emails.append(ContactEmailPayload(label=_label_from_params(params), email=email, is_primary=_is_pref(params) or not draft.emails)) + draft.emails.append( + ContactEmailPayload( + label=_label_from_params(params), + email=email, + is_primary=_is_pref(params) or not draft.emails, + ) + ) def _append_card_phone(draft: _VCardDraft, params: dict[str, list[str]], value: str) -> None: phone = _unescape_text(value) if phone: - draft.phones.append(ContactPhonePayload(label=_label_from_params(params), phone=phone, is_primary=_is_pref(params) or not draft.phones)) + draft.phones.append( + ContactPhonePayload( + label=_label_from_params(params), + phone=phone, + is_primary=_is_pref(params) or not draft.phones, + ) + ) def _append_card_address(draft: _VCardDraft, params: dict[str, list[str]], value: str) -> None: @@ -388,8 +425,14 @@ def _draft_contact_payload(draft: _VCardDraft) -> ContactCreateRequest: return payload -def parse_vcards_with_issues(content: str) -> VCardParseResult: - blocks = _card_blocks_with_issues(content) +def parse_vcards_with_issues( + content: str, + *, + max_cards: int = MAX_VCARD_CARDS, +) -> VCardParseResult: + if max_cards < 1 or max_cards > MAX_VCARD_CARDS: + raise VCardError(f"max_cards must be between 1 and {MAX_VCARD_CARDS}.") + blocks = _card_blocks_with_issues(content, max_cards=max_cards) parsed: list[ParsedVCard] = [] issues = list(blocks.issues) skipped = 0 @@ -411,8 +454,8 @@ def parse_vcards(content: str) -> list[ParsedVCard]: return result.cards -def contact_to_vcard(contact: Contact) -> str: - lines = _contact_identity_lines(contact) +def contact_to_vcard(contact: Contact, *, version: Literal["3.0", "4.0"] = "4.0") -> str: + lines = _contact_identity_lines(contact, version=version) lines.extend(_contact_email_lines(contact)) lines.extend(_contact_phone_lines(contact)) lines.extend(_contact_address_lines(contact)) @@ -422,10 +465,14 @@ def contact_to_vcard(contact: Contact) -> str: return "\r\n".join(lines) + "\r\n" -def _contact_identity_lines(contact: Contact) -> list[str]: +def _contact_identity_lines( + contact: Contact, + *, + version: Literal["3.0", "4.0"], +) -> list[str]: lines = [ "BEGIN:VCARD", - "VERSION:4.0", + f"VERSION:{version}", f"FN:{_escape_text(contact.display_name)}", f"N:{_escape_text(contact.family_name)};{_escape_text(contact.given_name)};;;", ] @@ -460,11 +507,7 @@ def _contact_address_lines(contact: Contact) -> list[str]: lines: list[str] = [] for address in contact.postal_addresses: label = f";TYPE={_escape_text(address.label)}" if address.label else "" - lines.append( - "ADR" - f"{label}:;;{_escape_text(address.street)};{_escape_text(address.locality)};" - f"{_escape_text(address.region)};{_escape_text(address.postal_code)};{_escape_text(address.country)}" - ) + lines.append(f"ADR{label}:;;{_escape_text(address.street)};{_escape_text(address.locality)};{_escape_text(address.region)};{_escape_text(address.postal_code)};{_escape_text(address.country)}") return lines @@ -496,5 +539,9 @@ def _contact_vcard_urls(contact: Contact) -> object: return vcard.get("urls") -def contacts_to_vcard(contacts: list[Contact]) -> str: - return "".join(contact_to_vcard(contact) for contact in contacts) +def contacts_to_vcard( + contacts: list[Contact], + *, + version: Literal["3.0", "4.0"] = "4.0", +) -> str: + return "".join(contact_to_vcard(contact, version=version) for contact in contacts) diff --git a/src/govoplan_addresses/backend/vcard_batch_schemas.py b/src/govoplan_addresses/backend/vcard_batch_schemas.py new file mode 100644 index 0000000..0a19ca7 --- /dev/null +++ b/src/govoplan_addresses/backend/vcard_batch_schemas.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Literal + +from pydantic import BaseModel, Field, model_validator + + +VCardPlanAction = Literal["create", "update", "ignore", "unchanged", "conflict"] +VCardCommitAction = Literal["create", "update", "ignore"] + + +class VCardBatchFilePayload(BaseModel): + filename: str = Field(min_length=1, max_length=500) + content_base64: str = Field(min_length=1, max_length=14_000_000) + + +class VCardBatchPreviewRequest(BaseModel): + files: list[VCardBatchFilePayload] = Field(min_length=1, max_length=50) + duplicate_card_policy: Literal["reject", "first", "last"] = "reject" + existing_contact_policy: Literal["update", "ignore", "reject"] = "update" + + +class VCardDuplicateSuggestion(BaseModel): + contact_id: str + display_name: str + reasons: list[str] = Field(default_factory=list) + + +class VCardBatchPlanItemResponse(BaseModel): + source_key: str + source_filename: str + card_index: int + action: VCardPlanAction + allowed_actions: list[VCardCommitAction] = Field(default_factory=list) + contact_id: str | None = None + display_name: str | None = None + changed_fields: list[str] = Field(default_factory=list) + duplicate_suggestions: list[VCardDuplicateSuggestion] = Field(default_factory=list) + message: str | None = None + + +class VCardBatchDiagnosticResponse(BaseModel): + severity: Literal["info", "warning", "error"] + code: str + message: str + source_filename: str | None = None + card_index: int | None = None + field: str | None = None + details: dict[str, Any] = Field(default_factory=dict) + + +class VCardBatchProgressResponse(BaseModel): + total: int + completed: int + created: int = 0 + updated: int = 0 + ignored: int = 0 + failed: int = 0 + + +class VCardBatchRunResponse(BaseModel): + id: str + address_book_id: str + status: str + input_hash: str + plan_hash: str + parser_version: str + execution_mode: Literal["bounded_sync", "persisted_batch"] + file_count: int + card_count: int + statistics: dict[str, int | str] = Field(default_factory=dict) + diagnostics: list[VCardBatchDiagnosticResponse] = Field(default_factory=list) + plan: list[VCardBatchPlanItemResponse] = Field(default_factory=list) + progress: VCardBatchProgressResponse + can_apply: bool + can_cancel: bool + commit_hash: str | None = None + created_at: datetime + updated_at: datetime + applied_at: datetime | None = None + + +class VCardBatchSelection(BaseModel): + source_key: str = Field(min_length=1, max_length=1000) + action: VCardCommitAction + + +class VCardBatchCommitRequest(BaseModel): + expected_plan_hash: str = Field(min_length=64, max_length=64) + selections: list[VCardBatchSelection] = Field( + default_factory=list, max_length=10_000 + ) + + +class VCardBatchCancelRequest(BaseModel): + expected_plan_hash: str = Field(min_length=64, max_length=64) + reason: str = Field(min_length=3, max_length=2000) + + +class VCardExportRequest(BaseModel): + scope: Literal["address_book", "address_list", "contacts"] = "address_book" + address_list_id: str | None = Field(default=None, max_length=36) + contact_ids: list[str] = Field(default_factory=list, max_length=10_000) + version: Literal["3.0", "4.0"] = "4.0" + + @model_validator(mode="after") + def validate_scope(self) -> "VCardExportRequest": + if self.scope == "address_list" and not self.address_list_id: + raise ValueError("Address-list export requires address_list_id.") + if self.scope == "contacts" and not self.contact_ids: + raise ValueError( + "Selected-contact export requires at least one contact id." + ) + if self.scope != "address_list" and self.address_list_id: + raise ValueError("address_list_id is only valid for address-list export.") + if self.scope != "contacts" and self.contact_ids: + raise ValueError("contact_ids are only valid for selected-contact export.") + if len(set(self.contact_ids)) != len(self.contact_ids): + raise ValueError("Selected contact ids must be unique.") + return self + + +class VCardExportResponse(BaseModel): + filename: str + media_type: str = "text/vcard" + scope: str + version: str + ordering: str + contact_count: int + content_hash: str + content: str + + +__all__ = [ + "VCardBatchCancelRequest", + "VCardBatchCommitRequest", + "VCardBatchFilePayload", + "VCardBatchPreviewRequest", + "VCardBatchRunResponse", + "VCardBatchSelection", + "VCardExportRequest", + "VCardExportResponse", +] diff --git a/src/govoplan_addresses/backend/vcard_batches.py b/src/govoplan_addresses/backend/vcard_batches.py new file mode 100644 index 0000000..8984977 --- /dev/null +++ b/src/govoplan_addresses/backend/vcard_batches.py @@ -0,0 +1,901 @@ +from __future__ import annotations + +import base64 +import binascii +from collections import Counter, defaultdict +import hashlib +import json +import os +from typing import Any + +from sqlalchemy.orm import Session, selectinload + +from govoplan_addresses.backend.db.models import ( + AddressImportRun, + AddressListEntry, + Contact, +) +from govoplan_addresses.backend.schemas import ( + ContactCreateRequest, + ContactUpdateRequest, +) +from govoplan_addresses.backend.service import ( + AddressBookError, + create_contact, + get_visible_address_book, + get_visible_address_list, + get_visible_contact, + update_contact, +) +from govoplan_addresses.backend.vcard import ( + MAX_VCARD_CARDS, + VCARD_PARSER_VERSION, + contacts_to_vcard, + parse_vcards_with_issues, +) +from govoplan_addresses.backend.vcard_batch_schemas import ( + VCardBatchCancelRequest, + VCardBatchCommitRequest, + VCardBatchPreviewRequest, + VCardExportRequest, +) +from govoplan_core.auth import ApiPrincipal +from govoplan_core.db.base import utcnow + + +MAX_VCARD_BATCH_BYTES = 10_000_000 +DEFAULT_PERSISTED_BATCH_THRESHOLD = 500 + + +def preview_vcard_batch( + session: Session, + principal: ApiPrincipal, + address_book_id: str, + payload: VCardBatchPreviewRequest, +) -> AddressImportRun: + book = get_visible_address_book(session, principal, address_book_id) + if book.read_only: + raise AddressBookError("Static vCard imports require a writable address book.") + + decoded = _decode_files(payload) + input_hash = _hash_json( + { + "files": [ + { + "filename": filename, + "sha256": hashlib.sha256(raw).hexdigest(), + "size": len(raw), + } + for filename, raw in decoded + ] + } + ) + parsed_cards, diagnostics = _parse_files(decoded) + plan = _plan_cards( + session, + book.id, + parsed_cards, + duplicate_card_policy=payload.duplicate_card_policy, + existing_contact_policy=payload.existing_contact_policy, + ) + statistics: dict[str, int | str] = dict( + Counter(str(item["action"]) for item in plan) + ) + statistics.update( + { + "files": len(decoded), + "cards": len(parsed_cards), + "errors": sum(item["severity"] == "error" for item in diagnostics), + "warnings": sum(item["severity"] == "warning" for item in diagnostics), + "parser_version": VCARD_PARSER_VERSION, + "execution_mode": _execution_mode(len(parsed_cards)), + } + ) + plan_hash = _hash_json( + { + "address_book_id": book.id, + "input_hash": input_hash, + "parser_version": VCARD_PARSER_VERSION, + "duplicate_card_policy": payload.duplicate_card_policy, + "existing_contact_policy": payload.existing_contact_policy, + "plan": plan, + } + ) + source_filename = decoded[0][0] + if len(decoded) > 1: + source_filename = f"{source_filename} (+{len(decoded) - 1} files)" + run = AddressImportRun( + tenant_id=book.tenant_id, + address_book_id=book.id, + profile_id=None, + source_filename=source_filename[:500], + source_format="vcard", + input_hash=input_hash, + plan_hash=plan_hash, + status="previewed", + row_count=len(parsed_cards), + statistics=statistics, + diagnostics=diagnostics, + plan_data=plan, + result_evidence={ + "parser_version": VCARD_PARSER_VERSION, + "file_manifest": [ + { + "filename": filename, + "sha256": hashlib.sha256(raw).hexdigest(), + "size": len(raw), + } + for filename, raw in decoded + ], + "progress": _progress(len(plan)), + }, + created_by_account_id=principal.account_id, + ) + session.add(run) + session.flush() + return run + + +def get_vcard_batch_run( + session: Session, + principal: ApiPrincipal, + run_id: str, +) -> AddressImportRun: + book_ids = [book.id for book in _visible_books(session, principal)] + if not book_ids: + raise AddressBookError("vCard batch run not found.") + item = ( + session.query(AddressImportRun) + .filter( + AddressImportRun.id == run_id, + AddressImportRun.address_book_id.in_(book_ids), + AddressImportRun.source_format == "vcard", + ) + .one_or_none() + ) + if item is None: + raise AddressBookError("vCard batch run not found.") + return item + + +def apply_vcard_batch( + session: Session, + principal: ApiPrincipal, + run_id: str, + payload: VCardBatchCommitRequest, +) -> AddressImportRun: + run = get_vcard_batch_run(session, principal, run_id) + if run.plan_hash != payload.expected_plan_hash: + raise AddressBookError("The reviewed vCard plan changed; create a new preview.") + selections = _selection_map(payload) + commit_hash = _hash_json( + { + "plan_hash": run.plan_hash, + "selections": [ + {"source_key": key, "action": selections[key]} + for key in sorted(selections) + ], + } + ) + evidence = dict(run.result_evidence or {}) + if run.status == "applied": + if evidence.get("commit_hash") != commit_hash: + raise AddressBookError( + "This vCard batch was already applied with a different selection." + ) + return run + if run.status != "previewed": + raise AddressBookError( + f"vCard batch cannot be applied from status {run.status!r}." + ) + if not selections: + raise AddressBookError( + "Select at least one vCard action before applying the batch." + ) + + plan_by_key = {str(item["source_key"]): item for item in run.plan_data or []} + unknown = sorted(set(selections).difference(plan_by_key)) + if unknown: + raise AddressBookError( + "The selection contains cards that are not part of the reviewed plan." + ) + created_ids: list[str] = [] + updated_ids: list[str] = [] + ignored = 0 + for source_key in sorted(plan_by_key): + item = plan_by_key[source_key] + action = selections.get(source_key, "ignore") + allowed = set(item.get("allowed_actions") or []) + if action not in allowed: + raise AddressBookError( + f'Action {action!r} is not allowed for vCard "{item.get("display_name") or source_key}".' + ) + if action == "ignore": + ignored += 1 + continue + contact = _apply_plan_item( + session, + principal, + run=run, + item=item, + action=action, + ) + if action == "create": + created_ids.append(contact.id) + else: + updated_ids.append(contact.id) + + run.status = "applied" + run.applied_at = utcnow() + run.result_evidence = { + **evidence, + "commit_hash": commit_hash, + "selection_count": len(selections), + "created_contact_ids": created_ids, + "updated_contact_ids": updated_ids, + "ignored_count": ignored, + "applied_by_account_id": principal.account_id, + "applied_at": run.applied_at.isoformat(), + "progress": { + "total": len(plan_by_key), + "completed": len(plan_by_key), + "created": len(created_ids), + "updated": len(updated_ids), + "ignored": ignored, + "failed": 0, + }, + } + run.statistics = { + **dict(run.statistics or {}), + "applied_create": len(created_ids), + "applied_update": len(updated_ids), + "applied_ignore": ignored, + } + return run + + +def cancel_vcard_batch( + session: Session, + principal: ApiPrincipal, + run_id: str, + payload: VCardBatchCancelRequest, +) -> AddressImportRun: + run = get_vcard_batch_run(session, principal, run_id) + if run.plan_hash != payload.expected_plan_hash: + raise AddressBookError("The reviewed vCard plan changed; reload the batch.") + if run.status == "cancelled": + return run + if run.status != "previewed": + raise AddressBookError("Only a previewed vCard batch can be cancelled.") + run.status = "cancelled" + run.result_evidence = { + **dict(run.result_evidence or {}), + "cancel_reason": payload.reason.strip(), + "cancelled_by_account_id": principal.account_id, + "cancelled_at": utcnow().isoformat(), + } + return run + + +def vcard_batch_payload(run: AddressImportRun) -> dict[str, Any]: + evidence = dict(run.result_evidence or {}) + progress = dict(evidence.get("progress") or _progress(run.row_count)) + return { + "id": run.id, + "address_book_id": run.address_book_id, + "status": run.status, + "input_hash": run.input_hash, + "plan_hash": run.plan_hash, + "parser_version": str(evidence.get("parser_version") or VCARD_PARSER_VERSION), + "execution_mode": str( + (run.statistics or {}).get("execution_mode") or "bounded_sync" + ), + "file_count": int((run.statistics or {}).get("files") or 0), + "card_count": run.row_count, + "statistics": dict(run.statistics or {}), + "diagnostics": list(run.diagnostics or []), + "plan": [_public_plan_item(item) for item in run.plan_data or []], + "progress": progress, + "can_apply": run.status == "previewed" and bool(run.plan_data), + "can_cancel": run.status == "previewed", + "commit_hash": evidence.get("commit_hash"), + "created_at": run.created_at, + "updated_at": run.updated_at, + "applied_at": run.applied_at, + } + + +def vcard_diagnostics_payload(run: AddressImportRun) -> dict[str, Any]: + return { + "schema_version": "1.0", + "run_id": run.id, + "status": run.status, + "input_hash": run.input_hash, + "plan_hash": run.plan_hash, + "parser_version": (run.result_evidence or {}).get("parser_version"), + "statistics": dict(run.statistics or {}), + "diagnostics": list(run.diagnostics or []), + "effects": [_public_plan_item(item) for item in run.plan_data or []], + } + + +def export_vcards( + session: Session, + principal: ApiPrincipal, + address_book_id: str, + payload: VCardExportRequest, +) -> dict[str, Any]: + book = get_visible_address_book(session, principal, address_book_id) + contacts = _export_contacts(session, principal, book.id, payload) + contacts.sort(key=lambda item: (item.display_name.casefold(), item.id)) + content = contacts_to_vcard(contacts, version=payload.version) + scope_label = { + "address_book": book.name, + "address_list": "address-list", + "contacts": "selected-contacts", + }[payload.scope] + return { + "filename": f"{_safe_filename(scope_label)}-{payload.version.replace('.', '')}.vcf", + "media_type": "text/vcard", + "scope": payload.scope, + "version": payload.version, + "ordering": "display_name_casefold_then_contact_id", + "contact_count": len(contacts), + "content_hash": hashlib.sha256(content.encode("utf-8")).hexdigest(), + "content": content, + } + + +def _decode_files(payload: VCardBatchPreviewRequest) -> list[tuple[str, bytes]]: + decoded: list[tuple[str, bytes]] = [] + total = 0 + for item in payload.files: + filename = item.filename.strip() + if not filename.casefold().endswith(".vcf"): + raise AddressBookError("vCard batch uploads accept only .vcf files.") + try: + raw = base64.b64decode(item.content_base64, validate=True) + except (binascii.Error, ValueError) as exc: + raise AddressBookError( + f'vCard file "{filename}" is not valid base64.' + ) from exc + if not raw: + raise AddressBookError(f'vCard file "{filename}" is empty.') + total += len(raw) + if total > MAX_VCARD_BATCH_BYTES: + raise AddressBookError( + f"Combined vCard uploads are limited to {MAX_VCARD_BATCH_BYTES} bytes." + ) + decoded.append((filename, raw)) + return decoded + + +def _parse_files( + decoded: list[tuple[str, bytes]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + cards: list[dict[str, Any]] = [] + diagnostics: list[dict[str, Any]] = [] + for file_index, (filename, raw) in enumerate(decoded): + try: + content = raw.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise AddressBookError( + f'vCard file "{filename}" is not valid UTF-8: {exc}.' + ) from exc + remaining = MAX_VCARD_CARDS - len(cards) + if remaining < 1: + raise AddressBookError( + f"vCard batches are limited to {MAX_VCARD_CARDS} cards." + ) + result = parse_vcards_with_issues(content, max_cards=remaining) + for issue in result.issues: + diagnostics.append( + { + "severity": issue.severity, + "code": "vcard_parse_error" + if issue.severity == "error" + else "vcard_parse_warning", + "message": issue.message, + "source_filename": filename, + "card_index": issue.index or None, + "field": issue.field, + "details": {"line": issue.line} if issue.line is not None else {}, + } + ) + for card_index, parsed in enumerate(result.cards, start=1): + raw_hash = hashlib.sha256(parsed.raw.encode("utf-8")).hexdigest() + identity = ( + f"uid:{parsed.source_ref.strip()}" + if parsed.source_ref and parsed.source_ref.strip() + else f"sha256:{raw_hash}" + ) + source_key = hashlib.sha256( + f"{file_index}:{filename}:{card_index}:{raw_hash}".encode("utf-8") + ).hexdigest() + cards.append( + { + "source_key": source_key, + "source_identity": identity, + "source_filename": filename, + "card_index": card_index, + "raw": parsed.raw, + "source_ref": parsed.source_ref.strip() + if parsed.source_ref + else None, + "source_revision": parsed.source_revision.strip() + if parsed.source_revision + else None, + "payload": parsed.payload.model_dump(mode="json"), + } + ) + return cards, diagnostics + + +def _plan_cards( + session: Session, + address_book_id: str, + cards: list[dict[str, Any]], + *, + duplicate_card_policy: str, + existing_contact_policy: str, +) -> list[dict[str, Any]]: + contacts = ( + session.query(Contact) + .options( + selectinload(Contact.emails), + selectinload(Contact.phones), + selectinload(Contact.postal_addresses), + ) + .filter( + Contact.address_book_id == address_book_id, Contact.deleted_at.is_(None) + ) + .all() + ) + by_source: dict[str, list[Contact]] = defaultdict(list) + by_email: dict[str, list[Contact]] = defaultdict(list) + for contact in contacts: + if contact.source_ref: + by_source[contact.source_ref.strip()].append(contact) + for email in contact.emails: + normalized = (email.normalized_email or email.email).strip().casefold() + if normalized: + by_email[normalized].append(contact) + + identity_positions: dict[str, list[int]] = defaultdict(list) + for index, card in enumerate(cards): + identity_positions[str(card["source_identity"])].append(index) + result: list[dict[str, Any]] = [] + for index, card in enumerate(cards): + positions = identity_positions[str(card["source_identity"])] + if len(positions) > 1: + chosen = positions[0] if duplicate_card_policy == "first" else positions[-1] + if duplicate_card_policy == "reject": + result.append( + _planned_card( + card, + action="conflict", + allowed=["ignore"], + message="Duplicate UID or identical card appears in this batch.", + ) + ) + continue + if index != chosen: + result.append( + _planned_card( + card, + action="ignore", + allowed=["ignore"], + message=f"Duplicate card ignored by {duplicate_card_policy} policy.", + ) + ) + continue + + suggestions = _duplicate_candidates( + card, by_source=by_source, by_email=by_email + ) + exact_source = [item for item in suggestions if "source_uid" in item["reasons"]] + candidates = exact_source or suggestions + if len(candidates) > 1: + result.append( + _planned_card( + card, + action="conflict", + allowed=["create", "ignore"], + suggestions=suggestions, + message="Multiple existing contacts match this card; create explicitly or ignore it.", + ) + ) + continue + existing = next( + ( + contact + for contact in contacts + if candidates and contact.id == candidates[0]["contact_id"] + ), + None, + ) + if existing is None: + result.append( + _planned_card( + card, + action="create", + allowed=["create", "ignore"], + suggestions=suggestions, + ) + ) + continue + changed = _changed_fields(existing, card["payload"]) + if not changed: + result.append( + _planned_card( + card, + action="unchanged", + allowed=["ignore"], + contact=existing, + suggestions=suggestions, + message="Existing contact already matches the parsed card.", + ) + ) + elif existing_contact_policy == "reject": + result.append( + _planned_card( + card, + action="conflict", + allowed=["ignore"], + contact=existing, + suggestions=suggestions, + changed=changed, + message="An existing contact matches and the preview policy rejects updates.", + ) + ) + elif existing_contact_policy == "ignore": + result.append( + _planned_card( + card, + action="ignore", + allowed=["update", "ignore"], + contact=existing, + suggestions=suggestions, + changed=changed, + message="Existing contact is ignored by preview policy.", + ) + ) + else: + result.append( + _planned_card( + card, + action="update", + allowed=["update", "ignore"], + contact=existing, + suggestions=suggestions, + changed=changed, + ) + ) + return result + + +def _planned_card( + card: dict[str, Any], + *, + action: str, + allowed: list[str], + contact: Contact | None = None, + suggestions: list[dict[str, Any]] | None = None, + changed: list[str] | None = None, + message: str | None = None, +) -> dict[str, Any]: + return { + **card, + "row_number": int(card["card_index"]), + "action": action, + "allowed_actions": allowed, + "contact_id": contact.id if contact is not None else None, + "expected_contact_hash": _contact_hash(contact) + if contact is not None + else None, + "display_name": card["payload"].get("display_name"), + "changed_fields": changed or [], + "duplicate_suggestions": suggestions or [], + "message": message, + } + + +def _duplicate_candidates( + card: dict[str, Any], + *, + by_source: dict[str, list[Contact]], + by_email: dict[str, list[Contact]], +) -> list[dict[str, Any]]: + reasons: dict[str, set[str]] = defaultdict(set) + contacts: dict[str, Contact] = {} + source_ref = card.get("source_ref") + if source_ref: + for contact in by_source.get(str(source_ref), []): + contacts[contact.id] = contact + reasons[contact.id].add("source_uid") + for item in card["payload"].get("emails") or []: + normalized = str(item.get("email") or "").strip().casefold() + for contact in by_email.get(normalized, []): + contacts[contact.id] = contact + reasons[contact.id].add("email") + return [ + { + "contact_id": contact_id, + "display_name": contacts[contact_id].display_name, + "reasons": sorted(reasons[contact_id]), + } + for contact_id in sorted( + contacts, key=lambda item: (contacts[item].display_name.casefold(), item) + )[:5] + ] + + +def _apply_plan_item( + session: Session, + principal: ApiPrincipal, + *, + run: AddressImportRun, + item: dict[str, Any], + action: str, +) -> Contact: + contact_payload = ContactCreateRequest.model_validate(item["payload"]) + if action == "create": + if item.get("source_ref"): + appeared = ( + session.query(Contact) + .filter( + Contact.address_book_id == run.address_book_id, + Contact.source_ref == item["source_ref"], + Contact.deleted_at.is_(None), + ) + .first() + ) + if appeared is not None: + raise AddressBookError( + "A matching vCard UID appeared after preview; preview the batch again." + ) + contact = create_contact( + session, principal, run.address_book_id, contact_payload + ) + else: + contact_id = str(item.get("contact_id") or "") + if not contact_id: + raise AddressBookError( + "The reviewed vCard update has no stable target contact." + ) + current = get_visible_contact(session, principal, contact_id) + if _contact_hash(current) != item.get("expected_contact_hash"): + raise AddressBookError( + f'Contact "{current.display_name}" changed after preview; preview the batch again.' + ) + contact = update_contact( + session, + principal, + current.id, + ContactUpdateRequest.model_validate(item["payload"]), + ) + contact.source_kind = "vcard" + contact.source_ref = ( + item.get("source_ref") + or f"vcard-sha256:{str(item['source_identity']).split(':', 1)[-1]}" + ) + contact.source_payload_kind = "vcard" + contact.source_payload_raw = item["raw"] + contact.source_revision = item.get("source_revision") + provenance = dict(contact.provenance or {}) + provenance["vcard_batch"] = { + "run_id": run.id, + "input_hash": run.input_hash, + "plan_hash": run.plan_hash, + "parser_version": VCARD_PARSER_VERSION, + "source_filename": item["source_filename"], + "card_index": item["card_index"], + } + contact.provenance = provenance + session.flush() + return contact + + +def _changed_fields(contact: Contact, payload: dict[str, Any]) -> list[str]: + current = _contact_projection(contact) + incoming = _payload_projection(payload) + return sorted(key for key in incoming if current.get(key) != incoming.get(key)) + + +def _contact_hash(contact: Contact) -> str: + return _hash_json(_contact_projection(contact)) + + +def _contact_projection(contact: Contact) -> dict[str, Any]: + return { + "display_name": contact.display_name, + "given_name": contact.given_name, + "family_name": contact.family_name, + "organization": contact.organization, + "role_title": contact.role_title, + "note": contact.note, + "tags": list(contact.tags or []), + "emails": [ + {"label": item.label, "email": item.email, "is_primary": item.is_primary} + for item in contact.emails + ], + "phones": [ + {"label": item.label, "phone": item.phone, "is_primary": item.is_primary} + for item in contact.phones + ], + "postal_addresses": [ + { + "label": item.label, + "street": item.street, + "postal_code": item.postal_code, + "locality": item.locality, + "region": item.region, + "country": item.country, + "is_primary": item.is_primary, + } + for item in contact.postal_addresses + ], + } + + +def _payload_projection(payload: dict[str, Any]) -> dict[str, Any]: + return { + key: payload.get(key) + for key in ( + "display_name", + "given_name", + "family_name", + "organization", + "role_title", + "note", + "tags", + "emails", + "phones", + "postal_addresses", + ) + } + + +def _selection_map(payload: VCardBatchCommitRequest) -> dict[str, str]: + result: dict[str, str] = {} + for selection in payload.selections: + if selection.source_key in result: + raise AddressBookError("Each vCard may be selected only once.") + result[selection.source_key] = selection.action + return result + + +def _public_plan_item(item: dict[str, Any]) -> dict[str, Any]: + return { + key: item.get(key) + for key in ( + "source_key", + "source_filename", + "card_index", + "action", + "allowed_actions", + "contact_id", + "display_name", + "changed_fields", + "duplicate_suggestions", + "message", + ) + } + + +def _export_contacts( + session: Session, + principal: ApiPrincipal, + address_book_id: str, + payload: VCardExportRequest, +) -> list[Contact]: + if payload.scope == "address_book": + return _loaded_contacts(session, address_book_id=address_book_id) + if payload.scope == "contacts": + contacts = [ + get_visible_contact(session, principal, contact_id) + for contact_id in payload.contact_ids + ] + if any(contact.address_book_id != address_book_id for contact in contacts): + raise AddressBookError( + "Every selected contact must belong to the exported address book." + ) + return contacts + address_list = get_visible_address_list( + session, principal, str(payload.address_list_id) + ) + if address_list.address_book_id != address_book_id: + raise AddressBookError( + "The selected address list does not belong to the exported address book." + ) + contact_ids = [ + item.contact_id + for item in ( + session.query(AddressListEntry) + .filter(AddressListEntry.address_list_id == address_list.id) + .order_by(AddressListEntry.order_index.asc(), AddressListEntry.id.asc()) + .all() + ) + ] + if not contact_ids: + return [] + return _loaded_contacts( + session, address_book_id=address_book_id, contact_ids=set(contact_ids) + ) + + +def _loaded_contacts( + session: Session, + *, + address_book_id: str, + contact_ids: set[str] | None = None, +) -> list[Contact]: + query = ( + session.query(Contact) + .options( + selectinload(Contact.emails), + selectinload(Contact.phones), + selectinload(Contact.postal_addresses), + ) + .filter( + Contact.address_book_id == address_book_id, Contact.deleted_at.is_(None) + ) + ) + if contact_ids is not None: + query = query.filter(Contact.id.in_(contact_ids)) + return query.all() + + +def _visible_books(session: Session, principal: ApiPrincipal): + from govoplan_addresses.backend.service import list_address_books + + return list_address_books(session, principal) + + +def _execution_mode(card_count: int) -> str: + raw = os.getenv( + "GOVOPLAN_ADDRESSES_VCARD_JOB_THRESHOLD", str(DEFAULT_PERSISTED_BATCH_THRESHOLD) + ) + try: + threshold = max(1, min(MAX_VCARD_CARDS, int(raw))) + except ValueError: + threshold = DEFAULT_PERSISTED_BATCH_THRESHOLD + return "persisted_batch" if card_count >= threshold else "bounded_sync" + + +def _progress(total: int) -> dict[str, int]: + return { + "total": total, + "completed": 0, + "created": 0, + "updated": 0, + "ignored": 0, + "failed": 0, + } + + +def _hash_json(value: Any) -> str: + return hashlib.sha256( + json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + ).hexdigest() + + +def _safe_filename(value: str) -> str: + safe = "".join( + character if character.isalnum() or character in {"-", "_"} else "-" + for character in value.strip() + ) + return safe.strip("-")[:120] or "contacts" + + +__all__ = [ + "apply_vcard_batch", + "cancel_vcard_batch", + "export_vcards", + "get_vcard_batch_run", + "preview_vcard_batch", + "vcard_batch_payload", + "vcard_diagnostics_payload", +] diff --git a/tests/test_migrations.py b/tests/test_migrations.py index e55fbab..0094506 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -35,7 +35,7 @@ class AddressesMigrationTests(unittest.TestCase): try: with engine.connect() as connection: self.assertIn( - "c5d7e8f9a0b1", + "d6e8f9a0b1c2", set(MigrationContext.configure(connection).get_current_heads()), ) tables = set(inspect(connection).get_table_names()) diff --git a/tests/test_vcard_batches.py b/tests/test_vcard_batches.py new file mode 100644 index 0000000..b1cd904 --- /dev/null +++ b/tests/test_vcard_batches.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +import base64 +import unittest + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from govoplan_addresses.backend.db.models import ( + AddressBook, + AddressList, + AddressListEntry, + Contact, +) +from govoplan_addresses.backend.schemas import ContactCreateRequest +from govoplan_addresses.backend.service import AddressBookError, create_contact +from govoplan_addresses.backend.vcard import ( + MAX_VCARD_UNFOLDED_LINE_CHARS, + parse_vcards_with_issues, +) +from govoplan_addresses.backend.vcard_batch_schemas import ( + VCardBatchCancelRequest, + VCardBatchCommitRequest, + VCardBatchFilePayload, + VCardBatchPreviewRequest, + VCardBatchSelection, + VCardExportRequest, +) +from govoplan_addresses.backend.vcard_batches import ( + apply_vcard_batch, + cancel_vcard_batch, + export_vcards, + preview_vcard_batch, + vcard_batch_payload, +) +from govoplan_core.db.base import Base + + +class Principal: + account_id = "account-1" + group_ids = frozenset() + + @property + def tenant_id(self) -> str: + return "tenant-1" + + def has(self, scope: str) -> bool: + return scope in { + "addresses:address_book:read", + "addresses:address_book:write", + "addresses:contact:read", + "addresses:contact:write", + } + + +def vcard(uid: str, name: str, email: str) -> str: + return ( + "BEGIN:VCARD\r\n" + "VERSION:4.0\r\n" + f"UID:{uid}\r\n" + f"FN:{name}\r\n" + f"EMAIL:{email}\r\n" + "END:VCARD\r\n" + ) + + +def batch_file(filename: str, content: str) -> VCardBatchFilePayload: + return VCardBatchFilePayload( + filename=filename, + content_base64=base64.b64encode(content.encode()).decode(), + ) + + +class VCardBatchTests(unittest.TestCase): + def setUp(self) -> None: + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + self.session = sessionmaker(bind=engine, expire_on_commit=False)() + self.principal = Principal() + self.book = AddressBook( + tenant_id="tenant-1", + scope_type="tenant", + scope_id="tenant-1", + name="Batch contacts", + source_kind="local", + read_only=False, + ) + self.session.add(self.book) + self.session.flush() + + def test_multifile_preview_selective_apply_and_repeat_are_idempotent(self) -> None: + run = preview_vcard_batch( + self.session, + self.principal, + self.book.id, + VCardBatchPreviewRequest( + files=[ + batch_file( + "ada.vcf", vcard("ada-1", "Ada Lovelace", "ada@example.test") + ), + batch_file( + "grace.vcf", + vcard("grace-1", "Grace Hopper", "grace@example.test"), + ), + ] + ), + ) + + self.assertEqual(0, self.session.query(Contact).count()) + self.assertEqual("previewed", run.status) + self.assertEqual(2, run.row_count) + self.assertEqual("govoplan-vcard/2", run.result_evidence["parser_version"]) + self.assertNotIn("ada@example.test", repr(vcard_batch_payload(run))) + + selections = [ + VCardBatchSelection( + source_key=run.plan_data[0]["source_key"], action="create" + ), + VCardBatchSelection( + source_key=run.plan_data[1]["source_key"], action="ignore" + ), + ] + request = VCardBatchCommitRequest( + expected_plan_hash=run.plan_hash, selections=selections + ) + applied = apply_vcard_batch(self.session, self.principal, run.id, request) + repeated = apply_vcard_batch(self.session, self.principal, run.id, request) + + self.assertIs(applied, repeated) + self.assertEqual(1, self.session.query(Contact).count()) + self.assertEqual("Ada Lovelace", self.session.query(Contact).one().display_name) + self.assertEqual(1, applied.result_evidence["progress"]["ignored"]) + with self.assertRaisesRegex(AddressBookError, "different selection"): + apply_vcard_batch( + self.session, + self.principal, + run.id, + VCardBatchCommitRequest( + expected_plan_hash=run.plan_hash, + selections=[ + VCardBatchSelection( + source_key=run.plan_data[1]["source_key"], action="create" + ) + ], + ), + ) + + def test_duplicate_uid_policy_and_cancellation(self) -> None: + run = preview_vcard_batch( + self.session, + self.principal, + self.book.id, + VCardBatchPreviewRequest( + files=[ + batch_file( + "duplicates.vcf", + vcard("same", "First", "first@example.test") + + vcard("same", "Last", "last@example.test"), + ) + ], + duplicate_card_policy="reject", + ), + ) + self.assertEqual( + ["conflict", "conflict"], [item["action"] for item in run.plan_data] + ) + cancelled = cancel_vcard_batch( + self.session, + self.principal, + run.id, + VCardBatchCancelRequest( + expected_plan_hash=run.plan_hash, + reason="Operator rejected duplicate source UIDs.", + ), + ) + self.assertEqual("cancelled", cancelled.status) + self.assertEqual(0, self.session.query(Contact).count()) + + last = preview_vcard_batch( + self.session, + self.principal, + self.book.id, + VCardBatchPreviewRequest( + files=[ + batch_file( + "duplicates.vcf", + vcard("same", "First", "first@example.test") + + vcard("same", "Last", "last@example.test"), + ) + ], + duplicate_card_policy="last", + ), + ) + self.assertEqual( + ["ignore", "create"], [item["action"] for item in last.plan_data] + ) + + def test_deterministic_scoped_export_supports_vcard_versions(self) -> None: + grace = create_contact( + self.session, + self.principal, + self.book.id, + ContactCreateRequest(display_name="Grace Hopper"), + ) + ada = create_contact( + self.session, + self.principal, + self.book.id, + ContactCreateRequest(display_name="Ada Lovelace"), + ) + address_list = AddressList( + tenant_id="tenant-1", + address_book_id=self.book.id, + name="Selected", + source_kind="local", + read_only=False, + ) + self.session.add(address_list) + self.session.flush() + self.session.add( + AddressListEntry( + address_list_id=address_list.id, contact_id=grace.id, order_index=0 + ) + ) + self.session.flush() + + selected = export_vcards( + self.session, + self.principal, + self.book.id, + VCardExportRequest( + scope="contacts", contact_ids=[grace.id, ada.id], version="3.0" + ), + ) + repeated = export_vcards( + self.session, + self.principal, + self.book.id, + VCardExportRequest( + scope="contacts", contact_ids=[ada.id, grace.id], version="3.0" + ), + ) + listed = export_vcards( + self.session, + self.principal, + self.book.id, + VCardExportRequest(scope="address_list", address_list_id=address_list.id), + ) + + self.assertEqual(selected["content_hash"], repeated["content_hash"]) + self.assertLess( + selected["content"].index("Ada Lovelace"), + selected["content"].index("Grace Hopper"), + ) + self.assertIn("VERSION:3.0", selected["content"]) + self.assertEqual(1, listed["contact_count"]) + self.assertIn("Grace Hopper", listed["content"]) + + def test_parser_rejects_pathological_unfolded_lines(self) -> None: + result = parse_vcards_with_issues( + "BEGIN:VCARD\nFN:" + + ("a" * (MAX_VCARD_UNFOLDED_LINE_CHARS + 1)) + + "\nEND:VCARD" + ) + self.assertEqual([], result.cards) + self.assertIn("unfolded lines", result.issues[0].message) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/src/api/addresses.ts b/webui/src/api/addresses.ts index 900398f..5fa494c 100644 --- a/webui/src/api/addresses.ts +++ b/webui/src/api/addresses.ts @@ -354,6 +354,60 @@ export type VCardImportResult = { issues: Array<{ index: number; message: string; severity: "warning" | "error"; field?: string | null; line?: number | null }>; }; +export type VCardBatchPlanItem = { + source_key: string; + source_filename: string; + card_index: number; + action: "create" | "update" | "ignore" | "unchanged" | "conflict"; + allowed_actions: Array<"create" | "update" | "ignore">; + contact_id?: string | null; + display_name?: string | null; + changed_fields: string[]; + duplicate_suggestions: Array<{ contact_id: string; display_name: string; reasons: string[] }>; + message?: string | null; +}; + +export type VCardBatchRun = { + id: string; + address_book_id: string; + status: string; + input_hash: string; + plan_hash: string; + parser_version: string; + execution_mode: "bounded_sync" | "persisted_batch"; + file_count: number; + card_count: number; + statistics: Record; + diagnostics: Array<{ + severity: "info" | "warning" | "error"; + code: string; + message: string; + source_filename?: string | null; + card_index?: number | null; + field?: string | null; + details: Record; + }>; + plan: VCardBatchPlanItem[]; + progress: { total: number; completed: number; created: number; updated: number; ignored: number; failed: number }; + can_apply: boolean; + can_cancel: boolean; + commit_hash?: string | null; + created_at: string; + updated_at: string; + applied_at?: string | null; +}; + +export type VCardExportResult = { + filename: string; + media_type: string; + scope: "address_book" | "address_list" | "contacts"; + version: "3.0" | "4.0"; + ordering: string; + contact_count: number; + content_hash: string; + content: string; +}; + export type AddressSyncSource = { id: string; tenant_id?: string | null; @@ -538,7 +592,7 @@ export type AddressImportDiagnostic = { export type AddressImportRun = { id: string; address_book_id: string; - profile_id: string; + profile_id: string | null; source_filename: string; source_format: string; input_hash: string; @@ -1057,6 +1111,47 @@ export function importAddressBookVcards(settings: ApiSettings, addressBookId: st }); } +export function previewVCardBatch( + settings: ApiSettings, + addressBookId: string, + payload: { + files: Array<{ filename: string; content_base64: string }>; + duplicate_card_policy?: "reject" | "first" | "last"; + existing_contact_policy?: "update" | "ignore" | "reject"; + } +): Promise { + return apiFetch(settings, `/api/v1/addresses/address-books/${addressBookId}/vcard-batches/preview`, { + method: "POST", + body: JSON.stringify(payload) + }); +} + +export function getVCardBatch(settings: ApiSettings, runId: string): Promise { + return apiFetch(settings, `/api/v1/addresses/vcard-batches/${encodeURIComponent(runId)}`); +} + +export function applyVCardBatch( + settings: ApiSettings, + run: VCardBatchRun, + selections: Array<{ source_key: string; action: "create" | "update" | "ignore" }> +): Promise { + return apiFetch(settings, `/api/v1/addresses/vcard-batches/${encodeURIComponent(run.id)}/apply`, { + method: "POST", + body: JSON.stringify({ expected_plan_hash: run.plan_hash, selections }) + }); +} + +export function cancelVCardBatch(settings: ApiSettings, run: VCardBatchRun, reason: string): Promise { + return apiFetch(settings, `/api/v1/addresses/vcard-batches/${encodeURIComponent(run.id)}/cancel`, { + method: "POST", + body: JSON.stringify({ expected_plan_hash: run.plan_hash, reason }) + }); +} + +export function exportVCardBatchDiagnostics(settings: ApiSettings, runId: string): Promise { + return apiFetch(settings, `/api/v1/addresses/vcard-batches/${encodeURIComponent(runId)}/diagnostics`); +} + export async function listAddressImportProfiles(settings: ApiSettings): Promise { const response = await apiFetch(settings, "/api/v1/addresses/import-profiles"); return response.profiles; @@ -1123,6 +1218,22 @@ export function exportAddressBookVcards(settings: ApiSettings, addressBookId: st return apiFetch(settings, `/api/v1/addresses/address-books/${addressBookId}/vcards/export`); } +export function exportScopedVcards( + settings: ApiSettings, + addressBookId: string, + payload: { + scope: "address_book" | "address_list" | "contacts"; + address_list_id?: string | null; + contact_ids?: string[]; + version?: "3.0" | "4.0"; + } +): Promise { + return apiFetch(settings, `/api/v1/addresses/address-books/${addressBookId}/vcards/export`, { + method: "POST", + body: JSON.stringify(payload) + }); +} + export function exportContactVcard(settings: ApiSettings, contactId: string): Promise { return apiFetch(settings, `/api/v1/addresses/contacts/${contactId}/vcard`); } diff --git a/webui/src/features/addressbook/AddressBookPage.tsx b/webui/src/features/addressbook/AddressBookPage.tsx index 059f050..2bbbd26 100644 --- a/webui/src/features/addressbook/AddressBookPage.tsx +++ b/webui/src/features/addressbook/AddressBookPage.tsx @@ -42,6 +42,9 @@ import { createContact, createContactChannelRule, createContactQualityDecision, + applyAddressImport, + applyVCardBatch, + cancelVCardBatch, deleteAddressBook, deleteAddressList, deleteAddressListEntry, @@ -50,11 +53,9 @@ import { discoverCardDavAddressBooks, discoverLdapBaseDns, endContactChannelRule, - exportAddressBookVcards, - exportContactVcard, + exportScopedVcards, getAddressImportRun, - importAddressBookVcards, - applyAddressImport, + getVCardBatch, getAddressQualitySummary, listAddressBooks, listAddressImportProfiles, @@ -73,6 +74,7 @@ import { listContactProvenance, previewAddressSyncSource, previewAddressImport, + previewVCardBatch, rollbackAddressImport, mergeContacts, recoverContactMerge, @@ -109,7 +111,8 @@ import { type ContactFieldProvenance, type ContactMergeRecord, type ContactPointQualityDecision, - type ContactPointQualityState + type ContactPointQualityState, + type VCardBatchRun } from "../../api/addresses"; import { ADDRESS_FIELDS_DOCUMENTATION, @@ -905,10 +908,6 @@ function contactFormHasIdentity(form: ContactFormState): boolean { ); } -function safeFilename(value: string): string { - return (value.trim().replace(/[^A-Za-z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "") || "address-book") + ".vcf"; -} - function downloadText(filename: string, content: string, type = "text/vcard;charset=utf-8") { const blob = new Blob([content], { type }); const url = window.URL.createObjectURL(blob); @@ -1003,7 +1002,12 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) const [dropTargetListId, setDropTargetListId] = useState(""); const [importOpen, setImportOpen] = useState(false); const [importMode, setImportMode] = useState("vcard"); - const [vcardContent, setVcardContent] = useState(""); + const [vcardFiles, setVcardFiles] = useState([]); + const [vcardRun, setVcardRun] = useState(null); + const [vcardSelections, setVcardSelections] = useState>({}); + const [vcardDuplicatePolicy, setVcardDuplicatePolicy] = useState<"reject" | "first" | "last">("reject"); + const [vcardExistingPolicy, setVcardExistingPolicy] = useState<"update" | "ignore" | "reject">("update"); + const [vcardExportVersion, setVcardExportVersion] = useState<"3.0" | "4.0">("4.0"); const [importProfiles, setImportProfiles] = useState([]); const [selectedImportProfileId, setSelectedImportProfileId] = useState(""); const [creatingImportProfile, setCreatingImportProfile] = useState(false); @@ -1305,10 +1309,16 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) ] ); const dialogCancelReason = disabledReason([saving, savingReason]); - const vcardImportReason = disabledReason( + const vcardPreviewReason = disabledReason( [saving, savingReason], [!selectedBook, "Select an address book before importing vCards."], - [!vcardContent.trim(), "Paste vCard content before importing."] + [vcardFiles.length === 0, "Select one or more .vcf files before previewing."] + ); + const vcardApplyReason = disabledReason( + [saving, savingReason], + [!vcardRun, "Preview the vCard files before applying."], + [!vcardRun?.can_apply, "This vCard batch is no longer pending."], + [Object.keys(vcardSelections).length === 0, "Select an action for at least one card."] ); const importProfileSaveReason = disabledReason( [saving, savingReason], @@ -2288,8 +2298,11 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) setError(""); setNotice(""); try { - const content = await exportAddressBookVcards(settings, selectedBook.id); - downloadText(safeFilename(selectedBook.name), content); + const result = await exportScopedVcards(settings, selectedBook.id, selectedList + ? { scope: "address_list", address_list_id: selectedList.id, version: vcardExportVersion } + : { scope: "address_book", version: vcardExportVersion }); + downloadText(result.filename, result.content); + setNotice(`Exported ${result.contact_count} contact${result.contact_count === 1 ? "" : "s"} as vCard ${result.version}.`); } catch (err) { setError(errorMessage(err)); } finally { @@ -2302,8 +2315,12 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) setError(""); setNotice(""); try { - const content = await exportContactVcard(settings, contact.id); - downloadText(safeFilename(contact.display_name), content); + const result = await exportScopedVcards(settings, contact.address_book_id, { + scope: "contacts", + contact_ids: [contact.id], + version: vcardExportVersion + }); + downloadText(result.filename, result.content); } catch (err) { setError(errorMessage(err)); } finally { @@ -2311,18 +2328,58 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) } } - async function submitVcardImport(event: FormEvent) { - event.preventDefault(); + async function previewSelectedVcards() { if (!selectedBook) return; setSaving(true); setError(""); setNotice(""); try { - const result = await importAddressBookVcards(settings, selectedBook.id, vcardContent); - setImportOpen(false); - setVcardContent(""); - const issueCount = result.issues.length; - setNotice(`Imported ${result.imported} contact${result.imported === 1 ? "" : "s"}${result.skipped ? `, skipped ${result.skipped}` : ""}${issueCount ? ` (${issueCount} import issue${issueCount === 1 ? "" : "s"})` : ""}.`); + const files = await Promise.all(vcardFiles.map(async (file) => ({ + filename: file.name, + content_base64: await fileAsBase64(file) + }))); + const run = await previewVCardBatch(settings, selectedBook.id, { + files, + duplicate_card_policy: vcardDuplicatePolicy, + existing_contact_policy: vcardExistingPolicy + }); + setVcardRun(run); + setVcardSelections(Object.fromEntries(run.plan.map((item) => { + const action = item.allowed_actions.includes(item.action as "create" | "update" | "ignore") + ? item.action as "create" | "update" | "ignore" + : "ignore"; + return [item.source_key, action]; + }))); + setNotice(`Previewed ${run.card_count} vCard${run.card_count === 1 ? "" : "s"} without changing contacts.`); + } catch (err) { + setError(errorMessage(err)); + } finally { + setSaving(false); + } + } + + async function reloadVcardRun() { + if (!vcardRun) return; + setSaving(true); + setError(""); + try { + setVcardRun(await getVCardBatch(settings, vcardRun.id)); + } catch (err) { + setError(errorMessage(err)); + } finally { + setSaving(false); + } + } + + async function applySelectedVcards() { + if (!selectedBook || !vcardRun) return; + setSaving(true); + setError(""); + setNotice(""); + try { + const run = await applyVCardBatch(settings, vcardRun, Object.entries(vcardSelections).map(([source_key, action]) => ({ source_key, action }))); + setVcardRun(run); + setNotice(`Applied vCard batch: ${run.progress.created} created, ${run.progress.updated} updated, ${run.progress.ignored} ignored.`); await refreshBooks(); await refreshContacts(selectedBook.id, query); } catch (err) { @@ -2332,9 +2389,27 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) } } + async function cancelSelectedVcardBatch() { + if (!vcardRun) return; + setSaving(true); + setError(""); + try { + const run = await cancelVCardBatch(settings, vcardRun, "Cancelled by operator before commit."); + setVcardRun(run); + setNotice("Cancelled the pending vCard batch without changing contacts."); + } catch (err) { + setError(errorMessage(err)); + } finally { + setSaving(false); + } + } + function openImportDialog() { setSearchParams(withImportRunSearch(searchParams, null), { replace: true }); setImportMode("vcard"); + setVcardFiles([]); + setVcardRun(null); + setVcardSelections({}); setImportRun(null); setImportRunUnavailable(""); setImportFile(null); @@ -2346,6 +2421,9 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) function closeImportDialog() { setImportOpen(false); + setVcardFiles([]); + setVcardRun(null); + setVcardSelections({}); setImportRun(null); setImportRunUnavailable(""); setImportRollbackOpen(false); @@ -2882,7 +2960,11 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) - + + @@ -3804,7 +3886,11 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) <> {importMode === "vcard" && - } + } + {importMode === "vcard" && vcardRun?.can_apply && + } + {importMode === "vcard" && vcardRun?.can_cancel && + } {importMode === "tabular" && creatingImportProfile && } {importMode === "tabular" && !creatingImportProfile && @@ -3825,18 +3911,74 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) onChange={(mode) => { setImportMode(mode); clearRetainedImportRun(); }} /> {importMode === "vcard" && - void submitVcardImport(event)}> -

Paste one or more vCard entries into the selected address book.

- -