feat: add selective vCard batch workflows

This commit is contained in:
2026-08-20 17:24:42 +02:00
parent 147f34c5c1
commit 42fe262376
14 changed files with 2116 additions and 158 deletions
+8 -2
View File
@@ -20,8 +20,14 @@ tenant summaries, and uninstall guards.
The first UI supports user, group, tenant, and system-scoped address books,
multi-value contact methods, soft deletion, restore, read-only lookup/search,
and vCard import/export for common contact fields. Imported vCards preserve
source payload and revision metadata for later sync/conflict work.
and vCard import/export for common contact fields. Multi-file vCard imports now
create a persisted preview before mutation, expose duplicate suggestions and
per-card create/update/ignore choices, reject stale plans, and make identical
commit retries idempotent. Pending batches can be reloaded or cancelled.
Address-book, address-list, and selected-contact exports explicitly support
vCard 3.0 or 4.0 with deterministic ordering and a recorded content hash.
Imported vCards preserve source payload and revision metadata for later
sync/conflict work, while batch diagnostics expose only bounded metadata.
The backend and WebUI also support classical address lists: reusable groupings
of contacts or specific contact methods within one address book. Campaigns can
+5
View File
@@ -26,6 +26,11 @@ imports, synchronization, quality review, and reversible merge operations.
source diagnostics and conflict state as explicitly stated.
- Imports and synchronization separate preview from apply; incomplete external
reads never infer deletions.
- vCard batch upload accepts multiple files, persists a non-mutating preview,
and requires an explicit create/update/ignore choice for every reviewed card.
Reload and cancellation preserve the pending plan; only applying the matching
plan hash mutates contacts. Scoped exports name the selected vCard version and
use deterministic ordering.
- Merge and communication-governance operations append auditable evidence and
never silently erase prior state.
- Request feedback is rendered as a compact shared alert over the full-height
+3 -3
View File
@@ -587,9 +587,9 @@ class AddressImportRun(Base, TimestampMixin):
nullable=False,
index=True,
)
profile_id: Mapped[str] = mapped_column(
profile_id: Mapped[str | None] = mapped_column(
ForeignKey("addresses_import_profiles.id", ondelete="RESTRICT"),
nullable=False,
nullable=True,
index=True,
)
source_filename: Mapped[str] = mapped_column(String(500), nullable=False)
@@ -607,7 +607,7 @@ class AddressImportRun(Base, TimestampMixin):
rolled_back_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
address_book: Mapped[AddressBook] = relationship()
profile: Mapped[AddressImportProfile] = relationship()
profile: Mapped[AddressImportProfile | None] = relationship()
__all__ = [
@@ -129,7 +129,7 @@ class AddressImportDiagnosticResponse(BaseModel):
class AddressImportRunResponse(BaseModel):
id: str
address_book_id: str
profile_id: str
profile_id: str | None
source_filename: str
source_format: str
input_hash: str
+27 -2
View File
@@ -460,6 +460,31 @@ manifest = ModuleManifest(
related_modules=("connectors", "datasources", "dataflow", "files", "audit"),
order=33,
),
DocumentationTopic(
id="addresses.vcard-batches",
title="Selective vCard batch import and export",
summary="Preview multiple vCard files, choose each card's effect, and export deterministic scoped files.",
body=(
"One or more UTF-8 .vcf files are parsed into a persisted, non-mutating preview with bounded diagnostics, "
"duplicate suggestions, an input hash, a parser version, and a deterministic plan hash. Operators choose "
"create, update, or ignore only where the reviewed plan permits it. Apply rejects stale contact targets and "
"is idempotent for the same selection; a different retry is rejected. Pending runs can be reloaded or cancelled "
"without changing contacts. Upload size, file count, card count, line count, and unfolded-line length are bounded. "
"Exports can target a complete address book, one address list, or explicit contacts; vCard 3.0 or 4.0 is selected "
"explicitly and contacts use deterministic display-name and stable-ID ordering. Export and import evidence records "
"hashes and counts, while diagnostics never disclose raw contact payloads. Large previews remain persisted and expose "
"their batch execution mode so a runtime job capability can execute them asynchronously when available."
),
layer="configured",
documentation_types=("admin", "user"),
audience=("tenant_admin", "operator", "module_admin", "power_user"),
related_modules=("files", "audit", "connectors"),
order=34,
metadata={
"seed": True,
"help_contexts": ["addresses.action.import", "addresses.contacts", "addresses.sources"],
},
),
DocumentationTopic(
id="addresses.ldap-directory",
title="LDAP and Active Directory address sources",
@@ -475,7 +500,7 @@ manifest = ModuleManifest(
documentation_types=("admin", "user"),
audience=("tenant_admin", "operator", "module_admin"),
related_modules=("connectors", "idm", "access", "policy", "audit"),
order=34,
order=35,
),
DocumentationTopic(
id="addresses.quality-and-merge",
@@ -513,7 +538,7 @@ manifest = ModuleManifest(
documentation_types=("admin", "user"),
audience=("tenant_admin", "operator", "module_admin", "power_user"),
related_modules=("dist_lists", "connectors", "datasources", "campaigns", "policy", "audit"),
order=35,
order=36,
metadata={
"seed": True,
"help_contexts": [
@@ -0,0 +1,34 @@
"""Allow profile-free persisted vCard batch runs.
Revision ID: d6e8f9a0b1c2
Revises: c5d7e8f9a0b1
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "d6e8f9a0b1c2"
down_revision = "c5d7e8f9a0b1"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("addresses_import_runs") as batch:
batch.alter_column(
"profile_id",
existing_type=sa.String(length=36),
nullable=True,
)
def downgrade() -> None:
with op.batch_alter_table("addresses_import_runs") as batch:
batch.alter_column(
"profile_id",
existing_type=sa.String(length=36),
nullable=False,
)
+350 -77
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from dataclasses import asdict
import json
import re
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
@@ -56,6 +57,23 @@ from govoplan_addresses.backend.imports import (
rollback_address_import,
update_import_profile,
)
from govoplan_addresses.backend.vcard_batch_schemas import (
VCardBatchCancelRequest,
VCardBatchCommitRequest,
VCardBatchPreviewRequest,
VCardBatchRunResponse,
VCardExportRequest,
VCardExportResponse,
)
from govoplan_addresses.backend.vcard_batches import (
apply_vcard_batch,
cancel_vcard_batch,
export_vcards,
get_vcard_batch_run,
preview_vcard_batch,
vcard_batch_payload,
vcard_diagnostics_payload,
)
from govoplan_addresses.backend.capabilities import (
AddressesContactPointResolutionCapability,
AddressesContactWriterCapability,
@@ -238,9 +256,7 @@ def _contact_response(
decision = quality.get((channel, point_id)) or quality.get((channel, None))
return {
"quality_state": decision.state if decision is not None else "valid",
"quality_reason_code": (
decision.reason_code if decision is not None else None
),
"quality_reason_code": (decision.reason_code if decision is not None else None),
}
return ContactResponse.model_validate(
@@ -323,9 +339,7 @@ def _contact_point_audit_details(
"postal": [item.id for item in contact.postal_addresses],
}
return {
f"{key_prefix}contact_point_counts": {
channel: len(ids) for channel, ids in point_ids.items()
},
f"{key_prefix}contact_point_counts": {channel: len(ids) for channel, ids in point_ids.items()},
f"{key_prefix}contact_point_ids": point_ids,
}
@@ -454,7 +468,9 @@ def _sync_source_response(sync_source: AddressSyncSource) -> AddressSyncSourceRe
)
def _sync_diagnostic_response(diagnostic: AddressSyncDiagnostic) -> AddressSyncDiagnosticResponse:
def _sync_diagnostic_response(
diagnostic: AddressSyncDiagnostic,
) -> AddressSyncDiagnosticResponse:
return AddressSyncDiagnosticResponse.model_validate(
{
"id": diagnostic.id,
@@ -470,7 +486,9 @@ def _sync_diagnostic_response(diagnostic: AddressSyncDiagnostic) -> AddressSyncD
)
def _sync_tombstone_response(tombstone: AddressSyncTombstone) -> AddressSyncTombstoneResponse:
def _sync_tombstone_response(
tombstone: AddressSyncTombstone,
) -> AddressSyncTombstoneResponse:
return AddressSyncTombstoneResponse.model_validate(
{
"id": tombstone.id,
@@ -490,7 +508,9 @@ def _sync_tombstone_response(tombstone: AddressSyncTombstone) -> AddressSyncTomb
)
def _sync_conflict_response(conflict: AddressSyncConflict) -> AddressSyncConflictResponse:
def _sync_conflict_response(
conflict: AddressSyncConflict,
) -> AddressSyncConflictResponse:
return AddressSyncConflictResponse.model_validate(
{
"id": conflict.id,
@@ -583,7 +603,11 @@ def api_list_address_books(
return AddressBookListResponse(address_books=[_book_response(book, contact_count=counts.get(book.id, 0)) for book in books])
@router.post("/address-books", response_model=AddressBookResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/address-books",
response_model=AddressBookResponse,
status_code=status.HTTP_201_CREATED,
)
def api_create_address_book(
payload: AddressBookCreateRequest,
principal: ApiPrincipal = Depends(get_api_principal),
@@ -591,7 +615,12 @@ def api_create_address_book(
):
_require_scope(principal, "addresses:address_book:write")
try:
book = create_address_book(session, principal, payload, allow_system=has_scope(principal, "addresses:address_book:admin"))
book = create_address_book(
session,
principal,
payload,
allow_system=has_scope(principal, "addresses:address_book:admin"),
)
session.commit()
session.refresh(book)
return _book_response(book)
@@ -738,10 +767,7 @@ def api_suggest_duplicate_contacts(
right=_contact_response(item.right),
score=item.score,
confidence=item.confidence,
features=[
ContactDuplicateFeatureResponse(**asdict(feature))
for feature in item.features
],
features=[ContactDuplicateFeatureResponse(**asdict(feature)) for feature in item.features],
)
for item in scan.suggestions
],
@@ -778,10 +804,7 @@ def api_address_quality_summary(
quality_counts=summary.quality_counts,
duplicate_suggestion_count=summary.duplicate_suggestion_count,
correction_count=summary.correction_count,
corrections=[
AddressQualityCorrectionResponse(**asdict(item))
for item in summary.corrections
],
corrections=[AddressQualityCorrectionResponse(**asdict(item)) for item in summary.corrections],
truncated=summary.truncated,
)
except AddressBookError as exc:
@@ -895,9 +918,7 @@ def api_resolve_contact_redirect(
):
_require_scope(principal, "addresses:contact:read")
try:
return ContactRedirectResponse.model_validate(
asdict(resolve_contact_redirect(session, principal, contact_id))
)
return ContactRedirectResponse.model_validate(asdict(resolve_contact_redirect(session, principal, contact_id)))
except AddressBookError as exc:
raise _error(exc) from exc
@@ -1167,16 +1188,23 @@ def api_list_address_lists(
):
_require_scope(principal, "addresses:address_list:read")
try:
address_lists = list_address_lists(session, principal, address_book_id=address_book_id, include_deleted=include_deleted)
counts = address_list_entry_counts(session, [address_list.id for address_list in address_lists])
return AddressListListResponse(
address_lists=[_address_list_response(address_list, entry_count=counts.get(address_list.id, 0)) for address_list in address_lists]
address_lists = list_address_lists(
session,
principal,
address_book_id=address_book_id,
include_deleted=include_deleted,
)
counts = address_list_entry_counts(session, [address_list.id for address_list in address_lists])
return AddressListListResponse(address_lists=[_address_list_response(address_list, entry_count=counts.get(address_list.id, 0)) for address_list in address_lists])
except AddressBookError as exc:
raise _error(exc) from exc
@router.post("/address-books/{book_id}/address-lists", response_model=AddressListResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/address-books/{book_id}/address-lists",
response_model=AddressListResponse,
status_code=status.HTTP_201_CREATED,
)
def api_create_address_list(
book_id: str,
payload: AddressListCreateRequest,
@@ -1247,7 +1275,10 @@ def api_restore_address_list(
raise _error(exc) from exc
@router.get("/address-lists/{address_list_id}/entries", response_model=AddressListEntryListResponse)
@router.get(
"/address-lists/{address_list_id}/entries",
response_model=AddressListEntryListResponse,
)
def api_list_address_list_entries(
address_list_id: str,
principal: ApiPrincipal = Depends(get_api_principal),
@@ -1261,7 +1292,11 @@ def api_list_address_list_entries(
raise _error(exc) from exc
@router.post("/address-lists/{address_list_id}/entries", response_model=AddressListEntryResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/address-lists/{address_list_id}/entries",
response_model=AddressListEntryResponse,
status_code=status.HTTP_201_CREATED,
)
def api_create_address_list_entry(
address_list_id: str,
payload: AddressListEntryCreateRequest,
@@ -1332,7 +1367,10 @@ def api_list_write_targets(
return AddressBookWriteTargetsResponse(targets=[_write_decision_response(decision) for decision in decisions])
@router.get("/address-books/{book_id}/write-decision", response_model=AddressBookWriteDecisionResponse)
@router.get(
"/address-books/{book_id}/write-decision",
response_model=AddressBookWriteDecisionResponse,
)
def api_get_address_book_write_decision(
book_id: str,
operation: str = Query(default="create_contact"),
@@ -1378,9 +1416,7 @@ def api_discover_ldap_base_dns(
):
_require_scope(principal, "addresses:sync:write")
try:
return AddressLdapDiscoveryResponse(
base_dns=list(discover_ldap_base_dns(session, principal, payload))
)
return AddressLdapDiscoveryResponse(base_dns=list(discover_ldap_base_dns(session, principal, payload)))
except (AddressBookError, AddressLdapError) as exc:
raise _error(AddressBookError(str(exc))) from exc
@@ -1438,7 +1474,11 @@ def api_list_address_credentials(
)
@router.post("/address-books/{book_id}/carddav/sources", response_model=AddressSyncSourceResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/address-books/{book_id}/carddav/sources",
response_model=AddressSyncSourceResponse,
status_code=status.HTTP_201_CREATED,
)
def api_create_carddav_sync_source(
book_id: str,
payload: AddressCardDavSourceCreateRequest,
@@ -1454,7 +1494,11 @@ def api_create_carddav_sync_source(
action="addresses.sync_source_created",
object_type="address_sync_source",
object_id=sync_source.id,
details={"address_book_id": book_id, "connector_type": "carddav", "sync_direction": sync_source.sync_direction},
details={
"address_book_id": book_id,
"connector_type": "carddav",
"sync_direction": sync_source.sync_direction,
},
)
session.commit()
session.refresh(sync_source)
@@ -1473,7 +1517,12 @@ def api_list_sync_sources(
):
_require_scope(principal, "addresses:sync:read")
try:
sync_sources = list_sync_sources(session, principal, address_book_id=address_book_id, include_disabled=include_disabled)
sync_sources = list_sync_sources(
session,
principal,
address_book_id=address_book_id,
include_disabled=include_disabled,
)
return AddressSyncSourceListResponse(sync_sources=[_sync_source_response(sync_source) for sync_source in sync_sources])
except AddressBookError as exc:
raise _error(exc) from exc
@@ -1575,7 +1624,11 @@ def api_run_sync_source(
raise _error(AddressBookError(message)) from exc
@router.post("/address-books/{book_id}/sync-sources", response_model=AddressSyncSourceResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/address-books/{book_id}/sync-sources",
response_model=AddressSyncSourceResponse,
status_code=status.HTTP_201_CREATED,
)
def api_create_sync_source(
book_id: str,
payload: AddressSyncSourceCreateRequest,
@@ -1591,7 +1644,11 @@ def api_create_sync_source(
action="addresses.sync_source_created",
object_type="address_sync_source",
object_id=sync_source.id,
details={"address_book_id": book_id, "connector_type": sync_source.connector_type, "sync_direction": sync_source.sync_direction},
details={
"address_book_id": book_id,
"connector_type": sync_source.connector_type,
"sync_direction": sync_source.sync_direction,
},
)
session.commit()
session.refresh(sync_source)
@@ -1617,7 +1674,11 @@ def api_update_sync_source(
action="addresses.sync_source_updated",
object_type="address_sync_source",
object_id=sync_source.id,
details={"connector_type": sync_source.connector_type, "sync_direction": sync_source.sync_direction, "enabled": sync_source.enabled},
details={
"connector_type": sync_source.connector_type,
"sync_direction": sync_source.sync_direction,
"enabled": sync_source.enabled,
},
)
session.commit()
session.refresh(sync_source)
@@ -1651,7 +1712,10 @@ def api_delete_sync_source(
raise _error(exc) from exc
@router.post("/sync-sources/{sync_source_id}/attempts/start", response_model=AddressSyncSourceResponse)
@router.post(
"/sync-sources/{sync_source_id}/attempts/start",
response_model=AddressSyncSourceResponse,
)
def api_start_sync_attempt(
sync_source_id: str,
principal: ApiPrincipal = Depends(get_api_principal),
@@ -1676,7 +1740,10 @@ def api_start_sync_attempt(
raise _error(exc) from exc
@router.post("/sync-sources/{sync_source_id}/attempts/finish", response_model=AddressSyncSourceResponse)
@router.post(
"/sync-sources/{sync_source_id}/attempts/finish",
response_model=AddressSyncSourceResponse,
)
def api_finish_sync_attempt(
sync_source_id: str,
payload: AddressSyncAttemptFinishRequest,
@@ -1692,7 +1759,11 @@ def api_finish_sync_attempt(
action="addresses.sync_finished",
object_type="address_sync_source",
object_id=sync_source.id,
details={"connector_type": sync_source.connector_type, "status": sync_source.status, "error": sync_source.last_error},
details={
"connector_type": sync_source.connector_type,
"status": sync_source.status,
"error": sync_source.last_error,
},
)
session.commit()
session.refresh(sync_source)
@@ -1702,7 +1773,10 @@ def api_finish_sync_attempt(
raise _error(exc) from exc
@router.get("/sync-sources/{sync_source_id}/diagnostics", response_model=AddressSyncDiagnosticListResponse)
@router.get(
"/sync-sources/{sync_source_id}/diagnostics",
response_model=AddressSyncDiagnosticListResponse,
)
def api_list_sync_diagnostics(
sync_source_id: str,
limit: int = Query(default=100, ge=1, le=500),
@@ -1717,7 +1791,11 @@ def api_list_sync_diagnostics(
raise _error(exc) from exc
@router.post("/sync-sources/{sync_source_id}/diagnostics", response_model=AddressSyncDiagnosticResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/sync-sources/{sync_source_id}/diagnostics",
response_model=AddressSyncDiagnosticResponse,
status_code=status.HTTP_201_CREATED,
)
def api_record_sync_diagnostic(
sync_source_id: str,
payload: AddressSyncDiagnosticCreateRequest,
@@ -1735,7 +1813,10 @@ def api_record_sync_diagnostic(
raise _error(exc) from exc
@router.get("/sync-sources/{sync_source_id}/tombstones", response_model=AddressSyncTombstoneListResponse)
@router.get(
"/sync-sources/{sync_source_id}/tombstones",
response_model=AddressSyncTombstoneListResponse,
)
def api_list_sync_tombstones(
sync_source_id: str,
limit: int = Query(default=200, ge=1, le=1000),
@@ -1750,7 +1831,11 @@ def api_list_sync_tombstones(
raise _error(exc) from exc
@router.post("/sync-sources/{sync_source_id}/tombstones", response_model=AddressSyncTombstoneResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/sync-sources/{sync_source_id}/tombstones",
response_model=AddressSyncTombstoneResponse,
status_code=status.HTTP_201_CREATED,
)
def api_record_sync_tombstone(
sync_source_id: str,
payload: AddressSyncTombstoneCreateRequest,
@@ -1768,7 +1853,10 @@ def api_record_sync_tombstone(
raise _error(exc) from exc
@router.get("/sync-sources/{sync_source_id}/conflicts", response_model=AddressSyncConflictListResponse)
@router.get(
"/sync-sources/{sync_source_id}/conflicts",
response_model=AddressSyncConflictListResponse,
)
def api_list_sync_conflicts(
sync_source_id: str,
status_filter: str | None = Query(default="open", alias="status"),
@@ -1784,7 +1872,11 @@ def api_list_sync_conflicts(
raise _error(exc) from exc
@router.post("/sync-sources/{sync_source_id}/conflicts", response_model=AddressSyncConflictResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/sync-sources/{sync_source_id}/conflicts",
response_model=AddressSyncConflictResponse,
status_code=status.HTTP_201_CREATED,
)
def api_record_sync_conflict(
sync_source_id: str,
payload: AddressSyncConflictCreateRequest,
@@ -1818,7 +1910,11 @@ def api_resolve_sync_conflict(
action="addresses.sync_conflict_resolved",
object_type="address_sync_conflict",
object_id=conflict.id,
details={"sync_source_id": conflict.sync_source_id, "status": conflict.status, "resolution": conflict.resolution},
details={
"sync_source_id": conflict.sync_source_id,
"status": conflict.status,
"resolution": conflict.resolution,
},
)
session.commit()
session.refresh(conflict)
@@ -1828,7 +1924,11 @@ def api_resolve_sync_conflict(
raise _error(exc) from exc
@router.post("/address-books/{book_id}/contacts", response_model=ContactResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/address-books/{book_id}/contacts",
response_model=ContactResponse,
status_code=status.HTTP_201_CREATED,
)
def api_create_contact(
book_id: str,
payload: ContactCreateRequest,
@@ -1869,11 +1969,7 @@ def api_update_contact(
_require_scope(principal, "addresses:contact:write")
try:
previous_contact = session.get(Contact, contact_id)
previous_point_details = (
_contact_point_audit_details(previous_contact, prefix="previous")
if previous_contact is not None
else {}
)
previous_point_details = _contact_point_audit_details(previous_contact, prefix="previous") if previous_contact is not None else {}
contact = update_contact(session, principal, contact_id, payload)
audit_from_principal(
session,
@@ -1897,7 +1993,10 @@ def api_update_contact(
raise _error(exc) from exc
@router.get("/contacts/{contact_id}/channel-rules", response_model=ContactChannelRuleListResponse)
@router.get(
"/contacts/{contact_id}/channel-rules",
response_model=ContactChannelRuleListResponse,
)
def api_list_contact_channel_rules(
contact_id: str,
principal: ApiPrincipal = Depends(get_api_principal),
@@ -1905,12 +2004,7 @@ def api_list_contact_channel_rules(
):
_require_scope(principal, "addresses:governance:read")
try:
return ContactChannelRuleListResponse(
rules=[
_channel_rule_response(rule)
for rule in list_contact_channel_rules(session, principal, contact_id)
]
)
return ContactChannelRuleListResponse(rules=[_channel_rule_response(rule) for rule in list_contact_channel_rules(session, principal, contact_id)])
except AddressBookError as exc:
raise _error(exc) from exc
@@ -2044,7 +2138,11 @@ def api_restore_contact(
raise _error(exc) from exc
@router.post("/address-books/{book_id}/vcards/import", response_model=VCardImportResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/address-books/{book_id}/vcards/import",
response_model=VCardImportResponse,
status_code=status.HTTP_201_CREATED,
)
def api_import_address_book_vcards(
book_id: str,
payload: VCardImportRequest,
@@ -2091,6 +2189,136 @@ def api_import_address_book_vcards(
raise _error(AddressBookError(str(exc))) from exc
@router.post(
"/address-books/{book_id}/vcard-batches/preview",
response_model=VCardBatchRunResponse,
status_code=status.HTTP_201_CREATED,
)
def api_preview_vcard_batch(
book_id: str,
payload: VCardBatchPreviewRequest,
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_scope(principal, "addresses:contact:write")
try:
run = preview_vcard_batch(session, principal, book_id, payload)
audit_from_principal(
session,
principal,
action="addresses.vcard_batch_previewed",
object_type="address_import_run",
object_id=run.id,
details={
"address_book_id": book_id,
"input_hash": run.input_hash,
"plan_hash": run.plan_hash,
"statistics": run.statistics,
},
)
session.commit()
session.refresh(run)
return VCardBatchRunResponse.model_validate(vcard_batch_payload(run))
except AddressBookError as exc:
session.rollback()
raise _error(exc) from exc
@router.get("/vcard-batches/{run_id}", response_model=VCardBatchRunResponse)
def api_get_vcard_batch(
run_id: str,
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_scope(principal, "addresses:contact:read")
try:
return VCardBatchRunResponse.model_validate(vcard_batch_payload(get_vcard_batch_run(session, principal, run_id)))
except AddressBookError as exc:
raise _error(exc) from exc
@router.post("/vcard-batches/{run_id}/apply", response_model=VCardBatchRunResponse)
def api_apply_vcard_batch(
run_id: str,
payload: VCardBatchCommitRequest,
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_scope(principal, "addresses:contact:write")
try:
run = apply_vcard_batch(session, principal, run_id, payload)
session.flush()
audit_from_principal(
session,
principal,
action="addresses.vcard_batch_applied",
object_type="address_import_run",
object_id=run.id,
details={
"address_book_id": run.address_book_id,
"input_hash": run.input_hash,
"plan_hash": run.plan_hash,
"statistics": run.statistics,
},
)
session.commit()
session.refresh(run)
return VCardBatchRunResponse.model_validate(vcard_batch_payload(run))
except AddressBookError as exc:
session.rollback()
raise _error(exc) from exc
@router.post("/vcard-batches/{run_id}/cancel", response_model=VCardBatchRunResponse)
def api_cancel_vcard_batch(
run_id: str,
payload: VCardBatchCancelRequest,
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_scope(principal, "addresses:contact:write")
try:
run = cancel_vcard_batch(session, principal, run_id, payload)
audit_from_principal(
session,
principal,
action="addresses.vcard_batch_cancelled",
object_type="address_import_run",
object_id=run.id,
details={"address_book_id": run.address_book_id, "reason": payload.reason},
)
session.commit()
session.refresh(run)
return VCardBatchRunResponse.model_validate(vcard_batch_payload(run))
except AddressBookError as exc:
session.rollback()
raise _error(exc) from exc
@router.get("/vcard-batches/{run_id}/diagnostics")
def api_export_vcard_batch_diagnostics(
run_id: str,
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_scope(principal, "addresses:contact:read")
try:
run = get_vcard_batch_run(session, principal, run_id)
content = json.dumps(
vcard_diagnostics_payload(run),
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
return Response(
content=content,
media_type="application/json",
headers={"Content-Disposition": f'attachment; filename="vcard-batch-{run.id}.json"'},
)
except AddressBookError as exc:
raise _error(exc) from exc
@router.get("/import-profiles", response_model=AddressImportProfileListResponse)
def api_list_address_import_profiles(
include_history: bool = Query(default=False),
@@ -2098,15 +2326,14 @@ def api_list_address_import_profiles(
session: Session = Depends(get_session),
):
_require_scope(principal, "addresses:contact:read")
return AddressImportProfileListResponse(
profiles=[
AddressImportProfileResponse.model_validate(item)
for item in list_import_profiles(session, principal, include_history=include_history)
]
)
return AddressImportProfileListResponse(profiles=[AddressImportProfileResponse.model_validate(item) for item in list_import_profiles(session, principal, include_history=include_history)])
@router.post("/import-profiles", response_model=AddressImportProfileResponse, status_code=status.HTTP_201_CREATED)
@router.post(
"/import-profiles",
response_model=AddressImportProfileResponse,
status_code=status.HTTP_201_CREATED,
)
def api_create_address_import_profile(
payload: AddressImportProfileCreateRequest,
principal: ApiPrincipal = Depends(get_api_principal),
@@ -2122,7 +2349,11 @@ def api_create_address_import_profile(
action="addresses.import_profile_created",
object_type="address_import_profile",
object_id=profile.profile_key,
details={"version": profile.version, "source_format": profile.source_format, "scope_type": profile.scope_type},
details={
"version": profile.version,
"source_format": profile.source_format,
"scope_type": profile.scope_type,
},
)
session.commit()
session.refresh(profile)
@@ -2149,7 +2380,10 @@ def api_update_address_import_profile(
action="addresses.import_profile_versioned",
object_type="address_import_profile",
object_id=profile.profile_key,
details={"version": profile.version, "source_format": profile.source_format},
details={
"version": profile.version,
"source_format": profile.source_format,
},
)
session.commit()
session.refresh(profile)
@@ -2202,7 +2436,11 @@ def api_preview_address_import(
action="addresses.import_previewed",
object_type="address_import_run",
object_id=run.id,
details={"input_hash": run.input_hash, "plan_hash": run.plan_hash, "statistics": run.statistics},
details={
"input_hash": run.input_hash,
"plan_hash": run.plan_hash,
"statistics": run.statistics,
},
)
session.commit()
session.refresh(run)
@@ -2220,9 +2458,7 @@ def api_get_address_import_run(
):
_require_scope(principal, "addresses:contact:read")
try:
return AddressImportRunResponse.model_validate(
import_run_payload(get_import_run(session, principal, run_id))
)
return AddressImportRunResponse.model_validate(import_run_payload(get_import_run(session, principal, run_id)))
except AddressBookError as exc:
raise _error(exc) from exc
@@ -2244,7 +2480,11 @@ def api_apply_address_import(
action="addresses.import_applied",
object_type="address_import_run",
object_id=run.id,
details={"input_hash": run.input_hash, "plan_hash": run.plan_hash, "statistics": run.statistics},
details={
"input_hash": run.input_hash,
"plan_hash": run.plan_hash,
"statistics": run.statistics,
},
)
session.commit()
session.refresh(run)
@@ -2300,6 +2540,39 @@ def api_export_address_book_vcards(
raise _error(exc) from exc
@router.post(
"/address-books/{book_id}/vcards/export",
response_model=VCardExportResponse,
)
def api_export_selected_vcards(
book_id: str,
payload: VCardExportRequest,
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_scope(principal, "addresses:contact:read")
try:
result = export_vcards(session, principal, book_id, payload)
audit_from_principal(
session,
principal,
action="addresses.vcards_exported",
object_type="address_book",
object_id=book_id,
details={
"scope": result["scope"],
"version": result["version"],
"contact_count": result["contact_count"],
"content_hash": result["content_hash"],
},
)
session.commit()
return VCardExportResponse.model_validate(result)
except AddressBookError as exc:
session.rollback()
raise _error(exc) from exc
@router.get("/contacts/{contact_id}/vcard")
def api_export_contact_vcard(
contact_id: str,
+82 -35
View File
@@ -16,6 +16,12 @@ class VCardError(ValueError):
pass
VCARD_PARSER_VERSION = "govoplan-vcard/2"
MAX_VCARD_CARDS = 10_000
MAX_VCARD_LINES = 200_000
MAX_VCARD_UNFOLDED_LINE_CHARS = 16_384
@dataclass(frozen=True, slots=True)
class ParsedVCard:
payload: ContactCreateRequest
@@ -60,12 +66,16 @@ class _VCardDraft:
def _normalize_lines(content: str) -> list[str]:
raw_lines = content.replace("\r\n", "\n").replace("\r", "\n").split("\n")
if len(raw_lines) > MAX_VCARD_LINES:
raise VCardError(f"vCard input exceeds the {MAX_VCARD_LINES}-line parser limit.")
lines: list[str] = []
for line in raw_lines:
if line.startswith((" ", "\t")) and lines:
lines[-1] += line[1:]
elif line:
lines.append(line)
if lines and len(lines[-1]) > MAX_VCARD_UNFOLDED_LINE_CHARS:
raise VCardError(f"vCard unfolded lines are limited to {MAX_VCARD_UNFOLDED_LINE_CHARS} characters.")
return lines
@@ -90,27 +100,13 @@ def _split_unescaped(value: str, separator: str) -> list[str]:
def _unescape_text(value: str) -> str:
return (
value.replace("\\n", "\n")
.replace("\\N", "\n")
.replace("\\,", ",")
.replace("\\;", ";")
.replace("\\\\", "\\")
.strip()
)
return value.replace("\\n", "\n").replace("\\N", "\n").replace("\\,", ",").replace("\\;", ";").replace("\\\\", "\\").strip()
def _escape_text(value: str | None) -> str:
if value is None:
return ""
return (
value.replace("\\", "\\\\")
.replace("\r\n", "\n")
.replace("\r", "\n")
.replace("\n", "\\n")
.replace(";", "\\;")
.replace(",", "\\,")
)
return value.replace("\\", "\\\\").replace("\r\n", "\n").replace("\r", "\n").replace("\n", "\\n").replace(";", "\\;").replace(",", "\\,")
def _parse_head(head: str) -> tuple[str, dict[str, list[str]]]:
@@ -154,7 +150,7 @@ def _is_pref(params: dict[str, list[str]]) -> bool:
def _card_blocks(content: str) -> list[list[str]]:
result = _card_blocks_with_issues(content)
result = _card_blocks_with_issues(content, max_cards=MAX_VCARD_CARDS)
if result.issues:
raise VCardError(result.issues[0].message)
return result.cards
@@ -166,8 +162,14 @@ class _CardBlockResult:
issues: list[ParsedVCardIssue]
def _card_blocks_with_issues(content: str) -> _CardBlockResult:
def _card_blocks_with_issues(content: str, *, max_cards: int) -> _CardBlockResult:
try:
lines = _normalize_lines(content)
except VCardError as exc:
return _CardBlockResult(
cards=[],
issues=[ParsedVCardIssue(index=0, message=str(exc))],
)
blocks: list[list[str]] = []
issues: list[ParsedVCardIssue] = []
current: list[str] | None = None
@@ -189,6 +191,15 @@ def _card_blocks_with_issues(content: str) -> _CardBlockResult:
issues.append(ParsedVCardIssue(index=card_index, message="vCard END appears before BEGIN."))
continue
current.append(line)
if len(blocks) >= max_cards:
issues.append(
ParsedVCardIssue(
index=len(blocks) + 1,
message=f"vCard input exceeds the configured {max_cards}-card limit.",
)
)
current = None
break
blocks.append(current)
current = None
elif current is not None:
@@ -258,7 +269,14 @@ def _apply_card_metadata(
if name == "VERSION":
draft.version = value.strip()
if draft.version and draft.version not in {"3.0", "4.0"}:
issues.append(ParsedVCardIssue(index=index, severity="warning", field="VERSION", message=f"vCard version {draft.version} is not fully supported."))
issues.append(
ParsedVCardIssue(
index=index,
severity="warning",
field="VERSION",
message=f"vCard version {draft.version} is not fully supported.",
)
)
return True
if name == "UID":
draft.uid = _unescape_text(value) or draft.uid
@@ -325,15 +343,34 @@ def _append_card_email(
if not email:
return
if "@" not in email:
issues.append(ParsedVCardIssue(index=index, severity="warning", field="EMAIL", message=f"Skipped invalid email address: {email}"))
issues.append(
ParsedVCardIssue(
index=index,
severity="warning",
field="EMAIL",
message=f"Skipped invalid email address: {email}",
)
)
return
draft.emails.append(ContactEmailPayload(label=_label_from_params(params), email=email, is_primary=_is_pref(params) or not draft.emails))
draft.emails.append(
ContactEmailPayload(
label=_label_from_params(params),
email=email,
is_primary=_is_pref(params) or not draft.emails,
)
)
def _append_card_phone(draft: _VCardDraft, params: dict[str, list[str]], value: str) -> None:
phone = _unescape_text(value)
if phone:
draft.phones.append(ContactPhonePayload(label=_label_from_params(params), phone=phone, is_primary=_is_pref(params) or not draft.phones))
draft.phones.append(
ContactPhonePayload(
label=_label_from_params(params),
phone=phone,
is_primary=_is_pref(params) or not draft.phones,
)
)
def _append_card_address(draft: _VCardDraft, params: dict[str, list[str]], value: str) -> None:
@@ -388,8 +425,14 @@ def _draft_contact_payload(draft: _VCardDraft) -> ContactCreateRequest:
return payload
def parse_vcards_with_issues(content: str) -> VCardParseResult:
blocks = _card_blocks_with_issues(content)
def parse_vcards_with_issues(
content: str,
*,
max_cards: int = MAX_VCARD_CARDS,
) -> VCardParseResult:
if max_cards < 1 or max_cards > MAX_VCARD_CARDS:
raise VCardError(f"max_cards must be between 1 and {MAX_VCARD_CARDS}.")
blocks = _card_blocks_with_issues(content, max_cards=max_cards)
parsed: list[ParsedVCard] = []
issues = list(blocks.issues)
skipped = 0
@@ -411,8 +454,8 @@ def parse_vcards(content: str) -> list[ParsedVCard]:
return result.cards
def contact_to_vcard(contact: Contact) -> str:
lines = _contact_identity_lines(contact)
def contact_to_vcard(contact: Contact, *, version: Literal["3.0", "4.0"] = "4.0") -> str:
lines = _contact_identity_lines(contact, version=version)
lines.extend(_contact_email_lines(contact))
lines.extend(_contact_phone_lines(contact))
lines.extend(_contact_address_lines(contact))
@@ -422,10 +465,14 @@ def contact_to_vcard(contact: Contact) -> str:
return "\r\n".join(lines) + "\r\n"
def _contact_identity_lines(contact: Contact) -> list[str]:
def _contact_identity_lines(
contact: Contact,
*,
version: Literal["3.0", "4.0"],
) -> list[str]:
lines = [
"BEGIN:VCARD",
"VERSION:4.0",
f"VERSION:{version}",
f"FN:{_escape_text(contact.display_name)}",
f"N:{_escape_text(contact.family_name)};{_escape_text(contact.given_name)};;;",
]
@@ -460,11 +507,7 @@ def _contact_address_lines(contact: Contact) -> list[str]:
lines: list[str] = []
for address in contact.postal_addresses:
label = f";TYPE={_escape_text(address.label)}" if address.label else ""
lines.append(
"ADR"
f"{label}:;;{_escape_text(address.street)};{_escape_text(address.locality)};"
f"{_escape_text(address.region)};{_escape_text(address.postal_code)};{_escape_text(address.country)}"
)
lines.append(f"ADR{label}:;;{_escape_text(address.street)};{_escape_text(address.locality)};{_escape_text(address.region)};{_escape_text(address.postal_code)};{_escape_text(address.country)}")
return lines
@@ -496,5 +539,9 @@ def _contact_vcard_urls(contact: Contact) -> object:
return vcard.get("urls")
def contacts_to_vcard(contacts: list[Contact]) -> str:
return "".join(contact_to_vcard(contact) for contact in contacts)
def contacts_to_vcard(
contacts: list[Contact],
*,
version: Literal["3.0", "4.0"] = "4.0",
) -> str:
return "".join(contact_to_vcard(contact, version=version) for contact in contacts)
@@ -0,0 +1,144 @@
from __future__ import annotations
from datetime import datetime
from typing import Any, Literal
from pydantic import BaseModel, Field, model_validator
VCardPlanAction = Literal["create", "update", "ignore", "unchanged", "conflict"]
VCardCommitAction = Literal["create", "update", "ignore"]
class VCardBatchFilePayload(BaseModel):
filename: str = Field(min_length=1, max_length=500)
content_base64: str = Field(min_length=1, max_length=14_000_000)
class VCardBatchPreviewRequest(BaseModel):
files: list[VCardBatchFilePayload] = Field(min_length=1, max_length=50)
duplicate_card_policy: Literal["reject", "first", "last"] = "reject"
existing_contact_policy: Literal["update", "ignore", "reject"] = "update"
class VCardDuplicateSuggestion(BaseModel):
contact_id: str
display_name: str
reasons: list[str] = Field(default_factory=list)
class VCardBatchPlanItemResponse(BaseModel):
source_key: str
source_filename: str
card_index: int
action: VCardPlanAction
allowed_actions: list[VCardCommitAction] = Field(default_factory=list)
contact_id: str | None = None
display_name: str | None = None
changed_fields: list[str] = Field(default_factory=list)
duplicate_suggestions: list[VCardDuplicateSuggestion] = Field(default_factory=list)
message: str | None = None
class VCardBatchDiagnosticResponse(BaseModel):
severity: Literal["info", "warning", "error"]
code: str
message: str
source_filename: str | None = None
card_index: int | None = None
field: str | None = None
details: dict[str, Any] = Field(default_factory=dict)
class VCardBatchProgressResponse(BaseModel):
total: int
completed: int
created: int = 0
updated: int = 0
ignored: int = 0
failed: int = 0
class VCardBatchRunResponse(BaseModel):
id: str
address_book_id: str
status: str
input_hash: str
plan_hash: str
parser_version: str
execution_mode: Literal["bounded_sync", "persisted_batch"]
file_count: int
card_count: int
statistics: dict[str, int | str] = Field(default_factory=dict)
diagnostics: list[VCardBatchDiagnosticResponse] = Field(default_factory=list)
plan: list[VCardBatchPlanItemResponse] = Field(default_factory=list)
progress: VCardBatchProgressResponse
can_apply: bool
can_cancel: bool
commit_hash: str | None = None
created_at: datetime
updated_at: datetime
applied_at: datetime | None = None
class VCardBatchSelection(BaseModel):
source_key: str = Field(min_length=1, max_length=1000)
action: VCardCommitAction
class VCardBatchCommitRequest(BaseModel):
expected_plan_hash: str = Field(min_length=64, max_length=64)
selections: list[VCardBatchSelection] = Field(
default_factory=list, max_length=10_000
)
class VCardBatchCancelRequest(BaseModel):
expected_plan_hash: str = Field(min_length=64, max_length=64)
reason: str = Field(min_length=3, max_length=2000)
class VCardExportRequest(BaseModel):
scope: Literal["address_book", "address_list", "contacts"] = "address_book"
address_list_id: str | None = Field(default=None, max_length=36)
contact_ids: list[str] = Field(default_factory=list, max_length=10_000)
version: Literal["3.0", "4.0"] = "4.0"
@model_validator(mode="after")
def validate_scope(self) -> "VCardExportRequest":
if self.scope == "address_list" and not self.address_list_id:
raise ValueError("Address-list export requires address_list_id.")
if self.scope == "contacts" and not self.contact_ids:
raise ValueError(
"Selected-contact export requires at least one contact id."
)
if self.scope != "address_list" and self.address_list_id:
raise ValueError("address_list_id is only valid for address-list export.")
if self.scope != "contacts" and self.contact_ids:
raise ValueError("contact_ids are only valid for selected-contact export.")
if len(set(self.contact_ids)) != len(self.contact_ids):
raise ValueError("Selected contact ids must be unique.")
return self
class VCardExportResponse(BaseModel):
filename: str
media_type: str = "text/vcard"
scope: str
version: str
ordering: str
contact_count: int
content_hash: str
content: str
__all__ = [
"VCardBatchCancelRequest",
"VCardBatchCommitRequest",
"VCardBatchFilePayload",
"VCardBatchPreviewRequest",
"VCardBatchRunResponse",
"VCardBatchSelection",
"VCardExportRequest",
"VCardExportResponse",
]
@@ -0,0 +1,901 @@
from __future__ import annotations
import base64
import binascii
from collections import Counter, defaultdict
import hashlib
import json
import os
from typing import Any
from sqlalchemy.orm import Session, selectinload
from govoplan_addresses.backend.db.models import (
AddressImportRun,
AddressListEntry,
Contact,
)
from govoplan_addresses.backend.schemas import (
ContactCreateRequest,
ContactUpdateRequest,
)
from govoplan_addresses.backend.service import (
AddressBookError,
create_contact,
get_visible_address_book,
get_visible_address_list,
get_visible_contact,
update_contact,
)
from govoplan_addresses.backend.vcard import (
MAX_VCARD_CARDS,
VCARD_PARSER_VERSION,
contacts_to_vcard,
parse_vcards_with_issues,
)
from govoplan_addresses.backend.vcard_batch_schemas import (
VCardBatchCancelRequest,
VCardBatchCommitRequest,
VCardBatchPreviewRequest,
VCardExportRequest,
)
from govoplan_core.auth import ApiPrincipal
from govoplan_core.db.base import utcnow
MAX_VCARD_BATCH_BYTES = 10_000_000
DEFAULT_PERSISTED_BATCH_THRESHOLD = 500
def preview_vcard_batch(
session: Session,
principal: ApiPrincipal,
address_book_id: str,
payload: VCardBatchPreviewRequest,
) -> AddressImportRun:
book = get_visible_address_book(session, principal, address_book_id)
if book.read_only:
raise AddressBookError("Static vCard imports require a writable address book.")
decoded = _decode_files(payload)
input_hash = _hash_json(
{
"files": [
{
"filename": filename,
"sha256": hashlib.sha256(raw).hexdigest(),
"size": len(raw),
}
for filename, raw in decoded
]
}
)
parsed_cards, diagnostics = _parse_files(decoded)
plan = _plan_cards(
session,
book.id,
parsed_cards,
duplicate_card_policy=payload.duplicate_card_policy,
existing_contact_policy=payload.existing_contact_policy,
)
statistics: dict[str, int | str] = dict(
Counter(str(item["action"]) for item in plan)
)
statistics.update(
{
"files": len(decoded),
"cards": len(parsed_cards),
"errors": sum(item["severity"] == "error" for item in diagnostics),
"warnings": sum(item["severity"] == "warning" for item in diagnostics),
"parser_version": VCARD_PARSER_VERSION,
"execution_mode": _execution_mode(len(parsed_cards)),
}
)
plan_hash = _hash_json(
{
"address_book_id": book.id,
"input_hash": input_hash,
"parser_version": VCARD_PARSER_VERSION,
"duplicate_card_policy": payload.duplicate_card_policy,
"existing_contact_policy": payload.existing_contact_policy,
"plan": plan,
}
)
source_filename = decoded[0][0]
if len(decoded) > 1:
source_filename = f"{source_filename} (+{len(decoded) - 1} files)"
run = AddressImportRun(
tenant_id=book.tenant_id,
address_book_id=book.id,
profile_id=None,
source_filename=source_filename[:500],
source_format="vcard",
input_hash=input_hash,
plan_hash=plan_hash,
status="previewed",
row_count=len(parsed_cards),
statistics=statistics,
diagnostics=diagnostics,
plan_data=plan,
result_evidence={
"parser_version": VCARD_PARSER_VERSION,
"file_manifest": [
{
"filename": filename,
"sha256": hashlib.sha256(raw).hexdigest(),
"size": len(raw),
}
for filename, raw in decoded
],
"progress": _progress(len(plan)),
},
created_by_account_id=principal.account_id,
)
session.add(run)
session.flush()
return run
def get_vcard_batch_run(
session: Session,
principal: ApiPrincipal,
run_id: str,
) -> AddressImportRun:
book_ids = [book.id for book in _visible_books(session, principal)]
if not book_ids:
raise AddressBookError("vCard batch run not found.")
item = (
session.query(AddressImportRun)
.filter(
AddressImportRun.id == run_id,
AddressImportRun.address_book_id.in_(book_ids),
AddressImportRun.source_format == "vcard",
)
.one_or_none()
)
if item is None:
raise AddressBookError("vCard batch run not found.")
return item
def apply_vcard_batch(
session: Session,
principal: ApiPrincipal,
run_id: str,
payload: VCardBatchCommitRequest,
) -> AddressImportRun:
run = get_vcard_batch_run(session, principal, run_id)
if run.plan_hash != payload.expected_plan_hash:
raise AddressBookError("The reviewed vCard plan changed; create a new preview.")
selections = _selection_map(payload)
commit_hash = _hash_json(
{
"plan_hash": run.plan_hash,
"selections": [
{"source_key": key, "action": selections[key]}
for key in sorted(selections)
],
}
)
evidence = dict(run.result_evidence or {})
if run.status == "applied":
if evidence.get("commit_hash") != commit_hash:
raise AddressBookError(
"This vCard batch was already applied with a different selection."
)
return run
if run.status != "previewed":
raise AddressBookError(
f"vCard batch cannot be applied from status {run.status!r}."
)
if not selections:
raise AddressBookError(
"Select at least one vCard action before applying the batch."
)
plan_by_key = {str(item["source_key"]): item for item in run.plan_data or []}
unknown = sorted(set(selections).difference(plan_by_key))
if unknown:
raise AddressBookError(
"The selection contains cards that are not part of the reviewed plan."
)
created_ids: list[str] = []
updated_ids: list[str] = []
ignored = 0
for source_key in sorted(plan_by_key):
item = plan_by_key[source_key]
action = selections.get(source_key, "ignore")
allowed = set(item.get("allowed_actions") or [])
if action not in allowed:
raise AddressBookError(
f'Action {action!r} is not allowed for vCard "{item.get("display_name") or source_key}".'
)
if action == "ignore":
ignored += 1
continue
contact = _apply_plan_item(
session,
principal,
run=run,
item=item,
action=action,
)
if action == "create":
created_ids.append(contact.id)
else:
updated_ids.append(contact.id)
run.status = "applied"
run.applied_at = utcnow()
run.result_evidence = {
**evidence,
"commit_hash": commit_hash,
"selection_count": len(selections),
"created_contact_ids": created_ids,
"updated_contact_ids": updated_ids,
"ignored_count": ignored,
"applied_by_account_id": principal.account_id,
"applied_at": run.applied_at.isoformat(),
"progress": {
"total": len(plan_by_key),
"completed": len(plan_by_key),
"created": len(created_ids),
"updated": len(updated_ids),
"ignored": ignored,
"failed": 0,
},
}
run.statistics = {
**dict(run.statistics or {}),
"applied_create": len(created_ids),
"applied_update": len(updated_ids),
"applied_ignore": ignored,
}
return run
def cancel_vcard_batch(
session: Session,
principal: ApiPrincipal,
run_id: str,
payload: VCardBatchCancelRequest,
) -> AddressImportRun:
run = get_vcard_batch_run(session, principal, run_id)
if run.plan_hash != payload.expected_plan_hash:
raise AddressBookError("The reviewed vCard plan changed; reload the batch.")
if run.status == "cancelled":
return run
if run.status != "previewed":
raise AddressBookError("Only a previewed vCard batch can be cancelled.")
run.status = "cancelled"
run.result_evidence = {
**dict(run.result_evidence or {}),
"cancel_reason": payload.reason.strip(),
"cancelled_by_account_id": principal.account_id,
"cancelled_at": utcnow().isoformat(),
}
return run
def vcard_batch_payload(run: AddressImportRun) -> dict[str, Any]:
evidence = dict(run.result_evidence or {})
progress = dict(evidence.get("progress") or _progress(run.row_count))
return {
"id": run.id,
"address_book_id": run.address_book_id,
"status": run.status,
"input_hash": run.input_hash,
"plan_hash": run.plan_hash,
"parser_version": str(evidence.get("parser_version") or VCARD_PARSER_VERSION),
"execution_mode": str(
(run.statistics or {}).get("execution_mode") or "bounded_sync"
),
"file_count": int((run.statistics or {}).get("files") or 0),
"card_count": run.row_count,
"statistics": dict(run.statistics or {}),
"diagnostics": list(run.diagnostics or []),
"plan": [_public_plan_item(item) for item in run.plan_data or []],
"progress": progress,
"can_apply": run.status == "previewed" and bool(run.plan_data),
"can_cancel": run.status == "previewed",
"commit_hash": evidence.get("commit_hash"),
"created_at": run.created_at,
"updated_at": run.updated_at,
"applied_at": run.applied_at,
}
def vcard_diagnostics_payload(run: AddressImportRun) -> dict[str, Any]:
return {
"schema_version": "1.0",
"run_id": run.id,
"status": run.status,
"input_hash": run.input_hash,
"plan_hash": run.plan_hash,
"parser_version": (run.result_evidence or {}).get("parser_version"),
"statistics": dict(run.statistics or {}),
"diagnostics": list(run.diagnostics or []),
"effects": [_public_plan_item(item) for item in run.plan_data or []],
}
def export_vcards(
session: Session,
principal: ApiPrincipal,
address_book_id: str,
payload: VCardExportRequest,
) -> dict[str, Any]:
book = get_visible_address_book(session, principal, address_book_id)
contacts = _export_contacts(session, principal, book.id, payload)
contacts.sort(key=lambda item: (item.display_name.casefold(), item.id))
content = contacts_to_vcard(contacts, version=payload.version)
scope_label = {
"address_book": book.name,
"address_list": "address-list",
"contacts": "selected-contacts",
}[payload.scope]
return {
"filename": f"{_safe_filename(scope_label)}-{payload.version.replace('.', '')}.vcf",
"media_type": "text/vcard",
"scope": payload.scope,
"version": payload.version,
"ordering": "display_name_casefold_then_contact_id",
"contact_count": len(contacts),
"content_hash": hashlib.sha256(content.encode("utf-8")).hexdigest(),
"content": content,
}
def _decode_files(payload: VCardBatchPreviewRequest) -> list[tuple[str, bytes]]:
decoded: list[tuple[str, bytes]] = []
total = 0
for item in payload.files:
filename = item.filename.strip()
if not filename.casefold().endswith(".vcf"):
raise AddressBookError("vCard batch uploads accept only .vcf files.")
try:
raw = base64.b64decode(item.content_base64, validate=True)
except (binascii.Error, ValueError) as exc:
raise AddressBookError(
f'vCard file "{filename}" is not valid base64.'
) from exc
if not raw:
raise AddressBookError(f'vCard file "{filename}" is empty.')
total += len(raw)
if total > MAX_VCARD_BATCH_BYTES:
raise AddressBookError(
f"Combined vCard uploads are limited to {MAX_VCARD_BATCH_BYTES} bytes."
)
decoded.append((filename, raw))
return decoded
def _parse_files(
decoded: list[tuple[str, bytes]],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
cards: list[dict[str, Any]] = []
diagnostics: list[dict[str, Any]] = []
for file_index, (filename, raw) in enumerate(decoded):
try:
content = raw.decode("utf-8-sig")
except UnicodeDecodeError as exc:
raise AddressBookError(
f'vCard file "{filename}" is not valid UTF-8: {exc}.'
) from exc
remaining = MAX_VCARD_CARDS - len(cards)
if remaining < 1:
raise AddressBookError(
f"vCard batches are limited to {MAX_VCARD_CARDS} cards."
)
result = parse_vcards_with_issues(content, max_cards=remaining)
for issue in result.issues:
diagnostics.append(
{
"severity": issue.severity,
"code": "vcard_parse_error"
if issue.severity == "error"
else "vcard_parse_warning",
"message": issue.message,
"source_filename": filename,
"card_index": issue.index or None,
"field": issue.field,
"details": {"line": issue.line} if issue.line is not None else {},
}
)
for card_index, parsed in enumerate(result.cards, start=1):
raw_hash = hashlib.sha256(parsed.raw.encode("utf-8")).hexdigest()
identity = (
f"uid:{parsed.source_ref.strip()}"
if parsed.source_ref and parsed.source_ref.strip()
else f"sha256:{raw_hash}"
)
source_key = hashlib.sha256(
f"{file_index}:{filename}:{card_index}:{raw_hash}".encode("utf-8")
).hexdigest()
cards.append(
{
"source_key": source_key,
"source_identity": identity,
"source_filename": filename,
"card_index": card_index,
"raw": parsed.raw,
"source_ref": parsed.source_ref.strip()
if parsed.source_ref
else None,
"source_revision": parsed.source_revision.strip()
if parsed.source_revision
else None,
"payload": parsed.payload.model_dump(mode="json"),
}
)
return cards, diagnostics
def _plan_cards(
session: Session,
address_book_id: str,
cards: list[dict[str, Any]],
*,
duplicate_card_policy: str,
existing_contact_policy: str,
) -> list[dict[str, Any]]:
contacts = (
session.query(Contact)
.options(
selectinload(Contact.emails),
selectinload(Contact.phones),
selectinload(Contact.postal_addresses),
)
.filter(
Contact.address_book_id == address_book_id, Contact.deleted_at.is_(None)
)
.all()
)
by_source: dict[str, list[Contact]] = defaultdict(list)
by_email: dict[str, list[Contact]] = defaultdict(list)
for contact in contacts:
if contact.source_ref:
by_source[contact.source_ref.strip()].append(contact)
for email in contact.emails:
normalized = (email.normalized_email or email.email).strip().casefold()
if normalized:
by_email[normalized].append(contact)
identity_positions: dict[str, list[int]] = defaultdict(list)
for index, card in enumerate(cards):
identity_positions[str(card["source_identity"])].append(index)
result: list[dict[str, Any]] = []
for index, card in enumerate(cards):
positions = identity_positions[str(card["source_identity"])]
if len(positions) > 1:
chosen = positions[0] if duplicate_card_policy == "first" else positions[-1]
if duplicate_card_policy == "reject":
result.append(
_planned_card(
card,
action="conflict",
allowed=["ignore"],
message="Duplicate UID or identical card appears in this batch.",
)
)
continue
if index != chosen:
result.append(
_planned_card(
card,
action="ignore",
allowed=["ignore"],
message=f"Duplicate card ignored by {duplicate_card_policy} policy.",
)
)
continue
suggestions = _duplicate_candidates(
card, by_source=by_source, by_email=by_email
)
exact_source = [item for item in suggestions if "source_uid" in item["reasons"]]
candidates = exact_source or suggestions
if len(candidates) > 1:
result.append(
_planned_card(
card,
action="conflict",
allowed=["create", "ignore"],
suggestions=suggestions,
message="Multiple existing contacts match this card; create explicitly or ignore it.",
)
)
continue
existing = next(
(
contact
for contact in contacts
if candidates and contact.id == candidates[0]["contact_id"]
),
None,
)
if existing is None:
result.append(
_planned_card(
card,
action="create",
allowed=["create", "ignore"],
suggestions=suggestions,
)
)
continue
changed = _changed_fields(existing, card["payload"])
if not changed:
result.append(
_planned_card(
card,
action="unchanged",
allowed=["ignore"],
contact=existing,
suggestions=suggestions,
message="Existing contact already matches the parsed card.",
)
)
elif existing_contact_policy == "reject":
result.append(
_planned_card(
card,
action="conflict",
allowed=["ignore"],
contact=existing,
suggestions=suggestions,
changed=changed,
message="An existing contact matches and the preview policy rejects updates.",
)
)
elif existing_contact_policy == "ignore":
result.append(
_planned_card(
card,
action="ignore",
allowed=["update", "ignore"],
contact=existing,
suggestions=suggestions,
changed=changed,
message="Existing contact is ignored by preview policy.",
)
)
else:
result.append(
_planned_card(
card,
action="update",
allowed=["update", "ignore"],
contact=existing,
suggestions=suggestions,
changed=changed,
)
)
return result
def _planned_card(
card: dict[str, Any],
*,
action: str,
allowed: list[str],
contact: Contact | None = None,
suggestions: list[dict[str, Any]] | None = None,
changed: list[str] | None = None,
message: str | None = None,
) -> dict[str, Any]:
return {
**card,
"row_number": int(card["card_index"]),
"action": action,
"allowed_actions": allowed,
"contact_id": contact.id if contact is not None else None,
"expected_contact_hash": _contact_hash(contact)
if contact is not None
else None,
"display_name": card["payload"].get("display_name"),
"changed_fields": changed or [],
"duplicate_suggestions": suggestions or [],
"message": message,
}
def _duplicate_candidates(
card: dict[str, Any],
*,
by_source: dict[str, list[Contact]],
by_email: dict[str, list[Contact]],
) -> list[dict[str, Any]]:
reasons: dict[str, set[str]] = defaultdict(set)
contacts: dict[str, Contact] = {}
source_ref = card.get("source_ref")
if source_ref:
for contact in by_source.get(str(source_ref), []):
contacts[contact.id] = contact
reasons[contact.id].add("source_uid")
for item in card["payload"].get("emails") or []:
normalized = str(item.get("email") or "").strip().casefold()
for contact in by_email.get(normalized, []):
contacts[contact.id] = contact
reasons[contact.id].add("email")
return [
{
"contact_id": contact_id,
"display_name": contacts[contact_id].display_name,
"reasons": sorted(reasons[contact_id]),
}
for contact_id in sorted(
contacts, key=lambda item: (contacts[item].display_name.casefold(), item)
)[:5]
]
def _apply_plan_item(
session: Session,
principal: ApiPrincipal,
*,
run: AddressImportRun,
item: dict[str, Any],
action: str,
) -> Contact:
contact_payload = ContactCreateRequest.model_validate(item["payload"])
if action == "create":
if item.get("source_ref"):
appeared = (
session.query(Contact)
.filter(
Contact.address_book_id == run.address_book_id,
Contact.source_ref == item["source_ref"],
Contact.deleted_at.is_(None),
)
.first()
)
if appeared is not None:
raise AddressBookError(
"A matching vCard UID appeared after preview; preview the batch again."
)
contact = create_contact(
session, principal, run.address_book_id, contact_payload
)
else:
contact_id = str(item.get("contact_id") or "")
if not contact_id:
raise AddressBookError(
"The reviewed vCard update has no stable target contact."
)
current = get_visible_contact(session, principal, contact_id)
if _contact_hash(current) != item.get("expected_contact_hash"):
raise AddressBookError(
f'Contact "{current.display_name}" changed after preview; preview the batch again.'
)
contact = update_contact(
session,
principal,
current.id,
ContactUpdateRequest.model_validate(item["payload"]),
)
contact.source_kind = "vcard"
contact.source_ref = (
item.get("source_ref")
or f"vcard-sha256:{str(item['source_identity']).split(':', 1)[-1]}"
)
contact.source_payload_kind = "vcard"
contact.source_payload_raw = item["raw"]
contact.source_revision = item.get("source_revision")
provenance = dict(contact.provenance or {})
provenance["vcard_batch"] = {
"run_id": run.id,
"input_hash": run.input_hash,
"plan_hash": run.plan_hash,
"parser_version": VCARD_PARSER_VERSION,
"source_filename": item["source_filename"],
"card_index": item["card_index"],
}
contact.provenance = provenance
session.flush()
return contact
def _changed_fields(contact: Contact, payload: dict[str, Any]) -> list[str]:
current = _contact_projection(contact)
incoming = _payload_projection(payload)
return sorted(key for key in incoming if current.get(key) != incoming.get(key))
def _contact_hash(contact: Contact) -> str:
return _hash_json(_contact_projection(contact))
def _contact_projection(contact: Contact) -> dict[str, Any]:
return {
"display_name": contact.display_name,
"given_name": contact.given_name,
"family_name": contact.family_name,
"organization": contact.organization,
"role_title": contact.role_title,
"note": contact.note,
"tags": list(contact.tags or []),
"emails": [
{"label": item.label, "email": item.email, "is_primary": item.is_primary}
for item in contact.emails
],
"phones": [
{"label": item.label, "phone": item.phone, "is_primary": item.is_primary}
for item in contact.phones
],
"postal_addresses": [
{
"label": item.label,
"street": item.street,
"postal_code": item.postal_code,
"locality": item.locality,
"region": item.region,
"country": item.country,
"is_primary": item.is_primary,
}
for item in contact.postal_addresses
],
}
def _payload_projection(payload: dict[str, Any]) -> dict[str, Any]:
return {
key: payload.get(key)
for key in (
"display_name",
"given_name",
"family_name",
"organization",
"role_title",
"note",
"tags",
"emails",
"phones",
"postal_addresses",
)
}
def _selection_map(payload: VCardBatchCommitRequest) -> dict[str, str]:
result: dict[str, str] = {}
for selection in payload.selections:
if selection.source_key in result:
raise AddressBookError("Each vCard may be selected only once.")
result[selection.source_key] = selection.action
return result
def _public_plan_item(item: dict[str, Any]) -> dict[str, Any]:
return {
key: item.get(key)
for key in (
"source_key",
"source_filename",
"card_index",
"action",
"allowed_actions",
"contact_id",
"display_name",
"changed_fields",
"duplicate_suggestions",
"message",
)
}
def _export_contacts(
session: Session,
principal: ApiPrincipal,
address_book_id: str,
payload: VCardExportRequest,
) -> list[Contact]:
if payload.scope == "address_book":
return _loaded_contacts(session, address_book_id=address_book_id)
if payload.scope == "contacts":
contacts = [
get_visible_contact(session, principal, contact_id)
for contact_id in payload.contact_ids
]
if any(contact.address_book_id != address_book_id for contact in contacts):
raise AddressBookError(
"Every selected contact must belong to the exported address book."
)
return contacts
address_list = get_visible_address_list(
session, principal, str(payload.address_list_id)
)
if address_list.address_book_id != address_book_id:
raise AddressBookError(
"The selected address list does not belong to the exported address book."
)
contact_ids = [
item.contact_id
for item in (
session.query(AddressListEntry)
.filter(AddressListEntry.address_list_id == address_list.id)
.order_by(AddressListEntry.order_index.asc(), AddressListEntry.id.asc())
.all()
)
]
if not contact_ids:
return []
return _loaded_contacts(
session, address_book_id=address_book_id, contact_ids=set(contact_ids)
)
def _loaded_contacts(
session: Session,
*,
address_book_id: str,
contact_ids: set[str] | None = None,
) -> list[Contact]:
query = (
session.query(Contact)
.options(
selectinload(Contact.emails),
selectinload(Contact.phones),
selectinload(Contact.postal_addresses),
)
.filter(
Contact.address_book_id == address_book_id, Contact.deleted_at.is_(None)
)
)
if contact_ids is not None:
query = query.filter(Contact.id.in_(contact_ids))
return query.all()
def _visible_books(session: Session, principal: ApiPrincipal):
from govoplan_addresses.backend.service import list_address_books
return list_address_books(session, principal)
def _execution_mode(card_count: int) -> str:
raw = os.getenv(
"GOVOPLAN_ADDRESSES_VCARD_JOB_THRESHOLD", str(DEFAULT_PERSISTED_BATCH_THRESHOLD)
)
try:
threshold = max(1, min(MAX_VCARD_CARDS, int(raw)))
except ValueError:
threshold = DEFAULT_PERSISTED_BATCH_THRESHOLD
return "persisted_batch" if card_count >= threshold else "bounded_sync"
def _progress(total: int) -> dict[str, int]:
return {
"total": total,
"completed": 0,
"created": 0,
"updated": 0,
"ignored": 0,
"failed": 0,
}
def _hash_json(value: Any) -> str:
return hashlib.sha256(
json.dumps(
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
).encode("utf-8")
).hexdigest()
def _safe_filename(value: str) -> str:
safe = "".join(
character if character.isalnum() or character in {"-", "_"} else "-"
for character in value.strip()
)
return safe.strip("-")[:120] or "contacts"
__all__ = [
"apply_vcard_batch",
"cancel_vcard_batch",
"export_vcards",
"get_vcard_batch_run",
"preview_vcard_batch",
"vcard_batch_payload",
"vcard_diagnostics_payload",
]
+1 -1
View File
@@ -35,7 +35,7 @@ class AddressesMigrationTests(unittest.TestCase):
try:
with engine.connect() as connection:
self.assertIn(
"c5d7e8f9a0b1",
"d6e8f9a0b1c2",
set(MigrationContext.configure(connection).get_current_heads()),
)
tables = set(inspect(connection).get_table_names())
+270
View File
@@ -0,0 +1,270 @@
from __future__ import annotations
import base64
import unittest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from govoplan_addresses.backend.db.models import (
AddressBook,
AddressList,
AddressListEntry,
Contact,
)
from govoplan_addresses.backend.schemas import ContactCreateRequest
from govoplan_addresses.backend.service import AddressBookError, create_contact
from govoplan_addresses.backend.vcard import (
MAX_VCARD_UNFOLDED_LINE_CHARS,
parse_vcards_with_issues,
)
from govoplan_addresses.backend.vcard_batch_schemas import (
VCardBatchCancelRequest,
VCardBatchCommitRequest,
VCardBatchFilePayload,
VCardBatchPreviewRequest,
VCardBatchSelection,
VCardExportRequest,
)
from govoplan_addresses.backend.vcard_batches import (
apply_vcard_batch,
cancel_vcard_batch,
export_vcards,
preview_vcard_batch,
vcard_batch_payload,
)
from govoplan_core.db.base import Base
class Principal:
account_id = "account-1"
group_ids = frozenset()
@property
def tenant_id(self) -> str:
return "tenant-1"
def has(self, scope: str) -> bool:
return scope in {
"addresses:address_book:read",
"addresses:address_book:write",
"addresses:contact:read",
"addresses:contact:write",
}
def vcard(uid: str, name: str, email: str) -> str:
return (
"BEGIN:VCARD\r\n"
"VERSION:4.0\r\n"
f"UID:{uid}\r\n"
f"FN:{name}\r\n"
f"EMAIL:{email}\r\n"
"END:VCARD\r\n"
)
def batch_file(filename: str, content: str) -> VCardBatchFilePayload:
return VCardBatchFilePayload(
filename=filename,
content_base64=base64.b64encode(content.encode()).decode(),
)
class VCardBatchTests(unittest.TestCase):
def setUp(self) -> None:
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
self.session = sessionmaker(bind=engine, expire_on_commit=False)()
self.principal = Principal()
self.book = AddressBook(
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
name="Batch contacts",
source_kind="local",
read_only=False,
)
self.session.add(self.book)
self.session.flush()
def test_multifile_preview_selective_apply_and_repeat_are_idempotent(self) -> None:
run = preview_vcard_batch(
self.session,
self.principal,
self.book.id,
VCardBatchPreviewRequest(
files=[
batch_file(
"ada.vcf", vcard("ada-1", "Ada Lovelace", "ada@example.test")
),
batch_file(
"grace.vcf",
vcard("grace-1", "Grace Hopper", "grace@example.test"),
),
]
),
)
self.assertEqual(0, self.session.query(Contact).count())
self.assertEqual("previewed", run.status)
self.assertEqual(2, run.row_count)
self.assertEqual("govoplan-vcard/2", run.result_evidence["parser_version"])
self.assertNotIn("ada@example.test", repr(vcard_batch_payload(run)))
selections = [
VCardBatchSelection(
source_key=run.plan_data[0]["source_key"], action="create"
),
VCardBatchSelection(
source_key=run.plan_data[1]["source_key"], action="ignore"
),
]
request = VCardBatchCommitRequest(
expected_plan_hash=run.plan_hash, selections=selections
)
applied = apply_vcard_batch(self.session, self.principal, run.id, request)
repeated = apply_vcard_batch(self.session, self.principal, run.id, request)
self.assertIs(applied, repeated)
self.assertEqual(1, self.session.query(Contact).count())
self.assertEqual("Ada Lovelace", self.session.query(Contact).one().display_name)
self.assertEqual(1, applied.result_evidence["progress"]["ignored"])
with self.assertRaisesRegex(AddressBookError, "different selection"):
apply_vcard_batch(
self.session,
self.principal,
run.id,
VCardBatchCommitRequest(
expected_plan_hash=run.plan_hash,
selections=[
VCardBatchSelection(
source_key=run.plan_data[1]["source_key"], action="create"
)
],
),
)
def test_duplicate_uid_policy_and_cancellation(self) -> None:
run = preview_vcard_batch(
self.session,
self.principal,
self.book.id,
VCardBatchPreviewRequest(
files=[
batch_file(
"duplicates.vcf",
vcard("same", "First", "first@example.test")
+ vcard("same", "Last", "last@example.test"),
)
],
duplicate_card_policy="reject",
),
)
self.assertEqual(
["conflict", "conflict"], [item["action"] for item in run.plan_data]
)
cancelled = cancel_vcard_batch(
self.session,
self.principal,
run.id,
VCardBatchCancelRequest(
expected_plan_hash=run.plan_hash,
reason="Operator rejected duplicate source UIDs.",
),
)
self.assertEqual("cancelled", cancelled.status)
self.assertEqual(0, self.session.query(Contact).count())
last = preview_vcard_batch(
self.session,
self.principal,
self.book.id,
VCardBatchPreviewRequest(
files=[
batch_file(
"duplicates.vcf",
vcard("same", "First", "first@example.test")
+ vcard("same", "Last", "last@example.test"),
)
],
duplicate_card_policy="last",
),
)
self.assertEqual(
["ignore", "create"], [item["action"] for item in last.plan_data]
)
def test_deterministic_scoped_export_supports_vcard_versions(self) -> None:
grace = create_contact(
self.session,
self.principal,
self.book.id,
ContactCreateRequest(display_name="Grace Hopper"),
)
ada = create_contact(
self.session,
self.principal,
self.book.id,
ContactCreateRequest(display_name="Ada Lovelace"),
)
address_list = AddressList(
tenant_id="tenant-1",
address_book_id=self.book.id,
name="Selected",
source_kind="local",
read_only=False,
)
self.session.add(address_list)
self.session.flush()
self.session.add(
AddressListEntry(
address_list_id=address_list.id, contact_id=grace.id, order_index=0
)
)
self.session.flush()
selected = export_vcards(
self.session,
self.principal,
self.book.id,
VCardExportRequest(
scope="contacts", contact_ids=[grace.id, ada.id], version="3.0"
),
)
repeated = export_vcards(
self.session,
self.principal,
self.book.id,
VCardExportRequest(
scope="contacts", contact_ids=[ada.id, grace.id], version="3.0"
),
)
listed = export_vcards(
self.session,
self.principal,
self.book.id,
VCardExportRequest(scope="address_list", address_list_id=address_list.id),
)
self.assertEqual(selected["content_hash"], repeated["content_hash"])
self.assertLess(
selected["content"].index("Ada Lovelace"),
selected["content"].index("Grace Hopper"),
)
self.assertIn("VERSION:3.0", selected["content"])
self.assertEqual(1, listed["contact_count"])
self.assertIn("Grace Hopper", listed["content"])
def test_parser_rejects_pathological_unfolded_lines(self) -> None:
result = parse_vcards_with_issues(
"BEGIN:VCARD\nFN:"
+ ("a" * (MAX_VCARD_UNFOLDED_LINE_CHARS + 1))
+ "\nEND:VCARD"
)
self.assertEqual([], result.cards)
self.assertIn("unfolded lines", result.issues[0].message)
if __name__ == "__main__":
unittest.main()
+112 -1
View File
@@ -354,6 +354,60 @@ export type VCardImportResult = {
issues: Array<{ index: number; message: string; severity: "warning" | "error"; field?: string | null; line?: number | null }>;
};
export type VCardBatchPlanItem = {
source_key: string;
source_filename: string;
card_index: number;
action: "create" | "update" | "ignore" | "unchanged" | "conflict";
allowed_actions: Array<"create" | "update" | "ignore">;
contact_id?: string | null;
display_name?: string | null;
changed_fields: string[];
duplicate_suggestions: Array<{ contact_id: string; display_name: string; reasons: string[] }>;
message?: string | null;
};
export type VCardBatchRun = {
id: string;
address_book_id: string;
status: string;
input_hash: string;
plan_hash: string;
parser_version: string;
execution_mode: "bounded_sync" | "persisted_batch";
file_count: number;
card_count: number;
statistics: Record<string, number | string>;
diagnostics: Array<{
severity: "info" | "warning" | "error";
code: string;
message: string;
source_filename?: string | null;
card_index?: number | null;
field?: string | null;
details: Record<string, unknown>;
}>;
plan: VCardBatchPlanItem[];
progress: { total: number; completed: number; created: number; updated: number; ignored: number; failed: number };
can_apply: boolean;
can_cancel: boolean;
commit_hash?: string | null;
created_at: string;
updated_at: string;
applied_at?: string | null;
};
export type VCardExportResult = {
filename: string;
media_type: string;
scope: "address_book" | "address_list" | "contacts";
version: "3.0" | "4.0";
ordering: string;
contact_count: number;
content_hash: string;
content: string;
};
export type AddressSyncSource = {
id: string;
tenant_id?: string | null;
@@ -538,7 +592,7 @@ export type AddressImportDiagnostic = {
export type AddressImportRun = {
id: string;
address_book_id: string;
profile_id: string;
profile_id: string | null;
source_filename: string;
source_format: string;
input_hash: string;
@@ -1057,6 +1111,47 @@ export function importAddressBookVcards(settings: ApiSettings, addressBookId: st
});
}
export function previewVCardBatch(
settings: ApiSettings,
addressBookId: string,
payload: {
files: Array<{ filename: string; content_base64: string }>;
duplicate_card_policy?: "reject" | "first" | "last";
existing_contact_policy?: "update" | "ignore" | "reject";
}
): Promise<VCardBatchRun> {
return apiFetch<VCardBatchRun>(settings, `/api/v1/addresses/address-books/${addressBookId}/vcard-batches/preview`, {
method: "POST",
body: JSON.stringify(payload)
});
}
export function getVCardBatch(settings: ApiSettings, runId: string): Promise<VCardBatchRun> {
return apiFetch<VCardBatchRun>(settings, `/api/v1/addresses/vcard-batches/${encodeURIComponent(runId)}`);
}
export function applyVCardBatch(
settings: ApiSettings,
run: VCardBatchRun,
selections: Array<{ source_key: string; action: "create" | "update" | "ignore" }>
): Promise<VCardBatchRun> {
return apiFetch<VCardBatchRun>(settings, `/api/v1/addresses/vcard-batches/${encodeURIComponent(run.id)}/apply`, {
method: "POST",
body: JSON.stringify({ expected_plan_hash: run.plan_hash, selections })
});
}
export function cancelVCardBatch(settings: ApiSettings, run: VCardBatchRun, reason: string): Promise<VCardBatchRun> {
return apiFetch<VCardBatchRun>(settings, `/api/v1/addresses/vcard-batches/${encodeURIComponent(run.id)}/cancel`, {
method: "POST",
body: JSON.stringify({ expected_plan_hash: run.plan_hash, reason })
});
}
export function exportVCardBatchDiagnostics(settings: ApiSettings, runId: string): Promise<string> {
return apiFetch<string>(settings, `/api/v1/addresses/vcard-batches/${encodeURIComponent(runId)}/diagnostics`);
}
export async function listAddressImportProfiles(settings: ApiSettings): Promise<AddressImportProfile[]> {
const response = await apiFetch<AddressImportProfileListResponse>(settings, "/api/v1/addresses/import-profiles");
return response.profiles;
@@ -1123,6 +1218,22 @@ export function exportAddressBookVcards(settings: ApiSettings, addressBookId: st
return apiFetch<string>(settings, `/api/v1/addresses/address-books/${addressBookId}/vcards/export`);
}
export function exportScopedVcards(
settings: ApiSettings,
addressBookId: string,
payload: {
scope: "address_book" | "address_list" | "contacts";
address_list_id?: string | null;
contact_ids?: string[];
version?: "3.0" | "4.0";
}
): Promise<VCardExportResult> {
return apiFetch<VCardExportResult>(settings, `/api/v1/addresses/address-books/${addressBookId}/vcards/export`, {
method: "POST",
body: JSON.stringify(payload)
});
}
export function exportContactVcard(settings: ApiSettings, contactId: string): Promise<string> {
return apiFetch<string>(settings, `/api/v1/addresses/contacts/${contactId}/vcard`);
}
@@ -42,6 +42,9 @@ import {
createContact,
createContactChannelRule,
createContactQualityDecision,
applyAddressImport,
applyVCardBatch,
cancelVCardBatch,
deleteAddressBook,
deleteAddressList,
deleteAddressListEntry,
@@ -50,11 +53,9 @@ import {
discoverCardDavAddressBooks,
discoverLdapBaseDns,
endContactChannelRule,
exportAddressBookVcards,
exportContactVcard,
exportScopedVcards,
getAddressImportRun,
importAddressBookVcards,
applyAddressImport,
getVCardBatch,
getAddressQualitySummary,
listAddressBooks,
listAddressImportProfiles,
@@ -73,6 +74,7 @@ import {
listContactProvenance,
previewAddressSyncSource,
previewAddressImport,
previewVCardBatch,
rollbackAddressImport,
mergeContacts,
recoverContactMerge,
@@ -109,7 +111,8 @@ import {
type ContactFieldProvenance,
type ContactMergeRecord,
type ContactPointQualityDecision,
type ContactPointQualityState
type ContactPointQualityState,
type VCardBatchRun
} from "../../api/addresses";
import {
ADDRESS_FIELDS_DOCUMENTATION,
@@ -905,10 +908,6 @@ function contactFormHasIdentity(form: ContactFormState): boolean {
);
}
function safeFilename(value: string): string {
return (value.trim().replace(/[^A-Za-z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "") || "address-book") + ".vcf";
}
function downloadText(filename: string, content: string, type = "text/vcard;charset=utf-8") {
const blob = new Blob([content], { type });
const url = window.URL.createObjectURL(blob);
@@ -1003,7 +1002,12 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props)
const [dropTargetListId, setDropTargetListId] = useState("");
const [importOpen, setImportOpen] = useState(false);
const [importMode, setImportMode] = useState<ImportMode>("vcard");
const [vcardContent, setVcardContent] = useState("");
const [vcardFiles, setVcardFiles] = useState<File[]>([]);
const [vcardRun, setVcardRun] = useState<VCardBatchRun | null>(null);
const [vcardSelections, setVcardSelections] = useState<Record<string, "create" | "update" | "ignore">>({});
const [vcardDuplicatePolicy, setVcardDuplicatePolicy] = useState<"reject" | "first" | "last">("reject");
const [vcardExistingPolicy, setVcardExistingPolicy] = useState<"update" | "ignore" | "reject">("update");
const [vcardExportVersion, setVcardExportVersion] = useState<"3.0" | "4.0">("4.0");
const [importProfiles, setImportProfiles] = useState<AddressImportProfile[]>([]);
const [selectedImportProfileId, setSelectedImportProfileId] = useState("");
const [creatingImportProfile, setCreatingImportProfile] = useState(false);
@@ -1305,10 +1309,16 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props)
]
);
const dialogCancelReason = disabledReason([saving, savingReason]);
const vcardImportReason = disabledReason(
const vcardPreviewReason = disabledReason(
[saving, savingReason],
[!selectedBook, "Select an address book before importing vCards."],
[!vcardContent.trim(), "Paste vCard content before importing."]
[vcardFiles.length === 0, "Select one or more .vcf files before previewing."]
);
const vcardApplyReason = disabledReason(
[saving, savingReason],
[!vcardRun, "Preview the vCard files before applying."],
[!vcardRun?.can_apply, "This vCard batch is no longer pending."],
[Object.keys(vcardSelections).length === 0, "Select an action for at least one card."]
);
const importProfileSaveReason = disabledReason(
[saving, savingReason],
@@ -2288,8 +2298,11 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props)
setError("");
setNotice("");
try {
const content = await exportAddressBookVcards(settings, selectedBook.id);
downloadText(safeFilename(selectedBook.name), content);
const result = await exportScopedVcards(settings, selectedBook.id, selectedList
? { scope: "address_list", address_list_id: selectedList.id, version: vcardExportVersion }
: { scope: "address_book", version: vcardExportVersion });
downloadText(result.filename, result.content);
setNotice(`Exported ${result.contact_count} contact${result.contact_count === 1 ? "" : "s"} as vCard ${result.version}.`);
} catch (err) {
setError(errorMessage(err));
} finally {
@@ -2302,8 +2315,12 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props)
setError("");
setNotice("");
try {
const content = await exportContactVcard(settings, contact.id);
downloadText(safeFilename(contact.display_name), content);
const result = await exportScopedVcards(settings, contact.address_book_id, {
scope: "contacts",
contact_ids: [contact.id],
version: vcardExportVersion
});
downloadText(result.filename, result.content);
} catch (err) {
setError(errorMessage(err));
} finally {
@@ -2311,18 +2328,58 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props)
}
}
async function submitVcardImport(event: FormEvent<HTMLFormElement>) {
event.preventDefault();
async function previewSelectedVcards() {
if (!selectedBook) return;
setSaving(true);
setError("");
setNotice("");
try {
const result = await importAddressBookVcards(settings, selectedBook.id, vcardContent);
setImportOpen(false);
setVcardContent("");
const issueCount = result.issues.length;
setNotice(`Imported ${result.imported} contact${result.imported === 1 ? "" : "s"}${result.skipped ? `, skipped ${result.skipped}` : ""}${issueCount ? ` (${issueCount} import issue${issueCount === 1 ? "" : "s"})` : ""}.`);
const files = await Promise.all(vcardFiles.map(async (file) => ({
filename: file.name,
content_base64: await fileAsBase64(file)
})));
const run = await previewVCardBatch(settings, selectedBook.id, {
files,
duplicate_card_policy: vcardDuplicatePolicy,
existing_contact_policy: vcardExistingPolicy
});
setVcardRun(run);
setVcardSelections(Object.fromEntries(run.plan.map((item) => {
const action = item.allowed_actions.includes(item.action as "create" | "update" | "ignore")
? item.action as "create" | "update" | "ignore"
: "ignore";
return [item.source_key, action];
})));
setNotice(`Previewed ${run.card_count} vCard${run.card_count === 1 ? "" : "s"} without changing contacts.`);
} catch (err) {
setError(errorMessage(err));
} finally {
setSaving(false);
}
}
async function reloadVcardRun() {
if (!vcardRun) return;
setSaving(true);
setError("");
try {
setVcardRun(await getVCardBatch(settings, vcardRun.id));
} catch (err) {
setError(errorMessage(err));
} finally {
setSaving(false);
}
}
async function applySelectedVcards() {
if (!selectedBook || !vcardRun) return;
setSaving(true);
setError("");
setNotice("");
try {
const run = await applyVCardBatch(settings, vcardRun, Object.entries(vcardSelections).map(([source_key, action]) => ({ source_key, action })));
setVcardRun(run);
setNotice(`Applied vCard batch: ${run.progress.created} created, ${run.progress.updated} updated, ${run.progress.ignored} ignored.`);
await refreshBooks();
await refreshContacts(selectedBook.id, query);
} catch (err) {
@@ -2332,9 +2389,27 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props)
}
}
async function cancelSelectedVcardBatch() {
if (!vcardRun) return;
setSaving(true);
setError("");
try {
const run = await cancelVCardBatch(settings, vcardRun, "Cancelled by operator before commit.");
setVcardRun(run);
setNotice("Cancelled the pending vCard batch without changing contacts.");
} catch (err) {
setError(errorMessage(err));
} finally {
setSaving(false);
}
}
function openImportDialog() {
setSearchParams(withImportRunSearch(searchParams, null), { replace: true });
setImportMode("vcard");
setVcardFiles([]);
setVcardRun(null);
setVcardSelections({});
setImportRun(null);
setImportRunUnavailable("");
setImportFile(null);
@@ -2346,6 +2421,9 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props)
function closeImportDialog() {
setImportOpen(false);
setVcardFiles([]);
setVcardRun(null);
setVcardSelections({});
setImportRun(null);
setImportRunUnavailable("");
setImportRollbackOpen(false);
@@ -2882,7 +2960,11 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props)
<Button type="button" title="Add address book" aria-label="Add address book" variant="primary" onClick={openCreateBookDialog} disabledReason={createBookReason}><Plus size={15} /></Button>
<Button type="button" title="Add address list" aria-label="Add address list" onClick={openCreateListDialog} disabledReason={createListReason}><Plus size={15} /></Button>
<Button type="button" title="Import contacts" aria-label="Import contacts" onClick={openImportDialog} disabledReason={importBookReason}><Upload size={15} /></Button>
<Button type="button" title="Export selected address book as vCard" aria-label="Export selected address book as vCard" onClick={() => void exportSelectedBook()} disabledReason={exportBookReason}><Download size={15} /></Button>
<select aria-label="vCard export version" value={vcardExportVersion} onChange={(event) => setVcardExportVersion(event.target.value as "3.0" | "4.0")}>
<option value="4.0">vCard 4.0</option>
<option value="3.0">vCard 3.0</option>
</select>
<Button type="button" title={`Export selected ${selectedList ? "address list" : "address book"} as vCard ${vcardExportVersion}`} aria-label={`Export selected ${selectedList ? "address list" : "address book"} as vCard ${vcardExportVersion}`} onClick={() => void exportSelectedBook()} disabledReason={exportBookReason}><Download size={15} /></Button>
<Button type="button" title="Connect CardDAV" aria-label="Connect CardDAV" onClick={openCardDavDialog} disabledReason={connectCardDavReason}><Link2 size={15} /></Button>
<Button type="button" title="Connect LDAP or Active Directory" aria-label="Connect LDAP or Active Directory" onClick={openLdapDialog} disabledReason={connectLdapReason}><Network size={15} /></Button>
<Button type="button" title="Inspect sync source" aria-label="Inspect sync source" onClick={() => selectedSyncSource && void openSyncInspector(selectedSyncSource)} disabledReason={inspectSyncReason}><Search size={15} /></Button>
@@ -3804,7 +3886,11 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props)
<>
<Button type="button" onClick={closeImportDialog} disabledReason={dialogCancelReason}>Close</Button>
{importMode === "vcard" &&
<Button type="submit" form="address-vcard-import-form" variant="primary" disabledReason={vcardImportReason}><Upload size={16} /> Import</Button>}
<Button type="button" onClick={() => void previewSelectedVcards()} disabledReason={vcardPreviewReason}><Search size={16} /> Preview</Button>}
{importMode === "vcard" && vcardRun?.can_apply &&
<Button type="button" variant="primary" onClick={() => void applySelectedVcards()} disabledReason={vcardApplyReason}><Upload size={16} /> Apply selected</Button>}
{importMode === "vcard" && vcardRun?.can_cancel &&
<Button type="button" variant="danger" onClick={() => void cancelSelectedVcardBatch()} disabledReason={savingReason}><X size={16} /> Cancel batch</Button>}
{importMode === "tabular" && creatingImportProfile &&
<Button type="button" variant="primary" onClick={() => void saveImportProfile()} disabledReason={importProfileSaveReason}><Save size={16} /> {editingImportProfileId ? "Save new version" : "Save mapping"}</Button>}
{importMode === "tabular" && !creatingImportProfile &&
@@ -3825,18 +3911,74 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props)
onChange={(mode) => { setImportMode(mode); clearRetainedImportRun(); }}
/>
{importMode === "vcard" &&
<DialogForm id="address-vcard-import-form" className="address-dialog-form" onSubmit={(event) => void submitVcardImport(event)}>
<p className="muted">Paste one or more vCard entries into the selected address book.</p>
<FormField label="vCard content">
<textarea
className="address-vcard-textarea"
value={vcardContent}
onChange={(event) => setVcardContent(event.target.value)}
rows={14}
placeholder={"BEGIN:VCARD\nVERSION:4.0\nFN:Ada Lovelace\nEMAIL;TYPE=work:ada@example.local\nEND:VCARD"}
<div className="address-import-workspace">
<p className="muted">Select one or more .vcf files. Preview parses and validates them without changing contacts; only the reviewed actions are committed.</p>
<FormField label="vCard files">
<input
type="file"
accept=".vcf,text/vcard,text/x-vcard"
multiple
onChange={(event) => {
setVcardFiles(Array.from(event.target.files ?? []));
setVcardRun(null);
setVcardSelections({});
}}
/>
</FormField>
</DialogForm>}
<FormGrid columns={2} collapseAt="standard" className="">
<FormField label="Duplicate cards in upload">
<select value={vcardDuplicatePolicy} onChange={(event) => { setVcardDuplicatePolicy(event.target.value as typeof vcardDuplicatePolicy); setVcardRun(null); }}>
<option value="reject">Require manual rejection</option>
<option value="first">Use first occurrence</option>
<option value="last">Use last occurrence</option>
</select>
</FormField>
<FormField label="Existing contacts">
<select value={vcardExistingPolicy} onChange={(event) => { setVcardExistingPolicy(event.target.value as typeof vcardExistingPolicy); setVcardRun(null); }}>
<option value="update">Propose update</option>
<option value="ignore">Propose ignore</option>
<option value="reject">Require rejection</option>
</select>
</FormField>
</FormGrid>
{vcardRun &&
<div className="address-import-preview">
<div className="address-import-run-state">
<div>
<strong>Persisted vCard batch</strong>
<span className="muted block">{vcardRun.id} · {vcardRun.parser_version} · {vcardRun.execution_mode.replace("_", " ")}</span>
</div>
<StatusBadge status={vcardRun.status} />
<Button type="button" onClick={() => void reloadVcardRun()} disabledReason={savingReason}><RefreshCw size={15} /> Reload run</Button>
</div>
<div className="address-sync-plan-grid">
{(["create", "update", "ignore", "unchanged", "conflict", "errors"] as const).map((key) =>
<div key={key}><strong>{vcardRun.statistics[key] ?? 0}</strong><small>{key}</small></div>)}
</div>
{vcardRun.diagnostics.map((diagnostic, index) =>
<DismissibleAlert key={`${diagnostic.code}-${diagnostic.source_filename ?? index}-${diagnostic.card_index ?? index}`} tone={diagnostic.severity === "error" ? "danger" : diagnostic.severity === "warning" ? "warning" : "info"}>
{diagnostic.source_filename ? `${diagnostic.source_filename}${diagnostic.card_index ? ` card ${diagnostic.card_index}` : ""}: ` : ""}{diagnostic.message}
</DismissibleAlert>)}
<div className="address-sync-result-list">
{vcardRun.plan.map((item) =>
<div className="address-sync-plan-row" key={item.source_key}>
<StatusBadge status={item.action} />
<span>
<strong>{item.display_name || `Card ${item.card_index}`} · {item.source_filename}</strong>
<small>{item.changed_fields.join(", ") || item.message || "No field changes"}</small>
{item.duplicate_suggestions.length > 0 && <small>Possible match: {item.duplicate_suggestions.map((candidate) => candidate.display_name).join(", ")}</small>}
</span>
<select
aria-label={`Import action for ${item.display_name || `card ${item.card_index}`}`}
value={vcardSelections[item.source_key] ?? "ignore"}
disabled={!vcardRun.can_apply || item.allowed_actions.length < 2}
onChange={(event) => setVcardSelections((current) => ({ ...current, [item.source_key]: event.target.value as "create" | "update" | "ignore" }))}>
{item.allowed_actions.map((action) => <option value={action} key={action}>{action}</option>)}
</select>
</div>)}
</div>
</div>}
</div>}
{importMode === "tabular" &&
<div className="address-import-workspace">
{(requestedImportRunId || importRun) &&