diff --git a/README.md b/README.md index d868018..4a41cd8 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ Frontend package: Platform RBAC and governance rules are documented in `govoplan-core/docs/`. +## Operations + +- [Campaign delivery runbook](docs/CAMPAIGN_DELIVERY_RUNBOOK.md) covers queueing, local vs Celery operation, retries, reconciliation, reports, and the live SMTP/IMAP test checklist. + ## Release packaging The repository root includes a `package.json` for git-based WebUI installs. It exports the package `@govoplan/campaign-webui` from `webui/src` so release builds can depend on tagged git refs instead of local `file:` paths. Files and mail WebUI packages remain optional product-composition dependencies supplied by the core host build, not required campaign package dependencies. diff --git a/docs/CAMPAIGN_DELIVERY_RUNBOOK.md b/docs/CAMPAIGN_DELIVERY_RUNBOOK.md new file mode 100644 index 0000000..caa00f9 --- /dev/null +++ b/docs/CAMPAIGN_DELIVERY_RUNBOOK.md @@ -0,0 +1,77 @@ +# Campaign Delivery Runbook + +This runbook covers controlled campaign delivery after a campaign version has +been validated, built, reviewed, and locked. + +## Operating Modes + +- Local direct send: `CELERY_ENABLED=false`. Queueing stores jobs in the DB, and + small development runs can be processed with "Send queued now". +- Worker send: `CELERY_ENABLED=true` with Redis/Celery workers running. Queueing + publishes delivery tasks, and the Review & Send page polls summary counters. +- Mock send: use only for development review. It does not prove real SMTP/IMAP + credentials or server policy. + +## Before First Live Use + +- Use dedicated non-production SMTP/IMAP credentials. +- Use a dedicated mailbox/folder for append-to-Sent tests. +- Confirm policy allows the SMTP host, envelope sender, recipients, and optional + IMAP append target. +- Run one campaign each for no attachment, one attachment, and password-protected + ZIP before using production recipients. +- Keep the report page open during tests; it is the operational source of truth + for attempts, outcomes, and reconciliation. + +## Queue And Send + +1. Validate the version with file checks enabled. +2. Build the version and inspect all blocking review items. +3. Queue only after the selected version is the intended immutable execution + version. +4. In local mode, use "Send queued now" for small test runs. +5. In worker mode, verify queue counters move from queued/claimed/sending to a + terminal SMTP state. + +## Outcome Handling + +- `smtp_accepted`: Do not retry. If IMAP append is enabled and pending, run or + enqueue the append action. +- `failed_temporary`: Retry explicitly after checking the error and retry count. +- `failed_permanent`: Retry only if the operator has corrected the root cause and + intentionally includes permanent failures. +- `outcome_unknown`: Do not retry directly. Check SMTP logs, mailbox evidence, or + provider control panels, then reconcile as accepted or not sent. +- `claimed` or `sending` that does not progress: treat as a worker interruption. + Re-run worker handling or reconcile if SMTP may already have accepted the + message. + +## Reconciliation + +- Choose "Accepted" only with external evidence that SMTP accepted the message. + The job becomes protected from retry and may proceed to IMAP append. +- Choose "Not sent" only when SMTP did not accept the message. The job becomes a + temporary failure and can be selected by explicit retry. +- Add a note that identifies the evidence used, for example SMTP log line, + provider message ID, or operator ticket. + +## Fault Injection Checklist + +Use mock infrastructure first, then repeat against the non-production real test +bed where possible: + +- SMTP temporary failure. +- SMTP permanent failure. +- Recipient refusal after partial SMTP acceptance. +- Connection drop or worker interruption during SMTP. +- IMAP append failure after SMTP acceptance. +- Worker restart with queued, claimed, and sending jobs. + +## Reporting Checks + +- Partial delivery must show accepted, failed, and unknown counts separately. +- Accepted and unknown jobs must not appear in retry selections. +- Reconciled accepted jobs must remain protected from resend. +- Reconciled not-sent jobs must appear only as explicit retry candidates. +- The final report should include SMTP attempts, IMAP append attempts, and any + reconciliation notes before the campaign is considered operationally closed. diff --git a/package.json b/package.json index 5adf2a3..7eb8c9b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/campaign-webui", - "version": "0.1.1", + "version": "0.1.4", "private": true, "type": "module", "main": "webui/src/index.ts", @@ -18,8 +18,11 @@ "README.md", "LICENSE" ], + "dependencies": { + "read-excel-file": "^9.2.0" + }, "peerDependencies": { - "@govoplan/core-webui": "^0.1.1", + "@govoplan/core-webui": "^0.1.4", "lucide-react": "^0.555.0", "react": "^19.0.0", "react-dom": "^19.0.0", diff --git a/pyproject.toml b/pyproject.toml index 17574ec..ed043b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta" [project] name = "govoplan-campaign" -version = "0.1.3" +version = "0.1.4" description = "GovOPlaN campaigns module with backend and WebUI integration." readme = "README.md" requires-python = ">=3.12" license = { file = "LICENSE" } authors = [{ name = "GovOPlaN" }] dependencies = [ - "govoplan-core>=0.1.3", + "govoplan-core>=0.1.4", "jsonschema>=4,<5", "pydantic>=2,<3", "SQLAlchemy>=2,<3", diff --git a/src/govoplan_campaign/backend/campaign/models.py b/src/govoplan_campaign/backend/campaign/models.py index 2e9833b..87a7b0a 100644 --- a/src/govoplan_campaign/backend/campaign/models.py +++ b/src/govoplan_campaign/backend/campaign/models.py @@ -489,6 +489,35 @@ class EntryConfig(StrictModel): last_sent: str | None = None +class ImportColumnProvenance(StrictModel): + column_index: int = Field(ge=0) + header: str + kind: str + field_name: str | None = None + new_field_name: str | None = None + + +class ImportProvenance(StrictModel): + id: str + imported_at: str + mode: Literal["append", "replace"] + source_type: Literal["csv", "xlsx", "text"] + filename: str | None = None + sheet_name: str | None = None + encoding: str | None = None + delimiter: str | None = None + header_rows: int = Field(default=0, ge=0) + quoted: bool | None = None + value_separators: str | None = None + rows_total: int = Field(default=0, ge=0) + valid_rows: int = Field(default=0, ge=0) + invalid_rows: int = Field(default=0, ge=0) + imported_rows: int = Field(default=0, ge=0) + field_names_created: list[str] = Field(default_factory=list) + attachment_patterns: int = Field(default=0, ge=0) + mapping: list[ImportColumnProvenance] = Field(default_factory=list) + + class SourceConfig(StrictModel): type: SourceType path: str @@ -502,6 +531,7 @@ class EntriesConfig(StrictModel): source: SourceConfig | None = None mapping: dict[str, str] | None = None defaults: EntryConfig | None = None + imports: list[ImportProvenance] = Field(default_factory=list) @model_validator(mode="after") def inline_or_external(self) -> "EntriesConfig": diff --git a/src/govoplan_campaign/backend/db/models.py b/src/govoplan_campaign/backend/db/models.py index 68628ce..5cf38c0 100644 --- a/src/govoplan_campaign/backend/db/models.py +++ b/src/govoplan_campaign/backend/db/models.py @@ -144,6 +144,33 @@ class CampaignShare(Base, TimestampMixin): revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) +class RecipientImportMappingProfile(Base, TimestampMixin): + __tablename__ = "campaign_recipient_import_mapping_profiles" + __table_args__ = ( + Index("ix_recipient_import_profiles_owner", "tenant_id", "owner_user_id"), + Index("ix_recipient_import_profiles_ordered_fp", "tenant_id", "owner_user_id", "ordered_header_fingerprint"), + Index("ix_recipient_import_profiles_unordered_fp", "tenant_id", "owner_user_id", "unordered_header_fingerprint"), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) + owner_user_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + column_count: Mapped[int] = mapped_column(Integer, nullable=False) + headers: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) + normalized_headers: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) + ordered_header_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + unordered_header_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + delimiter: Mapped[str] = mapped_column(String(8), nullable=False) + header_rows: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + quoted: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + value_separators: Mapped[str] = mapped_column(String(50), default=",;|", nullable=False) + mappings: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False) + + tenant: Mapped[Tenant] = relationship() + owner: Mapped[User] = relationship() + + class CampaignVersion(Base, TimestampMixin): __tablename__ = "campaign_versions" __table_args__ = (UniqueConstraint("campaign_id", "version_number", name="uq_campaign_versions_campaign_number"),) diff --git a/src/govoplan_campaign/backend/manifest.py b/src/govoplan_campaign/backend/manifest.py index 2599a21..386b6b6 100644 --- a/src/govoplan_campaign/backend/manifest.py +++ b/src/govoplan_campaign/backend/manifest.py @@ -109,7 +109,7 @@ def _campaigns_router(context: ModuleContext): manifest = ModuleManifest( id="campaigns", name="Campaigns", - version="0.1.2", + version="0.1.4", dependencies=("access",), optional_dependencies=("files", "mail"), permissions=PERMISSIONS, diff --git a/src/govoplan_campaign/backend/migrations/versions/2c3d4e5f7081_recipient_import_mapping_profiles.py b/src/govoplan_campaign/backend/migrations/versions/2c3d4e5f7081_recipient_import_mapping_profiles.py new file mode 100644 index 0000000..5e12ed3 --- /dev/null +++ b/src/govoplan_campaign/backend/migrations/versions/2c3d4e5f7081_recipient_import_mapping_profiles.py @@ -0,0 +1,65 @@ +"""recipient import mapping profiles + +Revision ID: 2c3d4e5f7081 +Revises: 1b2c3d4e5f70 +Create Date: 2026-06-26 00:00:00.000000 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "2c3d4e5f7081" +down_revision = "1b2c3d4e5f70" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if "campaign_recipient_import_mapping_profiles" in inspector.get_table_names(): + return + op.create_table( + "campaign_recipient_import_mapping_profiles", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("owner_user_id", sa.String(length=36), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("column_count", sa.Integer(), nullable=False), + sa.Column("headers", sa.JSON(), nullable=False), + sa.Column("normalized_headers", sa.JSON(), nullable=False), + sa.Column("ordered_header_fingerprint", sa.String(length=64), nullable=False), + sa.Column("unordered_header_fingerprint", sa.String(length=64), nullable=False), + sa.Column("delimiter", sa.String(length=8), nullable=False), + sa.Column("header_rows", sa.Integer(), nullable=False), + sa.Column("quoted", sa.Boolean(), nullable=False), + sa.Column("value_separators", sa.String(length=50), nullable=False), + sa.Column("mappings", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.id"], name=op.f("fk_campaign_recipient_import_mapping_profiles_owner_user_id_users"), ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], name=op.f("fk_campaign_recipient_import_mapping_profiles_tenant_id_tenants"), ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id", name=op.f("pk_campaign_recipient_import_mapping_profiles")), + ) + op.create_index(op.f("ix_campaign_recipient_import_mapping_profiles_tenant_id"), "campaign_recipient_import_mapping_profiles", ["tenant_id"], unique=False) + op.create_index(op.f("ix_campaign_recipient_import_mapping_profiles_owner_user_id"), "campaign_recipient_import_mapping_profiles", ["owner_user_id"], unique=False) + op.create_index(op.f("ix_campaign_recipient_import_mapping_profiles_ordered_header_fingerprint"), "campaign_recipient_import_mapping_profiles", ["ordered_header_fingerprint"], unique=False) + op.create_index(op.f("ix_campaign_recipient_import_mapping_profiles_unordered_header_fingerprint"), "campaign_recipient_import_mapping_profiles", ["unordered_header_fingerprint"], unique=False) + op.create_index("ix_recipient_import_profiles_owner", "campaign_recipient_import_mapping_profiles", ["tenant_id", "owner_user_id"], unique=False) + op.create_index("ix_recipient_import_profiles_ordered_fp", "campaign_recipient_import_mapping_profiles", ["tenant_id", "owner_user_id", "ordered_header_fingerprint"], unique=False) + op.create_index("ix_recipient_import_profiles_unordered_fp", "campaign_recipient_import_mapping_profiles", ["tenant_id", "owner_user_id", "unordered_header_fingerprint"], unique=False) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if "campaign_recipient_import_mapping_profiles" not in inspector.get_table_names(): + return + op.drop_index("ix_recipient_import_profiles_unordered_fp", table_name="campaign_recipient_import_mapping_profiles") + op.drop_index("ix_recipient_import_profiles_ordered_fp", table_name="campaign_recipient_import_mapping_profiles") + op.drop_index("ix_recipient_import_profiles_owner", table_name="campaign_recipient_import_mapping_profiles") + op.drop_index(op.f("ix_campaign_recipient_import_mapping_profiles_unordered_header_fingerprint"), table_name="campaign_recipient_import_mapping_profiles") + op.drop_index(op.f("ix_campaign_recipient_import_mapping_profiles_ordered_header_fingerprint"), table_name="campaign_recipient_import_mapping_profiles") + op.drop_index(op.f("ix_campaign_recipient_import_mapping_profiles_owner_user_id"), table_name="campaign_recipient_import_mapping_profiles") + op.drop_index(op.f("ix_campaign_recipient_import_mapping_profiles_tenant_id"), table_name="campaign_recipient_import_mapping_profiles") + op.drop_table("campaign_recipient_import_mapping_profiles") diff --git a/src/govoplan_campaign/backend/router.py b/src/govoplan_campaign/backend/router.py index f7ffe55..5f198e7 100644 --- a/src/govoplan_campaign/backend/router.py +++ b/src/govoplan_campaign/backend/router.py @@ -19,6 +19,9 @@ from govoplan_campaign.backend.schemas import ( CampaignShareTargetsResponse, CampaignShareUpsertRequest, CampaignOwnerUpdateRequest, + RecipientImportMappingProfileListResponse, + RecipientImportMappingProfilePayload, + RecipientImportMappingProfileResponse, CampaignJobsResponse, CampaignJobDetailResponse, CampaignRetryJobsRequest, @@ -40,7 +43,7 @@ from govoplan_campaign.backend.schemas import ( ) from govoplan_core.auth.dependencies import ApiPrincipal, has_scope, require_scope from govoplan_core.audit.logging import audit_from_principal -from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignShare, CampaignVersion, Group, ImapAppendAttempt, SendAttempt, User, UserGroupMembership +from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignShare, CampaignVersion, Group, ImapAppendAttempt, RecipientImportMappingProfile, SendAttempt, User, UserGroupMembership from govoplan_core.db.session import get_session from govoplan_campaign.backend.reports.campaigns import CampaignReportError, generate_campaign_report, generate_jobs_csv from govoplan_campaign.backend.reports.emailing import CampaignReportEmailError, send_campaign_report_email @@ -181,6 +184,27 @@ def _require_permission(principal: ApiPrincipal, scope: str) -> None: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Missing scope: {scope}") +def _get_recipient_import_profile_for_principal(session: Session, profile_id: str, principal: ApiPrincipal) -> RecipientImportMappingProfile: + profile = session.get(RecipientImportMappingProfile, profile_id) + if not profile or profile.tenant_id != principal.tenant_id or profile.owner_user_id != principal.user.id: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Recipient import mapping profile not found") + return profile + + +def _apply_recipient_import_profile_payload(profile: RecipientImportMappingProfile, payload: RecipientImportMappingProfilePayload) -> None: + profile.name = payload.name.strip() + profile.column_count = payload.column_count + profile.headers = list(payload.headers) + profile.normalized_headers = list(payload.normalized_headers) + profile.ordered_header_fingerprint = payload.ordered_header_fingerprint + profile.unordered_header_fingerprint = payload.unordered_header_fingerprint + profile.delimiter = payload.delimiter + profile.header_rows = payload.header_rows + profile.quoted = payload.quoted + profile.value_separators = payload.value_separators + profile.mappings = [mapping.model_dump(mode="json") for mapping in payload.mappings] + + def _recipient_sections_changed(current: dict[str, object] | None, proposed: dict[str, object] | None) -> bool: if proposed is None: return False @@ -346,6 +370,109 @@ def list_campaigns( return CampaignListResponse(campaigns=[CampaignResponse.model_validate(item) for item in campaigns]) +@router.get("/recipient-import/mapping-profiles", response_model=RecipientImportMappingProfileListResponse) +def list_recipient_import_mapping_profiles( + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("campaigns:recipient:import")), +): + profiles = ( + session.query(RecipientImportMappingProfile) + .filter( + RecipientImportMappingProfile.tenant_id == principal.tenant_id, + RecipientImportMappingProfile.owner_user_id == principal.user.id, + ) + .order_by(RecipientImportMappingProfile.updated_at.desc(), RecipientImportMappingProfile.name.asc()) + .all() + ) + return RecipientImportMappingProfileListResponse( + profiles=[RecipientImportMappingProfileResponse.model_validate(profile) for profile in profiles] + ) + + +@router.post( + "/recipient-import/mapping-profiles", + response_model=RecipientImportMappingProfileResponse, + status_code=status.HTTP_201_CREATED, +) +def create_recipient_import_mapping_profile( + payload: RecipientImportMappingProfilePayload, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("campaigns:recipient:import")), +): + profile = RecipientImportMappingProfile( + tenant_id=principal.tenant_id, + owner_user_id=principal.user.id, + name=payload.name.strip(), + column_count=payload.column_count, + headers=list(payload.headers), + normalized_headers=list(payload.normalized_headers), + ordered_header_fingerprint=payload.ordered_header_fingerprint, + unordered_header_fingerprint=payload.unordered_header_fingerprint, + delimiter=payload.delimiter, + header_rows=payload.header_rows, + quoted=payload.quoted, + value_separators=payload.value_separators, + mappings=[mapping.model_dump(mode="json") for mapping in payload.mappings], + ) + session.add(profile) + session.flush() + audit_from_principal( + session, + principal, + action="campaign.recipient_import_mapping_profile_created", + object_type="recipient_import_mapping_profile", + object_id=profile.id, + details={"name": profile.name, "ordered_header_fingerprint": profile.ordered_header_fingerprint}, + commit=True, + ) + session.refresh(profile) + return RecipientImportMappingProfileResponse.model_validate(profile) + + +@router.put("/recipient-import/mapping-profiles/{profile_id}", response_model=RecipientImportMappingProfileResponse) +def update_recipient_import_mapping_profile( + profile_id: str, + payload: RecipientImportMappingProfilePayload, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("campaigns:recipient:import")), +): + profile = _get_recipient_import_profile_for_principal(session, profile_id, principal) + _apply_recipient_import_profile_payload(profile, payload) + session.add(profile) + audit_from_principal( + session, + principal, + action="campaign.recipient_import_mapping_profile_updated", + object_type="recipient_import_mapping_profile", + object_id=profile.id, + details={"name": profile.name, "ordered_header_fingerprint": profile.ordered_header_fingerprint}, + commit=True, + ) + session.refresh(profile) + return RecipientImportMappingProfileResponse.model_validate(profile) + + +@router.delete("/recipient-import/mapping-profiles/{profile_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_recipient_import_mapping_profile( + profile_id: str, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("campaigns:recipient:import")), +): + profile = _get_recipient_import_profile_for_principal(session, profile_id, principal) + details = {"name": profile.name, "ordered_header_fingerprint": profile.ordered_header_fingerprint} + session.delete(profile) + audit_from_principal( + session, + principal, + action="campaign.recipient_import_mapping_profile_deleted", + object_type="recipient_import_mapping_profile", + object_id=profile_id, + details=details, + commit=True, + ) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + @router.get("/{campaign_id}", response_model=CampaignResponse) def get_campaign( campaign_id: str, @@ -2008,4 +2135,3 @@ def preview_campaign_attachments( rules=rules, unused_shared_files=[_file_preview(session, asset) for asset in unused] if payload.include_unmatched else [], ) - diff --git a/src/govoplan_campaign/backend/schema/campaign.schema.json b/src/govoplan_campaign/backend/schema/campaign.schema.json index d30b5ca..9534a64 100644 --- a/src/govoplan_campaign/backend/schema/campaign.schema.json +++ b/src/govoplan_campaign/backend/schema/campaign.schema.json @@ -482,6 +482,13 @@ }, "defaults": { "$ref": "#/$defs/entry" + }, + "imports": { + "type": "array", + "items": { + "$ref": "#/$defs/import_provenance" + }, + "default": [] } }, "additionalProperties": false @@ -505,6 +512,13 @@ }, "defaults": { "$ref": "#/$defs/entry" + }, + "imports": { + "type": "array", + "items": { + "$ref": "#/$defs/import_provenance" + }, + "default": [] } }, "additionalProperties": false @@ -1027,6 +1041,151 @@ }, "additionalProperties": false }, + "import_provenance": { + "type": "object", + "required": [ + "id", + "imported_at", + "mode", + "source_type", + "header_rows", + "rows_total", + "valid_rows", + "invalid_rows", + "imported_rows" + ], + "properties": { + "id": { + "type": "string" + }, + "imported_at": { + "type": "string", + "format": "date-time" + }, + "mode": { + "type": "string", + "enum": [ + "append", + "replace" + ] + }, + "source_type": { + "type": "string", + "enum": [ + "csv", + "xlsx", + "text" + ] + }, + "filename": { + "type": [ + "string", + "null" + ] + }, + "sheet_name": { + "type": [ + "string", + "null" + ] + }, + "encoding": { + "type": [ + "string", + "null" + ] + }, + "delimiter": { + "type": [ + "string", + "null" + ] + }, + "header_rows": { + "type": "integer", + "minimum": 0, + "default": 0 + }, + "quoted": { + "type": [ + "boolean", + "null" + ] + }, + "value_separators": { + "type": [ + "string", + "null" + ] + }, + "rows_total": { + "type": "integer", + "minimum": 0 + }, + "valid_rows": { + "type": "integer", + "minimum": 0 + }, + "invalid_rows": { + "type": "integer", + "minimum": 0 + }, + "imported_rows": { + "type": "integer", + "minimum": 0 + }, + "field_names_created": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "attachment_patterns": { + "type": "integer", + "minimum": 0, + "default": 0 + }, + "mapping": { + "type": "array", + "items": { + "type": "object", + "required": [ + "column_index", + "header", + "kind" + ], + "properties": { + "column_index": { + "type": "integer", + "minimum": 0 + }, + "header": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "field_name": { + "type": [ + "string", + "null" + ] + }, + "new_field_name": { + "type": [ + "string", + "null" + ] + } + }, + "additionalProperties": false + }, + "default": [] + } + }, + "additionalProperties": false + }, "attachment_base_path": { "type": "object", "required": [ diff --git a/src/govoplan_campaign/backend/schemas.py b/src/govoplan_campaign/backend/schemas.py index 0be51a1..ce46aad 100644 --- a/src/govoplan_campaign/backend/schemas.py +++ b/src/govoplan_campaign/backend/schemas.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from govoplan_core.mail.config import ImapConfig, SmtpConfig @@ -186,6 +186,70 @@ class CampaignOwnerUpdateRequest(BaseModel): owner_group_id: str | None = None +RecipientImportColumnKind = Literal[ + "ignore", + "id", + "active", + "name", + "from", + "to", + "cc", + "bcc", + "reply_to", + "field", + "new_field", + "attachment_pattern", +] + + +class RecipientImportColumnMappingPayload(BaseModel): + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + column_index: int = Field(ge=0, alias="columnIndex") + kind: RecipientImportColumnKind + field_name: str | None = Field(default=None, max_length=255, alias="fieldName") + new_field_name: str | None = Field(default=None, max_length=255, alias="newFieldName") + + +class RecipientImportMappingProfilePayload(BaseModel): + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + name: str = Field(min_length=1, max_length=255) + column_count: int = Field(ge=0, le=500, alias="columnCount") + headers: list[str] = Field(default_factory=list, max_length=500) + normalized_headers: list[str] = Field(default_factory=list, max_length=500, alias="normalizedHeaders") + ordered_header_fingerprint: str = Field(min_length=1, max_length=64, alias="orderedHeaderFingerprint") + unordered_header_fingerprint: str = Field(min_length=1, max_length=64, alias="unorderedHeaderFingerprint") + delimiter: Literal[",", ";", "\t"] + header_rows: int = Field(ge=0, le=10, alias="headerRows") + quoted: bool = True + value_separators: str = Field(default=",;|", max_length=50, alias="valueSeparators") + mappings: list[RecipientImportColumnMappingPayload] = Field(default_factory=list, max_length=500) + + @model_validator(mode="after") + def validate_column_shape(self) -> "RecipientImportMappingProfilePayload": + if len(self.headers) != self.column_count: + raise ValueError("headers length must match columnCount") + if len(self.normalized_headers) != self.column_count: + raise ValueError("normalizedHeaders length must match columnCount") + for mapping in self.mappings: + if mapping.column_index >= self.column_count: + raise ValueError("mapping columnIndex exceeds columnCount") + return self + + +class RecipientImportMappingProfileResponse(RecipientImportMappingProfilePayload): + model_config = ConfigDict(from_attributes=True, populate_by_name=True) + + id: str + created_at: datetime = Field(alias="createdAt") + updated_at: datetime = Field(alias="updatedAt") + + +class RecipientImportMappingProfileListResponse(BaseModel): + profiles: list[RecipientImportMappingProfileResponse] = Field(default_factory=list) + + class CampaignJobsResponse(BaseModel): jobs: list[dict[str, Any]] page: int = 1 diff --git a/webui/package.json b/webui/package.json index 3891433..2729aca 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/campaign-webui", - "version": "0.1.2", + "version": "0.1.4", "private": true, "type": "module", "main": "src/index.ts", @@ -13,8 +13,11 @@ }, "./styles/campaign-workspace.css": "./src/styles/campaign-workspace.css" }, + "dependencies": { + "read-excel-file": "^9.2.0" + }, "peerDependencies": { - "@govoplan/core-webui": "^0.1.2", + "@govoplan/core-webui": "^0.1.4", "lucide-react": "^0.555.0", "react": "^19.0.0", "react-dom": "^19.0.0", diff --git a/webui/src/api/campaigns.ts b/webui/src/api/campaigns.ts index 5200349..3a0bd8f 100644 --- a/webui/src/api/campaigns.ts +++ b/webui/src/api/campaigns.ts @@ -92,6 +92,38 @@ export type CampaignWorkspaceQuery = { includeVersions?: boolean; }; +export type RecipientImportColumnKind = "ignore" | "id" | "active" | "name" | "from" | "to" | "cc" | "bcc" | "reply_to" | "field" | "new_field" | "attachment_pattern"; + +export type RecipientImportColumnMappingProfileItem = { + columnIndex: number; + kind: RecipientImportColumnKind; + fieldName?: string | null; + newFieldName?: string | null; +}; + +export type RecipientImportMappingProfile = { + id: string; + name: string; + createdAt: string; + updatedAt: string; + columnCount: number; + headers: string[]; + normalizedHeaders: string[]; + orderedHeaderFingerprint: string; + unorderedHeaderFingerprint: string; + delimiter: "," | ";" | "\t"; + headerRows: number; + quoted: boolean; + valueSeparators: string; + mappings: RecipientImportColumnMappingProfileItem[]; +}; + +export type RecipientImportMappingProfilePayload = Omit; + +export type RecipientImportMappingProfileListResponse = { + profiles: RecipientImportMappingProfile[]; +}; + export type CampaignVersionUpdatePayload = { campaign_json?: Record | null; current_flow?: string | null; @@ -298,6 +330,36 @@ export async function listCampaigns(settings: ApiSettings): Promise { + const response = await apiFetch(settings, "/api/v1/campaigns/recipient-import/mapping-profiles"); + return response.profiles ?? []; +} + +export async function createRecipientImportMappingProfile( + settings: ApiSettings, + payload: RecipientImportMappingProfilePayload +): Promise { + return apiFetch(settings, "/api/v1/campaigns/recipient-import/mapping-profiles", { + method: "POST", + body: JSON.stringify(payload) + }); +} + +export async function updateRecipientImportMappingProfile( + settings: ApiSettings, + profileId: string, + payload: RecipientImportMappingProfilePayload +): Promise { + return apiFetch(settings, `/api/v1/campaigns/recipient-import/mapping-profiles/${encodeURIComponent(profileId)}`, { + method: "PUT", + body: JSON.stringify(payload) + }); +} + +export async function deleteRecipientImportMappingProfile(settings: ApiSettings, profileId: string): Promise { + await apiFetch(settings, `/api/v1/campaigns/recipient-import/mapping-profiles/${encodeURIComponent(profileId)}`, { method: "DELETE" }); +} + export async function getCampaign(settings: ApiSettings, campaignId: string): Promise { return apiFetch(settings, `/api/v1/campaigns/${campaignId}`); } diff --git a/webui/src/api/files.ts b/webui/src/api/files.ts deleted file mode 100644 index f3927c5..0000000 --- a/webui/src/api/files.ts +++ /dev/null @@ -1,100 +0,0 @@ -import type { ApiSettings } from "../types"; -import { apiFetch } from "./client"; - -export type FileSpace = { - id: string; - label: string; - owner_type: "user" | "group"; - owner_id: string; - description?: string | null; -}; - -export type FileShare = { - id: string; - target_type: string; - target_id: string; - permission: string; - created_at: string; - revoked_at?: string | null; -}; - -export type ManagedFile = { - id: string; - tenant_id: string; - owner_type: "user" | "group"; - owner_id: string; - display_path: string; - filename: string; - description?: string | null; - size_bytes: number; - content_type?: string | null; - checksum_sha256: string; - version_id: string; - created_at: string; - updated_at: string; - deleted_at?: string | null; - audit_relevant: boolean; - metadata?: Record | null; - shares?: FileShare[]; -}; - -export type FileListResponse = { files: ManagedFile[] }; -export type FileSpacesResponse = { spaces: FileSpace[] }; -export type FileFolder = { - id: string; - tenant_id: string; - owner_type: "user" | "group"; - owner_id: string; - path: string; - created_at: string; - updated_at: string; - deleted_at?: string | null; -}; -export type FileFoldersResponse = { folders: FileFolder[] }; -export type PatternResolveResponse = { - patterns: { pattern: string; matches: ManagedFile[] }[]; - unmatched: ManagedFile[]; -}; - -export function listFileSpaces(settings: ApiSettings): Promise { - return apiFetch(settings, "/api/v1/files/spaces"); -} - -export function listFolders(settings: ApiSettings, params: { owner_type: "user" | "group"; owner_id: string }): Promise { - const search = new URLSearchParams(); - search.set("owner_type", params.owner_type); - search.set("owner_id", params.owner_id); - return apiFetch(settings, `/api/v1/files/folders?${search.toString()}`); -} - -export function listFiles(settings: ApiSettings, params: { owner_type?: string; owner_id?: string; campaign_id?: string; path_prefix?: string } = {}): Promise { - const search = new URLSearchParams(); - for (const [key, value] of Object.entries(params)) { - if (value) search.set(key, value); - } - const suffix = search.toString() ? `?${search.toString()}` : ""; - return apiFetch(settings, `/api/v1/files${suffix}`); -} - -export type FileBulkShareResponse = { shares: FileShare[]; shared_count: number }; - -export function shareFilesWithCampaign(settings: ApiSettings, fileIds: string[], campaignId: string): Promise { - return apiFetch(settings, "/api/v1/files/bulk-shares", { - method: "POST", - body: JSON.stringify({ file_ids: fileIds, target_type: "campaign", target_id: campaignId, permission: "read" }) - }); -} - -export async function shareFileWithCampaign(settings: ApiSettings, fileId: string, campaignId: string): Promise { - const response = await shareFilesWithCampaign(settings, [fileId], campaignId); - const share = response.shares[0]; - if (!share) throw new Error("File share was not created."); - return share; -} - -export function resolveFilePatterns( - settings: ApiSettings, - payload: { patterns: string[]; owner_type?: "user" | "group"; owner_id?: string; campaign_id?: string; path_prefix?: string; include_unmatched?: boolean; case_sensitive?: boolean } -): Promise { - return apiFetch(settings, "/api/v1/files/resolve-patterns", { method: "POST", body: JSON.stringify(payload) }); -} diff --git a/webui/src/features/campaigns/AttachmentsDataPage.tsx b/webui/src/features/campaigns/AttachmentsDataPage.tsx index c6a6163..9e43e8d 100644 --- a/webui/src/features/campaigns/AttachmentsDataPage.tsx +++ b/webui/src/features/campaigns/AttachmentsDataPage.tsx @@ -1,8 +1,7 @@ import { useEffect, useMemo, useState } from "react"; import { Pencil } from "lucide-react"; -import { usePlatformModuleInstalled } from "@govoplan/core-webui"; +import { usePlatformModuleInstalled, usePlatformUiCapability, type FilesFileExplorerUiCapability, type FilesFileSpace } from "@govoplan/core-webui"; import type { ApiSettings } from "../../types"; -import { listFileSpaces, type FileSpace } from "../../api/files"; import { Button } from "@govoplan/core-webui"; import { Card } from "@govoplan/core-webui"; import { PageTitle } from "@govoplan/core-webui"; @@ -18,7 +17,6 @@ import { useCampaignDraftEditor } from "./hooks/useCampaignDraftEditor"; import { asArray, asRecord, isAuditLockedVersion } from "./utils/campaignView"; import { updateNested } from "./utils/draftEditor"; import { AttachmentRulesDataGrid } from "./components/AttachmentRulesOverlay"; -import ManagedFileChooser from "./components/ManagedFileChooser"; import TemplateExpressionEditorDialog from "./components/TemplateExpressionEditorDialog"; import { countIndividualAttachmentRules, countIndividualAttachmentRulesForBasePath, createAttachmentBasePath, ensureAttachmentBasePaths, normalizeAttachmentBasePaths, normalizeAttachmentRules, normalizeAttachmentZipCollection, createAttachmentZipArchive, parseManagedAttachmentSource, removeIndividualAttachmentRulesForBasePath, summarizeAttachmentRules, type AttachmentBasePath, type AttachmentZipArchive, type AttachmentZipCollection } from "./utils/attachments"; import { insertAfter, moveArrayItem } from "@govoplan/core-webui"; @@ -30,9 +28,13 @@ type IndividualDisableState = { index: number; usageCount: number }; export default function AttachmentsDataPage({ settings, campaignId }: { settings: ApiSettings; campaignId: string }) { const filesModuleInstalled = usePlatformModuleInstalled("files"); + const filesFileExplorer = usePlatformUiCapability("files.fileExplorer"); + const ManagedFileChooser = filesModuleInstalled ? filesFileExplorer?.ManagedFileChooser : null; + const managedFilesAvailable = Boolean(ManagedFileChooser); + const listManagedFileSpaces = filesModuleInstalled ? filesFileExplorer?.listFileSpaces : undefined; const { data, loading, error, reload, setError } = useCampaignWorkspaceData(settings, campaignId); const [pathChooser, setPathChooser] = useState(null); - const [fileSpaces, setFileSpaces] = useState([]); + const [fileSpaces, setFileSpaces] = useState([]); const [individualDisable, setIndividualDisable] = useState(null); const [zipNameEditorIndex, setZipNameEditorIndex] = useState(null); const version = data.currentVersion; @@ -71,16 +73,16 @@ export default function AttachmentsDataPage({ settings, campaignId }: { settings ); useEffect(() => { - if (!filesModuleInstalled) { + if (!listManagedFileSpaces) { setFileSpaces([]); return; } let cancelled = false; - void listFileSpaces(settings) + void listManagedFileSpaces(settings) .then((response) => { if (!cancelled) setFileSpaces(response.spaces); }) .catch(() => { if (!cancelled) setFileSpaces([]); }); return () => { cancelled = true; }; - }, [filesModuleInstalled, settings.apiBaseUrl, settings.apiKey, settings.accessToken]); + }, [listManagedFileSpaces, settings.apiBaseUrl, settings.apiKey, settings.accessToken]); function patchBasePaths(paths: AttachmentBasePath[]) { if (locked) return; @@ -251,7 +253,7 @@ export default function AttachmentsDataPage({ settings, campaignId }: { settings basePath.id} emptyText="No attachment sources configured." emptyAction={ addBasePath(-1)} disabled={locked} label="Add first attachment source" />} @@ -310,7 +312,7 @@ export default function AttachmentsDataPage({ settings, campaignId }: { settings settings={settings} campaignId={campaignId} zipConfig={zipConfig} - filesModuleInstalled={filesModuleInstalled} + filesModuleInstalled={managedFilesAvailable} previewContext={attachmentPreviewContext} onChange={(rules) => patch(["attachments", "global"], rules)} /> @@ -321,9 +323,13 @@ export default function AttachmentsDataPage({ settings, campaignId }: { settings
Base paths
{basePaths.length}
Global attachments
direct: {globalSummary.direct} / rules: {globalSummary.rules}
Per-recipient patterns
{individualRulesCount}
-
Upload support
{filesModuleInstalled ? "Connected through Files" : "Manual paths"}
+
Upload support
{managedFilesAvailable ? "Connected through Files" : "Manual paths"}
-

{filesModuleInstalled ? "Files are managed in the top-level Files module. Upload there, share files with campaigns, then use these rules to resolve concrete attachments during review/build." : "The Files module is not installed. Use manual paths and patterns; managed file browsing and sharing are unavailable."}

+

{managedFilesAvailable + ? "Files are managed in the top-level Files module. Upload there, share files with campaigns, then use these rules to resolve concrete attachments during review/build." + : filesModuleInstalled + ? "Managed file browsing is unavailable. Use manual paths and patterns until the Files module exposes the file explorer capability." + : "The Files module is not installed. Use manual paths and patterns; managed file browsing and sharing are unavailable."}

@@ -357,11 +363,11 @@ export default function AttachmentsDataPage({ settings, campaignId }: { settings canAddField={(name) => !getDraftFields(displayDraft).some((field) => field.name === name)} /> - {pathChooser && filesModuleInstalled && ( + {pathChooser && ManagedFileChooser && ( ) => void; setIndividualEligibility: (index: number, checked: boolean) => void; addBasePath: (afterIndex?: number) => void; @@ -572,7 +578,7 @@ type AttachmentSourceColumnContext = { setPathChooser: (state: PathChooserState | null) => void; }; -function attachmentSourceColumns({ locked, basePaths, fileSpaces, filesModuleInstalled, patchBasePath, setIndividualEligibility, addBasePath, moveBasePath, removeBasePath, setPathChooser }: AttachmentSourceColumnContext): DataGridColumn[] { +function attachmentSourceColumns({ locked, basePaths, fileSpaces, managedFilesAvailable, patchBasePath, setIndividualEligibility, addBasePath, moveBasePath, removeBasePath, setPathChooser }: AttachmentSourceColumnContext): DataGridColumn[] { return [ { id: "name", header: "Name", width: 220, resizable: true, sortable: true, filterable: true, sticky: "start", render: (basePath, index) => patchBasePath(index, { name: event.target.value })} />, value: (basePath) => basePath.name }, { @@ -586,23 +592,23 @@ function attachmentSourceColumns({ locked, basePaths, fileSpaces, filesModuleIns
{ - if (!filesModuleInstalled) patchBasePath(index, { path: event.target.value, source: "" }); + if (!managedFilesAvailable) patchBasePath(index, { path: event.target.value, source: "" }); }} - onClick={() => filesModuleInstalled && !locked && setPathChooser({ index })} + onClick={() => managedFilesAvailable && !locked && setPathChooser({ index })} onKeyDown={(event) => { - if (filesModuleInstalled && !locked && (event.key === "Enter" || event.key === " ")) { + if (managedFilesAvailable && !locked && (event.key === "Enter" || event.key === " ")) { event.preventDefault(); setPathChooser({ index }); } }} /> - {filesModuleInstalled && } + {managedFilesAvailable && }
), value: (basePath) => formatAttachmentSourcePath(basePath, fileSpaces) @@ -631,7 +637,7 @@ function attachmentSourceColumns({ locked, basePaths, fileSpaces, filesModuleIns } ]; } -function formatAttachmentSourcePath(basePath: AttachmentBasePath, spaces: FileSpace[]): string { +function formatAttachmentSourcePath(basePath: AttachmentBasePath, spaces: FilesFileSpace[]): string { const parsedSource = parseManagedAttachmentSource(basePath.source); const matchingSpace = parsedSource ? spaces.find((space) => space.owner_type === parsedSource.ownerType && space.owner_id === parsedSource.ownerId) diff --git a/webui/src/features/campaigns/RecipientDataPage.tsx b/webui/src/features/campaigns/RecipientDataPage.tsx index 9d874ff..002d6bc 100644 --- a/webui/src/features/campaigns/RecipientDataPage.tsx +++ b/webui/src/features/campaigns/RecipientDataPage.tsx @@ -1,11 +1,18 @@ import { useEffect, useMemo, useState } from "react"; import type { ApiSettings } from "../../types"; +import { + createRecipientImportMappingProfile, + listRecipientImportMappingProfiles, + updateRecipientImportMappingProfile, + type RecipientImportMappingProfilePayload +} from "../../api/campaigns"; import { Button } from "@govoplan/core-webui"; import { Card } from "@govoplan/core-webui"; import { FileDropZone } from "@govoplan/core-webui"; import { FormField } from "@govoplan/core-webui"; import { PageTitle } from "@govoplan/core-webui"; import { LoadingFrame } from "@govoplan/core-webui"; +import { usePlatformUiCapability, type FilesFileExplorerUiCapability } from "@govoplan/core-webui"; import LockedVersionNotice from "./components/LockedVersionNotice"; import VersionLine from "./components/VersionLine"; import { ToggleSwitch } from "@govoplan/core-webui"; @@ -20,20 +27,33 @@ import { getBool } from "./utils/draftEditor"; import { getDraftFields } from "./utils/fieldDefinitions"; import { normalizeAttachmentBasePaths, type AttachmentBasePath } from "./utils/attachments"; import { + applyRecipientMappingProfile, buildImportTable, + buildImportTableFromRows, buildRecipientImportPreview, + createRecipientImportProvenance, + createRecipientMappingProfile as buildRecipientMappingProfile, defaultColumnMappings, importedRowsNeedAttachmentSource, + matchRecipientMappingProfiles, materializeRecipientImport, + recipientImportHeaderFingerprints, + xlsxSheetsFromArrayBuffer, type CsvDelimiter, type ImportedAddress, type RecipientColumnKind, type RecipientColumnMapping, type RecipientImportMode, - type RecipientImportPreview + type RecipientImportPreview, + type RecipientImportProvenance, + type RecipientImportSourceType, + type RecipientImportSpreadsheetSheet, + type RecipientMappingProfile, + type RecipientMappingProfileMatch } from "./utils/bulkImport"; import { bulkLinkFilesToCampaign, + type ImportFileLinkCapability, renderedImportPatterns, resolveImportedAttachmentLinks, type ImportFileLinkResolution @@ -178,7 +198,7 @@ export default function RecipientDataPage({ settings, campaignId }: { settings: replaceInlineEntries(moveArrayItem(inlineEntries, index, targetIndex)); } - function applyRecipientImport(preview: RecipientImportPreview, mode: RecipientImportMode) { + function applyRecipientImport(preview: RecipientImportPreview, mode: RecipientImportMode, provenance?: RecipientImportProvenance | null) { if (locked || !draft) return; const needsAttachmentSource = importedRowsNeedAttachmentSource(preview); const currentAttachments = asRecord(draft.attachments); @@ -194,7 +214,7 @@ export default function RecipientDataPage({ settings, campaignId }: { settings: attachmentBasePath = nextBasePaths[selectedIndex] ?? null; } - const importedDraft = materializeRecipientImport(draft, preview, { mode, attachmentBasePath }); + const importedDraft = materializeRecipientImport(draft, preview, { mode, attachmentBasePath, provenance }); const nextDraft = needsAttachmentSource ? { ...importedDraft, @@ -359,12 +379,12 @@ type RecipientImportDialogProps = { existingFields: ReturnType; defaultAttachmentBasePath: AttachmentBasePath | null; onCancel: () => void; - onImport: (preview: RecipientImportPreview, mode: RecipientImportMode) => void; + onImport: (preview: RecipientImportPreview, mode: RecipientImportMode, provenance?: RecipientImportProvenance | null) => void; }; type RecipientImportStepId = "upload" | "parse" | "map" | "preview" | "files"; -const recipientImportSteps: Array<{ id: RecipientImportStepId; label: string }> = [ +const allRecipientImportSteps: Array<{ id: RecipientImportStepId; label: string }> = [ { id: "upload", label: "Upload" }, { id: "parse", label: "Parse" }, { id: "map", label: "Map" }, @@ -387,28 +407,74 @@ const recipientColumnKindOptions: Array<{ value: RecipientColumnKind; label: str { value: "attachment_pattern", label: "Attachment pattern" } ]; +const AUTOMATIC_MAPPING_PROFILE_MIN_SCORE = 0.55; + +const RECIPIENT_IMPORT_ENCODINGS = [ + { value: "utf-8", label: "UTF-8" }, + { value: "windows-1252", label: "Windows-1252" }, + { value: "iso-8859-1", label: "ISO-8859-1" }, + { value: "utf-16le", label: "UTF-16 LE" }, + { value: "utf-16be", label: "UTF-16 BE" } +]; + export function RecipientImportDialog({ settings, campaignId, existingEntries, existingFields, defaultAttachmentBasePath, onCancel, onImport }: RecipientImportDialogProps) { + const filesFileExplorer = usePlatformUiCapability("files.fileExplorer"); + const fileLinkCapability = useMemo(() => { + if (!filesFileExplorer?.listFiles || !filesFileExplorer.resolveFilePatterns || !filesFileExplorer.shareFilesWithTarget) return null; + return { + listFiles: filesFileExplorer.listFiles, + resolveFilePatterns: filesFileExplorer.resolveFilePatterns, + shareFilesWithTarget: filesFileExplorer.shareFilesWithTarget + }; + }, [filesFileExplorer]); + const recipientImportSteps = useMemo( + () => fileLinkCapability ? allRecipientImportSteps : allRecipientImportSteps.filter((step) => step.id !== "files"), + [fileLinkCapability] + ); const [activeStep, setActiveStep] = useState("upload"); const [csvText, setCsvText] = useState(""); + const [sourceType, setSourceType] = useState("text"); const [filename, setFilename] = useState(""); + const [fileBuffer, setFileBuffer] = useState(null); + const [encoding, setEncoding] = useState("utf-8"); + const [workbookSheets, setWorkbookSheets] = useState([]); + const [selectedSheetName, setSelectedSheetName] = useState(""); const [mode, setMode] = useState("append"); const [delimiter, setDelimiter] = useState("auto"); const [headerRows, setHeaderRows] = useState(1); const [quoted, setQuoted] = useState(true); const [valueSeparators, setValueSeparators] = useState(",;|"); const [mappings, setMappings] = useState([]); + const [mappingProfiles, setMappingProfiles] = useState([]); + const [mappingManuallyChanged, setMappingManuallyChanged] = useState(false); + const [mappingProfileError, setMappingProfileError] = useState(""); + const [mappingProfileNotice, setMappingProfileNotice] = useState(""); const [fileError, setFileError] = useState(""); const [fileLinkResolution, setFileLinkResolution] = useState(null); const [fileLinkResolving, setFileLinkResolving] = useState(false); const [fileLinking, setFileLinking] = useState(false); const [fileLinkError, setFileLinkError] = useState(""); const [fileLinkNotice, setFileLinkNotice] = useState(""); - const hasContent = csvText.trim().length > 0; + const selectedSheet = useMemo( + () => workbookSheets.find((sheet) => sheet.name === selectedSheetName) ?? workbookSheets[0] ?? null, + [selectedSheetName, workbookSheets] + ); + const hasContent = sourceType === "xlsx" + ? Boolean(selectedSheet && selectedSheet.rows.some((row) => row.some((cell) => cell.trim()))) + : csvText.trim().length > 0; const table = useMemo( - () => hasContent ? buildImportTable(csvText, { delimiter, headerRows, quoted }) : null, - [csvText, delimiter, hasContent, headerRows, quoted] + () => { + if (!hasContent) return null; + if (sourceType === "xlsx") return selectedSheet ? buildImportTableFromRows(selectedSheet.rows, { headerRows }) : null; + return buildImportTable(csvText, { delimiter, headerRows, quoted }); + }, + [csvText, delimiter, hasContent, headerRows, quoted, selectedSheet, sourceType] ); const tableHeaderKey = table ? `${table.delimiter}:${table.firstDataRowNumber}:${table.headers.join("\u001f")}` : ""; + const mappingProfileMatches = useMemo( + () => table ? matchRecipientMappingProfiles(table, mappingProfiles) : [], + [mappingProfiles, table] + ); const parseReady = Boolean(table && table.dataRows.some((row) => row.some((cell) => cell.trim()))); const preview = useMemo( () => table ? buildRecipientImportPreview(table, mappings, { existingFields, existingEntries, valueSeparators }) : null, @@ -417,33 +483,93 @@ export function RecipientImportDialog({ settings, campaignId, existingEntries, e const activeStepIndex = recipientImportSteps.findIndex((step) => step.id === activeStep); const nextStep = recipientImportSteps[activeStepIndex + 1]?.id ?? null; const previousStep = recipientImportSteps[activeStepIndex - 1]?.id ?? null; + const isLastStep = activeStepIndex === recipientImportSteps.length - 1; const mappedColumnCount = mappings.filter((mapping) => mapping.kind !== "ignore").length; const patternRows = preview?.rows.filter((row) => row.patterns.length > 0).length ?? 0; const renderedPatterns = useMemo(() => preview ? renderedImportPatterns(preview) : [], [preview]); + useEffect(() => { + let cancelled = false; + setMappingProfileError(""); + void listRecipientImportMappingProfiles(settings) + .then((profiles) => { + if (!cancelled) setMappingProfiles(profiles); + }) + .catch((err) => { + if (!cancelled) setMappingProfileError(err instanceof Error ? err.message : String(err)); + }); + return () => { cancelled = true; }; + }, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]); + + useEffect(() => { + if (!fileBuffer || sourceType !== "csv") return; + try { + setCsvText(decodeImportText(fileBuffer, encoding)); + setFileError(""); + } catch (err) { + setCsvText(""); + setFileError(err instanceof Error ? err.message : String(err)); + } + }, [encoding, fileBuffer, sourceType]); + + useEffect(() => { + setMappingManuallyChanged(false); + }, [tableHeaderKey]); + useEffect(() => { if (!table) { setMappings([]); + setMappingProfileNotice(""); + return; + } + if (mappingManuallyChanged) return; + const automaticMatch = mappingProfileMatches.find(isAutomaticMappingProfileMatch); + if (automaticMatch) { + setMappings(applyRecipientMappingProfile(table, automaticMatch.profile, existingFields)); + setValueSeparators(automaticMatch.profile.valueSeparators || ",;|"); + setMappingProfileNotice(mappingProfileMatchNotice(automaticMatch)); return; } setMappings(defaultColumnMappings(table.headers, existingFields)); - }, [existingFields, tableHeaderKey]); + setMappingProfileNotice(""); + }, [existingFields, mappingManuallyChanged, mappingProfileMatches, table, tableHeaderKey]); useEffect(() => { if (!hasContent) setActiveStep("upload"); }, [hasContent]); useEffect(() => { - if (activeStep === "files") void refreshFileLinks(); + if (activeStep === "files" && !fileLinkCapability) setActiveStep("preview"); + }, [activeStep, fileLinkCapability]); + + useEffect(() => { + if (activeStep === "files" && fileLinkCapability) void refreshFileLinks(); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [activeStep, defaultAttachmentBasePath?.id, defaultAttachmentBasePath?.path, defaultAttachmentBasePath?.source, renderedPatterns.length]); + }, [activeStep, defaultAttachmentBasePath?.id, defaultAttachmentBasePath?.path, defaultAttachmentBasePath?.source, fileLinkCapability, renderedPatterns.length]); async function readFile(file: File | undefined) { if (!file) return; setFilename(file.name); setFileError(""); + setWorkbookSheets([]); + setSelectedSheetName(""); try { - setCsvText(await file.text()); + const buffer = await file.arrayBuffer(); + if (isXlsxFile(file)) { + const sheets = await xlsxSheetsFromArrayBuffer(buffer); + if (sheets.length === 0) { + throw new Error("Workbook contains no readable sheets."); + } + setSourceType("xlsx"); + setFileBuffer(null); + setCsvText(""); + setWorkbookSheets(sheets); + setSelectedSheetName(sheets[0]?.name ?? ""); + } else { + setSourceType("csv"); + setFileBuffer(buffer); + setCsvText(decodeImportText(buffer, encoding)); + } setActiveStep("parse"); } catch (err) { setFileError(err instanceof Error ? err.message : String(err)); @@ -454,6 +580,7 @@ export function RecipientImportDialog({ settings, campaignId, existingEntries, e if (stepId === "upload") return true; if (stepId === "parse") return hasContent; if (stepId === "map") return parseReady; + if (stepId === "files" && !fileLinkCapability) return false; return parseReady && mappings.length > 0; } @@ -475,6 +602,11 @@ export function RecipientImportDialog({ settings, campaignId, existingEntries, e async function refreshFileLinks() { setFileLinkNotice(""); + if (!fileLinkCapability) { + setFileLinkResolution(null); + setFileLinkError(""); + return; + } if (!preview || preview.patternCount === 0) { setFileLinkResolution(null); setFileLinkError(""); @@ -488,7 +620,7 @@ export function RecipientImportDialog({ settings, campaignId, existingEntries, e setFileLinkResolving(true); setFileLinkError(""); try { - setFileLinkResolution(await resolveImportedAttachmentLinks(settings, campaignId, defaultAttachmentBasePath, preview)); + setFileLinkResolution(await resolveImportedAttachmentLinks(fileLinkCapability, settings, campaignId, defaultAttachmentBasePath, preview)); } catch (err) { setFileLinkResolution(null); setFileLinkError(err instanceof Error ? err.message : String(err)); @@ -498,12 +630,12 @@ export function RecipientImportDialog({ settings, campaignId, existingEntries, e } async function linkImportedFiles() { - if (!fileLinkResolution || fileLinkResolution.linkableFiles.length === 0) return; + if (!fileLinkCapability || !fileLinkResolution || fileLinkResolution.linkableFiles.length === 0) return; setFileLinking(true); setFileLinkError(""); setFileLinkNotice(""); try { - const linkedCount = await bulkLinkFilesToCampaign(settings, campaignId, fileLinkResolution.linkableFiles); + const linkedCount = await bulkLinkFilesToCampaign(fileLinkCapability, settings, campaignId, fileLinkResolution.linkableFiles); await refreshFileLinks(); setFileLinkNotice(`Linked ${linkedCount} file${linkedCount === 1 ? "" : "s"} to this campaign.`); } catch (err) { @@ -513,9 +645,60 @@ export function RecipientImportDialog({ settings, campaignId, existingEntries, e } } + function rememberCurrentMappingProfile() { + if (!table || mappings.length === 0) return; + const tableSnapshot = table; + const mappingsSnapshot = mappings; + const fallbackProfiles = mappingProfiles; + const filenameSnapshot = filename; + const parseOptions = { headerRows, quoted }; + const valueSeparatorsSnapshot = valueSeparators; + + void (async () => { + try { + const latestProfiles = await listRecipientImportMappingProfiles(settings).catch(() => fallbackProfiles); + const reusableProfile = findReusableMappingProfile(tableSnapshot, latestProfiles); + const draftProfile = buildRecipientMappingProfile({ + id: reusableProfile?.id, + name: reusableProfile?.name ?? defaultMappingProfileName(filenameSnapshot, tableSnapshot), + table: tableSnapshot, + mappings: mappingsSnapshot, + parseOptions, + valueSeparators: valueSeparatorsSnapshot, + createdAt: reusableProfile?.createdAt, + updatedAt: new Date().toISOString() + }); + const payload = mappingProfilePayload(draftProfile); + await (reusableProfile + ? updateRecipientImportMappingProfile(settings, reusableProfile.id, payload) + : createRecipientImportMappingProfile(settings, payload)); + } catch { + // Import acceptance must not be blocked by passive profile learning. + } + })(); + } + + function confirmRecipientImport(nextPreview: RecipientImportPreview) { + rememberCurrentMappingProfile(); + const provenance = createRecipientImportProvenance({ + preview: nextPreview, + mappings, + mode, + sourceType, + filename, + sheetName: sourceType === "xlsx" ? selectedSheet?.name ?? null : null, + encoding: sourceType === "xlsx" ? null : encoding, + delimiter: sourceType === "xlsx" ? null : formatDelimiter(nextPreview.table.delimiter), + headerRows, + quoted: sourceType === "xlsx" ? null : quoted, + valueSeparators + }); + onImport(nextPreview, mode, provenance); + } function replaceColumnMapping(nextMapping: RecipientColumnMapping) { const columnCount = table?.headers.length ?? 0; + setMappingManuallyChanged(true); setMappings((current) => { const byColumn = new Map(current.map((mapping) => [mapping.columnIndex, mapping])); byColumn.set(nextMapping.columnIndex, nextMapping); @@ -537,11 +720,11 @@ export function RecipientImportDialog({ settings, campaignId, existingEntries, e
readFile(files[0])} /> @@ -553,21 +736,38 @@ export function RecipientImportDialog({ settings, campaignId, existingEntries, e
{fileError && {fileError}} - -