From aa9f90c71e58690d8a209a563937217938984e55 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Thu, 6 Aug 2026 12:42:20 +0200 Subject: [PATCH] Add purpose-bound form evidence storage --- docs/FILES_HANDBOOK.md | 19 +- src/govoplan_files/backend/db/models.py | 412 ++++++++++--- src/govoplan_files/backend/form_evidence.py | 449 ++++++++++++++ src/govoplan_files/backend/manifest.py | 582 +++++++++++++++--- ...2b3c4d5e6f8_form_evidence_upload_grants.py | 147 +++++ src/govoplan_files/backend/router.py | 2 + .../backend/routes/form_evidence.py | 172 ++++++ tests/test_form_evidence.py | 333 ++++++++++ tests/test_manifest_documentation.py | 11 +- tests/test_migrations.py | 42 ++ tests/test_router_contract.py | 27 +- tests/test_tenant_summary_batch.py | 21 +- 12 files changed, 2039 insertions(+), 178 deletions(-) create mode 100644 src/govoplan_files/backend/form_evidence.py create mode 100644 src/govoplan_files/backend/migrations/versions/a2b3c4d5e6f8_form_evidence_upload_grants.py create mode 100644 src/govoplan_files/backend/routes/form_evidence.py create mode 100644 tests/test_form_evidence.py create mode 100644 tests/test_migrations.py diff --git a/docs/FILES_HANDBOOK.md b/docs/FILES_HANDBOOK.md index 5ae558e..51aa49b 100644 --- a/docs/FILES_HANDBOOK.md +++ b/docs/FILES_HANDBOOK.md @@ -60,6 +60,7 @@ The main domain objects are: | Connector policy | Allow and deny rules inherited from system through tenant to one leaf scope | Evaluated before configuration and connector use | | Connector space | A read-only, manually synchronized remote folder/library linked to a user or group space | Created, updated, disabled, and soft-deleted | | Campaign attachment use | Evidence connecting a campaign job or entry to an exact asset, version, blob, checksum, and stage | Retained for campaign execution evidence | +| Form evidence upload grant | A one-use, hash-only bearer grant tied to an exact Form instance/revision, purpose, custodian, size, and media-type policy | Issued for at most 15 minutes, consumed by one managed upload, then retained as evidence provenance | ## User tasks @@ -649,6 +650,7 @@ public HTTP API. They must not import Files ORM models or storage helpers. | `files.access` (`0.1.6`) | Explain resource access provenance for managed files, explicit folders, and virtual folders | | `files.campaign_attachments` (`0.1.6`) | Resolve managed attachment matches, prepare frozen campaign snapshots, annotate built messages, share assets with a campaign, and record/mark exact attachment use | | `records.source.files` (`1.0.0`) | Recheck current Files access and resolve one exact, integrity-approved managed file version for Records filing | +| `forms_runtime.evidence.files` (`1.0.0`) | Issue a one-time managed attachment grant and re-verify the exact file/version/checksum at Form submission | Files requires Core principal resolution and permission evaluation. Campaign is an optional dependency; when installed, Files consumes the optional @@ -659,6 +661,14 @@ requested `FileVersion` identity, path snapshot, content metadata, SHA-256, integrity/protection state, and launch link. It rejects mutable aliases, cross-tenant requests, missing access, and quarantined or failed blobs. Records stores the filing decision; Files continues to own the version and bytes. +Forms Runtime is optional as well. Its public or authenticated participant never +receives general Files access through this integration. Forms Runtime asks for a +purpose-bound grant, Files verifies an active same-tenant user custodian, stores +only the token digest, accepts one bounded upload, and returns an immutable +`EvidenceReference`. Draft save and final submit independently recheck the exact +Form instance/revision, grant, asset, version, checksum, deletion state, and +integrity state. An idempotent replay returns the existing grant without +reproducing its bearer secret. ### API families @@ -678,6 +688,7 @@ All routes below are under `/api/v1/files`. | Credentials | `GET/POST /connectors/credentials`, `GET/PATCH/DELETE /connectors/credentials/{credential_id}` | | Policy | `GET/PUT /connectors/policies/{scope_type}`, `POST /connector-policy/evaluate` | | Incremental connector settings | `GET /connectors/settings/delta` | +| Form evidence | `POST /form-evidence/upload` with a short-lived `X-Form-Evidence-Token` issued by Forms Runtime | Consumers should use cursor/watermark contracts instead of assuming an unbounded complete list. The default full-list page size is 500 and public page @@ -715,6 +726,9 @@ Files baseline indiscriminately. - Upload, archive extraction, connector response, and S3 stream code use bounded reads. Archive previews are sealed and short-lived; ZIP passwords are request-only. +- Public Form evidence uploads require a custom-header bearer grant that is + stored only as SHA-256, expires after at most 15 minutes, is bound to one + exact submission and user custodian, and can create only one managed file. - Connector HTTP sockets use connection-time DNS/IP validation and pinning, redirects are refused, and unsafe SDK transports fail before client creation. - Database-managed connector passwords/tokens are encrypted; responses redact @@ -894,6 +908,7 @@ returning different content or credentials. | Audit | Connector discovery/import/sync/access and connector deletion; campaign exact-use evidence | Dedicated canonical audit events for every ordinary Files mutation | | Preview | File metadata and attachment download | Dedicated safe content-preview service | | Campaign | Stable capability-based frozen attachments and sent-use evidence | Campaign-specific process state remains in Campaign | +| Forms Runtime | One-time managed attachment grants plus exact-version, checksum, deletion, tenant, submission, and integrity verification | Malware scanning and advanced/qualified signature providers remain separate assurance depth | | Collaboration | Governed input/output snapshots | Co-editing, comments, review, presence, locks, and semantic document versions belong to Documents/workflow/provider modules | ## Release and change checklist @@ -917,7 +932,9 @@ Before releasing Files: checksum after the current file changes. 10. Exercise a committed upload, a rolled-back upload, object tamper detection, and applied orphan cleanup; inspect their `files` operations in Ops. -11. Update the implemented/planned table whenever a boundary changes. +11. Exercise a Form evidence grant, token replay, wrong-submission reference, + expired token, unsupported media type, and quarantined-file rejection. +12. Update the implemented/planned table whenever a boundary changes. ## Related documents diff --git a/src/govoplan_files/backend/db/models.py b/src/govoplan_files/backend/db/models.py index bb3feed..a09052c 100644 --- a/src/govoplan_files/backend/db/models.py +++ b/src/govoplan_files/backend/db/models.py @@ -4,7 +4,18 @@ import uuid from datetime import datetime from typing import Any -from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint, text +from sqlalchemy import ( + Boolean, + DateTime, + ForeignKey, + Index, + Integer, + JSON, + String, + Text, + UniqueConstraint, + text, +) from sqlalchemy.orm import Mapped, mapped_column from govoplan_core.db.base import Base, TimestampMixin @@ -33,17 +44,29 @@ class FileBlob(Base, TimestampMixin): storage_key: Mapped[str] = mapped_column(String(1000), nullable=False) checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True) size_bytes: Mapped[int] = mapped_column(Integer, nullable=False) - protection_discriminator: Mapped[str] = mapped_column(String(320), default="plaintext", nullable=False, index=True) - encryption_envelope_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True) - storage_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) + protection_discriminator: Mapped[str] = mapped_column( + String(320), default="plaintext", nullable=False, index=True + ) + encryption_envelope_id: Mapped[str | None] = mapped_column( + String(255), nullable=True, index=True + ) + storage_checksum_sha256: Mapped[str | None] = mapped_column( + String(64), nullable=True + ) storage_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) content_type: Mapped[str | None] = mapped_column(String(255)) ref_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False) retained_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - integrity_status: Mapped[str] = mapped_column(String(30), default="unchecked", nullable=False, index=True) - integrity_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) + integrity_status: Mapped[str] = mapped_column( + String(30), default="unchecked", nullable=False, index=True + ) + integrity_checked_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) integrity_failure: Mapped[str | None] = mapped_column(String(100), nullable=True) - quarantined_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) + quarantined_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) class FileIntegrityScan(Base, TimestampMixin): @@ -53,21 +76,35 @@ class FileIntegrityScan(Base, TimestampMixin): tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) storage_backend: Mapped[str] = mapped_column(String(50), nullable=False) storage_prefix: Mapped[str] = mapped_column(String(1000), nullable=False) - status: Mapped[str] = mapped_column(String(30), default="pending", nullable=False, index=True) + status: Mapped[str] = mapped_column( + String(30), default="pending", nullable=False, index=True + ) revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False) phase: Mapped[str] = mapped_column(String(30), default="blobs", nullable=False) - verify_checksums: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + verify_checksums: Mapped[bool] = mapped_column( + Boolean, default=True, nullable=False + ) batch_size: Mapped[int] = mapped_column(Integer, default=100, nullable=False) blob_cursor: Mapped[str | None] = mapped_column(String(36), nullable=True) object_cursor: Mapped[str | None] = mapped_column(String(1000), nullable=True) scanned_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) verified_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) - quarantined_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) - scanned_object_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + quarantined_blob_count: Mapped[int] = mapped_column( + Integer, default=0, nullable=False + ) + scanned_object_count: Mapped[int] = mapped_column( + Integer, default=0, nullable=False + ) orphan_object_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + started_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + completed_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) last_error: Mapped[str | None] = mapped_column(String(255), nullable=True) @@ -78,19 +115,35 @@ class FileIntegrityFinding(Base, TimestampMixin): ) id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - scan_id: Mapped[str] = mapped_column(ForeignKey("file_integrity_scans.id", ondelete="CASCADE"), nullable=False, index=True) + scan_id: Mapped[str] = mapped_column( + ForeignKey("file_integrity_scans.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True) - state: Mapped[str] = mapped_column(String(30), default="open", nullable=False, index=True) + state: Mapped[str] = mapped_column( + String(30), default="open", nullable=False, index=True + ) revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False) - blob_id: Mapped[str | None] = mapped_column(ForeignKey("file_blobs.id", ondelete="SET NULL"), nullable=True, index=True) + blob_id: Mapped[str | None] = mapped_column( + ForeignKey("file_blobs.id", ondelete="SET NULL"), nullable=True, index=True + ) storage_key: Mapped[str] = mapped_column(String(1000), nullable=False) expected_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) observed_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) - expected_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) - observed_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) - resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - resolved_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + expected_checksum_sha256: Mapped[str | None] = mapped_column( + String(64), nullable=True + ) + observed_checksum_sha256: Mapped[str | None] = mapped_column( + String(64), nullable=True + ) + resolved_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + resolved_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) class FileFolder(Base, TimestampMixin): @@ -98,14 +151,18 @@ class FileFolder(Base, TimestampMixin): __table_args__ = ( Index( "uq_file_folders_active_user_path", - "tenant_id", "owner_user_id", "path", + "tenant_id", + "owner_user_id", + "path", unique=True, sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"), postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"), ), Index( "uq_file_folders_active_group_path", - "tenant_id", "owner_group_id", "path", + "tenant_id", + "owner_group_id", + "path", unique=True, sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"), postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"), @@ -115,12 +172,22 @@ class FileFolder(Base, TimestampMixin): id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) - owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True) + owner_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + owner_group_id: Mapped[str | None] = mapped_column( + ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True + ) path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) - metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + deleted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) + metadata_: Mapped[dict[str, Any] | None] = mapped_column( + "metadata", JSON, nullable=True + ) class FileAsset(Base, TimestampMixin): @@ -129,48 +196,149 @@ class FileAsset(Base, TimestampMixin): id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) - owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True) - current_version_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) + owner_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + owner_group_id: Mapped[str | None] = mapped_column( + ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True + ) + current_version_id: Mapped[str | None] = mapped_column( + String(36), nullable=True, index=True + ) display_path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True) filename: Mapped[str] = mapped_column(String(500), nullable=False, index=True) description: Mapped[str | None] = mapped_column(Text) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) - metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + deleted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) + metadata_: Mapped[dict[str, Any] | None] = mapped_column( + "metadata", JSON, nullable=True + ) class FileVersion(Base, TimestampMixin): __tablename__ = "file_versions" - __table_args__ = (UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"),) + __table_args__ = ( + UniqueConstraint( + "file_asset_id", "version_number", name="uq_file_versions_asset_number" + ), + ) id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) - file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True) - blob_id: Mapped[str] = mapped_column(ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True) + file_asset_id: Mapped[str] = mapped_column( + ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True + ) + blob_id: Mapped[str] = mapped_column( + ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True + ) version_number: Mapped[int] = mapped_column(Integer, nullable=False) filename_at_upload: Mapped[str] = mapped_column(String(500), nullable=False) display_path_at_upload: Mapped[str] = mapped_column(String(1000), nullable=False) content_type: Mapped[str | None] = mapped_column(String(255)) size_bytes: Mapped[int] = mapped_column(Integer, nullable=False) checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + + +class FileFormEvidenceGrant(Base, TimestampMixin): + __tablename__ = "file_form_evidence_grants" + __table_args__ = ( + UniqueConstraint( + "tenant_id", + "idempotency_key", + name="uq_file_form_evidence_grants_idempotency", + ), + Index( + "ix_file_form_evidence_grants_form", + "tenant_id", + "form_instance_id", + "form_definition_id", + "form_definition_revision", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + form_instance_id: Mapped[str] = mapped_column( + String(36), nullable=False, index=True + ) + form_definition_id: Mapped[str] = mapped_column(String(255), nullable=False) + form_definition_revision: Mapped[str] = mapped_column(String(255), nullable=False) + token_sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True) + idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False) + request_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + custodian_user_id: Mapped[str] = mapped_column( + ForeignKey("access_users.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + evidence_kind: Mapped[str] = mapped_column(String(30), nullable=False) + purpose: Mapped[str] = mapped_column(String(500), nullable=False) + status: Mapped[str] = mapped_column( + String(30), default="issued", nullable=False, index=True + ) + expires_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, index=True + ) + max_size_bytes: Mapped[int] = mapped_column(Integer, nullable=False) + allowed_content_types: Mapped[list[str]] = mapped_column( + JSON, default=list, nullable=False + ) + file_asset_id: Mapped[str | None] = mapped_column( + ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=True, index=True + ) + file_version_id: Mapped[str | None] = mapped_column( + ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=True, index=True + ) + uploaded_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + revoked_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + metadata_: Mapped[dict[str, Any]] = mapped_column( + "metadata", JSON, default=dict, nullable=False + ) class FileShare(Base, TimestampMixin): __tablename__ = "file_shares" - __table_args__ = (UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"),) + __table_args__ = ( + UniqueConstraint( + "file_asset_id", + "target_type", + "target_id", + "revoked_at", + name="uq_file_shares_active_target", + ), + ) id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) - file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True) + file_asset_id: Mapped[str] = mapped_column( + ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True + ) target_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) target_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) permission: Mapped[str] = mapped_column(String(20), default="read", nullable=False) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) - revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) - revoked_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + expires_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) + revoked_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) + revoked_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) class FileConnectorProfile(Base, TimestampMixin): @@ -181,15 +349,23 @@ class FileConnectorProfile(Base, TimestampMixin): id: Mapped[str] = mapped_column(String(255), primary_key=True) tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) - scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True) + scope_type: Mapped[str] = mapped_column( + String(20), default="tenant", nullable=False, index=True + ) scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) label: Mapped[str] = mapped_column(String(255), nullable=False) provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True) endpoint_url: Mapped[str | None] = mapped_column(String(1000), nullable=True) base_path: Mapped[str | None] = mapped_column(String(1000), nullable=True) - enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True) - credential_profile_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True) - credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False) + enabled: Mapped[bool] = mapped_column( + Boolean, default=True, nullable=False, index=True + ) + credential_profile_id: Mapped[str | None] = mapped_column( + String(255), nullable=True, index=True + ) + credential_mode: Mapped[str] = mapped_column( + String(30), default="none", nullable=False + ) username: Mapped[str | None] = mapped_column(String(320), nullable=True) password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True) token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -198,9 +374,15 @@ class FileConnectorProfile(Base, TimestampMixin): secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True) capabilities: Mapped[list[str] | None] = mapped_column(JSON, nullable=True) policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) - metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + metadata_: Mapped[dict[str, Any] | None] = mapped_column( + "metadata", JSON, nullable=True + ) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + updated_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) class FileConnectorCredential(Base, TimestampMixin): @@ -211,12 +393,18 @@ class FileConnectorCredential(Base, TimestampMixin): id: Mapped[str] = mapped_column(String(255), primary_key=True) tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) - scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True) + scope_type: Mapped[str] = mapped_column( + String(20), default="tenant", nullable=False, index=True + ) scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) label: Mapped[str] = mapped_column(String(255), nullable=False) provider: Mapped[str | None] = mapped_column(String(50), nullable=True, index=True) - enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True) - credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False) + enabled: Mapped[bool] = mapped_column( + Boolean, default=True, nullable=False, index=True + ) + credential_mode: Mapped[str] = mapped_column( + String(30), default="none", nullable=False + ) username: Mapped[str | None] = mapped_column(String(320), nullable=True) password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True) token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -224,15 +412,26 @@ class FileConnectorCredential(Base, TimestampMixin): token_env: Mapped[str | None] = mapped_column(String(255), nullable=True) secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True) policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) - metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + metadata_: Mapped[dict[str, Any] | None] = mapped_column( + "metadata", JSON, nullable=True + ) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + updated_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) class FileConnectorPolicy(Base, TimestampMixin): __tablename__ = "file_connector_policies" __table_args__ = ( - UniqueConstraint("tenant_id", "scope_type", "scope_id", name="uq_file_connector_policies_scope"), + UniqueConstraint( + "tenant_id", + "scope_type", + "scope_id", + name="uq_file_connector_policies_scope", + ), Index("ix_file_connector_policies_scope", "scope_type", "scope_id"), ) @@ -241,24 +440,38 @@ class FileConnectorPolicy(Base, TimestampMixin): scope_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + updated_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) class FileConnectorSpace(Base, TimestampMixin): __tablename__ = "file_connector_spaces" __table_args__ = ( - Index("ix_file_connector_spaces_owner", "tenant_id", "owner_type", "owner_user_id", "owner_group_id"), + Index( + "ix_file_connector_spaces_owner", + "tenant_id", + "owner_type", + "owner_user_id", + "owner_group_id", + ), Index( "uq_file_connector_spaces_active_user_label", - "tenant_id", "owner_user_id", "label", + "tenant_id", + "owner_user_id", + "label", unique=True, sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"), postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"), ), Index( "uq_file_connector_spaces_active_group_label", - "tenant_id", "owner_group_id", "label", + "tenant_id", + "owner_group_id", + "label", unique=True, sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"), postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"), @@ -268,41 +481,81 @@ class FileConnectorSpace(Base, TimestampMixin): id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) - owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True) + owner_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + owner_group_id: Mapped[str | None] = mapped_column( + ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True + ) label: Mapped[str] = mapped_column(String(255), nullable=False) - connector_profile_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + connector_profile_id: Mapped[str] = mapped_column( + String(255), nullable=False, index=True + ) provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True) library_id: Mapped[str | None] = mapped_column(String(255), nullable=True) remote_path: Mapped[str] = mapped_column(String(1000), default="", nullable=False) sync_mode: Mapped[str] = mapped_column(String(30), default="manual", nullable=False) read_only: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) - deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) - metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) + is_active: Mapped[bool] = mapped_column( + Boolean, default=True, nullable=False, index=True + ) + created_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True + ) + deleted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) + metadata_: Mapped[dict[str, Any] | None] = mapped_column( + "metadata", JSON, nullable=True + ) class CampaignAttachmentUse(Base, TimestampMixin): __tablename__ = "campaign_attachment_uses" - __table_args__ = (UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"),) + __table_args__ = ( + UniqueConstraint( + "campaign_job_id", + "file_version_id", + "filename_used", + "use_stage", + name="uq_campaign_attachment_uses_job_file_stage", + ), + ) id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) - campaign_id: Mapped[str] = mapped_column(ForeignKey("campaigns.id", ondelete="CASCADE"), nullable=False, index=True) - campaign_version_id: Mapped[str] = mapped_column(ForeignKey("campaign_versions.id", ondelete="CASCADE"), nullable=False, index=True) - campaign_job_id: Mapped[str | None] = mapped_column(ForeignKey("campaign_jobs.id", ondelete="SET NULL"), nullable=True, index=True) + campaign_id: Mapped[str] = mapped_column( + ForeignKey("campaigns.id", ondelete="CASCADE"), nullable=False, index=True + ) + campaign_version_id: Mapped[str] = mapped_column( + ForeignKey("campaign_versions.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + campaign_job_id: Mapped[str | None] = mapped_column( + ForeignKey("campaign_jobs.id", ondelete="SET NULL"), nullable=True, index=True + ) entry_index: Mapped[int | None] = mapped_column(Integer) entry_id: Mapped[str | None] = mapped_column(String(255), index=True) - file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=False, index=True) - file_version_id: Mapped[str] = mapped_column(ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=False, index=True) - file_blob_id: Mapped[str] = mapped_column(ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True) + file_asset_id: Mapped[str] = mapped_column( + ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=False, index=True + ) + file_version_id: Mapped[str] = mapped_column( + ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=False, index=True + ) + file_blob_id: Mapped[str] = mapped_column( + ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True + ) filename_used: Mapped[str] = mapped_column(String(500), nullable=False) checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False) size_bytes: Mapped[int] = mapped_column(Integer, nullable=False) content_type: Mapped[str | None] = mapped_column(String(255)) - use_stage: Mapped[str] = mapped_column(String(20), default="built", nullable=False, index=True) - used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) + use_stage: Mapped[str] = mapped_column( + String(20), default="built", nullable=False, index=True + ) + used_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) __all__ = [ @@ -314,6 +567,7 @@ __all__ = [ "FileConnectorProfile", "FileConnectorSpace", "FileFolder", + "FileFormEvidenceGrant", "FileShare", "FileVersion", ] diff --git a/src/govoplan_files/backend/form_evidence.py b/src/govoplan_files/backend/form_evidence.py new file mode 100644 index 0000000..47001ce --- /dev/null +++ b/src/govoplan_files/backend/form_evidence.py @@ -0,0 +1,449 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime, timedelta +import hashlib +import json +import secrets + +from sqlalchemy.orm import Session + +from govoplan_core.core.access import CAPABILITY_ACCESS_DIRECTORY, AccessDirectory +from govoplan_core.core.form_evidence import ( + FormEvidenceContractError, + FormEvidenceGrant, + FormEvidenceGrantRequest, + FormEvidenceInspection, + FormEvidenceInspectionRequest, + FormEvidenceState, +) +from govoplan_core.core.institutional import EvidenceReference +from govoplan_core.core.modules import ModuleContext +from govoplan_files.backend.db.models import ( + FileAsset, + FileBlob, + FileFormEvidenceGrant, + FileVersion, +) +from govoplan_files.backend.runtime import configure_runtime + + +PROVIDER_ID = "files" +CAPABILITY_FORM_EVIDENCE_FILES = "forms_runtime.evidence.files" +MAX_GRANT_TTL = timedelta(minutes=15) + + +class FilesFormEvidenceProvider: + provider_id = PROVIDER_ID + + def __init__(self, registry: object | None, settings: object) -> None: + self._registry = registry + self._settings = settings + + def supported_kinds(self) -> Sequence[str]: + return ("document",) + + def create_upload_grant( + self, + session: object, + principal: object, + *, + request: FormEvidenceGrantRequest, + ) -> FormEvidenceGrant: + db = _session(session) + _assert_tenant(principal, request.tenant_id) + if request.evidence_kind != "document": + raise FormEvidenceContractError( + "Files can accept only document evidence for Forms Runtime." + ) + custodian_user_id = _custodian_user_id(request.custodian_ref) + self._assert_active_custodian( + tenant_id=request.tenant_id, + user_id=custodian_user_id, + ) + now = datetime.now(UTC) + if request.expires_at <= now: + raise FormEvidenceContractError( + "Form evidence upload grant expiry must be in the future." + ) + allowed_content_types = _content_types(request.allowed_content_types) + configured_max = int(getattr(self._settings, "file_upload_max_bytes")) + max_size_bytes = min(request.max_size_bytes or configured_max, configured_max) + request_sha256 = _request_sha256( + request, + custodian_user_id=custodian_user_id, + max_size_bytes=max_size_bytes, + allowed_content_types=allowed_content_types, + ) + existing = ( + db.query(FileFormEvidenceGrant) + .filter( + FileFormEvidenceGrant.tenant_id == request.tenant_id, + FileFormEvidenceGrant.idempotency_key == request.idempotency_key, + ) + .one_or_none() + ) + if existing is not None: + if not secrets.compare_digest(existing.request_sha256, request_sha256): + raise FormEvidenceContractError( + "Form evidence grant idempotency conflict." + ) + if ( + existing.status in {"expired", "revoked"} + or _aware(existing.expires_at) <= now + ): + raise FormEvidenceContractError( + "The existing Form evidence upload grant is no longer usable; " + "request a new grant with a new idempotency key." + ) + return _grant_response(existing, upload_token=None, replayed=True) + + metadata = _bounded_metadata(request.metadata) + remaining_attachments = metadata.get("remaining_attachments") + if isinstance(remaining_attachments, int): + existing_attachment_ids = set(metadata.get("existing_attachment_ids", ())) + active_grants = ( + db.query(FileFormEvidenceGrant) + .filter( + FileFormEvidenceGrant.tenant_id == request.tenant_id, + FileFormEvidenceGrant.form_instance_id == request.instance_id, + FileFormEvidenceGrant.form_definition_id + == request.definition_ref.object_id, + FileFormEvidenceGrant.form_definition_revision + == str(request.definition_ref.version), + FileFormEvidenceGrant.status.in_(("issued", "uploaded")), + FileFormEvidenceGrant.expires_at > now, + ) + .with_for_update() + .all() + ) + outstanding = sum( + 1 + for item in active_grants + if not item.file_asset_id + or item.file_asset_id not in existing_attachment_ids + ) + if outstanding >= remaining_attachments: + raise FormEvidenceContractError( + "This Form already has the maximum number of active attachment uploads." + ) + + upload_token = secrets.token_urlsafe(32) + grant = FileFormEvidenceGrant( + tenant_id=request.tenant_id, + form_instance_id=request.instance_id, + form_definition_id=request.definition_ref.object_id, + form_definition_revision=str(request.definition_ref.version), + token_sha256=_token_sha256(upload_token), + idempotency_key=request.idempotency_key, + request_sha256=request_sha256, + custodian_user_id=custodian_user_id, + evidence_kind=request.evidence_kind, + purpose=request.purpose, + status="issued", + expires_at=min(request.expires_at, now + MAX_GRANT_TTL), + max_size_bytes=max_size_bytes, + allowed_content_types=list(allowed_content_types), + metadata_=metadata, + ) + db.add(grant) + db.flush() + return _grant_response(grant, upload_token=upload_token, replayed=False) + + def inspect_evidence( + self, + session: object, + principal: object, + *, + request: FormEvidenceInspectionRequest, + ) -> FormEvidenceInspection: + db = _session(session) + _assert_tenant(principal, request.tenant_id) + now = datetime.now(UTC) + if request.evidence.owner_module != PROVIDER_ID: + return _inspection( + request.evidence, + state="rejected", + observed_at=now, + reason="The evidence owner does not match the Files provider.", + ) + if request.evidence.kind != "document" or not request.evidence.version: + return _inspection( + request.evidence, + state="rejected", + observed_at=now, + reason="Files evidence requires an exact document version.", + ) + grant = ( + db.query(FileFormEvidenceGrant) + .filter( + FileFormEvidenceGrant.tenant_id == request.tenant_id, + FileFormEvidenceGrant.form_instance_id == request.instance_id, + FileFormEvidenceGrant.form_definition_id + == request.definition_ref.object_id, + FileFormEvidenceGrant.form_definition_revision + == str(request.definition_ref.version), + FileFormEvidenceGrant.file_asset_id == request.evidence.evidence_id, + FileFormEvidenceGrant.file_version_id == request.evidence.version, + ) + .one_or_none() + ) + if grant is None: + return _inspection( + request.evidence, + state="rejected", + observed_at=now, + reason="The document was not captured for this exact Form submission.", + ) + if grant.status == "revoked": + return _inspection( + request.evidence, + state="revoked", + observed_at=now, + reason="The Form evidence grant was revoked.", + ) + if grant.status != "uploaded": + return _inspection( + request.evidence, + state="pending", + observed_at=now, + retryable=True, + reason="The Form evidence upload has not completed.", + ) + asset = db.get(FileAsset, grant.file_asset_id) + version = db.get(FileVersion, grant.file_version_id) + blob = db.get(FileBlob, version.blob_id) if version is not None else None + if asset is None or version is None or blob is None: + return _inspection( + request.evidence, + state="unavailable", + observed_at=now, + retryable=True, + reason="The managed document cannot currently be reconstructed.", + ) + if ( + asset.tenant_id != request.tenant_id + or version.tenant_id != request.tenant_id + or blob.tenant_id != request.tenant_id + or version.file_asset_id != asset.id + ): + return _inspection( + request.evidence, + state="rejected", + observed_at=now, + reason="The managed document crosses an evidence ownership boundary.", + ) + if asset.deleted_at is not None: + return _inspection( + request.evidence, + state="revoked", + observed_at=now, + reason="The managed document is no longer active.", + ) + if blob.quarantined_at is not None or blob.integrity_status == "quarantined": + return _inspection( + request.evidence, + state="rejected", + observed_at=now, + reason="The managed document failed its integrity gate.", + ) + if blob.integrity_status != "verified": + return _inspection( + request.evidence, + state="pending", + observed_at=now, + retryable=True, + reason="The managed document is awaiting integrity verification.", + ) + if ( + not request.evidence.checksum + or not secrets.compare_digest( + request.evidence.checksum, + version.checksum_sha256, + ) + or not secrets.compare_digest( + version.checksum_sha256, + blob.checksum_sha256, + ) + ): + return _inspection( + request.evidence, + state="rejected", + observed_at=now, + reason="The managed document checksum does not match the evidence.", + ) + return _inspection( + request.evidence, + state="accepted", + observed_at=now, + metadata={ + "content_type": version.content_type, + "size_bytes": version.size_bytes, + "integrity_status": blob.integrity_status, + "grant_id": grant.id, + }, + ) + + def _assert_active_custodian(self, *, tenant_id: str, user_id: str) -> None: + registry = self._registry + if registry is None or not hasattr(registry, "has_capability"): + raise FormEvidenceContractError( + "The Access directory is unavailable for Form evidence custody." + ) + if not registry.has_capability(CAPABILITY_ACCESS_DIRECTORY): + raise FormEvidenceContractError( + "The Access directory is unavailable for Form evidence custody." + ) + directory = registry.require_capability(CAPABILITY_ACCESS_DIRECTORY) + if not isinstance(directory, AccessDirectory): + raise FormEvidenceContractError( + "The Access directory capability is invalid." + ) + user = directory.get_user(user_id) + if user is None or user.tenant_id != tenant_id or user.status != "active": + raise FormEvidenceContractError( + "Form evidence requires an active same-tenant custodian." + ) + + +def create_files_form_evidence_provider( + context: ModuleContext, +) -> FilesFormEvidenceProvider: + configure_runtime(registry=context.registry, settings=context.settings) + return FilesFormEvidenceProvider(context.registry, context.settings) + + +def _grant_response( + grant: FileFormEvidenceGrant, + *, + upload_token: str | None, + replayed: bool, +) -> FormEvidenceGrant: + return FormEvidenceGrant( + provider_id=PROVIDER_ID, + grant_id=grant.id, + upload_token=upload_token, + upload_url="/api/v1/files/form-evidence/upload", + expires_at=_aware(grant.expires_at), + max_size_bytes=grant.max_size_bytes, + allowed_content_types=tuple(grant.allowed_content_types), + replayed=replayed, + ) + + +def _inspection( + reference: EvidenceReference, + *, + state: FormEvidenceState, + observed_at: datetime, + retryable: bool = False, + reason: str | None = None, + metadata: Mapping[str, object] | None = None, +) -> FormEvidenceInspection: + return FormEvidenceInspection( + provider_id=PROVIDER_ID, + reference=reference, + state=state, + observed_at=observed_at, + retryable=retryable, + reason=reason, + metadata=dict(metadata or {}), + ) + + +def _session(value: object) -> Session: + if not isinstance(value, Session): + raise TypeError("Files Form evidence requires a SQLAlchemy session.") + return value + + +def _assert_tenant(principal: object, tenant_id: str) -> None: + principal_tenant = str(getattr(principal, "tenant_id", "") or "").strip() + if not principal_tenant or principal_tenant != tenant_id: + raise PermissionError("Form evidence cannot cross tenants.") + + +def _custodian_user_id(value: str | None) -> str: + clean = str(value or "").strip() + if not clean.startswith("user:") or len(clean) <= len("user:"): + raise FormEvidenceContractError( + "Files Form evidence requires a user custodian." + ) + return clean.removeprefix("user:") + + +def _content_types(values: Sequence[str]) -> tuple[str, ...]: + cleaned = tuple( + dict.fromkeys( + value.split(";", 1)[0].strip().casefold() + for value in values + if value.strip() + ) + ) + if len(cleaned) > 50 or any(len(value) > 255 for value in cleaned): + raise FormEvidenceContractError( + "Form evidence content-type restrictions are too large." + ) + return cleaned + + +def _bounded_metadata(value: Mapping[str, object]) -> dict[str, object]: + remaining = value.get("remaining_attachments") + result: dict[str, object] = {} + if isinstance(remaining, int) and remaining >= 0: + result["remaining_attachments"] = remaining + raw_ids = value.get("existing_attachment_ids") + if isinstance(raw_ids, (list, tuple)): + clean_ids = tuple( + dict.fromkeys( + item.strip() + for item in raw_ids + if isinstance(item, str) and item.strip() and len(item.strip()) <= 255 + ) + ) + if len(clean_ids) <= 1000: + result["existing_attachment_ids"] = clean_ids + return result + + +def _request_sha256( + request: FormEvidenceGrantRequest, + *, + custodian_user_id: str, + max_size_bytes: int, + allowed_content_types: Sequence[str], +) -> str: + return hashlib.sha256( + json.dumps( + { + "tenant_id": request.tenant_id, + "instance_id": request.instance_id, + "definition_ref": request.definition_ref.to_dict(), + "evidence_kind": request.evidence_kind, + "purpose": request.purpose, + "expires_at": request.expires_at.isoformat(), + "custodian_user_id": custodian_user_id, + "max_size_bytes": max_size_bytes, + "allowed_content_types": list(allowed_content_types), + "metadata": _bounded_metadata(request.metadata), + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + + +def _token_sha256(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _aware(value: datetime) -> datetime: + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + +__all__ = [ + "CAPABILITY_FORM_EVIDENCE_FILES", + "FilesFormEvidenceProvider", + "PROVIDER_ID", + "create_files_form_evidence_provider", +] diff --git a/src/govoplan_files/backend/manifest.py b/src/govoplan_files/backend/manifest.py index 1772210..9e528fd 100644 --- a/src/govoplan_files/backend/manifest.py +++ b/src/govoplan_files/backend/manifest.py @@ -5,13 +5,19 @@ from pathlib import Path from sqlalchemy import inspect -from govoplan_core.core.access import CAPABILITY_AUTH_PERMISSION_EVALUATOR, CAPABILITY_AUTH_PRINCIPAL_RESOLVER +from govoplan_core.core.access import ( + CAPABILITY_AUTH_PERMISSION_EVALUATOR, + CAPABILITY_AUTH_PRINCIPAL_RESOLVER, +) from govoplan_core.core.encryption import CAPABILITY_ENCRYPTION_CONTENT_CIPHER from govoplan_core.core.files import ( CAPABILITY_FILES_ACCESS, CAPABILITY_FILES_ARTIFACT_STORE, ) -from govoplan_core.core.module_guards import drop_table_retirement_provider, persistent_table_uninstall_guard +from govoplan_core.core.module_guards import ( + drop_table_retirement_provider, + persistent_table_uninstall_guard, +) from govoplan_core.core.modules import ( CapabilityDocumentation, DocumentationCondition, @@ -42,6 +48,10 @@ from govoplan_core.db.base import Base from govoplan_files.backend.change_tracking import register_files_change_tracking from govoplan_files.backend.db import models as file_models # noqa: F401 - populate Files ORM metadata from govoplan_files.backend.documentation import documentation_topics +from govoplan_files.backend.form_evidence import ( + CAPABILITY_FORM_EVIDENCE_FILES, + create_files_form_evidence_provider, +) from govoplan_files.backend.provider_state import ( REMOTE_STORAGE_PROVIDER_ID, remote_storage_provider_states, @@ -62,6 +72,7 @@ _files_table_retirement_provider = drop_table_retirement_provider( file_models.FileFolder, file_models.FileAsset, file_models.FileVersion, + file_models.FileFormEvidenceGrant, file_models.FileShare, file_models.FileConnectorCredential, file_models.FileConnectorPolicy, @@ -79,8 +90,12 @@ def _files_retirement_provider(session: object | None, module_id: str): return plan def executor(execute_session: object, execute_module_id: str) -> None: - if not hasattr(execute_session, "get_bind") or not hasattr(execute_session, "query"): - raise RuntimeError("No database session is available for Files credential retirement.") + if not hasattr(execute_session, "get_bind") or not hasattr( + execute_session, "query" + ): + raise RuntimeError( + "No database session is available for Files credential retirement." + ) live_inspector = inspect(execute_session.get_bind()) if any( live_inspector.has_table(table_name) @@ -121,13 +136,37 @@ def _permission(scope: str, label: str, description: str) -> PermissionDefinitio PERMISSIONS = ( - _permission("files:file:read", "View files", "List, search and preview managed files."), - _permission("files:file:download", "Download files", "Download managed files and generated archives."), - _permission("files:file:upload", "Upload files", "Upload new managed file versions."), - _permission("files:file:organize", "Organize files", "Create folders, rename, move or copy managed files."), - _permission("files:file:share", "Share files", "List, grant, update, expire, and revoke managed file shares."), - _permission("files:file:delete", "Delete files", "Delete or hide managed files and folders where policy allows it."), - _permission("files:file:admin", "Administer file spaces", "Administer all file spaces in the tenant."), + _permission( + "files:file:read", "View files", "List, search and preview managed files." + ), + _permission( + "files:file:download", + "Download files", + "Download managed files and generated archives.", + ), + _permission( + "files:file:upload", "Upload files", "Upload new managed file versions." + ), + _permission( + "files:file:organize", + "Organize files", + "Create folders, rename, move or copy managed files.", + ), + _permission( + "files:file:share", + "Share files", + "List, grant, update, expire, and revoke managed file shares.", + ), + _permission( + "files:file:delete", + "Delete files", + "Delete or hide managed files and folders where policy allows it.", + ), + _permission( + "files:file:admin", + "Administer file spaces", + "Administer all file spaces in the tenant.", + ), ) ROLE_TEMPLATES = ( @@ -154,21 +193,48 @@ ROLE_TEMPLATES = ( def _tenant_summary(session, tenant_id: str) -> dict[str, int]: - from govoplan_files.backend.db.models import FileAsset, FileConnectorCredential, FileConnectorPolicy, FileConnectorProfile, FileConnectorSpace + from govoplan_files.backend.db.models import ( + FileAsset, + FileConnectorCredential, + FileConnectorPolicy, + FileConnectorProfile, + FileConnectorSpace, + FileFormEvidenceGrant, + ) return { - "files": session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id).count(), - "connector_credentials": session.query(FileConnectorCredential).filter(FileConnectorCredential.tenant_id == tenant_id).count(), - "connector_policies": session.query(FileConnectorPolicy).filter(FileConnectorPolicy.tenant_id == tenant_id).count(), - "connector_profiles": session.query(FileConnectorProfile).filter(FileConnectorProfile.tenant_id == tenant_id).count(), - "connector_spaces": session.query(FileConnectorSpace).filter(FileConnectorSpace.tenant_id == tenant_id).count(), + "files": session.query(FileAsset) + .filter(FileAsset.tenant_id == tenant_id) + .count(), + "connector_credentials": session.query(FileConnectorCredential) + .filter(FileConnectorCredential.tenant_id == tenant_id) + .count(), + "connector_policies": session.query(FileConnectorPolicy) + .filter(FileConnectorPolicy.tenant_id == tenant_id) + .count(), + "connector_profiles": session.query(FileConnectorProfile) + .filter(FileConnectorProfile.tenant_id == tenant_id) + .count(), + "connector_spaces": session.query(FileConnectorSpace) + .filter(FileConnectorSpace.tenant_id == tenant_id) + .count(), + "form_evidence_upload_grants": session.query(FileFormEvidenceGrant) + .filter(FileFormEvidenceGrant.tenant_id == tenant_id) + .count(), } def _tenant_summary_batch(session, tenant_ids) -> dict[str, dict[str, int]]: from sqlalchemy import func - from govoplan_files.backend.db.models import FileAsset, FileConnectorCredential, FileConnectorPolicy, FileConnectorProfile, FileConnectorSpace + from govoplan_files.backend.db.models import ( + FileAsset, + FileConnectorCredential, + FileConnectorPolicy, + FileConnectorProfile, + FileConnectorSpace, + FileFormEvidenceGrant, + ) ids = tuple(dict.fromkeys(str(tenant_id) for tenant_id in tenant_ids if tenant_id)) if not ids: @@ -180,6 +246,7 @@ def _tenant_summary_batch(session, tenant_ids) -> dict[str, dict[str, int]]: "connector_policies": 0, "connector_profiles": 0, "connector_spaces": 0, + "form_evidence_upload_grants": 0, } for tenant_id in ids } @@ -189,6 +256,7 @@ def _tenant_summary_batch(session, tenant_ids) -> dict[str, dict[str, int]]: ("connector_policies", FileConnectorPolicy), ("connector_profiles", FileConnectorProfile), ("connector_spaces", FileConnectorSpace), + ("form_evidence_upload_grants", FileFormEvidenceGrant), ) for count_key, model in models: rows = ( @@ -203,21 +271,51 @@ def _tenant_summary_batch(session, tenant_ids) -> dict[str, dict[str, int]]: def _veto_group_delete(session, tenant_id: str, group_id: str) -> None: - from govoplan_files.backend.db.models import FileAsset, FileConnectorSpace, FileFolder, FileShare + from govoplan_files.backend.db.models import ( + FileAsset, + FileConnectorSpace, + FileFolder, + FileShare, + ) - owned_asset_count = session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id, FileAsset.owner_group_id == group_id).count() - owned_folder_count = session.query(FileFolder).filter(FileFolder.tenant_id == tenant_id, FileFolder.owner_group_id == group_id).count() - owned_connector_space_count = session.query(FileConnectorSpace).filter( - FileConnectorSpace.tenant_id == tenant_id, - FileConnectorSpace.owner_group_id == group_id, - ).count() - shared_asset_count = session.query(FileShare).filter( - FileShare.tenant_id == tenant_id, - FileShare.target_type == "group", - FileShare.target_id == group_id, - ).count() - if owned_asset_count or owned_folder_count or owned_connector_space_count or shared_asset_count: - raise ValueError("Cannot remove the group while it owns files, folders, connector spaces or file shares.") + owned_asset_count = ( + session.query(FileAsset) + .filter(FileAsset.tenant_id == tenant_id, FileAsset.owner_group_id == group_id) + .count() + ) + owned_folder_count = ( + session.query(FileFolder) + .filter( + FileFolder.tenant_id == tenant_id, FileFolder.owner_group_id == group_id + ) + .count() + ) + owned_connector_space_count = ( + session.query(FileConnectorSpace) + .filter( + FileConnectorSpace.tenant_id == tenant_id, + FileConnectorSpace.owner_group_id == group_id, + ) + .count() + ) + shared_asset_count = ( + session.query(FileShare) + .filter( + FileShare.tenant_id == tenant_id, + FileShare.target_type == "group", + FileShare.target_id == group_id, + ) + .count() + ) + if ( + owned_asset_count + or owned_folder_count + or owned_connector_space_count + or shared_asset_count + ): + raise ValueError( + "Cannot remove the group while it owns files, folders, connector spaces or file shares." + ) def _files_router(context: ModuleContext): @@ -229,6 +327,29 @@ def _files_router(context: ModuleContext): return router +def _public_tenant_resolver(request: object, session: object) -> str | None: + path = str(getattr(getattr(request, "url", None), "path", "")) + if not path.endswith("/files/form-evidence/upload") or not hasattr( + session, "query" + ): + return None + headers = getattr(request, "headers", {}) + token = str(headers.get("X-Form-Evidence-Token") or "").strip() + if len(token) < 32: + return None + import hashlib + + row = ( + session.query(file_models.FileFormEvidenceGrant) + .filter( + file_models.FileFormEvidenceGrant.token_sha256 + == hashlib.sha256(token.encode("utf-8")).hexdigest() + ) + .one_or_none() + ) + return row.tenant_id if row is not None else None + + REMOTE_STORAGE_PROVIDER = ExternalProviderDeclaration( id=REMOTE_STORAGE_PROVIDER_ID, module_id="files", @@ -285,13 +406,17 @@ manifest = ModuleManifest( id="files", name="Files", version="0.1.18", - required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR), + required_capabilities=( + CAPABILITY_AUTH_PRINCIPAL_RESOLVER, + CAPABILITY_AUTH_PERMISSION_EVALUATOR, + ), optional_dependencies=("campaigns", "encryption", "records", "search"), provides_interfaces=( ModuleInterfaceProvider(name="files.access", version="0.1.6"), ModuleInterfaceProvider(name="files.campaign_attachments", version="0.1.6"), ModuleInterfaceProvider(name=CAPABILITY_FILES_ARTIFACT_STORE, version="0.1.14"), ModuleInterfaceProvider(name=CAPABILITY_RECORD_SOURCE_FILES, version="1.0.0"), + ModuleInterfaceProvider(name=CAPABILITY_FORM_EVIDENCE_FILES, version="1.0.0"), ), requires_interfaces=( ModuleInterfaceRequirement( @@ -315,6 +440,7 @@ manifest = ModuleManifest( ), permissions=PERMISSIONS, route_factory=_files_router, + public_tenant_resolver=_public_tenant_resolver, role_templates=ROLE_TEMPLATES, tenant_summary_providers=(_tenant_summary,), tenant_summary_batch_providers=(_tenant_summary_batch,), @@ -325,7 +451,15 @@ manifest = ModuleManifest( ), ), delete_veto_providers={"group": (_veto_group_delete,)}, - nav_items=(NavItem(path="/files", label="Files", icon="folder", required_any=("files:file:read",), order=40),), + nav_items=( + NavItem( + path="/files", + label="Files", + icon="folder", + required_any=("files:file:read",), + order=40, + ), + ), frontend=FrontendModule( module_id="files", package_name="@govoplan/files-webui", @@ -337,18 +471,121 @@ manifest = ModuleManifest( order=40, ), ), - nav_items=(NavItem(path="/files", label="Files", icon="folder", required_any=("files:file:read",), order=40),), + nav_items=( + NavItem( + path="/files", + label="Files", + icon="folder", + required_any=("files:file:read",), + order=40, + ), + ), view_surfaces=( - ViewSurface(id="files.admin.system-connectors", module_id="files", kind="section", label="System file connections", order=75), - ViewSurface(id="files.admin.tenant-connectors", module_id="files", kind="section", label="Tenant file connections", order=65), - ViewSurface(id="files.admin.tenant-integrity", module_id="files", kind="section", label="File integrity", order=66), - ViewSurface(id="files.admin.group-connectors", module_id="files", kind="section", label="Group file connections", order=65), - ViewSurface(id="files.admin.user-connectors", module_id="files", kind="section", label="User file connections", order=65), - ViewSurface(id="files.settings.connectors", module_id="files", kind="section", label="Personal file connections", order=20), - ViewSurface(id="files.widget.spaces", module_id="files", kind="section", label="File spaces widget", order=35), + ViewSurface( + id="files.admin.system-connectors", + module_id="files", + kind="section", + label="System file connections", + order=75, + ), + ViewSurface( + id="files.admin.tenant-connectors", + module_id="files", + kind="section", + label="Tenant file connections", + order=65, + ), + ViewSurface( + id="files.admin.tenant-integrity", + module_id="files", + kind="section", + label="File integrity", + order=66, + ), + ViewSurface( + id="files.admin.group-connectors", + module_id="files", + kind="section", + label="Group file connections", + order=65, + ), + ViewSurface( + id="files.admin.user-connectors", + module_id="files", + kind="section", + label="User file connections", + order=65, + ), + ViewSurface( + id="files.settings.connectors", + module_id="files", + kind="section", + label="Personal file connections", + order=20, + ), + ViewSurface( + id="files.widget.spaces", + module_id="files", + kind="section", + label="File spaces widget", + order=35, + ), ), ), documentation=( + DocumentationTopic( + id="files.forms-runtime.managed-evidence", + title="Capture Form attachments as managed evidence", + summary="Issue short-lived upload grants and bind managed file versions to one exact Form submission.", + body=( + "When Forms Runtime requests document evidence, Files issues a purpose-bound grant for an active same-tenant user custodian. The bearer token is shown once, stored only as a SHA-256 digest, expires after at most 15 minutes, and can create one managed file version. Final Form submission rechecks the exact Form, grant, asset, version, checksum, deletion state, and Files integrity gate. Public intake therefore does not receive general Files permissions, and a document captured for one submission cannot be reused silently for another." + ), + layer="configured", + documentation_types=("admin", "user"), + audience=("form_participant", "form_admin", "file_manager", "auditor"), + related_modules=("forms_runtime", "forms"), + order=42, + conditions=( + DocumentationCondition( + required_modules=("files", "forms_runtime"), + any_scopes=( + "forms_runtime:instance:participate", + "forms_runtime:instance:write", + "files:file:admin", + ), + ), + ), + links=( + DocumentationLink(label="Forms", href="/forms-runtime", kind="runtime"), + DocumentationLink(label="Files", href="/files", kind="runtime"), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), + ), + metadata={ + "kind": "workflow", + "route": "/forms-runtime", + "help_contexts": ["forms_runtime.instance", "files.upload"], + "security_invariants": [ + "Bearer upload tokens are never retained in plaintext.", + "A grant is bound to one tenant, Form instance, definition revision, purpose, custodian, size limit, and optional media-type allowlist.", + "Submission fails closed unless the exact managed version passes the Files integrity gate.", + ], + "limitations": [ + "The first signature profile is an authenticated acknowledgement; advanced and qualified electronic signatures require a separate provider." + ], + "prerequisites": [ + "Forms Runtime and Files are enabled, and the intake profile has an active same-tenant user custodian." + ], + "steps": [ + "Request a Files evidence grant from the editable Form instance, upload through the returned one-time bearer route, then add the returned evidence reference to the draft." + ], + "outcome": "The draft references an immutable managed Files version with exact checksum evidence.", + "verification": "Submit the Form and verify that altered, deleted, quarantined, cross-tenant, and cross-submission references are rejected.", + }, + ), DocumentationTopic( id="files.records.exact-version-source", title="File exact versions into an eAkte", @@ -374,7 +611,11 @@ manifest = ModuleManifest( links=( DocumentationLink(label="Files", href="/files", kind="runtime"), DocumentationLink(label="Records", href="/records", kind="runtime"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), ), metadata={ "kind": "workflow", @@ -417,7 +658,11 @@ manifest = ModuleManifest( links=( DocumentationLink(label="Files", href="/files", kind="runtime"), DocumentationLink(label="Search", href="/search", kind="runtime"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), ), metadata={ "kind": "reference", @@ -446,10 +691,18 @@ manifest = ModuleManifest( ), links=( DocumentationLink(label="Files", href="/files", kind="runtime"), - DocumentationLink(label="Move or copy API", href="/api/v1/files/transfer", kind="api"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="Move or copy API", href="/api/v1/files/transfer", kind="api" + ), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), + ), + unlocks=( + "Managed content can be placed at stable logical paths without bypassing space access.", ), - unlocks=("Managed content can be placed at stable logical paths without bypassing space access.",), metadata={ "kind": "workflow", "route": "/files", @@ -494,15 +747,23 @@ manifest = ModuleManifest( ), links=( DocumentationLink(label="Files", href="/files", kind="runtime"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), + ), + unlocks=( + "Authorized users can retrieve governed current versions without gaining organization or sharing authority.", ), - unlocks=("Authorized users can retrieve governed current versions without gaining organization or sharing authority.",), metadata={ "kind": "workflow", "route": "/files", "screen": "Files", "help_contexts": ["files.list"], - "prerequisites": ["You may view and download managed files in the relevant space."], + "prerequisites": [ + "You may view and download managed files in the relevant space." + ], "steps": [ "Open Files and select the relevant personal or group space.", "Navigate folders, sort the list, or use a path/name pattern to find the intended content.", @@ -537,11 +798,21 @@ manifest = ModuleManifest( ), links=( DocumentationLink(label="Files", href="/files", kind="runtime"), - DocumentationLink(label="List and manage shares", href="/api/v1/files/{file_id}/shares", kind="api"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="List and manage shares", + href="/api/v1/files/{file_id}/shares", + kind="api", + ), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), ), related_modules=("campaigns",), - unlocks=("A supporting process can grant governed file access without changing file ownership.",), + unlocks=( + "A supporting process can grant governed file access without changing file ownership.", + ), metadata={ "kind": "workflow", "route": "/files", @@ -589,9 +860,15 @@ manifest = ModuleManifest( ), links=( DocumentationLink(label="Files", href="/files", kind="runtime"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), + ), + unlocks=( + "Authorized users can remove obsolete content from active file views while preserving the current soft-delete boundary.", ), - unlocks=("Authorized users can remove obsolete content from active file views while preserving the current soft-delete boundary.",), metadata={ "kind": "workflow", "route": "/files", @@ -644,14 +921,36 @@ manifest = ModuleManifest( ), ), links=( - DocumentationLink(label="Tenant connector administration", href="/admin?section=tenant-file-connectors", kind="runtime"), - DocumentationLink(label="System connector administration", href="/admin?section=system-file-connectors", kind="runtime"), - DocumentationLink(label="Tenant connector policy API", href="/api/v1/files/connectors/policies/tenant", kind="api"), - DocumentationLink(label="Connector credentials API", href="/api/v1/files/connectors/credentials", kind="api"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="Tenant connector administration", + href="/admin?section=tenant-file-connectors", + kind="runtime", + ), + DocumentationLink( + label="System connector administration", + href="/admin?section=system-file-connectors", + kind="runtime", + ), + DocumentationLink( + label="Tenant connector policy API", + href="/api/v1/files/connectors/policies/tenant", + kind="api", + ), + DocumentationLink( + label="Connector credentials API", + href="/api/v1/files/connectors/credentials", + kind="api", + ), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), ), related_modules=("access", "audit", "mail"), - unlocks=("Scoped, explainable external-file access without exposing credentials to consuming modules.",), + unlocks=( + "Scoped, explainable external-file access without exposing credentials to consuming modules.", + ), configuration_keys=( "GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON", "GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE", @@ -705,14 +1004,36 @@ manifest = ModuleManifest( ), ), links=( - DocumentationLink(label="System file connections", href="/admin?section=system-file-connectors", kind="runtime"), - DocumentationLink(label="File integrity operations", href="/admin?section=tenant-file-integrity", kind="runtime"), - DocumentationLink(label="Connector provider status", href="/api/v1/files/connectors/providers", kind="api"), - DocumentationLink(label="Create an integrity scan", href="/api/v1/files/integrity/scans", kind="api"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="System file connections", + href="/admin?section=system-file-connectors", + kind="runtime", + ), + DocumentationLink( + label="File integrity operations", + href="/admin?section=tenant-file-integrity", + kind="runtime", + ), + DocumentationLink( + label="Connector provider status", + href="/api/v1/files/connectors/providers", + kind="api", + ), + DocumentationLink( + label="Create an integrity scan", + href="/api/v1/files/integrity/scans", + kind="api", + ), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), ), related_modules=("audit", "encryption", "ops"), - unlocks=("Recoverable managed-file evidence without weakening outbound peer validation.",), + unlocks=( + "Recoverable managed-file evidence without weakening outbound peer validation.", + ), configuration_keys=( "FILE_STORAGE_BACKEND", "FILE_STORAGE_LOCAL_ROOT", @@ -734,7 +1055,13 @@ manifest = ModuleManifest( "route": "/admin?section=system-file-connectors", "screen": "System file connections and deployment operations", "section": "Storage integrity, backup/recovery, and fail-closed transports", - "recovery_unit": ["Files database rows", "Encryption envelope and wrapped-key rows", "managed blob namespace", "MASTER_KEY_B64", "deployment-owned connector configuration"], + "recovery_unit": [ + "Files database rows", + "Encryption envelope and wrapped-key rows", + "managed blob namespace", + "MASTER_KEY_B64", + "deployment-owned connector configuration", + ], "verification": "After restore, complete a checksum-enabled integrity scan, resolve every missing/corrupt finding, approve or retain every reported orphan, inspect Files recovery operations in Ops, verify authorized and denied access, and test configured pinned HTTP, S3, and SMB connectors against their recorded target topology.", "related_topic_ids": [ "files.governed-connectors-and-provenance", @@ -752,28 +1079,54 @@ manifest = ModuleManifest( ), layer="available", documentation_types=("admin", "user"), - audience=("module_integrator", "file_admin", "campaign_admin", "process_designer"), + audience=( + "module_integrator", + "file_admin", + "campaign_admin", + "process_designer", + ), order=52, conditions=( DocumentationCondition( required_modules=("files",), - any_scopes=("files:file:read", "files:file:upload", "files:file:admin"), + any_scopes=( + "files:file:read", + "files:file:upload", + "files:file:admin", + ), ), ), links=( DocumentationLink(label="Files", href="/files", kind="runtime"), DocumentationLink(label="Files API", href="/api/v1/files", kind="api"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), ), related_modules=("campaigns", "docs"), - unlocks=("Campaign, report, template, workflow, and document modules can exchange governed input/output snapshots.",), + unlocks=( + "Campaign, report, template, workflow, and document modules can exchange governed input/output snapshots.", + ), metadata={ "kind": "reference", "route": "/files", "screen": "Files and module integration", "section": "Managed snapshot provenance and capability boundaries", - "provided_interfaces": ["files.access@0.1.6", "files.campaign_attachments@0.1.6"], - "provenance_fields": ["connector_id", "provider", "external_id", "external_path", "revision", "metadata"], + "provided_interfaces": [ + "files.access@0.1.6", + "files.campaign_attachments@0.1.6", + "forms_runtime.evidence.files@1.0.0", + ], + "provenance_fields": [ + "connector_id", + "provider", + "external_id", + "external_path", + "revision", + "metadata", + ], "related_topic_ids": [ "files.workflow.import-managed-snapshot", "files.governed-connectors-and-provenance", @@ -791,7 +1144,13 @@ manifest = ModuleManifest( ), layer="configured", documentation_types=("admin", "user"), - audience=("process_owner", "release_manager", "file_admin", "operator", "security_auditor"), + audience=( + "process_owner", + "release_manager", + "file_admin", + "operator", + "security_auditor", + ), order=53, conditions=( DocumentationCondition( @@ -808,11 +1167,21 @@ manifest = ModuleManifest( links=( DocumentationLink(label="Files", href="/files", kind="runtime"), DocumentationLink(label="Files API", href="/api/v1/files", kind="api"), - DocumentationLink(label="Connector provider status", href="/api/v1/files/connectors/providers", kind="api"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="Connector provider status", + href="/api/v1/files/connectors/providers", + kind="api", + ), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), ), related_modules=("campaigns", "audit", "ops"), - unlocks=("Process owners and release managers can approve Files use against explicit controls, evidence, and known gaps.",), + unlocks=( + "Process owners and release managers can approve Files use against explicit controls, evidence, and known gaps.", + ), configuration_keys=( "FILE_STORAGE_BACKEND", "FILE_STORAGE_LOCAL_ROOT", @@ -877,7 +1246,11 @@ manifest = ModuleManifest( ), links=( DocumentationLink(label="Files", href="/files", kind="runtime"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), ), related_modules=("ops", "campaigns"), configuration_keys=( @@ -917,8 +1290,16 @@ manifest = ModuleManifest( ), links=( DocumentationLink(label="Files", href="/files", kind="runtime"), - DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), - DocumentationLink(label="Generated artifact contract", href="govoplan-files/docs/GENERATED_ARTIFACT_STORE.md", kind="repository"), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), + DocumentationLink( + label="Generated artifact contract", + href="govoplan-files/docs/GENERATED_ARTIFACT_STORE.md", + kind="repository", + ), ), related_modules=("templates", "campaigns", "reporting"), metadata={"kind": "reference", "route": "/files"}, @@ -939,6 +1320,7 @@ manifest = ModuleManifest( file_models.FileFolder, file_models.FileAsset, file_models.FileVersion, + file_models.FileFormEvidenceGrant, file_models.FileShare, file_models.FileConnectorCredential, file_models.FileConnectorPolicy, @@ -949,10 +1331,18 @@ manifest = ModuleManifest( ), ), capability_factories={ - CAPABILITY_FILES_ACCESS: lambda context: __import__("govoplan_files.backend.capabilities", fromlist=["access_capability"]).access_capability(context), - CAPABILITY_FILES_ARTIFACT_STORE: lambda context: __import__("govoplan_files.backend.capabilities", fromlist=["artifact_store_capability"]).artifact_store_capability(context), - "files.campaign_attachments": lambda context: __import__("govoplan_files.backend.capabilities", fromlist=["campaign_capability"]).campaign_capability(context), + CAPABILITY_FILES_ACCESS: lambda context: __import__( + "govoplan_files.backend.capabilities", fromlist=["access_capability"] + ).access_capability(context), + CAPABILITY_FILES_ARTIFACT_STORE: lambda context: __import__( + "govoplan_files.backend.capabilities", + fromlist=["artifact_store_capability"], + ).artifact_store_capability(context), + "files.campaign_attachments": lambda context: __import__( + "govoplan_files.backend.capabilities", fromlist=["campaign_capability"] + ).campaign_capability(context), CAPABILITY_RECORD_SOURCE_FILES: create_files_record_source, + CAPABILITY_FORM_EVIDENCE_FILES: create_files_form_evidence_provider, }, capability_documentation={ CAPABILITY_RECORD_SOURCE_FILES: CapabilityDocumentation( @@ -960,6 +1350,11 @@ manifest = ModuleManifest( summary="Resolves currently authorized immutable managed file versions for Records filing.", contract_version="1.0.0", ), + CAPABILITY_FORM_EVIDENCE_FILES: CapabilityDocumentation( + label="Files Form evidence provider", + summary="Issues one-time managed attachment grants and verifies exact Form evidence versions.", + contract_version="1.0.0", + ), }, operational_check_providers=( OperationalCheckProviderRegistration( @@ -986,14 +1381,27 @@ manifest = ModuleManifest( maturity="vertical_slice", documentation_ref="docs/FILES_HANDBOOK.md", test_ref="tests/test_storage_backends.py", - known_limits=("Target-environment multi-node recovery drills and writable remote connector effects are not reference-ready; hard purge and legal hold remain unimplemented.",), + known_limits=( + "Target-environment multi-node recovery drills and writable remote connector effects are not reference-ready; hard purge and legal hold remain unimplemented.", + ), supported_authority_modes=( "native_authoritative", "external_authoritative", "external_mirror", ), - owned_concepts=("file asset", "file version", "folder", "share", "connector space"), - non_owned_concepts=("record disposition", "campaign attachment rule", "external storage object"), + owned_concepts=( + "file asset", + "file version", + "folder", + "share", + "connector space", + "Form evidence upload grant", + ), + non_owned_concepts=( + "record disposition", + "campaign attachment rule", + "external storage object", + ), target_tested_providers=(REMOTE_STORAGE_PROVIDER_ID,), recovery_docs=("docs/FILES_HANDBOOK.md",), security_docs=("docs/CONNECTOR_BOUNDARY.md",), diff --git a/src/govoplan_files/backend/migrations/versions/a2b3c4d5e6f8_form_evidence_upload_grants.py b/src/govoplan_files/backend/migrations/versions/a2b3c4d5e6f8_form_evidence_upload_grants.py new file mode 100644 index 0000000..fa26f8b --- /dev/null +++ b/src/govoplan_files/backend/migrations/versions/a2b3c4d5e6f8_form_evidence_upload_grants.py @@ -0,0 +1,147 @@ +"""add purpose-bound Form evidence upload grants + +Revision ID: a2b3c4d5e6f8 +Revises: f1a2b3c4d5e7 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + + +revision = "a2b3c4d5e6f8" +down_revision = "f1a2b3c4d5e7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "file_form_evidence_grants", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("form_instance_id", sa.String(length=36), nullable=False), + sa.Column("form_definition_id", sa.String(length=255), nullable=False), + sa.Column("form_definition_revision", sa.String(length=255), nullable=False), + sa.Column("token_sha256", sa.String(length=64), nullable=False), + sa.Column("idempotency_key", sa.String(length=255), nullable=False), + sa.Column("request_sha256", sa.String(length=64), nullable=False), + sa.Column("custodian_user_id", sa.String(length=36), nullable=False), + sa.Column("evidence_kind", sa.String(length=30), nullable=False), + sa.Column("purpose", sa.String(length=500), nullable=False), + sa.Column("status", sa.String(length=30), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("max_size_bytes", sa.Integer(), nullable=False), + sa.Column("allowed_content_types", sa.JSON(), nullable=False), + sa.Column("file_asset_id", sa.String(length=36), nullable=True), + sa.Column("file_version_id", sa.String(length=36), nullable=True), + sa.Column("uploaded_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("metadata", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["custodian_user_id"], + ["access_users.id"], + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["file_asset_id"], + ["file_assets.id"], + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["file_version_id"], + ["file_versions.id"], + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "tenant_id", + "idempotency_key", + name="uq_file_form_evidence_grants_idempotency", + ), + sa.UniqueConstraint("token_sha256"), + ) + op.create_index( + "ix_file_form_evidence_grants_tenant_id", + "file_form_evidence_grants", + ["tenant_id"], + ) + op.create_index( + "ix_file_form_evidence_grants_form_instance_id", + "file_form_evidence_grants", + ["form_instance_id"], + ) + op.create_index( + "ix_file_form_evidence_grants_custodian_user_id", + "file_form_evidence_grants", + ["custodian_user_id"], + ) + op.create_index( + "ix_file_form_evidence_grants_status", + "file_form_evidence_grants", + ["status"], + ) + op.create_index( + "ix_file_form_evidence_grants_expires_at", + "file_form_evidence_grants", + ["expires_at"], + ) + op.create_index( + "ix_file_form_evidence_grants_file_asset_id", + "file_form_evidence_grants", + ["file_asset_id"], + ) + op.create_index( + "ix_file_form_evidence_grants_file_version_id", + "file_form_evidence_grants", + ["file_version_id"], + ) + op.create_index( + "ix_file_form_evidence_grants_form", + "file_form_evidence_grants", + [ + "tenant_id", + "form_instance_id", + "form_definition_id", + "form_definition_revision", + ], + ) + + +def downgrade() -> None: + op.drop_index( + "ix_file_form_evidence_grants_form", + table_name="file_form_evidence_grants", + ) + op.drop_index( + "ix_file_form_evidence_grants_file_version_id", + table_name="file_form_evidence_grants", + ) + op.drop_index( + "ix_file_form_evidence_grants_file_asset_id", + table_name="file_form_evidence_grants", + ) + op.drop_index( + "ix_file_form_evidence_grants_expires_at", + table_name="file_form_evidence_grants", + ) + op.drop_index( + "ix_file_form_evidence_grants_status", + table_name="file_form_evidence_grants", + ) + op.drop_index( + "ix_file_form_evidence_grants_custodian_user_id", + table_name="file_form_evidence_grants", + ) + op.drop_index( + "ix_file_form_evidence_grants_form_instance_id", + table_name="file_form_evidence_grants", + ) + op.drop_index( + "ix_file_form_evidence_grants_tenant_id", + table_name="file_form_evidence_grants", + ) + op.drop_table("file_form_evidence_grants") diff --git a/src/govoplan_files/backend/router.py b/src/govoplan_files/backend/router.py index 934b37b..0c630b5 100644 --- a/src/govoplan_files/backend/router.py +++ b/src/govoplan_files/backend/router.py @@ -11,6 +11,7 @@ from govoplan_files.backend.routes.connector_settings import ( router as connector_settings_router, ) from govoplan_files.backend.routes.folders import router as folders_router +from govoplan_files.backend.routes.form_evidence import router as form_evidence_router from govoplan_files.backend.routes.integrity import router as integrity_router from govoplan_files.backend.routes.listing import router as listing_router from govoplan_files.backend.routes.shares import router as shares_router @@ -23,6 +24,7 @@ router = APIRouter() for workflow_router in ( spaces_router, folders_router, + form_evidence_router, integrity_router, listing_router, uploads_router, diff --git a/src/govoplan_files/backend/routes/form_evidence.py b/src/govoplan_files/backend/routes/form_evidence.py new file mode 100644 index 0000000..bb86800 --- /dev/null +++ b/src/govoplan_files/backend/routes/form_evidence.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +from datetime import UTC, datetime +import hashlib + +from fastapi import ( + APIRouter, + Depends, + File as FastAPIFile, + Header, + HTTPException, + UploadFile, + status, +) +from sqlalchemy.orm import Session + +from govoplan_core.core.events import ( + EventActorRef, + EventObjectRef, + EventTenantRef, + PlatformEvent, + emit_platform_event, +) +from govoplan_core.core.institutional import EvidenceReference +from govoplan_core.db.session import get_session +from govoplan_files.backend.db.models import FileFormEvidenceGrant +from govoplan_files.backend.route_support import _read_limited_upload +from govoplan_files.backend.storage.common import FileStorageError +from govoplan_files.backend.storage.files import create_file_asset + + +router = APIRouter(prefix="/files", tags=["files"]) + + +@router.post( + "/form-evidence/upload", + response_model=dict[str, object], + status_code=status.HTTP_201_CREATED, +) +def upload_form_evidence( + file: UploadFile = FastAPIFile(...), + x_form_evidence_token: str = Header(alias="X-Form-Evidence-Token"), + session: Session = Depends(get_session), +) -> dict[str, object]: + now = datetime.now(UTC) + try: + grant = ( + session.query(FileFormEvidenceGrant) + .filter( + FileFormEvidenceGrant.token_sha256 + == _token_sha256(x_form_evidence_token) + ) + .with_for_update() + .one_or_none() + ) + if grant is None or grant.status != "issued" or _aware(grant.expires_at) <= now: + raise _upload_unavailable() + content_type = ( + str(file.content_type or "application/octet-stream") + .split(";", 1)[0] + .strip() + .casefold() + ) + if grant.allowed_content_types and content_type not in set( + grant.allowed_content_types + ): + raise HTTPException( + status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, + detail="This Form does not accept the uploaded document type.", + ) + data = _read_limited_upload(file, max_bytes=grant.max_size_bytes) + stored = create_file_asset( + session, + tenant_id=grant.tenant_id, + owner_type="user", + owner_id=grant.custodian_user_id, + user_id=grant.custodian_user_id, + filename=file.filename or "form-attachment", + data=data, + folder=f"Form submissions/{grant.form_instance_id}", + content_type=content_type, + description="Managed attachment captured through Forms Runtime.", + metadata={ + "form_evidence": { + "grant_id": grant.id, + "form_instance_id": grant.form_instance_id, + "form_definition_id": grant.form_definition_id, + "form_definition_revision": grant.form_definition_revision, + "purpose": grant.purpose, + }, + "source_provenance": { + "source_type": "form_evidence", + "connector_id": "forms_runtime.evidence.files", + "provider": "forms_runtime", + "external_id": grant.id, + "revision": grant.form_definition_revision, + }, + }, + conflict_strategy="rename", + is_admin=True, + ) + grant.status = "uploaded" + grant.file_asset_id = stored.asset.id + grant.file_version_id = stored.version.id + grant.uploaded_at = now + session.add(grant) + evidence = EvidenceReference( + kind="document", + owner_module="files", + evidence_id=stored.asset.id, + tenant_id=grant.tenant_id, + version=stored.version.id, + checksum=stored.version.checksum_sha256, + source_ref=(f"form_submission:{grant.form_instance_id}:grant:{grant.id}"), + responsible_actor_ref=f"user:{grant.custodian_user_id}", + captured_at=now, + ) + emit_platform_event( + session, + PlatformEvent( + type="files.form_evidence.uploaded", + module_id="files", + payload={ + "grant_id": grant.id, + "form_instance_id": grant.form_instance_id, + "file_asset_id": stored.asset.id, + "file_version_id": stored.version.id, + "checksum_sha256": stored.version.checksum_sha256, + "size_bytes": stored.version.size_bytes, + }, + occurred_at=now, + actor=EventActorRef(type="user", id=grant.custodian_user_id), + tenant=EventTenantRef(id=grant.tenant_id), + resource=EventObjectRef(type="file", id=stored.asset.id), + classification="confidential", + ), + ) + session.commit() + except HTTPException: + session.rollback() + raise + except (FileStorageError, ValueError) as exc: + session.rollback() + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail=str(exc), + ) from exc + return { + "grant_id": grant.id, + "evidence": evidence.to_dict(include_inspection=True), + } + + +def _token_sha256(token: str) -> str: + clean = str(token or "").strip() + if len(clean) < 32: + raise _upload_unavailable() + return hashlib.sha256(clean.encode("utf-8")).hexdigest() + + +def _upload_unavailable() -> HTTPException: + return HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="The Form evidence upload is unavailable.", + ) + + +def _aware(value: datetime) -> datetime: + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + +__all__ = ["router", "upload_form_evidence"] diff --git a/tests/test_form_evidence.py b/tests/test_form_evidence.py new file mode 100644 index 0000000..be6174e --- /dev/null +++ b/tests/test_form_evidence.py @@ -0,0 +1,333 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from io import BytesIO +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from fastapi import UploadFile +from sqlalchemy import create_engine +from sqlalchemy.orm import Session +from starlette.datastructures import Headers + +from govoplan_access.backend.db.models import Account, User +from govoplan_core.core.access import ( + CAPABILITY_ACCESS_DIRECTORY, + AccessSubjectRef, + UserRef, +) +from govoplan_core.core.change_sequence import ChangeSequenceEntry +from govoplan_core.core.form_evidence import ( + FormEvidenceContractError, + FormEvidenceGrantRequest, + FormEvidenceInspectionRequest, +) +from govoplan_core.core.institutional import EvidenceReference, InstitutionalReference +from govoplan_core.db.base import Base +from govoplan_files.backend.db.models import ( + FileAsset, + FileBlob, + FileFormEvidenceGrant, + FileVersion, +) +from govoplan_files.backend.form_evidence import FilesFormEvidenceProvider +from govoplan_files.backend.routes.form_evidence import upload_form_evidence + + +class _Directory: + def __init__(self, user: UserRef) -> None: + self._user = user + + def get_account(self, account_id: str): + del account_id + return None + + def get_user(self, user_id: str): + return self._user if user_id == self._user.id else None + + def get_users(self, user_ids): + return {user_id: self._user for user_id in user_ids if user_id == self._user.id} + + def users_for_tenant(self, tenant_id: str): + return (self._user,) if tenant_id == self._user.tenant_id else () + + def get_group(self, group_id: str): + del group_id + return None + + def get_groups(self, group_ids): + del group_ids + return {} + + def groups_for_tenant(self, tenant_id: str): + del tenant_id + return () + + def groups_for_user(self, user_id: str, *, tenant_id: str): + del user_id, tenant_id + return () + + def display_label(self, subject: AccessSubjectRef): + del subject + return self._user.display_name + + +class _Registry: + def __init__(self, directory: _Directory) -> None: + self._directory = directory + + def has_capability(self, name: str) -> bool: + return name == CAPABILITY_ACCESS_DIRECTORY + + def require_capability(self, name: str): + if not self.has_capability(name): + raise KeyError(name) + return self._directory + + +class FilesFormEvidenceTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all( + self.engine, + tables=[ + Account.__table__, + User.__table__, + FileBlob.__table__, + FileAsset.__table__, + FileVersion.__table__, + FileFormEvidenceGrant.__table__, + ChangeSequenceEntry.__table__, + ], + ) + self.session = Session(self.engine) + user_ref = UserRef( + id="user-1", + account_id="account-1", + tenant_id="tenant-1", + email="user@example.test", + display_name="Evidence Custodian", + ) + self.provider = FilesFormEvidenceProvider( + _Registry(_Directory(user_ref)), + SimpleNamespace(file_upload_max_bytes=2_000_000), + ) + self.principal = SimpleNamespace(tenant_id="tenant-1") + self.definition_ref = InstitutionalReference( + kind="form", + owner_module="forms", + object_id="form-1", + tenant_id="tenant-1", + version="3", + ) + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def request(self, **overrides: object) -> FormEvidenceGrantRequest: + values: dict[str, object] = { + "tenant_id": "tenant-1", + "instance_id": "instance-1", + "definition_ref": self.definition_ref, + "evidence_kind": "document", + "purpose": "supporting document", + "idempotency_key": "grant-1", + "expires_at": datetime.now(UTC) + timedelta(hours=1), + "custodian_ref": "user:user-1", + "max_size_bytes": 1_000_000, + "allowed_content_types": ("application/pdf",), + } + values.update(overrides) + return FormEvidenceGrantRequest(**values) # type: ignore[arg-type] + + def test_grant_persists_only_token_hash_and_replays_without_secret(self) -> None: + request = self.request() + issued = self.provider.create_upload_grant( + self.session, + self.principal, + request=request, + ) + row = self.session.query(FileFormEvidenceGrant).one() + + self.assertIsNotNone(issued.upload_token) + self.assertNotEqual(issued.upload_token, row.token_sha256) + self.assertLessEqual( + issued.expires_at, + datetime.now(UTC) + timedelta(minutes=15, seconds=1), + ) + replay = self.provider.create_upload_grant( + self.session, + self.principal, + request=request, + ) + self.assertTrue(replay.replayed) + self.assertIsNone(replay.upload_token) + self.assertEqual(issued.grant_id, replay.grant_id) + + def test_grant_capacity_counts_outstanding_uploads_but_allows_replay(self) -> None: + request = self.request(metadata={"remaining_attachments": 1}) + self.provider.create_upload_grant(self.session, self.principal, request=request) + + replay = self.provider.create_upload_grant( + self.session, + self.principal, + request=request, + ) + self.assertTrue(replay.replayed) + with self.assertRaisesRegex( + FormEvidenceContractError, + "maximum number of active attachment uploads", + ): + self.provider.create_upload_grant( + self.session, + self.principal, + request=self.request( + idempotency_key="grant-2", + metadata={"remaining_attachments": 1}, + ), + ) + + def test_idempotency_key_cannot_be_reused_for_another_request(self) -> None: + self.provider.create_upload_grant( + self.session, + self.principal, + request=self.request(), + ) + with self.assertRaisesRegex( + FormEvidenceContractError, + "idempotency conflict", + ): + self.provider.create_upload_grant( + self.session, + self.principal, + request=self.request(purpose="another purpose"), + ) + + def test_inspection_accepts_only_exact_verified_managed_version(self) -> None: + issued = self.provider.create_upload_grant( + self.session, + self.principal, + request=self.request(), + ) + grant = self.session.get(FileFormEvidenceGrant, issued.grant_id) + assert grant is not None + blob = FileBlob( + id="blob-1", + tenant_id="tenant-1", + storage_backend="local", + storage_key="tenant-1/blob-1", + checksum_sha256="a" * 64, + size_bytes=100, + integrity_status="verified", + ) + asset = FileAsset( + id="asset-1", + tenant_id="tenant-1", + owner_type="user", + owner_user_id="user-1", + current_version_id="version-1", + display_path="Form submissions/instance-1/evidence.pdf", + filename="evidence.pdf", + ) + version = FileVersion( + id="version-1", + tenant_id="tenant-1", + file_asset_id="asset-1", + blob_id="blob-1", + version_number=1, + filename_at_upload="evidence.pdf", + display_path_at_upload=asset.display_path, + content_type="application/pdf", + size_bytes=100, + checksum_sha256="a" * 64, + ) + self.session.add_all((blob, asset, version)) + grant.status = "uploaded" + grant.file_asset_id = asset.id + grant.file_version_id = version.id + self.session.flush() + reference = EvidenceReference( + kind="document", + owner_module="files", + evidence_id=asset.id, + tenant_id="tenant-1", + version=version.id, + checksum="a" * 64, + ) + + accepted = self.provider.inspect_evidence( + self.session, + self.principal, + request=FormEvidenceInspectionRequest( + tenant_id="tenant-1", + instance_id="instance-1", + definition_ref=self.definition_ref, + evidence=reference, + purpose="final submission", + final=True, + ), + ) + self.assertTrue(accepted.accepted) + + rejected = self.provider.inspect_evidence( + self.session, + self.principal, + request=FormEvidenceInspectionRequest( + tenant_id="tenant-1", + instance_id="another-instance", + definition_ref=self.definition_ref, + evidence=reference, + purpose="final submission", + final=True, + ), + ) + self.assertEqual("rejected", rejected.state) + + def test_public_upload_consumes_grant_and_returns_exact_evidence(self) -> None: + issued = self.provider.create_upload_grant( + self.session, + self.principal, + request=self.request(), + ) + assert issued.upload_token is not None + stored = SimpleNamespace( + asset=SimpleNamespace(id="asset-uploaded"), + version=SimpleNamespace( + id="version-uploaded", + checksum_sha256="c" * 64, + size_bytes=8, + ), + ) + upload = UploadFile( + filename="evidence.pdf", + file=BytesIO(b"evidence"), + headers=Headers({"content-type": "application/pdf"}), + ) + + with ( + patch( + "govoplan_files.backend.routes.form_evidence.create_file_asset", + return_value=stored, + ) as create, + patch("govoplan_files.backend.routes.form_evidence.emit_platform_event"), + ): + response = upload_form_evidence( + file=upload, + x_form_evidence_token=issued.upload_token, + session=self.session, + ) + + grant = self.session.get(FileFormEvidenceGrant, issued.grant_id) + assert grant is not None + self.assertEqual("uploaded", grant.status) + self.assertEqual("asset-uploaded", grant.file_asset_id) + self.assertEqual("version-uploaded", grant.file_version_id) + self.assertEqual("asset-uploaded", response["evidence"]["evidence_id"]) + self.assertEqual(b"evidence", create.call_args.kwargs["data"]) + self.assertEqual("user-1", create.call_args.kwargs["owner_id"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_manifest_documentation.py b/tests/test_manifest_documentation.py index a23ef82..832b8cd 100644 --- a/tests/test_manifest_documentation.py +++ b/tests/test_manifest_documentation.py @@ -15,6 +15,7 @@ STATIC_TOPIC_IDS = { "files.reference.generated-artifact-store", "files.reference.snapshot-provenance-and-capabilities", "files.records.exact-version-source", + "files.forms-runtime.managed-evidence", "files.assurance.process-and-release-readiness", } RUNTIME_TOPIC_IDS = { @@ -170,7 +171,9 @@ class FilesManifestDocumentationTests(unittest.TestCase): self.assertIn("bounded resumable integrity scan", topic.body) self.assertIn("quarantined", topic.body) self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"]) - self.assertIn("Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"]) + self.assertIn( + "Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"] + ) self.assertIn("lease-fenced Core recovery", topic.body) self.assertIn("Ops", topic.body) self.assertTrue(topic.metadata["verification"]) @@ -196,7 +199,11 @@ class FilesManifestDocumentationTests(unittest.TestCase): self.assertEqual(("admin", "user"), topic.documentation_types) self.assertEqual("reference", topic.metadata["kind"]) self.assertEqual( - ["files.access@0.1.6", "files.campaign_attachments@0.1.6"], + [ + "files.access@0.1.6", + "files.campaign_attachments@0.1.6", + "forms_runtime.evidence.files@1.0.0", + ], topic.metadata["provided_interfaces"], ) self.assertIn("revision", topic.metadata["provenance_fields"]) diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 0000000..3b510a8 --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from pathlib import Path +import tempfile +import unittest + +from alembic.runtime.migration import MigrationContext +from sqlalchemy import create_engine, inspect + +from govoplan_access.backend.manifest import get_manifest as get_access_manifest +from govoplan_core.db.migrations import migrate_database +from govoplan_files.backend.manifest import get_manifest as get_files_manifest + + +class FilesMigrationTests(unittest.TestCase): + def test_fresh_migration_creates_form_evidence_grants_and_head(self) -> None: + with tempfile.TemporaryDirectory( + prefix="govoplan-files-migration-" + ) as directory: + url = f"sqlite:///{Path(directory) / 'files.db'}" + migrate_database( + database_url=url, + enabled_modules=("access", "files"), + manifest_factories=(get_access_manifest, get_files_manifest), + ) + engine = create_engine(url) + try: + self.assertIn( + "file_form_evidence_grants", + inspect(engine).get_table_names(), + ) + with engine.connect() as connection: + self.assertIn( + "a2b3c4d5e6f8", + set(MigrationContext.configure(connection).get_current_heads()), + ) + finally: + engine.dispose() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_router_contract.py b/tests/test_router_contract.py index 5e66ca7..0f99e72 100644 --- a/tests/test_router_contract.py +++ b/tests/test_router_contract.py @@ -7,9 +7,14 @@ from inspect import signature from govoplan_files.backend.router import router from govoplan_files.backend.routes.assets import router as assets_router from govoplan_files.backend.routes.connector_io import router as connector_io_router -from govoplan_files.backend.routes.connector_profiles import router as connector_profiles_router -from govoplan_files.backend.routes.connector_settings import router as connector_settings_router +from govoplan_files.backend.routes.connector_profiles import ( + router as connector_profiles_router, +) +from govoplan_files.backend.routes.connector_settings import ( + router as connector_settings_router, +) from govoplan_files.backend.routes.folders import router as folders_router +from govoplan_files.backend.routes.form_evidence import router as form_evidence_router from govoplan_files.backend.routes.integrity import router as integrity_router from govoplan_files.backend.routes.listing import router as listing_router from govoplan_files.backend.routes.shares import router as shares_router @@ -31,6 +36,7 @@ class FilesRouterContractTests(unittest.TestCase): workflow_routers = ( spaces_router, folders_router, + form_evidence_router, integrity_router, listing_router, uploads_router, @@ -49,22 +55,23 @@ class FilesRouterContractTests(unittest.TestCase): actual = self._operation_keys(router) self.assertEqual(expected, actual) - self.assertEqual(52, len(actual)) + self.assertEqual(53, len(actual)) self.assertFalse( [operation for operation, count in Counter(actual).items() if count > 1] ) def test_archive_preview_and_confirmation_routes_are_exposed(self) -> None: routes = { - (tuple(sorted(route.methods or ())), route.path) - for route in router.routes + (tuple(sorted(route.methods or ())), route.path) for route in router.routes } self.assertIn((("POST",), "/files/archive-preview"), routes) self.assertIn((("POST",), "/files/archive-confirm"), routes) def test_connector_routes_keep_existing_api_paths(self) -> None: - routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes} + routes = { + (tuple(sorted(route.methods or ())), route.path) for route in router.routes + } expected = { (("GET",), "/files/connectors/providers"), @@ -83,13 +90,17 @@ class FilesRouterContractTests(unittest.TestCase): self.assertTrue(expected.issubset(routes)) def test_bulk_organize_routes_keep_existing_api_paths(self) -> None: - routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes} + routes = { + (tuple(sorted(route.methods or ())), route.path) for route in router.routes + } self.assertIn((("POST",), "/files/bulk-rename"), routes) self.assertIn((("POST",), "/files/transfer"), routes) def test_share_lifecycle_routes_are_exposed(self) -> None: - routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes} + routes = { + (tuple(sorted(route.methods or ())), route.path) for route in router.routes + } self.assertIn((("GET",), "/files/{file_id}/shares"), routes) self.assertIn((("POST",), "/files/{file_id}/shares"), routes) diff --git a/tests/test_tenant_summary_batch.py b/tests/test_tenant_summary_batch.py index 980d48d..2d3c6e9 100644 --- a/tests/test_tenant_summary_batch.py +++ b/tests/test_tenant_summary_batch.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import UTC, datetime import unittest from sqlalchemy import create_engine, event @@ -14,6 +15,7 @@ from govoplan_files.backend.db.models import ( FileConnectorPolicy, FileConnectorProfile, FileConnectorSpace, + FileFormEvidenceGrant, ) from govoplan_files.backend.manifest import _tenant_summary_batch @@ -32,6 +34,7 @@ class FilesTenantSummaryBatchTests(unittest.TestCase): FileConnectorPolicy.__table__, FileConnectorProfile.__table__, FileConnectorSpace.__table__, + FileFormEvidenceGrant.__table__, ChangeSequenceEntry.__table__, ], ) @@ -70,6 +73,21 @@ class FilesTenantSummaryBatchTests(unittest.TestCase): connector_profile_id="profile-1", provider="webdav", ), + FileFormEvidenceGrant( + id="grant-1", + tenant_id="tenant-1", + form_instance_id="form-instance-1", + form_definition_id="form-1", + form_definition_revision="1", + token_sha256="a" * 64, + idempotency_key="grant-1", + request_sha256="b" * 64, + custodian_user_id="user-1", + evidence_kind="document", + purpose="submission attachment", + expires_at=datetime.now(UTC), + max_size_bytes=1024, + ), ] ) session.commit() @@ -88,7 +106,7 @@ class FilesTenantSummaryBatchTests(unittest.TestCase): finally: event.remove(engine, "before_cursor_execute", count_query) - self.assertEqual(5, query_count) + self.assertEqual(6, query_count) self.assertEqual( { "files": 1, @@ -96,6 +114,7 @@ class FilesTenantSummaryBatchTests(unittest.TestCase): "connector_policies": 1, "connector_profiles": 1, "connector_spaces": 1, + "form_evidence_upload_grants": 1, }, counts["tenant-1"], )