GovOPlaN/govoplan-files
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Add purpose-bound form evidence storage

Browse Source
This commit is contained in:
zemion
2026-08-06 12:42:20 +02:00
parent 60c3d14f08
commit aa9f90c71e
12 changed files with 2039 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_form_evidence.py
+333
View File
@@ -0,0 +1,333 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from io import BytesIO
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from fastapi import UploadFile
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from starlette.datastructures import Headers
from govoplan_access.backend.db.models import Account, User
from govoplan_core.core.access import (
CAPABILITY_ACCESS_DIRECTORY,
AccessSubjectRef,
UserRef,
)
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.core.form_evidence import (
FormEvidenceContractError,
FormEvidenceGrantRequest,
FormEvidenceInspectionRequest,
)
from govoplan_core.core.institutional import EvidenceReference, InstitutionalReference
from govoplan_core.db.base import Base
from govoplan_files.backend.db.models import (
FileAsset,
FileBlob,
FileFormEvidenceGrant,
FileVersion,
)
from govoplan_files.backend.form_evidence import FilesFormEvidenceProvider
from govoplan_files.backend.routes.form_evidence import upload_form_evidence
class _Directory:
def __init__(self, user: UserRef) -> None:
self._user = user
def get_account(self, account_id: str):
del account_id
return None
def get_user(self, user_id: str):
return self._user if user_id == self._user.id else None
def get_users(self, user_ids):
return {user_id: self._user for user_id in user_ids if user_id == self._user.id}
def users_for_tenant(self, tenant_id: str):
return (self._user,) if tenant_id == self._user.tenant_id else ()
def get_group(self, group_id: str):
del group_id
return None
def get_groups(self, group_ids):
del group_ids
return {}
def groups_for_tenant(self, tenant_id: str):
del tenant_id
return ()
def groups_for_user(self, user_id: str, *, tenant_id: str):
del user_id, tenant_id
return ()
def display_label(self, subject: AccessSubjectRef):
del subject
return self._user.display_name
class _Registry:
def __init__(self, directory: _Directory) -> None:
self._directory = directory
def has_capability(self, name: str) -> bool:
return name == CAPABILITY_ACCESS_DIRECTORY
def require_capability(self, name: str):
if not self.has_capability(name):
raise KeyError(name)
return self._directory
class FilesFormEvidenceTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(
self.engine,
tables=[
Account.__table__,
User.__table__,
FileBlob.__table__,
FileAsset.__table__,
FileVersion.__table__,
FileFormEvidenceGrant.__table__,
ChangeSequenceEntry.__table__,
],
)
self.session = Session(self.engine)
user_ref = UserRef(
id="user-1",
account_id="account-1",
tenant_id="tenant-1",
email="user@example.test",
display_name="Evidence Custodian",
)
self.provider = FilesFormEvidenceProvider(
_Registry(_Directory(user_ref)),
SimpleNamespace(file_upload_max_bytes=2_000_000),
)
self.principal = SimpleNamespace(tenant_id="tenant-1")
self.definition_ref = InstitutionalReference(
kind="form",
owner_module="forms",
object_id="form-1",
tenant_id="tenant-1",
version="3",
)
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def request(self, **overrides: object) -> FormEvidenceGrantRequest:
values: dict[str, object] = {
"tenant_id": "tenant-1",
"instance_id": "instance-1",
"definition_ref": self.definition_ref,
"evidence_kind": "document",
"purpose": "supporting document",
"idempotency_key": "grant-1",
"expires_at": datetime.now(UTC) + timedelta(hours=1),
"custodian_ref": "user:user-1",
"max_size_bytes": 1_000_000,
"allowed_content_types": ("application/pdf",),
}
values.update(overrides)
return FormEvidenceGrantRequest(**values) # type: ignore[arg-type]
def test_grant_persists_only_token_hash_and_replays_without_secret(self) -> None:
request = self.request()
issued = self.provider.create_upload_grant(
self.session,
self.principal,
request=request,
)
row = self.session.query(FileFormEvidenceGrant).one()
self.assertIsNotNone(issued.upload_token)
self.assertNotEqual(issued.upload_token, row.token_sha256)
self.assertLessEqual(
issued.expires_at,
datetime.now(UTC) + timedelta(minutes=15, seconds=1),
)
replay = self.provider.create_upload_grant(
self.session,
self.principal,
request=request,
)
self.assertTrue(replay.replayed)
self.assertIsNone(replay.upload_token)
self.assertEqual(issued.grant_id, replay.grant_id)
def test_grant_capacity_counts_outstanding_uploads_but_allows_replay(self) -> None:
request = self.request(metadata={"remaining_attachments": 1})
self.provider.create_upload_grant(self.session, self.principal, request=request)
replay = self.provider.create_upload_grant(
self.session,
self.principal,
request=request,
)
self.assertTrue(replay.replayed)
with self.assertRaisesRegex(
FormEvidenceContractError,
"maximum number of active attachment uploads",
):
self.provider.create_upload_grant(
self.session,
self.principal,
request=self.request(
idempotency_key="grant-2",
metadata={"remaining_attachments": 1},
),
)
def test_idempotency_key_cannot_be_reused_for_another_request(self) -> None:
self.provider.create_upload_grant(
self.session,
self.principal,
request=self.request(),
)
with self.assertRaisesRegex(
FormEvidenceContractError,
"idempotency conflict",
):
self.provider.create_upload_grant(
self.session,
self.principal,
request=self.request(purpose="another purpose"),
)
def test_inspection_accepts_only_exact_verified_managed_version(self) -> None:
issued = self.provider.create_upload_grant(
self.session,
self.principal,
request=self.request(),
)
grant = self.session.get(FileFormEvidenceGrant, issued.grant_id)
assert grant is not None
blob = FileBlob(
id="blob-1",
tenant_id="tenant-1",
storage_backend="local",
storage_key="tenant-1/blob-1",
checksum_sha256="a" * 64,
size_bytes=100,
integrity_status="verified",
)
asset = FileAsset(
id="asset-1",
tenant_id="tenant-1",
owner_type="user",
owner_user_id="user-1",
current_version_id="version-1",
display_path="Form submissions/instance-1/evidence.pdf",
filename="evidence.pdf",
)
version = FileVersion(
id="version-1",
tenant_id="tenant-1",
file_asset_id="asset-1",
blob_id="blob-1",
version_number=1,
filename_at_upload="evidence.pdf",
display_path_at_upload=asset.display_path,
content_type="application/pdf",
size_bytes=100,
checksum_sha256="a" * 64,
)
self.session.add_all((blob, asset, version))
grant.status = "uploaded"
grant.file_asset_id = asset.id
grant.file_version_id = version.id
self.session.flush()
reference = EvidenceReference(
kind="document",
owner_module="files",
evidence_id=asset.id,
tenant_id="tenant-1",
version=version.id,
checksum="a" * 64,
)
accepted = self.provider.inspect_evidence(
self.session,
self.principal,
request=FormEvidenceInspectionRequest(
tenant_id="tenant-1",
instance_id="instance-1",
definition_ref=self.definition_ref,
evidence=reference,
purpose="final submission",
final=True,
),
)
self.assertTrue(accepted.accepted)
rejected = self.provider.inspect_evidence(
self.session,
self.principal,
request=FormEvidenceInspectionRequest(
tenant_id="tenant-1",
instance_id="another-instance",
definition_ref=self.definition_ref,
evidence=reference,
purpose="final submission",
final=True,
),
)
self.assertEqual("rejected", rejected.state)
def test_public_upload_consumes_grant_and_returns_exact_evidence(self) -> None:
issued = self.provider.create_upload_grant(
self.session,
self.principal,
request=self.request(),
)
assert issued.upload_token is not None
stored = SimpleNamespace(
asset=SimpleNamespace(id="asset-uploaded"),
version=SimpleNamespace(
id="version-uploaded",
checksum_sha256="c" * 64,
size_bytes=8,
),
)
upload = UploadFile(
filename="evidence.pdf",
file=BytesIO(b"evidence"),
headers=Headers({"content-type": "application/pdf"}),
)
with (
patch(
"govoplan_files.backend.routes.form_evidence.create_file_asset",
return_value=stored,
) as create,
patch("govoplan_files.backend.routes.form_evidence.emit_platform_event"),
):
response = upload_form_evidence(
file=upload,
x_form_evidence_token=issued.upload_token,
session=self.session,
)
grant = self.session.get(FileFormEvidenceGrant, issued.grant_id)
assert grant is not None
self.assertEqual("uploaded", grant.status)
self.assertEqual("asset-uploaded", grant.file_asset_id)
self.assertEqual("version-uploaded", grant.file_version_id)
self.assertEqual("asset-uploaded", response["evidence"]["evidence_id"])
self.assertEqual(b"evidence", create.call_args.kwargs["data"])
self.assertEqual("user-1", create.call_args.kwargs["owner_id"])
if __name__ == "__main__":
unittest.main()
tests/test_manifest_documentation.py
+9 -2
View File
@@ -15,6 +15,7 @@ STATIC_TOPIC_IDS = {
"files.reference.generated-artifact-store",
"files.reference.snapshot-provenance-and-capabilities",
"files.records.exact-version-source",
"files.forms-runtime.managed-evidence",
"files.assurance.process-and-release-readiness",
}
RUNTIME_TOPIC_IDS = {
@@ -170,7 +171,9 @@ class FilesManifestDocumentationTests(unittest.TestCase):
self.assertIn("bounded resumable integrity scan", topic.body)
self.assertIn("quarantined", topic.body)
self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"])
self.assertIn("Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"])
self.assertIn(
"Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"]
)
self.assertIn("lease-fenced Core recovery", topic.body)
self.assertIn("Ops", topic.body)
self.assertTrue(topic.metadata["verification"])
@@ -196,7 +199,11 @@ class FilesManifestDocumentationTests(unittest.TestCase):
self.assertEqual(("admin", "user"), topic.documentation_types)
self.assertEqual("reference", topic.metadata["kind"])
self.assertEqual(
["files.access@0.1.6", "files.campaign_attachments@0.1.6"],
[
"files.access@0.1.6",
"files.campaign_attachments@0.1.6",
"forms_runtime.evidence.files@1.0.0",
],
topic.metadata["provided_interfaces"],
)
self.assertIn("revision", topic.metadata["provenance_fields"])
tests/test_migrations.py
+42
View File
@@ -0,0 +1,42 @@
from __future__ import annotations
from pathlib import Path
import tempfile
import unittest
from alembic.runtime.migration import MigrationContext
from sqlalchemy import create_engine, inspect
from govoplan_access.backend.manifest import get_manifest as get_access_manifest
from govoplan_core.db.migrations import migrate_database
from govoplan_files.backend.manifest import get_manifest as get_files_manifest
class FilesMigrationTests(unittest.TestCase):
def test_fresh_migration_creates_form_evidence_grants_and_head(self) -> None:
with tempfile.TemporaryDirectory(
prefix="govoplan-files-migration-"
) as directory:
url = f"sqlite:///{Path(directory) / 'files.db'}"
migrate_database(
database_url=url,
enabled_modules=("access", "files"),
manifest_factories=(get_access_manifest, get_files_manifest),
)
engine = create_engine(url)
try:
self.assertIn(
"file_form_evidence_grants",
inspect(engine).get_table_names(),
)
with engine.connect() as connection:
self.assertIn(
"a2b3c4d5e6f8",
set(MigrationContext.configure(connection).get_current_heads()),
)
finally:
engine.dispose()
if __name__ == "__main__":
unittest.main()
tests/test_router_contract.py
+19 -8
View File
@@ -7,9 +7,14 @@ from inspect import signature
from govoplan_files.backend.router import router
from govoplan_files.backend.routes.assets import router as assets_router
from govoplan_files.backend.routes.connector_io import router as connector_io_router
from govoplan_files.backend.routes.connector_profiles import router as connector_profiles_router
from govoplan_files.backend.routes.connector_settings import router as connector_settings_router
from govoplan_files.backend.routes.connector_profiles import (
router as connector_profiles_router,
)
from govoplan_files.backend.routes.connector_settings import (
router as connector_settings_router,
)
from govoplan_files.backend.routes.folders import router as folders_router
from govoplan_files.backend.routes.form_evidence import router as form_evidence_router
from govoplan_files.backend.routes.integrity import router as integrity_router
from govoplan_files.backend.routes.listing import router as listing_router
from govoplan_files.backend.routes.shares import router as shares_router
@@ -31,6 +36,7 @@ class FilesRouterContractTests(unittest.TestCase):
workflow_routers = (
spaces_router,
folders_router,
form_evidence_router,
integrity_router,
listing_router,
uploads_router,
@@ -49,22 +55,23 @@ class FilesRouterContractTests(unittest.TestCase):
actual = self._operation_keys(router)
self.assertEqual(expected, actual)
self.assertEqual(52, len(actual))
self.assertEqual(53, len(actual))
self.assertFalse(
[operation for operation, count in Counter(actual).items() if count > 1]
)
def test_archive_preview_and_confirmation_routes_are_exposed(self) -> None:
routes = {
(tuple(sorted(route.methods or ())), route.path)
for route in router.routes
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
}
self.assertIn((("POST",), "/files/archive-preview"), routes)
self.assertIn((("POST",), "/files/archive-confirm"), routes)
def test_connector_routes_keep_existing_api_paths(self) -> None:
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
routes = {
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
}
expected = {
(("GET",), "/files/connectors/providers"),
@@ -83,13 +90,17 @@ class FilesRouterContractTests(unittest.TestCase):
self.assertTrue(expected.issubset(routes))
def test_bulk_organize_routes_keep_existing_api_paths(self) -> None:
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
routes = {
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
}
self.assertIn((("POST",), "/files/bulk-rename"), routes)
self.assertIn((("POST",), "/files/transfer"), routes)
def test_share_lifecycle_routes_are_exposed(self) -> None:
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
routes = {
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
}
self.assertIn((("GET",), "/files/{file_id}/shares"), routes)
self.assertIn((("POST",), "/files/{file_id}/shares"), routes)
tests/test_tenant_summary_batch.py
+20 -1
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from datetime import UTC, datetime
import unittest
from sqlalchemy import create_engine, event
@@ -14,6 +15,7 @@ from govoplan_files.backend.db.models import (
FileConnectorPolicy,
FileConnectorProfile,
FileConnectorSpace,
FileFormEvidenceGrant,
)
from govoplan_files.backend.manifest import _tenant_summary_batch
@@ -32,6 +34,7 @@ class FilesTenantSummaryBatchTests(unittest.TestCase):
FileConnectorPolicy.__table__,
FileConnectorProfile.__table__,
FileConnectorSpace.__table__,
FileFormEvidenceGrant.__table__,
ChangeSequenceEntry.__table__,
],
)
@@ -70,6 +73,21 @@ class FilesTenantSummaryBatchTests(unittest.TestCase):
connector_profile_id="profile-1",
provider="webdav",
),
FileFormEvidenceGrant(
id="grant-1",
tenant_id="tenant-1",
form_instance_id="form-instance-1",
form_definition_id="form-1",
form_definition_revision="1",
token_sha256="a" * 64,
idempotency_key="grant-1",
request_sha256="b" * 64,
custodian_user_id="user-1",
evidence_kind="document",
purpose="submission attachment",
expires_at=datetime.now(UTC),
max_size_bytes=1024,
),
]
)
session.commit()
@@ -88,7 +106,7 @@ class FilesTenantSummaryBatchTests(unittest.TestCase):
finally:
event.remove(engine, "before_cursor_execute", count_query)
self.assertEqual(5, query_count)
self.assertEqual(6, query_count)
self.assertEqual(
{
"files": 1,
@@ -96,6 +114,7 @@ class FilesTenantSummaryBatchTests(unittest.TestCase):
"connector_policies": 1,
"connector_profiles": 1,
"connector_spaces": 1,
"form_evidence_upload_grants": 1,
},
counts["tenant-1"],
)