diff --git a/package.json b/package.json index c59b76a..0889e46 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/files-webui", - "version": "0.1.18", + "version": "0.1.19", "private": true, "type": "module", "main": "webui/src/index.ts", diff --git a/pyproject.toml b/pyproject.toml index 348b70f..97f73e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta" [project] name = "govoplan-files" -version = "0.1.18" +version = "0.1.19" description = "GovOPlaN files module with backend and WebUI integration." readme = "README.md" requires-python = ">=3.12" license = { file = "LICENSE" } authors = [{ name = "GovOPlaN" }] dependencies = [ - "govoplan-core>=0.1.18", + "govoplan-core>=0.1.19", "defusedxml>=0.7,<1", "pyzipper>=0.3.6,<1", "python-multipart>=0.0.31,<1", diff --git a/src/govoplan_files/backend/capabilities.py b/src/govoplan_files/backend/capabilities.py index 09ef899..ad78b9d 100644 --- a/src/govoplan_files/backend/capabilities.py +++ b/src/govoplan_files/backend/capabilities.py @@ -5,10 +5,20 @@ import binascii from typing import Any from sqlalchemy import or_ +from sqlalchemy.orm import Session +from govoplan_core.audit.logging import audit_from_principal +from govoplan_core.auth import ApiPrincipal, has_scope from govoplan_core.core.access import AccessDecisionProvenance, PrincipalRef from govoplan_core.core.files import FileAccessProvider from govoplan_core.core.files import ( + ManagedTabularFile, + ManagedTabularFileAccessError, + ManagedTabularFileContent, + ManagedTabularFileNotFoundError, + ManagedTabularFileProvider, + ManagedTabularFileUnavailableError, + ManagedTabularFileValidationError, PostboxFileReferenceRef, PostboxFileReferenceRequest, PostboxFileReferenceProvider, @@ -33,6 +43,8 @@ from govoplan_files.backend.storage.files import ( create_file_asset, current_version_and_blob, get_asset_for_user, + list_recent_assets_for_user, + read_asset_version_bytes, sync_file_asset_from_source, ) from govoplan_files.backend.storage.paths import normalize_folder @@ -420,6 +432,228 @@ def access_capability(context: ModuleContext) -> FilesAccessService: return FilesAccessService() +class FilesManagedTabularFileService(ManagedTabularFileProvider): + """Expose authorized immutable CSV/XLSX versions to optional consumers.""" + + def list_tabular_files( + self, + session: object, + principal: object, + *, + query: str = "", + limit: int = 100, + ) -> tuple[ManagedTabularFile, ...]: + db, api_principal, user_id = _tabular_context( + session, + principal, + required_scope="files:file:read", + ) + normalized_query = str(query or "").strip().casefold() + requested_limit = max(1, min(int(limit), 100)) + assets = list_recent_assets_for_user( + db, + tenant_id=api_principal.tenant_id, + user_id=user_id, + limit=min(500, requested_limit * 5), + is_admin=has_scope(api_principal, "files:file:admin"), + ) + results: list[ManagedTabularFile] = [] + for asset in assets: + if normalized_query and normalized_query not in ( + f"{asset.filename} {asset.display_path} {asset.description or ''}" + ).casefold(): + continue + try: + version, blob = current_version_and_blob(db, asset) + except FileStorageError: + continue + if not _is_tabular_file(version.filename_at_upload, version.content_type): + continue + results.append(_managed_tabular_file(asset, version, blob)) + if len(results) >= requested_limit: + break + return tuple(results) + + def get_tabular_file( + self, + session: object, + principal: object, + *, + file_asset_id: str, + file_version_id: str | None = None, + ) -> ManagedTabularFile | None: + db, api_principal, user_id = _tabular_context( + session, + principal, + required_scope="files:file:read", + ) + try: + asset = get_asset_for_user( + db, + tenant_id=api_principal.tenant_id, + user_id=user_id, + asset_id=file_asset_id, + is_admin=has_scope(api_principal, "files:file:admin"), + ) + except FileStorageError: + return None + if file_version_id: + version = db.get(FileVersion, file_version_id) + if ( + version is None + or version.file_asset_id != asset.id + or version.tenant_id != api_principal.tenant_id + ): + return None + blob = db.get(FileBlob, version.blob_id) + if blob is None or blob.tenant_id != api_principal.tenant_id: + return None + else: + try: + version, blob = current_version_and_blob(db, asset) + except FileStorageError as exc: + raise ManagedTabularFileUnavailableError( + "Managed file version metadata is unavailable." + ) from exc + if not _is_tabular_file(version.filename_at_upload, version.content_type): + return None + return _managed_tabular_file(asset, version, blob) + + def read_tabular_file( + self, + session: object, + principal: object, + *, + file_asset_id: str, + file_version_id: str, + max_bytes: int, + ) -> ManagedTabularFileContent: + db, api_principal, user_id = _tabular_context( + session, + principal, + required_scope="files:file:download", + ) + try: + asset = get_asset_for_user( + db, + tenant_id=api_principal.tenant_id, + user_id=user_id, + asset_id=file_asset_id, + is_admin=has_scope(api_principal, "files:file:admin"), + ) + except FileStorageError as exc: + raise ManagedTabularFileNotFoundError( + "Managed tabular file not found." + ) from exc + metadata = self.get_tabular_file( + db, + api_principal, + file_asset_id=asset.id, + file_version_id=file_version_id, + ) + if metadata is None: + raise ManagedTabularFileNotFoundError( + "Managed tabular file version not found." + ) + effective_max = max(1, int(max_bytes)) + if metadata.size_bytes > effective_max: + raise ManagedTabularFileValidationError( + f"Managed tabular files are limited to {effective_max:,} bytes for this operation." + ) + try: + payload, version, blob = read_asset_version_bytes( + db, + asset, + file_version_id, + ) + except FileStorageError as exc: + raise ManagedTabularFileUnavailableError( + "Managed tabular file content is unavailable or failed integrity verification." + ) from exc + if len(payload) > effective_max: + raise ManagedTabularFileValidationError( + f"Managed tabular files are limited to {effective_max:,} bytes for this operation." + ) + result = _managed_tabular_file(asset, version, blob) + audit_from_principal( + db, + api_principal, + action="files.tabular_content.read", + object_type="file_version", + object_id=version.id, + details={ + "file_asset_id": asset.id, + "size_bytes": version.size_bytes, + "checksum_sha256": version.checksum_sha256, + "consumer": "tabular_content", + }, + ) + return ManagedTabularFileContent(file=result, payload=payload) + + +def managed_tabular_file_capability( + context: ModuleContext, +) -> FilesManagedTabularFileService: + configure_runtime(registry=context.registry, settings=context.settings) + return FilesManagedTabularFileService() + + +def _tabular_context( + session: object, + principal: object, + *, + required_scope: str, +) -> tuple[Session, ApiPrincipal, str]: + if not isinstance(session, Session): + raise TypeError("Managed tabular file access requires a SQLAlchemy session.") + if not isinstance(principal, ApiPrincipal): + raise ManagedTabularFileAccessError( + "Managed tabular file access requires a tenant API principal." + ) + if not ( + has_scope(principal, required_scope) + or has_scope(principal, "files:file:admin") + ): + raise ManagedTabularFileAccessError( + f"Managed tabular file access requires {required_scope}." + ) + user_id = str(getattr(principal.user, "id", "") or "").strip() + if not principal.tenant_id or not user_id: + raise ManagedTabularFileAccessError( + "Managed tabular file access requires a tenant user principal." + ) + return session, principal, user_id + + +def _managed_tabular_file( + asset: FileAsset, + version: FileVersion, + blob: FileBlob, +) -> ManagedTabularFile: + return ManagedTabularFile( + file_asset_id=asset.id, + file_version_id=version.id, + filename=version.filename_at_upload or asset.filename, + display_path=version.display_path_at_upload or asset.display_path, + content_type=version.content_type or blob.content_type, + size_bytes=version.size_bytes, + sha256=version.checksum_sha256, + updated_at=version.created_at, + current_version=asset.current_version_id == version.id, + ) + + +def _is_tabular_file(filename: str, content_type: str | None) -> bool: + normalized_name = str(filename or "").strip().casefold() + normalized_type = str(content_type or "").split(";", 1)[0].strip().casefold() + return normalized_name.endswith((".csv", ".xlsx")) or normalized_type in { + "text/csv", + "application/csv", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + } + + def virtual_folder_resource_id(*, tenant_id: str, owner_type: str, owner_id: str, path: str) -> str: normalized_path = normalize_folder(path) encoded_path = base64.urlsafe_b64encode(normalized_path.encode("utf-8")).decode("ascii").rstrip("=") diff --git a/src/govoplan_files/backend/manifest.py b/src/govoplan_files/backend/manifest.py index 00056a9..aa7ad1d 100644 --- a/src/govoplan_files/backend/manifest.py +++ b/src/govoplan_files/backend/manifest.py @@ -14,6 +14,7 @@ from govoplan_core.core.files import ( CAPABILITY_FILES_ACCESS, CAPABILITY_FILES_ARTIFACT_STORE, CAPABILITY_FILES_POSTBOX_REFERENCES, + CAPABILITY_FILES_TABULAR_CONTENT, ) from govoplan_core.core.module_guards import ( drop_table_retirement_provider, @@ -451,7 +452,7 @@ def _dsar_provider(context: ModuleContext) -> object: manifest = ModuleManifest( id="files", name="Files", - version="0.1.18", + version="0.1.19", required_capabilities=( CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR, @@ -462,6 +463,7 @@ manifest = ModuleManifest( ModuleInterfaceProvider(name="files.campaign_attachments", version="0.1.6"), ModuleInterfaceProvider(name=CAPABILITY_FILES_ARTIFACT_STORE, version="0.1.14"), ModuleInterfaceProvider(name=CAPABILITY_FILES_POSTBOX_REFERENCES, version="1.0.0"), + ModuleInterfaceProvider(name=CAPABILITY_FILES_TABULAR_CONTENT, version="1.0.0"), ModuleInterfaceProvider(name=CAPABILITY_RECORD_SOURCE_FILES, version="1.0.0"), ModuleInterfaceProvider(name=CAPABILITY_FORM_EVIDENCE_FILES, version="1.0.0"), ModuleInterfaceProvider(name=FILES_DSAR_CAPABILITY, version="0.1.0"), @@ -616,6 +618,49 @@ manifest = ModuleManifest( ), ), documentation=( + DocumentationTopic( + id="files.tabular-content", + title="Use managed CSV and XLSX versions as governed data sources", + summary="Expose exact, authorized managed file versions to Connectors without bypassing Files controls.", + body=( + "Files lists only CSV and XLSX assets visible to the current tenant user. " + "Opening content additionally requires Files download permission and an exact immutable version reference. " + "Size ceilings are checked before storage access; checksum verification, quarantine, encryption envelopes, deletion, ownership, and shares remain authoritative in Files. " + "Connectors receives file metadata and verified bytes through the Core capability and never imports Files models, storage keys, or encryption internals. " + "A newer current version does not silently replace a pinned source version; Connectors reports the version change for explicit review." + ), + layer="available", + documentation_types=("admin", "user"), + audience=("administrator", "operator", "power_user"), + related_modules=("connectors", "datasources", "dataflow"), + conditions=( + DocumentationCondition( + required_modules=("files", "connectors"), + required_scopes=("files:file:read", "files:file:download"), + ), + ), + links=( + DocumentationLink( + label="Files", + href="/files", + kind="runtime", + ), + DocumentationLink( + label="Files handbook", + href="govoplan-files/docs/FILES_HANDBOOK.md", + kind="repository", + ), + ), + metadata={ + "kind": "guide", + "consequence_classes": { + "pin_exact_version": "A source keeps its reviewed immutable file version until explicitly refreshed.", + "preserve_file_controls": "Files access, integrity, encryption, retention, and legal-hold controls remain authoritative.", + "fail_closed": "Unavailable, oversized, quarantined, or unauthorized content is not parsed or previewed.", + }, + }, + order=29, + ), DocumentationTopic( id="files.postbox.exact-version-references", title="Resolve exact file versions for Postbox evidence", @@ -1702,6 +1747,10 @@ manifest = ModuleManifest( "govoplan_files.backend.capabilities", fromlist=["postbox_reference_capability"], ).postbox_reference_capability(context), + CAPABILITY_FILES_TABULAR_CONTENT: lambda context: __import__( + "govoplan_files.backend.capabilities", + fromlist=["managed_tabular_file_capability"], + ).managed_tabular_file_capability(context), "files.campaign_attachments": lambda context: __import__( "govoplan_files.backend.capabilities", fromlist=["campaign_capability"] ).campaign_capability(context), @@ -1710,6 +1759,11 @@ manifest = ModuleManifest( FILES_DSAR_CAPABILITY: _dsar_provider, }, capability_documentation={ + CAPABILITY_FILES_TABULAR_CONTENT: CapabilityDocumentation( + label="Managed tabular file content", + summary="Lists and opens authorized exact CSV/XLSX versions with Files integrity and access controls.", + contract_version="1.0.0", + ), CAPABILITY_RECORD_SOURCE_FILES: CapabilityDocumentation( label="Files record source", summary="Resolves currently authorized immutable managed file versions for Records filing.", diff --git a/tests/test_manifest_documentation.py b/tests/test_manifest_documentation.py index 0bf2c11..43cf87d 100644 --- a/tests/test_manifest_documentation.py +++ b/tests/test_manifest_documentation.py @@ -21,6 +21,7 @@ STATIC_TOPIC_IDS = { "files.records.exact-version-source", "files.forms-runtime.managed-evidence", "files.postbox.exact-version-references", + "files.tabular-content", "files.assurance.process-and-release-readiness", } RUNTIME_TOPIC_IDS = { diff --git a/tests/test_tabular_content_capability.py b/tests/test_tabular_content_capability.py new file mode 100644 index 0000000..6240eac --- /dev/null +++ b/tests/test_tabular_content_capability.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import unittest +from datetime import UTC, datetime +from types import SimpleNamespace +from unittest.mock import patch + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from govoplan_core.auth import ApiPrincipal +from govoplan_core.core.access import PrincipalRef +from govoplan_core.core.files import ( + ManagedTabularFileAccessError, + ManagedTabularFileValidationError, +) +from govoplan_files.backend.capabilities import FilesManagedTabularFileService +from govoplan_files.backend.storage.common import FileStorageError + + +def principal( + tenant_id: str = "tenant-1", + *, + scopes: tuple[str, ...] = ( + "files:file:read", + "files:file:download", + "files:file:admin", + ), +) -> ApiPrincipal: + return ApiPrincipal( + principal=PrincipalRef( + account_id="account-1", + membership_id="membership-1", + tenant_id=tenant_id, + scopes=frozenset(scopes), + ), + account=object(), + user=SimpleNamespace(id="user-1"), + ) + + +class FilesManagedTabularContentTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite+pysqlite:///:memory:") + self.session = sessionmaker(bind=self.engine)() + self.blob = SimpleNamespace( + id="blob-1", + tenant_id="tenant-1", + storage_backend="local", + storage_key="tenants/tenant-1/files/blob-1", + checksum_sha256="a" * 64, + size_bytes=8, + content_type="text/csv", + integrity_status="verified", + ) + self.asset = SimpleNamespace( + id="asset-1", + tenant_id="tenant-1", + owner_type="user", + owner_user_id="user-1", + current_version_id="version-1", + display_path="Imports/source.csv", + filename="source.csv", + description=None, + updated_at=datetime(2026, 8, 21, tzinfo=UTC), + ) + self.version = SimpleNamespace( + id="version-1", + tenant_id="tenant-1", + file_asset_id="asset-1", + blob_id="blob-1", + version_number=1, + filename_at_upload="source.csv", + display_path_at_upload="Imports/source.csv", + content_type="text/csv", + size_bytes=8, + checksum_sha256="a" * 64, + created_by_user_id="user-1", + created_at=datetime(2026, 8, 21, tzinfo=UTC), + ) + self.provider = FilesManagedTabularFileService() + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def test_lists_and_reads_only_exact_authorized_tabular_versions(self) -> None: + with ( + patch( + "govoplan_files.backend.capabilities.list_recent_assets_for_user", + return_value=[self.asset], + ), + patch( + "govoplan_files.backend.capabilities.current_version_and_blob", + return_value=(self.version, self.blob), + ), + patch( + "govoplan_files.backend.capabilities.get_asset_for_user", + return_value=self.asset, + ), + patch.object( + self.session, + "get", + side_effect=lambda model, object_id: { + "version-1": self.version, + "blob-1": self.blob, + }.get(object_id), + ), + patch( + "govoplan_files.backend.capabilities.read_asset_version_bytes", + return_value=(b"id\n1\n", self.version, self.blob), + ) as read_bytes, + patch("govoplan_files.backend.capabilities.audit_from_principal"), + ): + listed = self.provider.list_tabular_files(self.session, principal()) + result = self.provider.read_tabular_file( + self.session, + principal(), + file_asset_id="asset-1", + file_version_id="version-1", + max_bytes=100, + ) + + self.assertEqual(("version-1",), tuple(item.file_version_id for item in listed)) + self.assertEqual(b"id\n1\n", result.payload) + self.assertTrue(result.file.current_version) + read_bytes.assert_called_once() + + def test_tenant_scope_permissions_and_pre_read_size_limit_fail_closed(self) -> None: + with patch( + "govoplan_files.backend.capabilities.get_asset_for_user", + side_effect=FileStorageError("File not found"), + ): + self.assertIsNone( + self.provider.get_tabular_file( + self.session, + principal("tenant-2"), + file_asset_id="asset-1", + ) + ) + with self.assertRaises(ManagedTabularFileAccessError): + self.provider.list_tabular_files( + self.session, + principal(scopes=()), + ) + with ( + patch( + "govoplan_files.backend.capabilities.get_asset_for_user", + return_value=self.asset, + ), + patch.object( + self.session, + "get", + side_effect=lambda model, object_id: { + "version-1": self.version, + "blob-1": self.blob, + }.get(object_id), + ), + patch( + "govoplan_files.backend.capabilities.read_asset_version_bytes" + ) as read_bytes, + self.assertRaises(ManagedTabularFileValidationError), + ): + self.provider.read_tabular_file( + self.session, + principal(), + file_asset_id="asset-1", + file_version_id="version-1", + max_bytes=4, + ) + read_bytes.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/package.json b/webui/package.json index da083ea..b154b03 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/files-webui", - "version": "0.1.18", + "version": "0.1.19", "private": true, "type": "module", "main": "src/index.ts",