GovOPlaN/govoplan-files
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(files): expose governed tabular content

Browse Source
This commit is contained in:
zemion
2026-08-21 19:29:54 +02:00
parent ce4eaefbfd
commit 7258afdc46
7 changed files with 469 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@govoplan/files-webui",
"version": "0.1.18",
"version": "0.1.19",
"private": true,
"type": "module",
"main": "webui/src/index.ts",
pyproject.toml
+2 -2
View File
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
[project]
name = "govoplan-files"
version = "0.1.18"
version = "0.1.19"
description = "GovOPlaN files module with backend and WebUI integration."
readme = "README.md"
requires-python = ">=3.12"
license = { file = "LICENSE" }
authors = [{ name = "GovOPlaN" }]
dependencies = [
"govoplan-core>=0.1.18",
"govoplan-core>=0.1.19",
"defusedxml>=0.7,<1",
"pyzipper>=0.3.6,<1",
"python-multipart>=0.0.31,<1",
src/govoplan_files/backend/capabilities.py
+234
View File
@@ -5,10 +5,20 @@ import binascii
from typing import Any
from sqlalchemy import or_
from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_from_principal
from govoplan_core.auth import ApiPrincipal, has_scope
from govoplan_core.core.access import AccessDecisionProvenance, PrincipalRef
from govoplan_core.core.files import FileAccessProvider
from govoplan_core.core.files import (
ManagedTabularFile,
ManagedTabularFileAccessError,
ManagedTabularFileContent,
ManagedTabularFileNotFoundError,
ManagedTabularFileProvider,
ManagedTabularFileUnavailableError,
ManagedTabularFileValidationError,
PostboxFileReferenceRef,
PostboxFileReferenceRequest,
PostboxFileReferenceProvider,
@@ -33,6 +43,8 @@ from govoplan_files.backend.storage.files import (
create_file_asset,
current_version_and_blob,
get_asset_for_user,
list_recent_assets_for_user,
read_asset_version_bytes,
sync_file_asset_from_source,
)
from govoplan_files.backend.storage.paths import normalize_folder
@@ -420,6 +432,228 @@ def access_capability(context: ModuleContext) -> FilesAccessService:
return FilesAccessService()
class FilesManagedTabularFileService(ManagedTabularFileProvider):
"""Expose authorized immutable CSV/XLSX versions to optional consumers."""
def list_tabular_files(
self,
session: object,
principal: object,
*,
query: str = "",
limit: int = 100,
) -> tuple[ManagedTabularFile, ...]:
db, api_principal, user_id = _tabular_context(
session,
principal,
required_scope="files:file:read",
)
normalized_query = str(query or "").strip().casefold()
requested_limit = max(1, min(int(limit), 100))
assets = list_recent_assets_for_user(
db,
tenant_id=api_principal.tenant_id,
user_id=user_id,
limit=min(500, requested_limit * 5),
is_admin=has_scope(api_principal, "files:file:admin"),
)
results: list[ManagedTabularFile] = []
for asset in assets:
if normalized_query and normalized_query not in (
f"{asset.filename} {asset.display_path} {asset.description or ''}"
).casefold():
continue
try:
version, blob = current_version_and_blob(db, asset)
except FileStorageError:
continue
if not _is_tabular_file(version.filename_at_upload, version.content_type):
continue
results.append(_managed_tabular_file(asset, version, blob))
if len(results) >= requested_limit:
break
return tuple(results)
def get_tabular_file(
self,
session: object,
principal: object,
*,
file_asset_id: str,
file_version_id: str | None = None,
) -> ManagedTabularFile | None:
db, api_principal, user_id = _tabular_context(
session,
principal,
required_scope="files:file:read",
)
try:
asset = get_asset_for_user(
db,
tenant_id=api_principal.tenant_id,
user_id=user_id,
asset_id=file_asset_id,
is_admin=has_scope(api_principal, "files:file:admin"),
)
except FileStorageError:
return None
if file_version_id:
version = db.get(FileVersion, file_version_id)
if (
version is None
or version.file_asset_id != asset.id
or version.tenant_id != api_principal.tenant_id
):
return None
blob = db.get(FileBlob, version.blob_id)
if blob is None or blob.tenant_id != api_principal.tenant_id:
return None
else:
try:
version, blob = current_version_and_blob(db, asset)
except FileStorageError as exc:
raise ManagedTabularFileUnavailableError(
"Managed file version metadata is unavailable."
) from exc
if not _is_tabular_file(version.filename_at_upload, version.content_type):
return None
return _managed_tabular_file(asset, version, blob)
def read_tabular_file(
self,
session: object,
principal: object,
*,
file_asset_id: str,
file_version_id: str,
max_bytes: int,
) -> ManagedTabularFileContent:
db, api_principal, user_id = _tabular_context(
session,
principal,
required_scope="files:file:download",
)
try:
asset = get_asset_for_user(
db,
tenant_id=api_principal.tenant_id,
user_id=user_id,
asset_id=file_asset_id,
is_admin=has_scope(api_principal, "files:file:admin"),
)
except FileStorageError as exc:
raise ManagedTabularFileNotFoundError(
"Managed tabular file not found."
) from exc
metadata = self.get_tabular_file(
db,
api_principal,
file_asset_id=asset.id,
file_version_id=file_version_id,
)
if metadata is None:
raise ManagedTabularFileNotFoundError(
"Managed tabular file version not found."
)
effective_max = max(1, int(max_bytes))
if metadata.size_bytes > effective_max:
raise ManagedTabularFileValidationError(
f"Managed tabular files are limited to {effective_max:,} bytes for this operation."
)
try:
payload, version, blob = read_asset_version_bytes(
db,
asset,
file_version_id,
)
except FileStorageError as exc:
raise ManagedTabularFileUnavailableError(
"Managed tabular file content is unavailable or failed integrity verification."
) from exc
if len(payload) > effective_max:
raise ManagedTabularFileValidationError(
f"Managed tabular files are limited to {effective_max:,} bytes for this operation."
)
result = _managed_tabular_file(asset, version, blob)
audit_from_principal(
db,
api_principal,
action="files.tabular_content.read",
object_type="file_version",
object_id=version.id,
details={
"file_asset_id": asset.id,
"size_bytes": version.size_bytes,
"checksum_sha256": version.checksum_sha256,
"consumer": "tabular_content",
},
)
return ManagedTabularFileContent(file=result, payload=payload)
def managed_tabular_file_capability(
context: ModuleContext,
) -> FilesManagedTabularFileService:
configure_runtime(registry=context.registry, settings=context.settings)
return FilesManagedTabularFileService()
def _tabular_context(
session: object,
principal: object,
*,
required_scope: str,
) -> tuple[Session, ApiPrincipal, str]:
if not isinstance(session, Session):
raise TypeError("Managed tabular file access requires a SQLAlchemy session.")
if not isinstance(principal, ApiPrincipal):
raise ManagedTabularFileAccessError(
"Managed tabular file access requires a tenant API principal."
)
if not (
has_scope(principal, required_scope)
or has_scope(principal, "files:file:admin")
):
raise ManagedTabularFileAccessError(
f"Managed tabular file access requires {required_scope}."
)
user_id = str(getattr(principal.user, "id", "") or "").strip()
if not principal.tenant_id or not user_id:
raise ManagedTabularFileAccessError(
"Managed tabular file access requires a tenant user principal."
)
return session, principal, user_id
def _managed_tabular_file(
asset: FileAsset,
version: FileVersion,
blob: FileBlob,
) -> ManagedTabularFile:
return ManagedTabularFile(
file_asset_id=asset.id,
file_version_id=version.id,
filename=version.filename_at_upload or asset.filename,
display_path=version.display_path_at_upload or asset.display_path,
content_type=version.content_type or blob.content_type,
size_bytes=version.size_bytes,
sha256=version.checksum_sha256,
updated_at=version.created_at,
current_version=asset.current_version_id == version.id,
)
def _is_tabular_file(filename: str, content_type: str | None) -> bool:
normalized_name = str(filename or "").strip().casefold()
normalized_type = str(content_type or "").split(";", 1)[0].strip().casefold()
return normalized_name.endswith((".csv", ".xlsx")) or normalized_type in {
"text/csv",
"application/csv",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
}
def virtual_folder_resource_id(*, tenant_id: str, owner_type: str, owner_id: str, path: str) -> str:
normalized_path = normalize_folder(path)
encoded_path = base64.urlsafe_b64encode(normalized_path.encode("utf-8")).decode("ascii").rstrip("=")
src/govoplan_files/backend/manifest.py
+55 -1
View File
@@ -14,6 +14,7 @@ from govoplan_core.core.files import (
CAPABILITY_FILES_ACCESS,
CAPABILITY_FILES_ARTIFACT_STORE,
CAPABILITY_FILES_POSTBOX_REFERENCES,
CAPABILITY_FILES_TABULAR_CONTENT,
)
from govoplan_core.core.module_guards import (
drop_table_retirement_provider,
@@ -451,7 +452,7 @@ def _dsar_provider(context: ModuleContext) -> object:
manifest = ModuleManifest(
id="files",
name="Files",
version="0.1.18",
version="0.1.19",
required_capabilities=(
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
@@ -462,6 +463,7 @@ manifest = ModuleManifest(
ModuleInterfaceProvider(name="files.campaign_attachments", version="0.1.6"),
ModuleInterfaceProvider(name=CAPABILITY_FILES_ARTIFACT_STORE, version="0.1.14"),
ModuleInterfaceProvider(name=CAPABILITY_FILES_POSTBOX_REFERENCES, version="1.0.0"),
ModuleInterfaceProvider(name=CAPABILITY_FILES_TABULAR_CONTENT, version="1.0.0"),
ModuleInterfaceProvider(name=CAPABILITY_RECORD_SOURCE_FILES, version="1.0.0"),
ModuleInterfaceProvider(name=CAPABILITY_FORM_EVIDENCE_FILES, version="1.0.0"),
ModuleInterfaceProvider(name=FILES_DSAR_CAPABILITY, version="0.1.0"),
@@ -616,6 +618,49 @@ manifest = ModuleManifest(
),
),
documentation=(
DocumentationTopic(
id="files.tabular-content",
title="Use managed CSV and XLSX versions as governed data sources",
summary="Expose exact, authorized managed file versions to Connectors without bypassing Files controls.",
body=(
"Files lists only CSV and XLSX assets visible to the current tenant user. "
"Opening content additionally requires Files download permission and an exact immutable version reference. "
"Size ceilings are checked before storage access; checksum verification, quarantine, encryption envelopes, deletion, ownership, and shares remain authoritative in Files. "
"Connectors receives file metadata and verified bytes through the Core capability and never imports Files models, storage keys, or encryption internals. "
"A newer current version does not silently replace a pinned source version; Connectors reports the version change for explicit review."
),
layer="available",
documentation_types=("admin", "user"),
audience=("administrator", "operator", "power_user"),
related_modules=("connectors", "datasources", "dataflow"),
conditions=(
DocumentationCondition(
required_modules=("files", "connectors"),
required_scopes=("files:file:read", "files:file:download"),
),
),
links=(
DocumentationLink(
label="Files",
href="/files",
kind="runtime",
),
DocumentationLink(
label="Files handbook",
href="govoplan-files/docs/FILES_HANDBOOK.md",
kind="repository",
),
),
metadata={
"kind": "guide",
"consequence_classes": {
"pin_exact_version": "A source keeps its reviewed immutable file version until explicitly refreshed.",
"preserve_file_controls": "Files access, integrity, encryption, retention, and legal-hold controls remain authoritative.",
"fail_closed": "Unavailable, oversized, quarantined, or unauthorized content is not parsed or previewed.",
},
},
order=29,
),
DocumentationTopic(
id="files.postbox.exact-version-references",
title="Resolve exact file versions for Postbox evidence",
@@ -1702,6 +1747,10 @@ manifest = ModuleManifest(
"govoplan_files.backend.capabilities",
fromlist=["postbox_reference_capability"],
).postbox_reference_capability(context),
CAPABILITY_FILES_TABULAR_CONTENT: lambda context: __import__(
"govoplan_files.backend.capabilities",
fromlist=["managed_tabular_file_capability"],
).managed_tabular_file_capability(context),
"files.campaign_attachments": lambda context: __import__(
"govoplan_files.backend.capabilities", fromlist=["campaign_capability"]
).campaign_capability(context),
@@ -1710,6 +1759,11 @@ manifest = ModuleManifest(
FILES_DSAR_CAPABILITY: _dsar_provider,
},
capability_documentation={
CAPABILITY_FILES_TABULAR_CONTENT: CapabilityDocumentation(
label="Managed tabular file content",
summary="Lists and opens authorized exact CSV/XLSX versions with Files integrity and access controls.",
contract_version="1.0.0",
),
CAPABILITY_RECORD_SOURCE_FILES: CapabilityDocumentation(
label="Files record source",
summary="Resolves currently authorized immutable managed file versions for Records filing.",
tests/test_manifest_documentation.py
+1
View File
@@ -21,6 +21,7 @@ STATIC_TOPIC_IDS = {
"files.records.exact-version-source",
"files.forms-runtime.managed-evidence",
"files.postbox.exact-version-references",
"files.tabular-content",
"files.assurance.process-and-release-readiness",
}
RUNTIME_TOPIC_IDS = {
tests/test_tabular_content_capability.py
+175
View File
@@ -0,0 +1,175 @@
from __future__ import annotations
import unittest
from datetime import UTC, datetime
from types import SimpleNamespace
from unittest.mock import patch
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.access import PrincipalRef
from govoplan_core.core.files import (
ManagedTabularFileAccessError,
ManagedTabularFileValidationError,
)
from govoplan_files.backend.capabilities import FilesManagedTabularFileService
from govoplan_files.backend.storage.common import FileStorageError
def principal(
tenant_id: str = "tenant-1",
*,
scopes: tuple[str, ...] = (
"files:file:read",
"files:file:download",
"files:file:admin",
),
) -> ApiPrincipal:
return ApiPrincipal(
principal=PrincipalRef(
account_id="account-1",
membership_id="membership-1",
tenant_id=tenant_id,
scopes=frozenset(scopes),
),
account=object(),
user=SimpleNamespace(id="user-1"),
)
class FilesManagedTabularContentTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite+pysqlite:///:memory:")
self.session = sessionmaker(bind=self.engine)()
self.blob = SimpleNamespace(
id="blob-1",
tenant_id="tenant-1",
storage_backend="local",
storage_key="tenants/tenant-1/files/blob-1",
checksum_sha256="a" * 64,
size_bytes=8,
content_type="text/csv",
integrity_status="verified",
)
self.asset = SimpleNamespace(
id="asset-1",
tenant_id="tenant-1",
owner_type="user",
owner_user_id="user-1",
current_version_id="version-1",
display_path="Imports/source.csv",
filename="source.csv",
description=None,
updated_at=datetime(2026, 8, 21, tzinfo=UTC),
)
self.version = SimpleNamespace(
id="version-1",
tenant_id="tenant-1",
file_asset_id="asset-1",
blob_id="blob-1",
version_number=1,
filename_at_upload="source.csv",
display_path_at_upload="Imports/source.csv",
content_type="text/csv",
size_bytes=8,
checksum_sha256="a" * 64,
created_by_user_id="user-1",
created_at=datetime(2026, 8, 21, tzinfo=UTC),
)
self.provider = FilesManagedTabularFileService()
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_lists_and_reads_only_exact_authorized_tabular_versions(self) -> None:
with (
patch(
"govoplan_files.backend.capabilities.list_recent_assets_for_user",
return_value=[self.asset],
),
patch(
"govoplan_files.backend.capabilities.current_version_and_blob",
return_value=(self.version, self.blob),
),
patch(
"govoplan_files.backend.capabilities.get_asset_for_user",
return_value=self.asset,
),
patch.object(
self.session,
"get",
side_effect=lambda model, object_id: {
"version-1": self.version,
"blob-1": self.blob,
}.get(object_id),
),
patch(
"govoplan_files.backend.capabilities.read_asset_version_bytes",
return_value=(b"id\n1\n", self.version, self.blob),
) as read_bytes,
patch("govoplan_files.backend.capabilities.audit_from_principal"),
):
listed = self.provider.list_tabular_files(self.session, principal())
result = self.provider.read_tabular_file(
self.session,
principal(),
file_asset_id="asset-1",
file_version_id="version-1",
max_bytes=100,
)
self.assertEqual(("version-1",), tuple(item.file_version_id for item in listed))
self.assertEqual(b"id\n1\n", result.payload)
self.assertTrue(result.file.current_version)
read_bytes.assert_called_once()
def test_tenant_scope_permissions_and_pre_read_size_limit_fail_closed(self) -> None:
with patch(
"govoplan_files.backend.capabilities.get_asset_for_user",
side_effect=FileStorageError("File not found"),
):
self.assertIsNone(
self.provider.get_tabular_file(
self.session,
principal("tenant-2"),
file_asset_id="asset-1",
)
)
with self.assertRaises(ManagedTabularFileAccessError):
self.provider.list_tabular_files(
self.session,
principal(scopes=()),
)
with (
patch(
"govoplan_files.backend.capabilities.get_asset_for_user",
return_value=self.asset,
),
patch.object(
self.session,
"get",
side_effect=lambda model, object_id: {
"version-1": self.version,
"blob-1": self.blob,
}.get(object_id),
),
patch(
"govoplan_files.backend.capabilities.read_asset_version_bytes"
) as read_bytes,
self.assertRaises(ManagedTabularFileValidationError),
):
self.provider.read_tabular_file(
self.session,
principal(),
file_asset_id="asset-1",
file_version_id="version-1",
max_bytes=4,
)
read_bytes.assert_not_called()
if __name__ == "__main__":
unittest.main()
webui/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@govoplan/files-webui",
"version": "0.1.18",
"version": "0.1.19",
"private": true,
"type": "module",
"main": "src/index.ts",