diff --git a/src/govoplan_files/backend/manifest.py b/src/govoplan_files/backend/manifest.py index 7bf1781..9c4ac9a 100644 --- a/src/govoplan_files/backend/manifest.py +++ b/src/govoplan_files/backend/manifest.py @@ -35,6 +35,7 @@ from govoplan_core.core.provider_governance import ( ProviderObjectDeclaration, declared_module_architecture, ) +from govoplan_core.core.search import SearchSourceProviderRegistration from govoplan_core.core.views import ViewSurface from govoplan_core.db.base import Base from govoplan_files.backend.change_tracking import register_files_change_tracking @@ -44,6 +45,7 @@ from govoplan_files.backend.provider_state import ( REMOTE_STORAGE_PROVIDER_ID, remote_storage_provider_states, ) +from govoplan_files.backend.search_source import create_files_search_source register_files_change_tracking() @@ -279,7 +281,7 @@ manifest = ModuleManifest( name="Files", version="0.1.9", required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR), - optional_dependencies=("campaigns", "encryption"), + optional_dependencies=("campaigns", "encryption", "search"), provides_interfaces=( ModuleInterfaceProvider(name="files.access", version="0.1.6"), ModuleInterfaceProvider(name="files.campaign_attachments", version="0.1.6"), @@ -298,12 +300,24 @@ manifest = ModuleManifest( version_max_exclusive="2.0.0", optional=True, ), + ModuleInterfaceRequirement( + name="search.source", + version_min="1.0.0", + version_max_exclusive="2.0.0", + optional=True, + ), ), permissions=PERMISSIONS, route_factory=_files_router, role_templates=ROLE_TEMPLATES, tenant_summary_providers=(_tenant_summary,), tenant_summary_batch_providers=(_tenant_summary_batch,), + search_sources=( + SearchSourceProviderRegistration( + id="files.objects", + factory=create_files_search_source, + ), + ), delete_veto_providers={"group": (_veto_group_delete,)}, nav_items=(NavItem(path="/files", label="Files", icon="folder", required_any=("files:file:read",), order=40),), frontend=FrontendModule( @@ -329,6 +343,23 @@ manifest = ModuleManifest( ), ), documentation=( + DocumentationTopic( + id="files.search.managed-content", + title="Search managed files and folders", + summary="Expose file names, logical paths, and descriptions to permission-aware platform Search.", + body=( + "When Search is installed, Files contributes managed files and folders to its derived index. " + "Every result is tenant-bounded and rechecks current ownership, group membership, direct shares, " + "expiry, revocation, deletion, and Files permissions before it is returned. Committed file and " + "share changes are delivered through the platform event outbox; an administrator can rebuild the " + "derived index without changing authoritative Files data." + ), + layer="configured", + documentation_types=("admin", "user"), + audience=("file_user", "file_manager", "administrator"), + related_modules=("search",), + order=41, + ), DocumentationTopic( id="files.workflow.organize-managed-files", title="Organize managed files and folders", diff --git a/src/govoplan_files/backend/search_source.py b/src/govoplan_files/backend/search_source.py new file mode 100644 index 0000000..8896c11 --- /dev/null +++ b/src/govoplan_files/backend/search_source.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from urllib.parse import quote + +from sqlalchemy import func, or_, select +from sqlalchemy.orm import Session + +from govoplan_core.auth import ApiPrincipal +from govoplan_core.core.events import PlatformEvent +from govoplan_core.core.modules import ModuleContext +from govoplan_core.core.search import ( + SearchAuthorizationRequest, + SearchBackfillPage, + SearchBackfillRequest, + SearchDocument, + SearchIndexChange, + SearchResourceReference, + SearchResourceType, +) +from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare +from govoplan_files.backend.storage.share_state import effective_file_share_clause + + +PROVIDER_ID = "files.objects" +RESOURCE_MODELS = { + "file": FileAsset, + "folder": FileFolder, +} +READ_SCOPE = "files:file:read" +ADMIN_SCOPE = "files:file:admin" + + +class FilesSearchSource: + def resource_types(self) -> Sequence[SearchResourceType]: + return ( + SearchResourceType( + provider_id=PROVIDER_ID, + module_id="files", + resource_type="file", + label="Files", + requires_authorization_recheck=True, + ), + SearchResourceType( + provider_id=PROVIDER_ID, + module_id="files", + resource_type="folder", + label="File folders", + requires_authorization_recheck=True, + ), + ) + + def backfill( + self, + session: object, + *, + request: SearchBackfillRequest, + ) -> SearchBackfillPage: + db = _session(session) + model = _model(request.provider_id, request.resource_type) + statement = select(model).where( + model.tenant_id == request.tenant_id, + model.deleted_at.is_(None), + ) + if request.cursor: + statement = statement.where(model.id > request.cursor) + rows = list( + db.scalars( + statement.order_by(model.id).limit(request.limit + 1) + ) + ) + has_more = len(rows) > request.limit + selected = rows[: request.limit] + shares = ( + _shares_by_asset(db, selected) + if request.resource_type == "file" + else {} + ) + high_watermark = db.scalar( + select(func.max(model.updated_at)).where( + model.tenant_id == request.tenant_id, + model.deleted_at.is_(None), + ) + ) + return SearchBackfillPage( + documents=tuple( + _document( + row, + resource_type=request.resource_type, + shares=shares.get(row.id, ()), + ) + for row in selected + ), + next_cursor=selected[-1].id if has_more and selected else None, + complete=not has_more, + high_watermark=( + high_watermark.isoformat() + if high_watermark is not None + else None + ), + ) + + def authorize( + self, + session: object, + principal: object, + *, + requests: Sequence[SearchAuthorizationRequest], + ) -> Mapping[str, bool]: + decisions = {item.reference.key: False for item in requests} + if not isinstance(principal, ApiPrincipal) or not ( + principal.has(READ_SCOPE) or principal.has(ADMIN_SCOPE) + ): + return decisions + db = _session(session) + for request in requests: + reference = request.reference + if ( + reference.tenant_id != principal.tenant_id + or reference.module_id != "files" + or reference.resource_type not in RESOURCE_MODELS + ): + continue + decisions[reference.key] = _can_read( + db, + principal, + reference=reference, + ) + return decisions + + def index_changes_for_event( + self, + session: object, + *, + event: PlatformEvent, + delivery_key: str, + ) -> Sequence[SearchIndexChange]: + if ( + event.module_id != "files" + or event.tenant is None + or event.resource is None + or event.resource.id is None + or event.resource.type not in RESOURCE_MODELS + ): + return () + db = _session(session) + reference = SearchResourceReference( + tenant_id=event.tenant.id, + module_id="files", + resource_type=event.resource.type, + resource_id=event.resource.id, + ) + model = RESOURCE_MODELS[event.resource.type] + row = db.get(model, event.resource.id) + deleted = row is None or row.tenant_id != event.tenant.id or row.deleted_at is not None + cursor = event.event_id + document = None + if not deleted: + shares = ( + tuple(_active_shares(db, row.id)) + if event.resource.type == "file" + else () + ) + document = _document( + row, + resource_type=event.resource.type, + shares=shares, + change_cursor=cursor, + ) + return ( + SearchIndexChange( + change_id=f"{delivery_key}:{PROVIDER_ID}:{event.resource.type}", + provider_id=PROVIDER_ID, + kind="delete" if deleted else "upsert", + reference=reference, + source_revision=( + document.source_revision if document is not None else cursor + ), + cursor=cursor, + document=document, + occurred_at=event.occurred_at, + ), + ) + + +def create_files_search_source(_context: ModuleContext) -> FilesSearchSource: + return FilesSearchSource() + + +def _model(provider_id: str, resource_type: str): + if provider_id != PROVIDER_ID or resource_type not in RESOURCE_MODELS: + raise ValueError("Unsupported Files search source.") + return RESOURCE_MODELS[resource_type] + + +def _document( + row: FileAsset | FileFolder, + *, + resource_type: str, + shares: Sequence[FileShare] = (), + change_cursor: str | None = None, +) -> SearchDocument: + is_file = isinstance(row, FileAsset) + title = row.filename if is_file else (row.path.rsplit("/", 1)[-1] or row.path) + path = row.display_path if is_file else row.path + owner_id = row.owner_user_id if row.owner_type == "user" else row.owner_group_id + tokens = [f"scope:{READ_SCOPE}", f"scope:{ADMIN_SCOPE}"] + if owner_id: + tokens.append( + f"membership:{owner_id}" + if row.owner_type == "user" + else f"group:{owner_id}" + ) + for share in shares: + prefix = "membership" if share.target_type == "user" else share.target_type + if prefix in {"membership", "group", "tenant"}: + tokens.append(f"{prefix}:{share.target_id}") + updated_at = row.updated_at or row.created_at + revision = ( + f"{row.current_version_id or 'none'}:{updated_at.isoformat()}" + if is_file + else updated_at.isoformat() + ) + return SearchDocument( + tenant_id=row.tenant_id, + module_id="files", + provider_id=PROVIDER_ID, + resource_type=resource_type, + resource_id=row.id, + title=title, + url=f"/files?{resource_type}Id={quote(row.id, safe='')}", + summary=((row.description or "") if is_file else path)[:4000] or None, + body=" ".join( + value for value in (path, row.description if is_file else None) if value + )[:200_000], + keywords=(path[:200], row.owner_type[:200]), + visibility="restricted", + acl_tokens=tuple(dict.fromkeys(tokens)), + metadata={ + "path": path, + "owner_type": row.owner_type, + "current_version_id": row.current_version_id if is_file else None, + }, + source_revision=revision, + change_cursor=change_cursor, + source_updated_at=updated_at, + requires_authorization_recheck=True, + ) + + +def _can_read( + session: Session, + principal: ApiPrincipal, + *, + reference: SearchResourceReference, +) -> bool: + model = RESOURCE_MODELS[reference.resource_type] + row = session.get(model, reference.resource_id) + if row is None or row.tenant_id != principal.tenant_id or row.deleted_at is not None: + return False + if principal.has(ADMIN_SCOPE): + return True + user_id = str(getattr(principal.user, "id", "") or principal.membership_id or "") + if row.owner_type == "user" and row.owner_user_id == user_id: + return True + if row.owner_type == "group" and row.owner_group_id in principal.group_ids: + return True + if reference.resource_type != "file": + return False + target_clauses = [ + (FileShare.target_type == "user") & (FileShare.target_id == user_id), + (FileShare.target_type == "tenant") + & (FileShare.target_id == principal.tenant_id), + ] + if principal.group_ids: + target_clauses.append( + (FileShare.target_type == "group") + & (FileShare.target_id.in_(tuple(principal.group_ids))) + ) + return session.scalar( + select(FileShare.id).where( + FileShare.tenant_id == principal.tenant_id, + FileShare.file_asset_id == row.id, + effective_file_share_clause(), + or_(*target_clauses), + ).limit(1) + ) is not None + + +def _active_shares(session: Session, asset_id: str) -> Sequence[FileShare]: + return tuple( + session.scalars( + select(FileShare).where( + FileShare.file_asset_id == asset_id, + effective_file_share_clause(), + ) + ) + ) + + +def _shares_by_asset( + session: Session, + rows: Sequence[FileAsset | FileFolder], +) -> dict[str, tuple[FileShare, ...]]: + asset_ids = [row.id for row in rows if isinstance(row, FileAsset)] + grouped: dict[str, list[FileShare]] = {asset_id: [] for asset_id in asset_ids} + if not asset_ids: + return {} + for share in session.scalars( + select(FileShare).where( + FileShare.file_asset_id.in_(asset_ids), + effective_file_share_clause(), + ) + ): + grouped.setdefault(share.file_asset_id, []).append(share) + return {key: tuple(value) for key, value in grouped.items()} + + +def _session(value: object) -> Session: + if not isinstance(value, Session): + raise TypeError("Files search requires a SQLAlchemy session.") + return value + + +__all__ = [ + "FilesSearchSource", + "PROVIDER_ID", + "create_files_search_source", +] diff --git a/tests/test_search_source.py b/tests/test_search_source.py new file mode 100644 index 0000000..6c47778 --- /dev/null +++ b/tests/test_search_source.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from types import SimpleNamespace +import unittest + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from govoplan_access.backend.db.models import Account, Group, User +from govoplan_core.auth import ApiPrincipal +from govoplan_core.core.access import PrincipalRef +from govoplan_core.core.events import EventObjectRef, EventTenantRef, PlatformEvent +from govoplan_core.core.search import ( + SearchAuthorizationRequest, + SearchBackfillRequest, + SearchResourceReference, +) +from govoplan_core.db.base import Base +from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare +from govoplan_files.backend.search_source import ( + FilesSearchSource, + PROVIDER_ID, +) + + +class FilesSearchSourceTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite://") + Base.metadata.create_all( + self.engine, + tables=( + Account.__table__, + User.__table__, + Group.__table__, + FileAsset.__table__, + FileFolder.__table__, + FileShare.__table__, + ), + ) + self.session = Session(self.engine) + self.session.add_all( + ( + Account( + id="account-1", + email="one@example.test", + normalized_email="one@example.test", + ), + User( + id="user-1", + tenant_id="tenant-1", + account_id="account-1", + email="one@example.test", + ), + FileAsset( + id="file-1", + tenant_id="tenant-1", + owner_type="user", + owner_user_id="user-1", + display_path="records/permit.pdf", + filename="permit.pdf", + description="Monthly permit evidence", + ), + FileAsset( + id="file-other", + tenant_id="tenant-2", + owner_type="user", + display_path="other.pdf", + filename="other.pdf", + ), + ) + ) + self.session.commit() + self.source = FilesSearchSource() + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def test_backfill_and_authorization_are_tenant_bounded(self) -> None: + page = self.source.backfill( + self.session, + request=SearchBackfillRequest( + tenant_id="tenant-1", + provider_id=PROVIDER_ID, + resource_type="file", + rebuild_id="rebuild-1", + ), + ) + self.assertEqual(("file-1",), tuple(doc.resource_id for doc in page.documents)) + reference = SearchResourceReference( + tenant_id="tenant-1", + module_id="files", + resource_type="file", + resource_id="file-1", + ) + request = SearchAuthorizationRequest(reference=reference, source_revision="1") + self.assertTrue( + self.source.authorize( + self.session, + _principal({"files:file:read"}), + requests=(request,), + )[reference.key] + ) + self.assertFalse( + self.source.authorize( + self.session, + _principal(set()), + requests=(request,), + )[reference.key] + ) + + def test_committed_file_event_produces_authoritative_upsert(self) -> None: + event = PlatformEvent( + type="files.file.updated", + module_id="files", + tenant=EventTenantRef(id="tenant-1"), + resource=EventObjectRef(type="file", id="file-1"), + ) + changes = self.source.index_changes_for_event( + self.session, + event=event, + delivery_key="delivery-1", + ) + self.assertEqual(1, len(changes)) + self.assertEqual("upsert", changes[0].kind) + self.assertEqual(event.event_id, changes[0].cursor) + + +def _principal(scopes: set[str]) -> ApiPrincipal: + return ApiPrincipal( + principal=PrincipalRef( + account_id="account-1", + membership_id="user-1", + tenant_id="tenant-1", + scopes=frozenset(scopes), + ), + account=SimpleNamespace(id="account-1"), + user=SimpleNamespace(id="user-1"), + ) + + +if __name__ == "__main__": + unittest.main()