from __future__ import annotations from collections.abc import Sequence from datetime import datetime, timezone from sqlalchemy import inspect, or_ from sqlalchemy.orm import Session from govoplan_core.core.dsar import ( DsarErasureActionRef, DsarExecutionResultRef, DsarRecordRef, DsarSubjectRef, dsar_capability_name, ) from govoplan_files.backend.db.models import ( CampaignAttachmentUse, FileAsset, FileConnectorCredential, FileConnectorPolicy, FileConnectorProfile, FileConnectorSpace, FileFolder, FileFormEvidenceGrant, FileIntegrityFinding, FileIntegrityScan, FileShare, FileVersion, ) FILES_DSAR_CAPABILITY = dsar_capability_name("files") _MAX_RECORDS = 5_000 class FilesDsarProvider: provider_id = "files" module_id = "files" def search_subject( self, session: object, *, tenant_id: str, subject: DsarSubjectRef, ) -> Sequence[DsarRecordRef]: db = _session(session) subject_user_id = _subject_user_id(subject) if subject_user_id is None: return () records: list[DsarRecordRef] = [] form_evidence_available = _has_table(db, FileFormEvidenceGrant) campaign_evidence_available = _has_table(db, CampaignAttachmentUse) retention_reasons: dict[str, str | None] = {} def retention_reason(asset: FileAsset) -> str | None: if asset.id not in retention_reasons: retention_reasons[asset.id] = _asset_retention_reason( db, asset, form_evidence_available=form_evidence_available, campaign_evidence_available=campaign_evidence_available, ) return retention_reasons[asset.id] def append(record: DsarRecordRef) -> None: if len(records) >= _MAX_RECORDS: raise ValueError( "Files DSAR match limit exceeded; narrow the subject selectors." ) records.append(record) assets = _bounded_rows( db.query(FileAsset) .filter( FileAsset.tenant_id == tenant_id, or_( FileAsset.owner_user_id == subject_user_id, FileAsset.created_by_user_id == subject_user_id, ), ) .order_by(FileAsset.id) ) for asset in assets: match_fields = _matching_fields( asset, subject_user_id, ("owner_user_id", "created_by_user_id"), ) asset_retention_reason = retention_reason(asset) append( _record( "file_asset", asset.id, "managed_file", asset.filename, { "match_fields": match_fields, "owner_type": asset.owner_type, "display_path": asset.display_path, "filename": asset.filename, "description": asset.description, "deleted_at": _iso(asset.deleted_at), "retained_until": _iso(asset.retained_until), "legal_hold": asset.legal_hold, "lifecycle_reason": asset.lifecycle_reason, "lifecycle_revision": asset.lifecycle_revision, }, observed_at=asset.updated_at, immutable=asset_retention_reason is not None, retention_reason=asset_retention_reason, source_path=f"/files?file={asset.id}", ) ) version_rows = _bounded_rows( db.query(FileVersion, FileAsset) .join(FileAsset, FileAsset.id == FileVersion.file_asset_id) .filter( FileVersion.tenant_id == tenant_id, FileAsset.tenant_id == tenant_id, or_( FileAsset.owner_user_id == subject_user_id, FileVersion.created_by_user_id == subject_user_id, ), ) .order_by(FileVersion.id) ) for version, asset in version_rows: asset_retention_reason = retention_reason(asset) append( _record( "file_version", version.id, "managed_file_version", version.filename_at_upload, { "match_fields": ( ["asset.owner_user_id"] if asset.owner_user_id == subject_user_id else [] ) + ( ["created_by_user_id"] if version.created_by_user_id == subject_user_id else [] ), "file_asset_id": version.file_asset_id, "version_number": version.version_number, "filename_at_upload": version.filename_at_upload, "display_path_at_upload": version.display_path_at_upload, "content_type": version.content_type, "size_bytes": version.size_bytes, "checksum_sha256": version.checksum_sha256, "created_at": _iso(version.created_at), }, observed_at=version.updated_at, immutable=asset_retention_reason is not None, retention_reason=asset_retention_reason, source_path=( f"/api/v1/files/{version.file_asset_id}/versions/" f"{version.id}/download" ), ) ) for folder in _bounded_rows( db.query(FileFolder) .filter( FileFolder.tenant_id == tenant_id, or_( FileFolder.owner_user_id == subject_user_id, FileFolder.created_by_user_id == subject_user_id, ), ) .order_by(FileFolder.id) ): append( _record( "file_folder", folder.id, "managed_folder", folder.path, { "match_fields": _matching_fields( folder, subject_user_id, ("owner_user_id", "created_by_user_id"), ), "owner_type": folder.owner_type, "path": folder.path, "deleted_at": _iso(folder.deleted_at), }, observed_at=folder.updated_at, source_path="/files", ) ) for share in _bounded_rows( db.query(FileShare) .filter( FileShare.tenant_id == tenant_id, or_( (FileShare.target_type == "user") & (FileShare.target_id == subject_user_id), FileShare.created_by_user_id == subject_user_id, FileShare.revoked_by_user_id == subject_user_id, ), ) .order_by(FileShare.id) ): target_matches = ( share.target_type == "user" and share.target_id == subject_user_id ) match_fields = _matching_fields( share, subject_user_id, ("created_by_user_id", "revoked_by_user_id"), ) if target_matches: match_fields.insert(0, "target_id") append( _record( "file_share", share.id, "file_access_evidence", f"Share for file {share.file_asset_id}", { "match_fields": match_fields, "file_asset_id": share.file_asset_id, "target_type": share.target_type, "target_is_subject": target_matches, "permission": share.permission, "expires_at": _iso(share.expires_at), "revoked_at": _iso(share.revoked_at), }, observed_at=share.updated_at, immutable=True, retention_reason=( "File-sharing history is institutional access evidence." ), source_path=f"/files?file={share.file_asset_id}", ) ) if form_evidence_available: evidence_rows = _bounded_rows( db.query(FileFormEvidenceGrant) .outerjoin( FileAsset, FileAsset.id == FileFormEvidenceGrant.file_asset_id, ) .filter( FileFormEvidenceGrant.tenant_id == tenant_id, or_( FileFormEvidenceGrant.custodian_user_id == subject_user_id, FileAsset.owner_user_id == subject_user_id, ), ) .order_by(FileFormEvidenceGrant.id) ) for grant in evidence_rows: append( _record( "file_form_evidence", grant.id, "form_evidence", f"Form evidence {grant.form_definition_id}", { "match_fields": ( ["custodian_user_id"] if grant.custodian_user_id == subject_user_id else ["asset.owner_user_id"] ), "form_instance_id": grant.form_instance_id, "form_definition_id": grant.form_definition_id, "form_definition_revision": grant.form_definition_revision, "evidence_kind": grant.evidence_kind, "purpose": grant.purpose, "status": grant.status, "expires_at": _iso(grant.expires_at), "file_asset_id": grant.file_asset_id, "file_version_id": grant.file_version_id, }, observed_at=grant.updated_at, immutable=True, retention_reason=( "Submitted Form attachment evidence follows the owning " "process retention and cannot be erased through Files alone." ), ) ) if campaign_evidence_available: campaign_rows = _bounded_rows( db.query(CampaignAttachmentUse) .join(FileAsset, FileAsset.id == CampaignAttachmentUse.file_asset_id) .filter( CampaignAttachmentUse.tenant_id == tenant_id, FileAsset.owner_user_id == subject_user_id, ) .order_by(CampaignAttachmentUse.id) ) for use in campaign_rows: append( _record( "campaign_attachment_use", use.id, "delivery_evidence", use.filename_used, { "match_fields": ["asset.owner_user_id"], "campaign_id": use.campaign_id, "campaign_version_id": use.campaign_version_id, "campaign_job_id": use.campaign_job_id, "file_asset_id": use.file_asset_id, "file_version_id": use.file_version_id, "filename_used": use.filename_used, "checksum_sha256": use.checksum_sha256, "size_bytes": use.size_bytes, "use_stage": use.use_stage, "used_at": _iso(use.used_at), }, observed_at=use.updated_at, immutable=True, retention_reason=( "Campaign attachment use is immutable delivery evidence." ), ) ) self._append_configuration_references( db, append=append, tenant_id=tenant_id, subject_user_id=subject_user_id, ) self._append_integrity_references( db, append=append, tenant_id=tenant_id, subject_user_id=subject_user_id, ) return tuple(records) def plan_erasure( self, session: object, *, tenant_id: str, subject: DsarSubjectRef, records: Sequence[DsarRecordRef], ) -> Sequence[DsarErasureActionRef]: del session subject_user_id = _subject_user_id(subject) if subject_user_id is None: return () actions: list[DsarErasureActionRef] = [] for record in records: if record.provider_id != self.provider_id or record.module_id != self.module_id: raise ValueError("Files DSAR received a foreign provider record.") match_fields = { str(value) for value in record.data.get("match_fields", ()) } if record.immutable_evidence: actions.append( _action( f"files:retain:{record.resource_type}:{record.resource_id}", "retain", record, f"Retain {record.title}", record.retention_reason or "Institutional evidence must be retained.", executable=False, ) ) elif record.resource_type in { "file_asset", "file_version", "file_folder", }: actions.append( _action( f"files:review:{record.resource_type}:{record.resource_id}", "manual_review", record, f"Review {record.title}", ( "Managed file content, names, paths, ownership, and shared " "references require a case decision. Approved byte erasure " "must use the separate Files purge workflow." ), executable=False, ) ) if ( record.resource_type == "file_share" and "target_id" in match_fields and record.data.get("revoked_at") is None ): actions.append( _action( f"files:revoke:file_share:{record.resource_id}", "revoke", record, "Revoke active file share", "The active user-targeted share can be revoked without deleting file evidence.", executable=True, metadata={"subject_user_id": subject_user_id}, ) ) for field_name in sorted( match_fields.intersection(_detachable_fields(record.resource_type)) ): actions.append( _action( ( f"files:detach:{record.resource_type}:" f"{field_name}:{record.resource_id}" ), "detach", record, f"Detach {field_name.replace('_', ' ')}", ( "Remove the mutable subject reference while preserving the " "governed resource and DSAR evidence." ), executable=True, metadata={ "field": field_name, "subject_user_id": subject_user_id, }, ) ) action_ids = [action.action_id for action in actions] if len(action_ids) != len(set(action_ids)): raise ValueError("Files DSAR produced duplicate action ids.") return tuple(actions) def execute_erasure( self, session: object, *, tenant_id: str, subject: DsarSubjectRef, actions: Sequence[DsarErasureActionRef], request_id: str, ) -> Sequence[DsarExecutionResultRef]: db = _session(session) subject_user_id = _subject_user_id(subject) if subject_user_id is None: return tuple( _blocked(action, "Files requires a direct membership subject reference.") for action in actions ) results: list[DsarExecutionResultRef] = [] for action in actions: if ( action.provider_id != self.provider_id or action.module_id != self.module_id or action.metadata.get("subject_user_id") != subject_user_id ): results.append(_blocked(action, "The Files DSAR action is stale or invalid.")) continue if action.action_id.startswith("files:revoke:file_share:"): results.append( _revoke_share( db, tenant_id=tenant_id, subject_user_id=subject_user_id, action=action, request_id=request_id, ) ) elif action.action_id.startswith("files:detach:"): results.append( _detach_reference( db, tenant_id=tenant_id, subject_user_id=subject_user_id, action=action, request_id=request_id, ) ) else: results.append(_blocked(action, "Files does not execute this action kind.")) db.flush() return tuple(results) def _append_configuration_references( self, db: Session, *, append: object, tenant_id: str, subject_user_id: str, ) -> None: configurations = ( (FileConnectorProfile, "connector_profile", ("created_by_user_id", "updated_by_user_id")), (FileConnectorCredential, "connector_credential", ("created_by_user_id", "updated_by_user_id")), (FileConnectorPolicy, "connector_policy", ("created_by_user_id", "updated_by_user_id")), (FileConnectorSpace, "connector_space", ("owner_user_id", "created_by_user_id")), ) for model, resource_type, fields in configurations: if not _has_table(db, model): continue conditions = [getattr(model, field) == subject_user_id for field in fields] query = db.query(model).filter(or_(*conditions)) if hasattr(model, "tenant_id"): query = query.filter(model.tenant_id == tenant_id) for row in _bounded_rows(query.order_by(model.id)): match_fields = _matching_fields(row, subject_user_id, fields) data: dict[str, object] = { "match_fields": match_fields, "label": getattr(row, "label", None), "provider": getattr(row, "provider", None), } if isinstance(row, FileConnectorCredential): data["credential_mode"] = row.credential_mode elif isinstance(row, FileConnectorSpace): data.update( { "remote_path": row.remote_path, "sync_mode": row.sync_mode, "read_only": row.read_only, "deleted_at": _iso(row.deleted_at), } ) append( # type: ignore[operator] _record( resource_type, row.id, "connector_configuration_evidence", getattr(row, "label", None) or resource_type.replace("_", " "), data, observed_at=row.updated_at, immutable=True, retention_reason=( "Connector configuration history is institutional evidence; " "credential secrets are excluded from the DSAR export." ), ) ) def _append_integrity_references( self, db: Session, *, append: object, tenant_id: str, subject_user_id: str, ) -> None: if _has_table(db, FileIntegrityScan): for scan in _bounded_rows( db.query(FileIntegrityScan) .filter( FileIntegrityScan.tenant_id == tenant_id, FileIntegrityScan.created_by_user_id == subject_user_id, ) .order_by(FileIntegrityScan.id) ): append( # type: ignore[operator] _record( "file_integrity_scan", scan.id, "storage_integrity_evidence", f"Integrity scan {scan.id}", { "match_fields": ["created_by_user_id"], "storage_backend": scan.storage_backend, "status": scan.status, "started_at": _iso(scan.started_at), "completed_at": _iso(scan.completed_at), }, observed_at=scan.updated_at, immutable=True, retention_reason="Storage integrity scans are operator evidence.", ) ) if _has_table(db, FileIntegrityFinding): for finding in _bounded_rows( db.query(FileIntegrityFinding) .filter( FileIntegrityFinding.tenant_id == tenant_id, FileIntegrityFinding.resolved_by_user_id == subject_user_id, ) .order_by(FileIntegrityFinding.id) ): append( # type: ignore[operator] _record( "file_integrity_finding", finding.id, "storage_integrity_evidence", f"Integrity finding {finding.kind}", { "match_fields": ["resolved_by_user_id"], "kind": finding.kind, "state": finding.state, "resolved_at": _iso(finding.resolved_at), }, observed_at=finding.updated_at, immutable=True, retention_reason="Integrity resolution is operator evidence.", ) ) def _asset_retention_reason( session: Session, asset: FileAsset, *, form_evidence_available: bool, campaign_evidence_available: bool, ) -> str | None: reasons: list[str] = [] if asset.legal_hold: reasons.append("The file is under legal hold.") retained_until = _aware(asset.retained_until) if retained_until is not None and retained_until > datetime.now(timezone.utc): reasons.append(f"The file is retained until {retained_until.isoformat()}.") if form_evidence_available: has_form_evidence = ( session.query(FileFormEvidenceGrant.id) .filter( FileFormEvidenceGrant.tenant_id == asset.tenant_id, FileFormEvidenceGrant.file_asset_id == asset.id, ) .first() is not None ) if has_form_evidence: reasons.append("The file is referenced by submitted Form evidence.") if campaign_evidence_available: has_campaign_evidence = ( session.query(CampaignAttachmentUse.id) .filter( CampaignAttachmentUse.tenant_id == asset.tenant_id, CampaignAttachmentUse.file_asset_id == asset.id, ) .first() is not None ) if has_campaign_evidence: reasons.append("The file is referenced by Campaign delivery evidence.") return " ".join(reasons) or None _DETACHABLE_MODELS: dict[str, tuple[type[object], frozenset[str]]] = { "file_asset": (FileAsset, frozenset({"created_by_user_id"})), "file_version": (FileVersion, frozenset({"created_by_user_id"})), "file_folder": (FileFolder, frozenset({"created_by_user_id"})), "file_share": ( FileShare, frozenset({"created_by_user_id", "revoked_by_user_id"}), ), "connector_profile": ( FileConnectorProfile, frozenset({"created_by_user_id", "updated_by_user_id"}), ), "connector_credential": ( FileConnectorCredential, frozenset({"created_by_user_id", "updated_by_user_id"}), ), "connector_policy": ( FileConnectorPolicy, frozenset({"created_by_user_id", "updated_by_user_id"}), ), "connector_space": (FileConnectorSpace, frozenset({"created_by_user_id"})), } def _detachable_fields(resource_type: str) -> frozenset[str]: entry = _DETACHABLE_MODELS.get(resource_type) return entry[1] if entry is not None else frozenset() def _revoke_share( session: Session, *, tenant_id: str, subject_user_id: str, action: DsarErasureActionRef, request_id: str, ) -> DsarExecutionResultRef: row = ( session.query(FileShare) .filter(FileShare.id == action.resource_id) .with_for_update() .one_or_none() ) if ( row is None or row.tenant_id != tenant_id or row.target_type != "user" or row.target_id != subject_user_id ): return _blocked(action, "The subject-targeted file share is no longer available.") if row.revoked_at is not None: return _result( action, "unchanged", "The file share was already revoked.", {"request_id": request_id, "revoked_at": _iso(row.revoked_at)}, ) row.revoked_at = datetime.now(timezone.utc) row.revoked_by_user_id = None return _result( action, "executed", "The subject-targeted file share was revoked.", {"request_id": request_id, "revoked_at": _iso(row.revoked_at)}, ) def _detach_reference( session: Session, *, tenant_id: str, subject_user_id: str, action: DsarErasureActionRef, request_id: str, ) -> DsarExecutionResultRef: entry = _DETACHABLE_MODELS.get(action.resource_type) field_name = str(action.metadata.get("field") or "") if entry is None or field_name not in entry[1]: return _blocked(action, "The requested Files subject reference is not detachable.") model = entry[0] row = ( session.query(model) .filter(getattr(model, "id") == action.resource_id) .with_for_update() .one_or_none() ) if row is None or getattr(row, "tenant_id", None) != tenant_id: return _blocked(action, "The Files resource is no longer available.") current = getattr(row, field_name) if current is None: return _result( action, "unchanged", "The subject reference was already detached.", {"request_id": request_id, "field": field_name}, ) if current != subject_user_id: return _blocked(action, "The Files subject reference changed after planning.") setattr(row, field_name, None) return _result( action, "executed", "The mutable subject reference was detached.", {"request_id": request_id, "field": field_name}, ) def _subject_user_id(subject: DsarSubjectRef) -> str | None: candidates: list[str] = [] if subject.membership_id: candidates.append(subject.membership_id) for key, value in subject.external_references.items(): if key in { "files.user", "files.membership", "access.membership", "membership_id", }: candidates.append(value) normalized = {value.strip() for value in candidates if value.strip()} if len(normalized) != 1: return None return normalized.pop() def _matching_fields( row: object, subject_user_id: str, fields: Sequence[str], ) -> list[str]: return [field for field in fields if getattr(row, field) == subject_user_id] def _record( resource_type: str, resource_id: str, category: str, title: str, data: dict[str, object], *, observed_at: datetime | None = None, immutable: bool = False, retention_reason: str | None = None, source_path: str | None = None, ) -> DsarRecordRef: return DsarRecordRef( provider_id="files", module_id="files", resource_type=resource_type, resource_id=resource_id, category=category, title=title, data=data, observed_at=observed_at, immutable_evidence=immutable, retention_reason=retention_reason, source_path=source_path, ) def _action( action_id: str, kind: str, record: DsarRecordRef, title: str, rationale: str, *, executable: bool, metadata: dict[str, object] | None = None, ) -> DsarErasureActionRef: return DsarErasureActionRef( action_id=action_id, provider_id="files", module_id="files", kind=kind, # type: ignore[arg-type] resource_type=record.resource_type, resource_id=record.resource_id, title=title, rationale=rationale, executable=executable, metadata=metadata or {}, ) def _result( action: DsarErasureActionRef, status: str, summary: str, evidence: dict[str, object] | None = None, ) -> DsarExecutionResultRef: return DsarExecutionResultRef( action_id=action.action_id, status=status, # type: ignore[arg-type] summary=summary, evidence=evidence or {}, ) def _blocked(action: DsarErasureActionRef, summary: str) -> DsarExecutionResultRef: return _result(action, "blocked", summary) def _session(value: object) -> Session: if not isinstance(value, Session): raise TypeError("Files DSAR provider requires a SQLAlchemy session.") return value def _bounded_rows(query: object) -> list[object]: rows = query.limit(_MAX_RECORDS + 1).all() # type: ignore[attr-defined] if len(rows) > _MAX_RECORDS: raise ValueError("Files DSAR match limit exceeded; narrow the subject selectors.") return rows def _has_table(session: Session, model: type[object]) -> bool: return inspect(session.connection()).has_table(model.__tablename__) def _aware(value: datetime | None) -> datetime | None: if value is None or value.tzinfo is not None: return value return value.replace(tzinfo=timezone.utc) def _iso(value: datetime | None) -> str | None: aware = _aware(value) return aware.isoformat() if aware else None __all__ = ["FILES_DSAR_CAPABILITY", "FilesDsarProvider"]