from __future__ import annotations import json import unittest from datetime import UTC, datetime, timedelta from sqlalchemy import create_engine from sqlalchemy.orm import Session from govoplan_core.core.dsar import ( DsarErasureActionRef, DsarProvider, DsarRecordRef, DsarSubjectRef, ) from govoplan_core.db.base import Base from govoplan_core.privacy.dsar_workflow import ( create_data_subject_request, search_data_subject_request, ) from govoplan_dataflow.backend.db.models import ( DataflowPipeline, DataflowPipelineDeployment, DataflowPipelineRevision, DataflowReconciliationDecision, DataflowReconciliationDecisionSet, DataflowRun, DataflowTrigger, DataflowTriggerDelivery, ) from govoplan_dataflow.backend.dsar_provider import ( DATAFLOW_DSAR_CAPABILITY, DataflowDsarProvider, ) from govoplan_dataflow.backend.manifest import manifest NOW = datetime(2026, 8, 21, 20, 0, tzinfo=UTC) SECRET = "private-dataflow-detail-do-not-export" class _Registry: def __init__(self, provider: DataflowDsarProvider, *, active: bool = True) -> None: self.provider = provider self.active = active def capability_names(self): return (DATAFLOW_DSAR_CAPABILITY,) def capability_owner(self, name): self._assert_capability(name) return "dataflow" def tenant_entitlement_resolver(self): active = self.active class _Resolver: @staticmethod def resolve(session, tenant_id): del session, tenant_id return type( "State", (), {"effective_modules": ("dataflow",) if active else ()}, )() return _Resolver() def require_tenant_capability(self, name, session, **kwargs): del session, kwargs self._assert_capability(name) return self.provider def manifests(self): return (type("Manifest", (), {"id": "dataflow"})(),) @staticmethod def _assert_capability(name: str) -> None: if name != DATAFLOW_DSAR_CAPABILITY: raise KeyError(name) class DataflowDsarProviderTests(unittest.TestCase): def setUp(self) -> None: self.engine = create_engine("sqlite+pysqlite:///:memory:") Base.metadata.create_all(self.engine) self.session = Session(self.engine) self.provider = DataflowDsarProvider() self.assertIsInstance(self.provider, DsarProvider) self._seed() self.session.commit() def tearDown(self) -> None: self.session.close() self.engine.dispose() def _seed(self) -> None: pipeline = DataflowPipeline( id="pipeline-1", tenant_id="tenant-1", scope_type="tenant", definition_kind="flow", name=SECRET, description=SECRET, status="active", current_revision=1, created_by="account-1", updated_by="account-1", metadata_={"secret": SECRET}, derivation_provenance={"secret": SECRET}, ) other = DataflowPipeline( id="pipeline-other", tenant_id="tenant-2", scope_type="tenant", definition_kind="flow", name=SECRET, status="active", current_revision=1, created_by="account-1", updated_by="account-1", ) self.session.add_all((pipeline, other)) self.session.flush() revision = DataflowPipelineRevision( id="revision-1", tenant_id="tenant-1", pipeline_id=pipeline.id, revision=1, schema_version=1, graph={"secret": SECRET}, sql_text=SECRET, editor_mode="graph", content_hash="a" * 64, created_by="account-1", ) self.session.add(revision) self.session.flush() decision_set = DataflowReconciliationDecisionSet( id="decision-set-1", tenant_id="tenant-1", pipeline_id=pipeline.id, name=SECRET, node_id="reconcile", created_by="account-1", updated_by="account-1", ) self.session.add(decision_set) self.session.flush() run = DataflowRun( id="run-1", tenant_id="tenant-1", pipeline_id=pipeline.id, pipeline_revision_id=revision.id, run_type="execute", status="succeeded", execution_backend="python", environment="production", executor_version="dataflow-v1", definition_hash="b" * 64, idempotency_key=SECRET, request_hash="c" * 64, request_={"secret": SECRET}, invocation_kind="manual", correlation_id=SECRET, causation_id=SECRET, source_fingerprints=[{"secret": SECRET}], result_schema=[{"secret": SECRET}], diagnostics=[{"secret": SECRET}], input_row_count=1, output_row_count=1, output_publication_ref=SECRET, output_datasource_ref=SECRET, output_materialization_ref=SECRET, attempts=1, progress_percent=100, progress_phase="complete", retention_until=NOW + timedelta(days=30), authorization_={"submitted_principal": {"secret": SECRET}}, resource_budget={"secret": SECRET}, started_at=NOW, finished_at=NOW, error=SECRET, created_by="account-1", ) trigger = DataflowTrigger( id="trigger-1", tenant_id="tenant-1", pipeline_id=pipeline.id, pipeline_revision_id=revision.id, name=SECRET, kind="event", status="active", revision=1, config_={"secret": SECRET}, publication_={"secret": SECRET}, row_limit=100, catch_up_policy="coalesce", max_concurrent_runs=1, next_fire_at=NOW + timedelta(hours=1), last_error=SECRET, authorization_account_id="account-1", authorization_membership_id="membership-1", authorization_ref=SECRET, grant_scopes=[SECRET], created_by="account-1", updated_by="account-1", ) self.session.add_all( ( DataflowReconciliationDecision( id="decision-1", tenant_id="tenant-1", decision_set_id=decision_set.id, revision=1, key_hash="d" * 64, input_hash="e" * 64, action="correct", reason=SECRET, correction={"secret": SECRET}, actor_ref="account-1", decided_at=NOW, ), run, DataflowPipelineDeployment( id="deployment-1", tenant_id="tenant-1", pipeline_id=pipeline.id, pipeline_revision_id=revision.id, environment="production", source_environment="staging", status="active", provenance={"secret": SECRET}, promoted_by="account-1", ), trigger, ) ) self.session.flush() self.session.add( DataflowTriggerDelivery( id="delivery-1", tenant_id="tenant-1", trigger_id=trigger.id, pipeline_id=pipeline.id, pipeline_revision_id=revision.id, source_key=SECRET, invocation_kind="event", status="succeeded", scheduled_for=NOW, event_={"secret": SECRET}, run_id=run.id, attempts=1, authorization_provenance={"secret": SECRET}, error=SECRET, finished_at=NOW, ) ) def test_canonical_selector_is_minimized_and_marks_automation_authority( self, ) -> None: records = self.provider.search_subject( self.session, tenant_id="tenant-1", subject=DsarSubjectRef( account_id="account-1", membership_id="membership-1", ), ) self.assertEqual(7, len(records)) trigger = next( item for item in records if item.resource_type == "dataflow_trigger" ) self.assertEqual("dataflow_automation_authority", trigger.category) exported = json.dumps([record.to_dict() for record in records]) self.assertNotIn(SECRET, exported) self.assertNotIn("account-1", exported) self.assertNotIn("membership-1", exported) self.assertNotIn("pipeline-other", exported) def test_exact_pipeline_package_is_review_only_and_minimized(self) -> None: records = self.provider.search_subject( self.session, tenant_id="tenant-1", subject=DsarSubjectRef( account_id="account-1", external_references={"dataflow.pipeline": "pipeline:pipeline-1"}, ), ) self.assertEqual(8, len(records)) exported = json.dumps([record.to_dict() for record in records]) self.assertNotIn(SECRET, exported) actions = self.provider.plan_erasure( self.session, tenant_id="tenant-1", subject=DsarSubjectRef( external_references={"dataflow.pipeline": "pipeline-1"} ), records=records, ) self.assertEqual({"manual_review"}, {action.kind for action in actions}) def test_every_exact_reference_and_fail_closed_correlation(self) -> None: references = { "dataflow.pipeline_revision": "revision-1", "dataflow.decision_set": "decision-set-1", "dataflow.decision": "decision-1", "dataflow.run": "dataflow-run:run-1", "dataflow.deployment": "deployment-1", "dataflow.trigger": "trigger-1", "dataflow.trigger_delivery": "delivery-1", } for key, value in references.items(): with self.subTest(key=key): records = self.provider.search_subject( self.session, tenant_id="tenant-1", subject=DsarSubjectRef(external_references={key: value}), ) self.assertEqual(1, len(records)) mismatch = self.provider.search_subject( self.session, tenant_id="tenant-1", subject=DsarSubjectRef( account_id="account-2", external_references={"dataflow.run": "run-1"}, ), ) wrong_tenant = self.provider.search_subject( self.session, tenant_id="tenant-2", subject=DsarSubjectRef(external_references={"dataflow.run": "run-1"}), ) conflict = self.provider.search_subject( self.session, tenant_id="tenant-1", subject=DsarSubjectRef( external_references={ "dataflow.pipeline_revision": "revision-1", "dataflow.revision": "different", } ), ) self.assertEqual((), mismatch) self.assertEqual((), wrong_tenant) self.assertEqual((), conflict) def test_terminal_run_and_delivery_minimization_is_idempotent(self) -> None: subject = DsarSubjectRef( external_references={ "dataflow.run": "run-1", "dataflow.delivery": "delivery-1", } ) records = self.provider.search_subject( self.session, tenant_id="tenant-1", subject=subject, ) actions = self.provider.plan_erasure( self.session, tenant_id="tenant-1", subject=subject, records=records, ) self.assertEqual({"anonymize"}, {action.kind for action in actions}) first = self.provider.execute_erasure( self.session, tenant_id="tenant-1", subject=subject, actions=actions, request_id="dsar-1", ) second = self.provider.execute_erasure( self.session, tenant_id="tenant-1", subject=subject, actions=actions, request_id="dsar-1-retry", ) self.assertTrue(all(result.status == "executed" for result in first)) self.assertTrue(all(result.status == "unchanged" for result in second)) run = self.session.get(DataflowRun, "run-1") delivery = self.session.get(DataflowTriggerDelivery, "delivery-1") self.assertEqual({}, run.request_) self.assertEqual([], run.diagnostics) self.assertIsNotNone(run.purged_at) self.assertIsNone(delivery.event_) self.assertEqual({}, delivery.authorization_provenance) self.assertEqual(SECRET, delivery.source_key) def test_automation_authority_is_revoked_with_retry_support(self) -> None: subject = DsarSubjectRef( account_id="account-1", membership_id="membership-1", ) records = self.provider.search_subject( self.session, tenant_id="tenant-1", subject=subject, ) actions = self.provider.plan_erasure( self.session, tenant_id="tenant-1", subject=subject, records=records, ) trigger_action = next( action for action in actions if action.resource_type == "dataflow_trigger" ) self.assertEqual("revoke", trigger_action.kind) self.assertTrue(trigger_action.executable) first = self.provider.execute_erasure( self.session, tenant_id="tenant-1", subject=subject, actions=(trigger_action,), request_id="dsar-2", ) second = self.provider.execute_erasure( self.session, tenant_id="tenant-1", subject=subject, actions=(trigger_action,), request_id="dsar-2-retry", ) self.assertEqual("executed", first[0].status) self.assertEqual("unchanged", second[0].status) trigger = self.session.get(DataflowTrigger, "trigger-1") self.assertEqual("disabled", trigger.status) self.assertEqual("redacted", trigger.authorization_account_id) self.assertEqual("redacted", trigger.authorization_membership_id) self.assertEqual([], trigger.grant_scopes) self.assertEqual({}, trigger.config_) def test_foreign_records_and_actions_are_rejected(self) -> None: subject = DsarSubjectRef(account_id="account-1") with self.assertRaisesRegex(ValueError, "foreign provider record"): self.provider.plan_erasure( self.session, tenant_id="tenant-1", subject=subject, records=( DsarRecordRef( provider_id="cases", module_id="cases", resource_type="case", resource_id="case-1", category="case", title="Case", ), ), ) with self.assertRaisesRegex(ValueError, "foreign provider action"): self.provider.execute_erasure( self.session, tenant_id="tenant-1", subject=subject, actions=( DsarErasureActionRef( action_id="cases:delete:case:case-1", provider_id="cases", module_id="cases", kind="delete", resource_type="case", resource_id="case-1", title="Delete case", rationale="Foreign", executable=True, ), ), request_id="dsar-3", ) def test_core_workflow_reports_active_and_inactive_provider(self) -> None: row = create_data_subject_request( self.session, tenant_id="tenant-1", reference="DSAR-DATAFLOW-1", request_kind="access_and_erasure", subject=DsarSubjectRef(account_id="account-1"), purpose="Respond to a verified request.", legal_basis="Article 15 and 17 GDPR", due_at=None, requested_by_account_id="privacy-officer", ) self.session.commit() search_data_subject_request( self.session, registry=_Registry(self.provider), row=row, expected_revision=1, ) self.assertEqual( [DATAFLOW_DSAR_CAPABILITY], row.coverage["provider_capabilities"], ) self.assertEqual(7, row.search_result["record_count"]) inactive = create_data_subject_request( self.session, tenant_id="tenant-1", reference="DSAR-DATAFLOW-2", request_kind="access", subject=DsarSubjectRef(account_id="account-1"), purpose="Respond to a verified request.", legal_basis="Article 15 GDPR", due_at=None, requested_by_account_id="privacy-officer", ) self.session.commit() search_data_subject_request( self.session, registry=_Registry(self.provider, active=False), row=inactive, expected_revision=1, ) self.assertEqual([], inactive.coverage["provider_capabilities"]) self.assertEqual( [DATAFLOW_DSAR_CAPABILITY], inactive.coverage["inactive_provider_capabilities"], ) self.assertEqual(0, inactive.search_result["record_count"]) def test_manifest_registers_and_documents_capability(self) -> None: self.assertIn(DATAFLOW_DSAR_CAPABILITY, manifest.capability_factories) self.assertIn(DATAFLOW_DSAR_CAPABILITY, manifest.capability_documentation) self.assertIn( DATAFLOW_DSAR_CAPABILITY, {item.name for item in manifest.provides_interfaces}, ) self.assertTrue( any( topic.id == "dataflow.data-subject-requests" and {"admin", "user"}.issubset(topic.documentation_types) for topic in manifest.documentation ) ) if __name__ == "__main__": unittest.main()