547 lines
19 KiB
Python
547 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import unittest
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_core.core.dsar import (
|
|
DsarErasureActionRef,
|
|
DsarProvider,
|
|
DsarRecordRef,
|
|
DsarSubjectRef,
|
|
)
|
|
from govoplan_core.db.base import Base
|
|
from govoplan_core.privacy.dsar_workflow import (
|
|
create_data_subject_request,
|
|
search_data_subject_request,
|
|
)
|
|
from govoplan_dataflow.backend.db.models import (
|
|
DataflowPipeline,
|
|
DataflowPipelineDeployment,
|
|
DataflowPipelineRevision,
|
|
DataflowReconciliationDecision,
|
|
DataflowReconciliationDecisionSet,
|
|
DataflowRun,
|
|
DataflowTrigger,
|
|
DataflowTriggerDelivery,
|
|
)
|
|
from govoplan_dataflow.backend.dsar_provider import (
|
|
DATAFLOW_DSAR_CAPABILITY,
|
|
DataflowDsarProvider,
|
|
)
|
|
from govoplan_dataflow.backend.manifest import manifest
|
|
|
|
|
|
NOW = datetime(2026, 8, 21, 20, 0, tzinfo=UTC)
|
|
SECRET = "private-dataflow-detail-do-not-export"
|
|
|
|
|
|
class _Registry:
|
|
def __init__(self, provider: DataflowDsarProvider, *, active: bool = True) -> None:
|
|
self.provider = provider
|
|
self.active = active
|
|
|
|
def capability_names(self):
|
|
return (DATAFLOW_DSAR_CAPABILITY,)
|
|
|
|
def capability_owner(self, name):
|
|
self._assert_capability(name)
|
|
return "dataflow"
|
|
|
|
def tenant_entitlement_resolver(self):
|
|
active = self.active
|
|
|
|
class _Resolver:
|
|
@staticmethod
|
|
def resolve(session, tenant_id):
|
|
del session, tenant_id
|
|
return type(
|
|
"State",
|
|
(),
|
|
{"effective_modules": ("dataflow",) if active else ()},
|
|
)()
|
|
|
|
return _Resolver()
|
|
|
|
def require_tenant_capability(self, name, session, **kwargs):
|
|
del session, kwargs
|
|
self._assert_capability(name)
|
|
return self.provider
|
|
|
|
def manifests(self):
|
|
return (type("Manifest", (), {"id": "dataflow"})(),)
|
|
|
|
@staticmethod
|
|
def _assert_capability(name: str) -> None:
|
|
if name != DATAFLOW_DSAR_CAPABILITY:
|
|
raise KeyError(name)
|
|
|
|
|
|
class DataflowDsarProviderTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(self.engine)
|
|
self.session = Session(self.engine)
|
|
self.provider = DataflowDsarProvider()
|
|
self.assertIsInstance(self.provider, DsarProvider)
|
|
self._seed()
|
|
self.session.commit()
|
|
|
|
def tearDown(self) -> None:
|
|
self.session.close()
|
|
self.engine.dispose()
|
|
|
|
def _seed(self) -> None:
|
|
pipeline = DataflowPipeline(
|
|
id="pipeline-1",
|
|
tenant_id="tenant-1",
|
|
scope_type="tenant",
|
|
definition_kind="flow",
|
|
name=SECRET,
|
|
description=SECRET,
|
|
status="active",
|
|
current_revision=1,
|
|
created_by="account-1",
|
|
updated_by="account-1",
|
|
metadata_={"secret": SECRET},
|
|
derivation_provenance={"secret": SECRET},
|
|
)
|
|
other = DataflowPipeline(
|
|
id="pipeline-other",
|
|
tenant_id="tenant-2",
|
|
scope_type="tenant",
|
|
definition_kind="flow",
|
|
name=SECRET,
|
|
status="active",
|
|
current_revision=1,
|
|
created_by="account-1",
|
|
updated_by="account-1",
|
|
)
|
|
self.session.add_all((pipeline, other))
|
|
self.session.flush()
|
|
revision = DataflowPipelineRevision(
|
|
id="revision-1",
|
|
tenant_id="tenant-1",
|
|
pipeline_id=pipeline.id,
|
|
revision=1,
|
|
schema_version=1,
|
|
graph={"secret": SECRET},
|
|
sql_text=SECRET,
|
|
editor_mode="graph",
|
|
content_hash="a" * 64,
|
|
created_by="account-1",
|
|
)
|
|
self.session.add(revision)
|
|
self.session.flush()
|
|
decision_set = DataflowReconciliationDecisionSet(
|
|
id="decision-set-1",
|
|
tenant_id="tenant-1",
|
|
pipeline_id=pipeline.id,
|
|
name=SECRET,
|
|
node_id="reconcile",
|
|
created_by="account-1",
|
|
updated_by="account-1",
|
|
)
|
|
self.session.add(decision_set)
|
|
self.session.flush()
|
|
run = DataflowRun(
|
|
id="run-1",
|
|
tenant_id="tenant-1",
|
|
pipeline_id=pipeline.id,
|
|
pipeline_revision_id=revision.id,
|
|
run_type="execute",
|
|
status="succeeded",
|
|
execution_backend="python",
|
|
environment="production",
|
|
executor_version="dataflow-v1",
|
|
definition_hash="b" * 64,
|
|
idempotency_key=SECRET,
|
|
request_hash="c" * 64,
|
|
request_={"secret": SECRET},
|
|
invocation_kind="manual",
|
|
correlation_id=SECRET,
|
|
causation_id=SECRET,
|
|
source_fingerprints=[{"secret": SECRET}],
|
|
result_schema=[{"secret": SECRET}],
|
|
diagnostics=[{"secret": SECRET}],
|
|
input_row_count=1,
|
|
output_row_count=1,
|
|
output_publication_ref=SECRET,
|
|
output_datasource_ref=SECRET,
|
|
output_materialization_ref=SECRET,
|
|
attempts=1,
|
|
progress_percent=100,
|
|
progress_phase="complete",
|
|
retention_until=NOW + timedelta(days=30),
|
|
authorization_={"submitted_principal": {"secret": SECRET}},
|
|
resource_budget={"secret": SECRET},
|
|
started_at=NOW,
|
|
finished_at=NOW,
|
|
error=SECRET,
|
|
created_by="account-1",
|
|
)
|
|
trigger = DataflowTrigger(
|
|
id="trigger-1",
|
|
tenant_id="tenant-1",
|
|
pipeline_id=pipeline.id,
|
|
pipeline_revision_id=revision.id,
|
|
name=SECRET,
|
|
kind="event",
|
|
status="active",
|
|
revision=1,
|
|
config_={"secret": SECRET},
|
|
publication_={"secret": SECRET},
|
|
row_limit=100,
|
|
catch_up_policy="coalesce",
|
|
max_concurrent_runs=1,
|
|
next_fire_at=NOW + timedelta(hours=1),
|
|
last_error=SECRET,
|
|
authorization_account_id="account-1",
|
|
authorization_membership_id="membership-1",
|
|
authorization_ref=SECRET,
|
|
grant_scopes=[SECRET],
|
|
created_by="account-1",
|
|
updated_by="account-1",
|
|
)
|
|
self.session.add_all(
|
|
(
|
|
DataflowReconciliationDecision(
|
|
id="decision-1",
|
|
tenant_id="tenant-1",
|
|
decision_set_id=decision_set.id,
|
|
revision=1,
|
|
key_hash="d" * 64,
|
|
input_hash="e" * 64,
|
|
action="correct",
|
|
reason=SECRET,
|
|
correction={"secret": SECRET},
|
|
actor_ref="account-1",
|
|
decided_at=NOW,
|
|
),
|
|
run,
|
|
DataflowPipelineDeployment(
|
|
id="deployment-1",
|
|
tenant_id="tenant-1",
|
|
pipeline_id=pipeline.id,
|
|
pipeline_revision_id=revision.id,
|
|
environment="production",
|
|
source_environment="staging",
|
|
status="active",
|
|
provenance={"secret": SECRET},
|
|
promoted_by="account-1",
|
|
),
|
|
trigger,
|
|
)
|
|
)
|
|
self.session.flush()
|
|
self.session.add(
|
|
DataflowTriggerDelivery(
|
|
id="delivery-1",
|
|
tenant_id="tenant-1",
|
|
trigger_id=trigger.id,
|
|
pipeline_id=pipeline.id,
|
|
pipeline_revision_id=revision.id,
|
|
source_key=SECRET,
|
|
invocation_kind="event",
|
|
status="succeeded",
|
|
scheduled_for=NOW,
|
|
event_={"secret": SECRET},
|
|
run_id=run.id,
|
|
attempts=1,
|
|
authorization_provenance={"secret": SECRET},
|
|
error=SECRET,
|
|
finished_at=NOW,
|
|
)
|
|
)
|
|
|
|
def test_canonical_selector_is_minimized_and_marks_automation_authority(
|
|
self,
|
|
) -> None:
|
|
records = self.provider.search_subject(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=DsarSubjectRef(
|
|
account_id="account-1",
|
|
membership_id="membership-1",
|
|
),
|
|
)
|
|
|
|
self.assertEqual(7, len(records))
|
|
trigger = next(
|
|
item for item in records if item.resource_type == "dataflow_trigger"
|
|
)
|
|
self.assertEqual("dataflow_automation_authority", trigger.category)
|
|
exported = json.dumps([record.to_dict() for record in records])
|
|
self.assertNotIn(SECRET, exported)
|
|
self.assertNotIn("account-1", exported)
|
|
self.assertNotIn("membership-1", exported)
|
|
self.assertNotIn("pipeline-other", exported)
|
|
|
|
def test_exact_pipeline_package_is_review_only_and_minimized(self) -> None:
|
|
records = self.provider.search_subject(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=DsarSubjectRef(
|
|
account_id="account-1",
|
|
external_references={"dataflow.pipeline": "pipeline:pipeline-1"},
|
|
),
|
|
)
|
|
|
|
self.assertEqual(8, len(records))
|
|
exported = json.dumps([record.to_dict() for record in records])
|
|
self.assertNotIn(SECRET, exported)
|
|
actions = self.provider.plan_erasure(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=DsarSubjectRef(
|
|
external_references={"dataflow.pipeline": "pipeline-1"}
|
|
),
|
|
records=records,
|
|
)
|
|
self.assertEqual({"manual_review"}, {action.kind for action in actions})
|
|
|
|
def test_every_exact_reference_and_fail_closed_correlation(self) -> None:
|
|
references = {
|
|
"dataflow.pipeline_revision": "revision-1",
|
|
"dataflow.decision_set": "decision-set-1",
|
|
"dataflow.decision": "decision-1",
|
|
"dataflow.run": "dataflow-run:run-1",
|
|
"dataflow.deployment": "deployment-1",
|
|
"dataflow.trigger": "trigger-1",
|
|
"dataflow.trigger_delivery": "delivery-1",
|
|
}
|
|
for key, value in references.items():
|
|
with self.subTest(key=key):
|
|
records = self.provider.search_subject(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=DsarSubjectRef(external_references={key: value}),
|
|
)
|
|
self.assertEqual(1, len(records))
|
|
|
|
mismatch = self.provider.search_subject(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=DsarSubjectRef(
|
|
account_id="account-2",
|
|
external_references={"dataflow.run": "run-1"},
|
|
),
|
|
)
|
|
wrong_tenant = self.provider.search_subject(
|
|
self.session,
|
|
tenant_id="tenant-2",
|
|
subject=DsarSubjectRef(external_references={"dataflow.run": "run-1"}),
|
|
)
|
|
conflict = self.provider.search_subject(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=DsarSubjectRef(
|
|
external_references={
|
|
"dataflow.pipeline_revision": "revision-1",
|
|
"dataflow.revision": "different",
|
|
}
|
|
),
|
|
)
|
|
self.assertEqual((), mismatch)
|
|
self.assertEqual((), wrong_tenant)
|
|
self.assertEqual((), conflict)
|
|
|
|
def test_terminal_run_and_delivery_minimization_is_idempotent(self) -> None:
|
|
subject = DsarSubjectRef(
|
|
external_references={
|
|
"dataflow.run": "run-1",
|
|
"dataflow.delivery": "delivery-1",
|
|
}
|
|
)
|
|
records = self.provider.search_subject(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=subject,
|
|
)
|
|
actions = self.provider.plan_erasure(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=subject,
|
|
records=records,
|
|
)
|
|
self.assertEqual({"anonymize"}, {action.kind for action in actions})
|
|
first = self.provider.execute_erasure(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=subject,
|
|
actions=actions,
|
|
request_id="dsar-1",
|
|
)
|
|
second = self.provider.execute_erasure(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=subject,
|
|
actions=actions,
|
|
request_id="dsar-1-retry",
|
|
)
|
|
self.assertTrue(all(result.status == "executed" for result in first))
|
|
self.assertTrue(all(result.status == "unchanged" for result in second))
|
|
run = self.session.get(DataflowRun, "run-1")
|
|
delivery = self.session.get(DataflowTriggerDelivery, "delivery-1")
|
|
self.assertEqual({}, run.request_)
|
|
self.assertEqual([], run.diagnostics)
|
|
self.assertIsNotNone(run.purged_at)
|
|
self.assertIsNone(delivery.event_)
|
|
self.assertEqual({}, delivery.authorization_provenance)
|
|
self.assertEqual(SECRET, delivery.source_key)
|
|
|
|
def test_automation_authority_is_revoked_with_retry_support(self) -> None:
|
|
subject = DsarSubjectRef(
|
|
account_id="account-1",
|
|
membership_id="membership-1",
|
|
)
|
|
records = self.provider.search_subject(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=subject,
|
|
)
|
|
actions = self.provider.plan_erasure(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=subject,
|
|
records=records,
|
|
)
|
|
trigger_action = next(
|
|
action for action in actions if action.resource_type == "dataflow_trigger"
|
|
)
|
|
self.assertEqual("revoke", trigger_action.kind)
|
|
self.assertTrue(trigger_action.executable)
|
|
first = self.provider.execute_erasure(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=subject,
|
|
actions=(trigger_action,),
|
|
request_id="dsar-2",
|
|
)
|
|
second = self.provider.execute_erasure(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=subject,
|
|
actions=(trigger_action,),
|
|
request_id="dsar-2-retry",
|
|
)
|
|
self.assertEqual("executed", first[0].status)
|
|
self.assertEqual("unchanged", second[0].status)
|
|
trigger = self.session.get(DataflowTrigger, "trigger-1")
|
|
self.assertEqual("disabled", trigger.status)
|
|
self.assertEqual("redacted", trigger.authorization_account_id)
|
|
self.assertEqual("redacted", trigger.authorization_membership_id)
|
|
self.assertEqual([], trigger.grant_scopes)
|
|
self.assertEqual({}, trigger.config_)
|
|
|
|
def test_foreign_records_and_actions_are_rejected(self) -> None:
|
|
subject = DsarSubjectRef(account_id="account-1")
|
|
with self.assertRaisesRegex(ValueError, "foreign provider record"):
|
|
self.provider.plan_erasure(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=subject,
|
|
records=(
|
|
DsarRecordRef(
|
|
provider_id="cases",
|
|
module_id="cases",
|
|
resource_type="case",
|
|
resource_id="case-1",
|
|
category="case",
|
|
title="Case",
|
|
),
|
|
),
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "foreign provider action"):
|
|
self.provider.execute_erasure(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
subject=subject,
|
|
actions=(
|
|
DsarErasureActionRef(
|
|
action_id="cases:delete:case:case-1",
|
|
provider_id="cases",
|
|
module_id="cases",
|
|
kind="delete",
|
|
resource_type="case",
|
|
resource_id="case-1",
|
|
title="Delete case",
|
|
rationale="Foreign",
|
|
executable=True,
|
|
),
|
|
),
|
|
request_id="dsar-3",
|
|
)
|
|
|
|
def test_core_workflow_reports_active_and_inactive_provider(self) -> None:
|
|
row = create_data_subject_request(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
reference="DSAR-DATAFLOW-1",
|
|
request_kind="access_and_erasure",
|
|
subject=DsarSubjectRef(account_id="account-1"),
|
|
purpose="Respond to a verified request.",
|
|
legal_basis="Article 15 and 17 GDPR",
|
|
due_at=None,
|
|
requested_by_account_id="privacy-officer",
|
|
)
|
|
self.session.commit()
|
|
search_data_subject_request(
|
|
self.session,
|
|
registry=_Registry(self.provider),
|
|
row=row,
|
|
expected_revision=1,
|
|
)
|
|
self.assertEqual(
|
|
[DATAFLOW_DSAR_CAPABILITY],
|
|
row.coverage["provider_capabilities"],
|
|
)
|
|
self.assertEqual(7, row.search_result["record_count"])
|
|
|
|
inactive = create_data_subject_request(
|
|
self.session,
|
|
tenant_id="tenant-1",
|
|
reference="DSAR-DATAFLOW-2",
|
|
request_kind="access",
|
|
subject=DsarSubjectRef(account_id="account-1"),
|
|
purpose="Respond to a verified request.",
|
|
legal_basis="Article 15 GDPR",
|
|
due_at=None,
|
|
requested_by_account_id="privacy-officer",
|
|
)
|
|
self.session.commit()
|
|
search_data_subject_request(
|
|
self.session,
|
|
registry=_Registry(self.provider, active=False),
|
|
row=inactive,
|
|
expected_revision=1,
|
|
)
|
|
self.assertEqual([], inactive.coverage["provider_capabilities"])
|
|
self.assertEqual(
|
|
[DATAFLOW_DSAR_CAPABILITY],
|
|
inactive.coverage["inactive_provider_capabilities"],
|
|
)
|
|
self.assertEqual(0, inactive.search_result["record_count"])
|
|
|
|
def test_manifest_registers_and_documents_capability(self) -> None:
|
|
self.assertIn(DATAFLOW_DSAR_CAPABILITY, manifest.capability_factories)
|
|
self.assertIn(DATAFLOW_DSAR_CAPABILITY, manifest.capability_documentation)
|
|
self.assertIn(
|
|
DATAFLOW_DSAR_CAPABILITY,
|
|
{item.name for item in manifest.provides_interfaces},
|
|
)
|
|
self.assertTrue(
|
|
any(
|
|
topic.id == "dataflow.data-subject-requests"
|
|
and {"admin", "user"}.issubset(topic.documentation_types)
|
|
for topic in manifest.documentation
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|