Add governed Dataflow audience outputs

This commit is contained in:
2026-07-31 21:00:02 +02:00
parent 8724591bf9
commit cef2ba9482
12 changed files with 1226 additions and 6 deletions
@@ -0,0 +1,70 @@
[
{
"recipient_key": "identity:alice",
"display_name": "Alice Example",
"identity_id": "identity-alice",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"email": "alice@example.test",
"postal_address": null,
"account_id": null,
"portal_target": null,
"contact_point_id": "contact-email-alice",
"selected_channel": "email",
"distribution_status": "usable",
"exclusion_reason": null,
"policy_decision": "preference.explicit_email",
"_reconciliation_status": "match"
},
{
"recipient_key": "identity:bob",
"display_name": "Bob Example",
"identity_id": "identity-bob",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"email": null,
"postal_address": "Example Street 2, 10000 Exampletown",
"account_id": null,
"portal_target": null,
"contact_point_id": "contact-postal-bob",
"selected_channel": "postal",
"distribution_status": "suppressed",
"exclusion_reason": "preference.suppressed",
"policy_decision": "preference.suppressed",
"_reconciliation_status": "match"
},
{
"recipient_key": "identity:grace",
"display_name": "Grace Example",
"identity_id": "identity-grace",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"email": null,
"postal_address": "Example Street 3, 10000 Exampletown",
"account_id": null,
"portal_target": null,
"contact_point_id": "contact-postal-grace",
"selected_channel": "postal",
"distribution_status": "usable",
"exclusion_reason": null,
"policy_decision": "preference.default_postal",
"_reconciliation_status": "match"
},
{
"recipient_key": "identity:frank",
"display_name": "Frank Example",
"identity_id": "identity-frank",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"email": null,
"postal_address": null,
"account_id": null,
"portal_target": "portal:frank",
"contact_point_id": "contact-portal-frank",
"selected_channel": "portal",
"distribution_status": "usable",
"exclusion_reason": null,
"policy_decision": "manual.include_with_reason",
"_reconciliation_status": "match"
}
]
@@ -0,0 +1,299 @@
{
"schema_version": 1,
"nodes": [
{
"id": "directory",
"type": "source.reference",
"label": "Frozen directory and contact snapshot",
"position": {"x": 40, "y": 60},
"config": {
"source_ref": "snapshot:directory-2026-01",
"source_name": "directory_snapshot",
"expected_fingerprint": "sha256:synthetic-directory-2026-01",
"consistency": "frozen",
"source_columns": [
"recipient_key", "display_name", "identity_id", "group_key",
"organization_unit_id", "function_id", "region", "purpose",
"effective", "effective_from", "effective_until",
"preferred_target", "preferred_contact_point_id", "preferred_channel",
"fallback_target", "fallback_contact_point_id", "fallback_channel",
"preference_decision", "sort_order"
]
}
},
{
"id": "group-filter",
"type": "filter",
"label": "Typed group",
"position": {"x": 260, "y": 60},
"config": {"column": "group_key", "operator": "eq", "value": {"$parameter": "group_key"}}
},
{
"id": "organization-filter",
"type": "filter",
"label": "Organization unit",
"position": {"x": 480, "y": 60},
"config": {"column": "organization_unit_id", "operator": "eq", "value": "district-north"}
},
{
"id": "function-filter",
"type": "filter",
"label": "Effective function",
"position": {"x": 700, "y": 60},
"config": {"column": "function_id", "operator": "eq", "value": "resident-service"}
},
{
"id": "region-filter",
"type": "filter",
"label": "Geography",
"position": {"x": 920, "y": 60},
"config": {"column": "region", "operator": "eq", "value": {"$parameter": "region"}}
},
{
"id": "purpose-filter",
"type": "filter",
"label": "Distribution purpose",
"position": {"x": 1140, "y": 60},
"config": {"column": "purpose", "operator": "eq", "value": {"$parameter": "purpose"}}
},
{
"id": "effective-filter",
"type": "filter",
"label": "Effective-dated assignment",
"position": {"x": 1360, "y": 60},
"config": {"column": "effective", "operator": "eq", "value": true}
},
{
"id": "contact-fallback",
"type": "calculate",
"label": "Resolve preferred or fallback contact",
"position": {"x": 1580, "y": 60},
"config": {
"calculations": [
{"target_column": "selected_target", "expression": "coalesce(preferred_target, fallback_target)", "result_type": "string"},
{"target_column": "selected_contact_point_id", "expression": "coalesce(preferred_contact_point_id, fallback_contact_point_id)", "result_type": "string"},
{"target_column": "selected_channel", "expression": "coalesce(preferred_channel, fallback_channel)", "result_type": "string"},
{"target_column": "email", "expression": "CASE WHEN selected_channel = 'email' THEN selected_target ELSE NULL END", "result_type": "string"},
{"target_column": "postal_address", "expression": "CASE WHEN selected_channel = 'postal' THEN selected_target ELSE NULL END", "result_type": "string"},
{"target_column": "account_id", "expression": "CASE WHEN selected_channel = 'internal_mail' THEN selected_target ELSE NULL END", "result_type": "string"},
{"target_column": "portal_target", "expression": "CASE WHEN selected_channel = 'portal' THEN selected_target ELSE NULL END", "result_type": "string"}
]
}
},
{
"id": "directory-fields",
"type": "select",
"label": "Canonical recipient candidate",
"position": {"x": 1800, "y": 60},
"config": {
"fields": [
"recipient_key", "display_name", "identity_id", "group_key",
"organization_unit_id", "function_id", "selected_target",
"selected_contact_point_id", "selected_channel", "email",
"postal_address", "account_id", "portal_target",
"preference_decision", "sort_order"
]
}
},
{
"id": "candidate-quality",
"type": "quality.rules",
"label": "Recipient quality gate",
"position": {"x": 2020, "y": 60},
"config": {
"rules": [
{"id": "recipient-key", "column": "recipient_key", "operator": "not_null"},
{"id": "delivery-target", "column": "selected_target", "operator": "not_null"},
{"id": "contact-point", "column": "selected_contact_point_id", "operator": "not_null"}
],
"action": "drop"
}
},
{
"id": "manual",
"type": "source.reference",
"label": "Frozen manual includes",
"position": {"x": 1580, "y": 300},
"config": {
"source_ref": "snapshot:manual-includes-2026-01",
"source_name": "manual_includes",
"expected_fingerprint": "sha256:synthetic-manual-includes-2026-01",
"consistency": "frozen",
"source_columns": [
"recipient_key", "display_name", "identity_id", "group_key",
"organization_unit_id", "function_id", "selected_target",
"selected_contact_point_id", "selected_channel", "email",
"postal_address", "account_id", "portal_target",
"preference_decision", "sort_order"
]
}
},
{
"id": "append-includes",
"type": "combine.union",
"label": "Append manual includes",
"position": {"x": 2240, "y": 170},
"config": {"mode": "all"}
},
{
"id": "deduplicate",
"type": "distinct",
"label": "One candidate per identity",
"position": {"x": 2460, "y": 170},
"config": {"columns": ["recipient_key"]}
},
{
"id": "suppressions",
"type": "source.reference",
"label": "Frozen channel suppressions",
"position": {"x": 2240, "y": 410},
"config": {
"source_ref": "snapshot:suppressions-2026-01",
"source_name": "channel_suppressions",
"expected_fingerprint": "sha256:synthetic-suppressions-2026-01",
"consistency": "frozen",
"source_columns": ["recipient_key", "suppression_reason"]
}
},
{
"id": "apply-suppressions",
"type": "combine.join",
"label": "Attach suppression decisions",
"position": {"x": 2680, "y": 250},
"config": {
"join_type": "left",
"left_keys": ["recipient_key"],
"right_keys": ["recipient_key"],
"right_prefix": "suppression_"
}
},
{
"id": "decision-columns",
"type": "calculate",
"label": "Classify channel decision",
"position": {"x": 2900, "y": 250},
"config": {
"calculations": [
{"target_column": "distribution_status", "expression": "CASE WHEN suppression_suppression_reason IS NOT NULL THEN 'suppressed' ELSE 'usable' END", "result_type": "string"},
{"target_column": "exclusion_reason", "expression": "suppression_suppression_reason", "result_type": "string"},
{"target_column": "policy_decision", "expression": "coalesce(suppression_suppression_reason, preference_decision)", "result_type": "string"}
]
}
},
{
"id": "decision-fields",
"type": "select",
"label": "Canonical audience decision",
"position": {"x": 3120, "y": 250},
"config": {
"fields": [
"recipient_key", "display_name", "identity_id",
"organization_unit_id", "function_id", "email", "postal_address",
"account_id", "portal_target",
{"column": "selected_contact_point_id", "alias": "contact_point_id"},
"selected_channel", "distribution_status", "exclusion_reason",
"policy_decision", "sort_order"
]
}
},
{
"id": "stable-order",
"type": "sort",
"label": "Stable recipient order",
"position": {"x": 3340, "y": 250},
"config": {
"fields": [
{"column": "sort_order", "direction": "asc"},
{"column": "recipient_key", "direction": "asc"}
]
}
},
{
"id": "expected",
"type": "source.reference",
"label": "Frozen expected audience",
"position": {"x": 3340, "y": 500},
"config": {
"source_ref": "snapshot:expected-audience-2026-01",
"source_name": "expected_audience",
"expected_fingerprint": "sha256:synthetic-expected-audience-2026-01",
"consistency": "frozen",
"source_columns": [
"recipient_key", "display_name", "identity_id",
"organization_unit_id", "function_id", "email", "postal_address",
"account_id", "portal_target", "contact_point_id",
"selected_channel", "distribution_status", "exclusion_reason",
"policy_decision"
]
}
},
{
"id": "reconcile",
"type": "reconcile.compare",
"label": "Reconcile expected audience",
"position": {"x": 3560, "y": 350},
"config": {
"left_keys": ["recipient_key"],
"right_keys": ["recipient_key"],
"compare_columns": [
"selected_channel", "contact_point_id", "distribution_status",
"exclusion_reason", "policy_decision"
],
"right_prefix": "actual_"
}
},
{
"id": "reconciled-only",
"type": "filter",
"label": "Require exact reconciliation",
"position": {"x": 3780, "y": 350},
"config": {"column": "_reconciliation_status", "operator": "eq", "value": "match"}
},
{
"id": "output-fields",
"type": "select",
"label": "Distribution-list dataset",
"position": {"x": 4000, "y": 350},
"config": {
"fields": [
"recipient_key", "display_name", "identity_id",
"organization_unit_id", "function_id", "email", "postal_address",
"account_id", "portal_target", "contact_point_id",
"selected_channel", "distribution_status", "exclusion_reason",
"policy_decision", "_reconciliation_status"
]
}
},
{
"id": "output",
"type": "output",
"label": "Pinned audience output",
"position": {"x": 4220, "y": 350},
"config": {}
}
],
"edges": [
{"id": "e01", "source": "directory", "target": "group-filter"},
{"id": "e02", "source": "group-filter", "target": "organization-filter"},
{"id": "e03", "source": "organization-filter", "target": "function-filter"},
{"id": "e04", "source": "function-filter", "target": "region-filter"},
{"id": "e05", "source": "region-filter", "target": "purpose-filter"},
{"id": "e06", "source": "purpose-filter", "target": "effective-filter"},
{"id": "e07", "source": "effective-filter", "target": "contact-fallback"},
{"id": "e08", "source": "contact-fallback", "target": "directory-fields"},
{"id": "e09", "source": "directory-fields", "target": "candidate-quality"},
{"id": "e10", "source": "candidate-quality", "target": "append-includes"},
{"id": "e11", "source": "manual", "target": "append-includes"},
{"id": "e12", "source": "append-includes", "target": "deduplicate"},
{"id": "e13", "source": "deduplicate", "target": "apply-suppressions", "target_port": "left"},
{"id": "e14", "source": "suppressions", "target": "apply-suppressions", "target_port": "right"},
{"id": "e15", "source": "apply-suppressions", "target": "decision-columns"},
{"id": "e16", "source": "decision-columns", "target": "decision-fields"},
{"id": "e17", "source": "decision-fields", "target": "stable-order"},
{"id": "e18", "source": "expected", "target": "reconcile", "target_port": "left"},
{"id": "e19", "source": "stable-order", "target": "reconcile", "target_port": "right"},
{"id": "e20", "source": "reconcile", "target": "reconciled-only"},
{"id": "e21", "source": "reconciled-only", "target": "output-fields"},
{"id": "e22", "source": "output-fields", "target": "output"}
]
}
@@ -0,0 +1,149 @@
[
{
"recipient_key": "identity:alice",
"display_name": "Alice Example",
"identity_id": "identity-alice",
"group_key": "residents",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"region": "north",
"purpose": "monthly-notice",
"effective": true,
"effective_from": "2025-01-01",
"effective_until": null,
"preferred_target": "alice@example.test",
"preferred_contact_point_id": "contact-email-alice",
"preferred_channel": "email",
"fallback_target": "Example Street 1, 10000 Exampletown",
"fallback_contact_point_id": "contact-postal-alice",
"fallback_channel": "postal",
"preference_decision": "preference.explicit_email",
"sort_order": 10
},
{
"recipient_key": "identity:bob",
"display_name": "Bob Example",
"identity_id": "identity-bob",
"group_key": "residents",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"region": "north",
"purpose": "monthly-notice",
"effective": true,
"effective_from": "2025-06-01",
"effective_until": "2027-01-01",
"preferred_target": null,
"preferred_contact_point_id": null,
"preferred_channel": null,
"fallback_target": "Example Street 2, 10000 Exampletown",
"fallback_contact_point_id": "contact-postal-bob",
"fallback_channel": "postal",
"preference_decision": "preference.default_postal",
"sort_order": 20
},
{
"recipient_key": "identity:grace",
"display_name": "Grace Example",
"identity_id": "identity-grace",
"group_key": "residents",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"region": "north",
"purpose": "monthly-notice",
"effective": true,
"effective_from": "2024-01-01",
"effective_until": null,
"preferred_target": null,
"preferred_contact_point_id": null,
"preferred_channel": null,
"fallback_target": "Example Street 3, 10000 Exampletown",
"fallback_contact_point_id": "contact-postal-grace",
"fallback_channel": "postal",
"preference_decision": "preference.default_postal",
"sort_order": 30
},
{
"recipient_key": "identity:carol",
"display_name": "Carol Example",
"identity_id": "identity-carol",
"group_key": "employees",
"organization_unit_id": "district-south",
"function_id": "finance-officer",
"region": "south",
"purpose": "monthly-notice",
"effective": true,
"effective_from": "2025-01-01",
"effective_until": null,
"preferred_target": "carol@example.test",
"preferred_contact_point_id": "contact-email-carol",
"preferred_channel": "email",
"fallback_target": null,
"fallback_contact_point_id": null,
"fallback_channel": null,
"preference_decision": "preference.explicit_email",
"sort_order": 40
},
{
"recipient_key": "identity:dana",
"display_name": "Dana Example",
"identity_id": "identity-dana",
"group_key": "residents",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"region": "north",
"purpose": "newsletter",
"effective": true,
"effective_from": "2025-01-01",
"effective_until": null,
"preferred_target": "dana@example.test",
"preferred_contact_point_id": "contact-email-dana",
"preferred_channel": "email",
"fallback_target": null,
"fallback_contact_point_id": null,
"fallback_channel": null,
"preference_decision": "preference.explicit_email",
"sort_order": 50
},
{
"recipient_key": "identity:erin",
"display_name": "Erin Example",
"identity_id": "identity-erin",
"group_key": "residents",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"region": "north",
"purpose": "monthly-notice",
"effective": false,
"effective_from": "2024-01-01",
"effective_until": "2025-01-01",
"preferred_target": "erin@example.test",
"preferred_contact_point_id": "contact-email-erin",
"preferred_channel": "email",
"fallback_target": null,
"fallback_contact_point_id": null,
"fallback_channel": null,
"preference_decision": "preference.explicit_email",
"sort_order": 60
},
{
"recipient_key": null,
"display_name": "Malformed Synthetic Row",
"identity_id": null,
"group_key": "residents",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"region": "north",
"purpose": "monthly-notice",
"effective": true,
"effective_from": "2025-01-01",
"effective_until": null,
"preferred_target": null,
"preferred_contact_point_id": null,
"preferred_channel": null,
"fallback_target": null,
"fallback_contact_point_id": null,
"fallback_channel": null,
"preference_decision": "preference.unresolved",
"sort_order": 70
}
]
@@ -0,0 +1,66 @@
[
{
"recipient_key": "identity:alice",
"display_name": "Alice Example",
"identity_id": "identity-alice",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"email": "alice@example.test",
"postal_address": null,
"account_id": null,
"portal_target": null,
"contact_point_id": "contact-email-alice",
"selected_channel": "email",
"distribution_status": "usable",
"exclusion_reason": null,
"policy_decision": "preference.explicit_email"
},
{
"recipient_key": "identity:bob",
"display_name": "Bob Example",
"identity_id": "identity-bob",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"email": null,
"postal_address": "Example Street 2, 10000 Exampletown",
"account_id": null,
"portal_target": null,
"contact_point_id": "contact-postal-bob",
"selected_channel": "postal",
"distribution_status": "suppressed",
"exclusion_reason": "preference.suppressed",
"policy_decision": "preference.suppressed"
},
{
"recipient_key": "identity:grace",
"display_name": "Grace Example",
"identity_id": "identity-grace",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"email": null,
"postal_address": "Example Street 3, 10000 Exampletown",
"account_id": null,
"portal_target": null,
"contact_point_id": "contact-postal-grace",
"selected_channel": "postal",
"distribution_status": "usable",
"exclusion_reason": null,
"policy_decision": "preference.default_postal"
},
{
"recipient_key": "identity:frank",
"display_name": "Frank Example",
"identity_id": "identity-frank",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"email": null,
"postal_address": null,
"account_id": null,
"portal_target": "portal:frank",
"contact_point_id": "contact-portal-frank",
"selected_channel": "portal",
"distribution_status": "usable",
"exclusion_reason": null,
"policy_decision": "manual.include_with_reason"
}
]
@@ -0,0 +1,36 @@
[
{
"recipient_key": "identity:alice",
"display_name": "Alice Example (duplicate manual include)",
"identity_id": "identity-alice",
"group_key": "residents",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"selected_target": "alice@example.test",
"selected_contact_point_id": "contact-email-alice",
"selected_channel": "email",
"email": "alice@example.test",
"postal_address": null,
"account_id": null,
"portal_target": null,
"preference_decision": "manual.include_duplicate",
"sort_order": 11
},
{
"recipient_key": "identity:frank",
"display_name": "Frank Example",
"identity_id": "identity-frank",
"group_key": "manual",
"organization_unit_id": "district-north",
"function_id": "resident-service",
"selected_target": "portal:frank",
"selected_contact_point_id": "contact-portal-frank",
"selected_channel": "portal",
"email": null,
"postal_address": null,
"account_id": null,
"portal_target": "portal:frank",
"preference_decision": "manual.include_with_reason",
"sort_order": 40
}
]
@@ -0,0 +1,6 @@
[
{
"recipient_key": "identity:bob",
"suppression_reason": "preference.suppressed"
}
]
@@ -0,0 +1,92 @@
{
"id": "adrema-audience-selection",
"title": "Synthetic governed AdreMa audience selection",
"issue": "GovOPlaN/govoplan-dataflow#18",
"tenant_id": "tenant-synthetic-adrema",
"graph": "graph.json",
"expected_output": "expected-output.json",
"parameters": {
"group_key": "residents",
"region": "north",
"purpose": "monthly-notice"
},
"sources": {
"snapshot:directory-2026-01": {
"fixture": "inputs/directory-snapshot.json",
"tenant_id": "tenant-synthetic-adrema",
"provider": "connectors.synthetic-directory",
"fingerprint": "sha256:synthetic-directory-2026-01"
},
"snapshot:manual-includes-2026-01": {
"fixture": "inputs/manual-includes.json",
"tenant_id": "tenant-synthetic-adrema",
"provider": "datasources.static",
"fingerprint": "sha256:synthetic-manual-includes-2026-01"
},
"snapshot:suppressions-2026-01": {
"fixture": "inputs/suppressions.json",
"tenant_id": "tenant-synthetic-adrema",
"provider": "addresses.synthetic-preferences",
"fingerprint": "sha256:synthetic-suppressions-2026-01"
},
"snapshot:expected-audience-2026-01": {
"fixture": "inputs/expected-audience.json",
"tenant_id": "tenant-synthetic-adrema",
"provider": "reporting.synthetic-baseline",
"fingerprint": "sha256:synthetic-expected-audience-2026-01"
}
},
"expected_definition_hash": "2fec104dfda522dbc708ce61b237c062919703d62d2912ba621d5873ee5b975e",
"expected_execution_graph_hash": "d1cbb51c837810300044ed5f660e295ace49411265108b102731174d5587a85d",
"expected_output_hash": "f2d744354279f80393c6f894f7854569aa686f8292e6256fe429569d4410a72e",
"expected_source_fingerprints": [
{
"node_id": "directory",
"source_ref": "snapshot:directory-2026-01",
"source_name": "directory_snapshot",
"kind": "datasource",
"provider": "connectors.synthetic-directory",
"fingerprint": "sha256:synthetic-directory-2026-01",
"row_count": 7,
"preview_rows": 7,
"truncated": false
},
{
"node_id": "manual",
"source_ref": "snapshot:manual-includes-2026-01",
"source_name": "manual_includes",
"kind": "datasource",
"provider": "datasources.static",
"fingerprint": "sha256:synthetic-manual-includes-2026-01",
"row_count": 2,
"preview_rows": 2,
"truncated": false
},
{
"node_id": "suppressions",
"source_ref": "snapshot:suppressions-2026-01",
"source_name": "channel_suppressions",
"kind": "datasource",
"provider": "addresses.synthetic-preferences",
"fingerprint": "sha256:synthetic-suppressions-2026-01",
"row_count": 1,
"preview_rows": 1,
"truncated": false
},
{
"node_id": "expected",
"source_ref": "snapshot:expected-audience-2026-01",
"source_name": "expected_audience",
"kind": "datasource",
"provider": "reporting.synthetic-baseline",
"fingerprint": "sha256:synthetic-expected-audience-2026-01",
"row_count": 4,
"preview_rows": 4,
"truncated": false
}
],
"expected_recipients": ["identity:alice", "identity:grace", "identity:frank"],
"expected_exclusions": {
"identity:bob": "preference.suppressed"
}
}