feat: add extensible operators and golden flows
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
# Golden Dataflows
|
||||
|
||||
Each directory is an executable product contract:
|
||||
|
||||
- `graph.json` contains the versioned pipeline graph.
|
||||
- `inputs/*.json` contains small, reviewable source fixtures.
|
||||
- `expected-output.json` is the exact normalized output.
|
||||
- `manifest.json` records the user story and intent.
|
||||
|
||||
Source nodes use a fixture filename in `config.fixture`; the golden-flow test
|
||||
injects those rows before validation and bounded execution.
|
||||
@@ -0,0 +1,29 @@
|
||||
[
|
||||
{
|
||||
"case_id": "A-1",
|
||||
"status": "open",
|
||||
"amount": 10,
|
||||
"monthly_case_id": "A-1",
|
||||
"monthly_status": "open",
|
||||
"monthly_amount": 10,
|
||||
"_reconciliation_status": "match",
|
||||
"_reconciliation_differences": []
|
||||
},
|
||||
{
|
||||
"case_id": "A-2",
|
||||
"status": "closed",
|
||||
"amount": 20,
|
||||
"monthly_case_id": "A-2",
|
||||
"monthly_status": "closed",
|
||||
"monthly_amount": 25,
|
||||
"_reconciliation_status": "changed",
|
||||
"_reconciliation_differences": ["amount"]
|
||||
},
|
||||
{
|
||||
"monthly_case_id": "A-3",
|
||||
"monthly_status": "open",
|
||||
"monthly_amount": 30,
|
||||
"_reconciliation_status": "missing_expected",
|
||||
"_reconciliation_differences": []
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,118 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"id": "existing",
|
||||
"type": "source.inline",
|
||||
"label": "Existing records",
|
||||
"position": {"x": 40, "y": 80},
|
||||
"config": {
|
||||
"source_name": "existing_records",
|
||||
"fixture": "existing-records.json",
|
||||
"rows": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "monthly",
|
||||
"type": "source.inline",
|
||||
"label": "Monthly file",
|
||||
"position": {"x": 40, "y": 280},
|
||||
"config": {
|
||||
"source_name": "monthly_file",
|
||||
"fixture": "monthly-file.json",
|
||||
"rows": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "convert-amount",
|
||||
"type": "convert",
|
||||
"label": "Convert amount",
|
||||
"position": {"x": 280, "y": 280},
|
||||
"config": {
|
||||
"source_column": "amount_text",
|
||||
"target_column": "amount",
|
||||
"target_type": "integer",
|
||||
"on_error": "fail"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "monthly-fields",
|
||||
"type": "select",
|
||||
"label": "Monthly fields",
|
||||
"position": {"x": 520, "y": 280},
|
||||
"config": {
|
||||
"fields": [
|
||||
{"column": "case_id", "alias": "case_id"},
|
||||
{"column": "status", "alias": "status"},
|
||||
{"column": "amount", "alias": "amount"}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "quality",
|
||||
"type": "quality.rules",
|
||||
"label": "Required monthly fields",
|
||||
"position": {"x": 760, "y": 280},
|
||||
"config": {
|
||||
"rules": [
|
||||
{"id": "case-id", "column": "case_id", "operator": "not_null"},
|
||||
{"id": "amount", "column": "amount", "operator": "not_null"}
|
||||
],
|
||||
"action": "fail"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "reconcile",
|
||||
"type": "reconcile.compare",
|
||||
"label": "Compare monthly state",
|
||||
"position": {"x": 1020, "y": 160},
|
||||
"config": {
|
||||
"left_keys": ["case_id"],
|
||||
"right_keys": ["case_id"],
|
||||
"compare_columns": ["status", "amount"],
|
||||
"right_prefix": "monthly_"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "output",
|
||||
"type": "output",
|
||||
"label": "Review differences",
|
||||
"position": {"x": 1260, "y": 160},
|
||||
"config": {}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"id": "e1",
|
||||
"source": "monthly",
|
||||
"target": "convert-amount"
|
||||
},
|
||||
{
|
||||
"id": "e2",
|
||||
"source": "convert-amount",
|
||||
"target": "monthly-fields"
|
||||
},
|
||||
{
|
||||
"id": "e3",
|
||||
"source": "monthly-fields",
|
||||
"target": "quality"
|
||||
},
|
||||
{
|
||||
"id": "e4",
|
||||
"source": "existing",
|
||||
"target": "reconcile",
|
||||
"target_port": "left"
|
||||
},
|
||||
{
|
||||
"id": "e5",
|
||||
"source": "quality",
|
||||
"target": "reconcile",
|
||||
"target_port": "right"
|
||||
},
|
||||
{
|
||||
"id": "e6",
|
||||
"source": "reconcile",
|
||||
"target": "output"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"case_id": "A-1", "status": "open", "amount": 10},
|
||||
{"case_id": "A-2", "status": "closed", "amount": 20}
|
||||
]
|
||||
@@ -0,0 +1,5 @@
|
||||
[
|
||||
{"case_id": "A-1", "status": "open", "amount_text": "10"},
|
||||
{"case_id": "A-2", "status": "closed", "amount_text": "25"},
|
||||
{"case_id": "A-3", "status": "open", "amount_text": "30"}
|
||||
]
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"id": "monthly-reconciliation",
|
||||
"title": "Monthly structured-file reconciliation",
|
||||
"user_story": "GovOPlaN/govoplan#8",
|
||||
"graph": "graph.json",
|
||||
"expected_output": "expected-output.json"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"subject_id": "S-1",
|
||||
"subject_name": " Example Trading GmbH ",
|
||||
"country": "DE",
|
||||
"list_entry_id": "EU-100",
|
||||
"program": "EU"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,101 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"id": "subjects",
|
||||
"type": "source.inline",
|
||||
"label": "Subjects",
|
||||
"position": {"x": 40, "y": 80},
|
||||
"config": {
|
||||
"source_name": "subjects",
|
||||
"fixture": "subjects.json",
|
||||
"rows": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "normalize-subject",
|
||||
"type": "expression",
|
||||
"label": "Normalize subject name",
|
||||
"position": {"x": 280, "y": 80},
|
||||
"config": {
|
||||
"target_column": "normalized_name",
|
||||
"expression": "lower(trim(name))",
|
||||
"result_type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "sanctions",
|
||||
"type": "source.inline",
|
||||
"label": "Sanctions list",
|
||||
"position": {"x": 40, "y": 300},
|
||||
"config": {
|
||||
"source_name": "sanctions",
|
||||
"fixture": "sanctions-list.json",
|
||||
"rows": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "normalize-sanction",
|
||||
"type": "expression",
|
||||
"label": "Normalize listed name",
|
||||
"position": {"x": 280, "y": 300},
|
||||
"config": {
|
||||
"target_column": "normalized_name",
|
||||
"expression": "lower(trim(name))",
|
||||
"result_type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "matches",
|
||||
"type": "combine.join",
|
||||
"label": "Exact normalized matches",
|
||||
"position": {"x": 540, "y": 190},
|
||||
"config": {
|
||||
"join_type": "inner",
|
||||
"left_keys": ["normalized_name"],
|
||||
"right_keys": ["normalized_name"],
|
||||
"right_prefix": "sanction_"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "fields",
|
||||
"type": "select",
|
||||
"label": "Screening result",
|
||||
"position": {"x": 800, "y": 190},
|
||||
"config": {
|
||||
"fields": [
|
||||
{"column": "subject_id", "alias": "subject_id"},
|
||||
{"column": "name", "alias": "subject_name"},
|
||||
{"column": "country", "alias": "country"},
|
||||
{"column": "sanction_list_entry_id", "alias": "list_entry_id"},
|
||||
{"column": "sanction_program", "alias": "program"}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "output",
|
||||
"type": "output",
|
||||
"label": "Potential matches",
|
||||
"position": {"x": 1040, "y": 190},
|
||||
"config": {}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{"id": "e1", "source": "subjects", "target": "normalize-subject"},
|
||||
{"id": "e2", "source": "sanctions", "target": "normalize-sanction"},
|
||||
{
|
||||
"id": "e3",
|
||||
"source": "normalize-subject",
|
||||
"target": "matches",
|
||||
"target_port": "left"
|
||||
},
|
||||
{
|
||||
"id": "e4",
|
||||
"source": "normalize-sanction",
|
||||
"target": "matches",
|
||||
"target_port": "right"
|
||||
},
|
||||
{"id": "e5", "source": "matches", "target": "fields"},
|
||||
{"id": "e6", "source": "fields", "target": "output"}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"list_entry_id": "EU-100", "name": "EXAMPLE TRADING GMBH", "program": "EU"},
|
||||
{"list_entry_id": "UN-200", "name": "Another Entity", "program": "UN"}
|
||||
]
|
||||
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"subject_id": "S-1", "name": " Example Trading GmbH ", "country": "DE"},
|
||||
{"subject_id": "S-2", "name": "Clear Services AG", "country": "CH"}
|
||||
]
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"id": "sanctions-screening",
|
||||
"title": "Sanctions-list screening",
|
||||
"user_story": "GovOPlaN/govoplan#12",
|
||||
"graph": "graph.json",
|
||||
"expected_output": "expected-output.json"
|
||||
}
|
||||
Reference in New Issue
Block a user