feat(dataflow): model production SQL flow patterns

2026-07-30 18:19:18 +02:00
parent 0664570607
commit e091042948
26 changed files with 2169 additions and 36 deletions
@@ -0,0 +1,16 @@
+[
+  {
+    "student_id": "S-100",
+    "full_name": "Ada Lovelace",
+    "birth_date": "10.12.2000",
+    "program": "CS",
+    "status_group": "current"
+  },
+  {
+    "student_id": "S-300",
+    "full_name": "Katherine Johnson",
+    "birth_date": "26.08.2001",
+    "program": "MATH",
+    "status_group": "current"
+  }
+]
@@ -0,0 +1,130 @@
+{
+  "schema_version": 1,
+  "nodes": [
+    {
+      "id": "statuses",
+      "type": "source.inline",
+      "label": "Student statuses",
+      "position": {"x": 40, "y": 80},
+      "config": {
+        "source_name": "student_statuses",
+        "fixture": "student-statuses.json",
+        "rows": []
+      }
+    },
+    {
+      "id": "exclusions",
+      "type": "source.inline",
+      "label": "Export exclusions",
+      "position": {"x": 520, "y": 300},
+      "config": {
+        "source_name": "export_exclusions",
+        "fixture": "export-exclusions.json",
+        "rows": []
+      }
+    },
+    {
+      "id": "latest-rank",
+      "type": "window.rank",
+      "label": "Latest status per student",
+      "position": {"x": 280, "y": 80},
+      "config": {
+        "method": "row_number",
+        "target_column": "status_rank",
+        "partition_by": ["student_id"],
+        "order_by": [
+          {"column": "status_date", "direction": "desc"}
+        ]
+      }
+    },
+    {
+      "id": "latest-only",
+      "type": "filter",
+      "label": "Keep latest status",
+      "position": {"x": 520, "y": 80},
+      "config": {
+        "column": "status_rank",
+        "operator": "eq",
+        "value": 1
+      }
+    },
+    {
+      "id": "eligible-only",
+      "type": "combine.join",
+      "label": "Exclude ineligible students",
+      "position": {"x": 760, "y": 160},
+      "config": {
+        "join_type": "anti",
+        "left_keys": ["student_id"],
+        "right_keys": ["student_id"],
+        "right_prefix": "excluded_"
+      }
+    },
+    {
+      "id": "export-calculations",
+      "type": "calculate",
+      "label": "Calculate export fields",
+      "position": {"x": 1000, "y": 160},
+      "config": {
+        "calculations": [
+          {
+            "target_column": "full_name",
+            "expression": "trim(first_name) || ' ' || trim(last_name)",
+            "result_type": "string"
+          },
+          {
+            "target_column": "birth_date_display",
+            "expression": "to_char(birth_date, 'DD.MM.YYYY')",
+            "result_type": "string"
+          },
+          {
+            "target_column": "status_group",
+            "expression": "case when status in ('active', 'enrolled') then 'current' else 'inactive' end",
+            "result_type": "string"
+          }
+        ]
+      }
+    },
+    {
+      "id": "export-fields",
+      "type": "select",
+      "label": "HEICO export",
+      "position": {"x": 1240, "y": 160},
+      "config": {
+        "fields": [
+          {"column": "student_id", "alias": "student_id"},
+          {"column": "full_name", "alias": "full_name"},
+          {"column": "birth_date_display", "alias": "birth_date"},
+          {"column": "program", "alias": "program"},
+          {"column": "status_group", "alias": "status_group"}
+        ]
+      }
+    },
+    {
+      "id": "output",
+      "type": "output",
+      "label": "Export rows",
+      "position": {"x": 1480, "y": 160},
+      "config": {}
+    }
+  ],
+  "edges": [
+    {"id": "e1", "source": "statuses", "target": "latest-rank"},
+    {"id": "e2", "source": "latest-rank", "target": "latest-only"},
+    {
+      "id": "e3",
+      "source": "latest-only",
+      "target": "eligible-only",
+      "target_port": "left"
+    },
+    {
+      "id": "e4",
+      "source": "exclusions",
+      "target": "eligible-only",
+      "target_port": "right"
+    },
+    {"id": "e5", "source": "eligible-only", "target": "export-calculations"},
+    {"id": "e6", "source": "export-calculations", "target": "export-fields"},
+    {"id": "e7", "source": "export-fields", "target": "output"}
+  ]
+}
@@ -0,0 +1,6 @@
+[
+  {
+    "student_id": "S-200",
+    "reason": "No active enrollment"
+  }
+]
@@ -0,0 +1,38 @@
+[
+  {
+    "student_id": "S-100",
+    "first_name": " Ada ",
+    "last_name": "Lovelace",
+    "birth_date": "2000-12-10",
+    "program": "CS",
+    "status": "active",
+    "status_date": "2026-04-01"
+  },
+  {
+    "student_id": "S-100",
+    "first_name": " Ada ",
+    "last_name": "Lovelace",
+    "birth_date": "2000-12-10",
+    "program": "CS",
+    "status": "enrolled",
+    "status_date": "2025-10-01"
+  },
+  {
+    "student_id": "S-200",
+    "first_name": "Grace",
+    "last_name": "Hopper",
+    "birth_date": "1999-12-09",
+    "program": "IS",
+    "status": "withdrawn",
+    "status_date": "2026-03-15"
+  },
+  {
+    "student_id": "S-300",
+    "first_name": "Katherine",
+    "last_name": "Johnson",
+    "birth_date": "2001-08-26",
+    "program": "MATH",
+    "status": "active",
+    "status_date": "2026-04-02"
+  }
+]
@@ -0,0 +1,7 @@
+{
+  "id": "heico-student-status",
+  "title": "HEICO-style current student status export",
+  "user_story": "GovOPlaN/govoplan-dataflow#17",
+  "graph": "graph.json",
+  "expected_output": "expected-output.json"
+}