feat(dataflow): model production SQL flow patterns

2026-07-30 18:19:18 +02:00
parent 0664570607
commit e091042948
26 changed files with 2169 additions and 36 deletions
@@ -0,0 +1,20 @@
+[
+  {
+    "period": "2026/01",
+    "booking_class": "personnel",
+    "booking_count": 2,
+    "net_amount": 70
+  },
+  {
+    "period": "2026/02",
+    "booking_class": "allocation",
+    "booking_count": 1,
+    "net_amount": 45
+  },
+  {
+    "period": "2026/02",
+    "booking_class": "other",
+    "booking_count": 1,
+    "net_amount": 20
+  }
+]
@@ -0,0 +1,104 @@
+{
+  "schema_version": 1,
+  "nodes": [
+    {
+      "id": "bookings",
+      "type": "source.inline",
+      "label": "Bookings",
+      "position": {"x": 40, "y": 120},
+      "config": {
+        "source_name": "bookings",
+        "fixture": "bookings.json",
+        "rows": []
+      }
+    },
+    {
+      "id": "booking-calculations",
+      "type": "calculate",
+      "label": "Normalize and classify",
+      "position": {"x": 280, "y": 120},
+      "config": {
+        "calculations": [
+          {
+            "target_column": "cost_center",
+            "expression": "cast(trim(cost_text) as integer)",
+            "result_type": "integer"
+          },
+          {
+            "target_column": "period",
+            "expression": "to_char(booking_date, 'YYYY/MM')",
+            "result_type": "string"
+          },
+          {
+            "target_column": "booking_class",
+            "expression": "case when cost_center between 62000 and 65999 then 'personnel' when cost_center between 97000000 and 97999999 then 'allocation' else 'other' end",
+            "result_type": "string"
+          },
+          {
+            "target_column": "signed_amount",
+            "expression": "case when entry_type = 'credit' then 0 - amount else amount end",
+            "result_type": "integer"
+          },
+          {
+            "target_column": "record_key",
+            "expression": "lpad(cast(sequence as text), 6, '0')",
+            "result_type": "string"
+          }
+        ]
+      }
+    },
+    {
+      "id": "required-fields",
+      "type": "quality.rules",
+      "label": "Required booking fields",
+      "position": {"x": 520, "y": 120},
+      "config": {
+        "rules": [
+          {"id": "cost-center", "column": "cost_center", "operator": "not_null"},
+          {"id": "period", "column": "period", "operator": "not_null"},
+          {"id": "amount", "column": "signed_amount", "operator": "not_null"}
+        ],
+        "action": "fail"
+      }
+    },
+    {
+      "id": "booking-summary",
+      "type": "aggregate",
+      "label": "Monthly booking summary",
+      "position": {"x": 760, "y": 120},
+      "config": {
+        "group_by": ["period", "booking_class"],
+        "aggregates": [
+          {"function": "count", "column": "*", "alias": "booking_count"},
+          {"function": "sum", "column": "signed_amount", "alias": "net_amount"}
+        ]
+      }
+    },
+    {
+      "id": "summary-order",
+      "type": "sort",
+      "label": "Order summary",
+      "position": {"x": 1000, "y": 120},
+      "config": {
+        "fields": [
+          {"column": "period", "direction": "asc"},
+          {"column": "booking_class", "direction": "asc"}
+        ]
+      }
+    },
+    {
+      "id": "output",
+      "type": "output",
+      "label": "Booking summary",
+      "position": {"x": 1240, "y": 120},
+      "config": {}
+    }
+  ],
+  "edges": [
+    {"id": "e1", "source": "bookings", "target": "booking-calculations"},
+    {"id": "e2", "source": "booking-calculations", "target": "required-fields"},
+    {"id": "e3", "source": "required-fields", "target": "booking-summary"},
+    {"id": "e4", "source": "booking-summary", "target": "summary-order"},
+    {"id": "e5", "source": "summary-order", "target": "output"}
+  ]
+}
@@ -0,0 +1,30 @@
+[
+  {
+    "sequence": 1,
+    "booking_date": "2026-01-12",
+    "cost_text": " 62150 ",
+    "entry_type": "debit",
+    "amount": 100
+  },
+  {
+    "sequence": 2,
+    "booking_date": "2026-01-13",
+    "cost_text": "62210",
+    "entry_type": "credit",
+    "amount": 30
+  },
+  {
+    "sequence": 3,
+    "booking_date": "2026-02-02",
+    "cost_text": "97020500",
+    "entry_type": "debit",
+    "amount": 45
+  },
+  {
+    "sequence": 4,
+    "booking_date": "2026-02-03",
+    "cost_text": "81000",
+    "entry_type": "debit",
+    "amount": 20
+  }
+]
@@ -0,0 +1,7 @@
+{
+  "id": "rele-booking-transform",
+  "title": "RELE-style booking normalization and aggregation",
+  "user_story": "GovOPlaN/govoplan-dataflow#17",
+  "graph": "graph.json",
+  "expected_output": "expected-output.json"
+}