Release v0.1.15

.gitea/workflows/module-package-release.yml

@@ -0,0 +1,270 @@
+name: Module Package Release
+
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: Existing protected version tag to publish
+        required: true
+        type: string
+
+jobs:
+  publish-packages:
+    runs-on: ubuntu-latest
+    env:
+      GITEA_REPOSITORY: ${{ gitea.repository }}
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: "3.12"
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
+        with:
+          node-version: "22"
+      - name: Select and validate protected release tag
+        shell: bash
+        env:
+          REQUESTED_TAG: ${{ inputs.release_tag }}
+          TRIGGER_TAG: ${{ gitea.ref_name }}
+        run: |
+          set -euo pipefail
+          tag="${REQUESTED_TAG:-$TRIGGER_TAG}"
+          case "$tag" in
+            v[0-9]*.[0-9]*.[0-9]*) ;;
+            *) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;;
+          esac
+          git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main
+          tag_commit="$(git rev-list -n 1 "$tag")"
+          git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || {
+            echo "Release tag is not contained in main" >&2
+            exit 1
+          }
+          git checkout --detach "$tag"
+          printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV"
+          printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV"
+      - name: Validate package versions
+        run: |
+          python - <<'PY'
+          import json
+          from pathlib import Path
+          import os
+          import re
+          import tomllib
+
+          tag = os.environ["RELEASE_TAG"]
+          expected = tag.removeprefix("v")
+          project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
+          if project.get("version") != expected:
+              raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}")
+          if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None:
+              raise SystemExit("Python distribution name must use the govoplan-* namespace")
+          webui = Path("webui/package.json")
+          if webui.is_file():
+              package = json.loads(webui.read_text(encoding="utf-8"))
+              if package.get("version") != expected:
+                  raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}")
+              if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None:
+                  raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace")
+              release = Path("webui/package.release.json")
+              if release.is_file():
+                  release_package = json.loads(release.read_text(encoding="utf-8"))
+                  if (
+                      release_package.get("name") != package.get("name")
+                      or release_package.get("version") != expected
+                  ):
+                      raise SystemExit("WebUI release package identity does not match package.json and the release tag")
+          PY
+      - name: Build immutable package artifacts
+        shell: bash
+        run: |
+          set -euo pipefail
+          python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
+          rm -rf dist .package-webui
+          python -m build --wheel --outdir dist
+          python -m twine check dist/*.whl
+          if [[ -f webui/package.json ]]; then
+            mkdir .package-webui
+            cp -a webui/. .package-webui/
+            rm -rf .package-webui/node_modules .package-webui/dist
+            if [[ -f .package-webui/package.release.json ]]; then
+              cp .package-webui/package.release.json .package-webui/package.json
+            fi
+          node <<'NODE'
+          const fs = require("node:fs");
+          const path = ".package-webui/package.json";
+          const packageJson = JSON.parse(fs.readFileSync(path, "utf8"));
+          const groups = ["dependencies", "optionalDependencies", "peerDependencies"];
+          for (const group of groups) {
+            for (const [name, specifier] of Object.entries(packageJson[group] || {})) {
+              if (!name.startsWith("@govoplan/")) continue;
+              if (typeof specifier !== "string") {
+                throw new Error(`${group}.${name} must use a string version`);
+              }
+              const packageSlug = name.slice("@govoplan/".length);
+              if (!packageSlug.endsWith("-webui")) {
+                throw new Error(`${group}.${name} is outside the WebUI package namespace`);
+              }
+              const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`;
+              const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+              const gitTag = specifier.match(
+                new RegExp(
+                  `^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/(?:GovOPlaN|add-ideas)/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`,
+                ),
+              );
+              if (gitTag) {
+                packageJson[group][name] = gitTag[1];
+                continue;
+              }
+              if (specifier.startsWith("file:") || specifier.startsWith("git+")) {
+                throw new Error(
+                  `${group}.${name} must resolve to an exact registry version for publication`,
+                );
+              }
+            }
+          }
+          delete packageJson.private;
+          fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`);
+          NODE
+            npm pkg delete private --prefix .package-webui
+            (cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist)
+          fi
+          python - <<'PY'
+          import hashlib
+          import json
+          from pathlib import Path
+          import os
+          import subprocess
+
+          artifacts = []
+          for path in sorted(Path("dist").iterdir()):
+              if path.suffix not in {".whl", ".tgz"}:
+                  continue
+              digest = hashlib.sha256(path.read_bytes()).hexdigest()
+              artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size})
+          payload = {
+              "schema_version": "1",
+              "repository": os.environ["GITEA_REPOSITORY"],
+              "tag": os.environ["RELEASE_TAG"],
+              "commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
+              "artifacts": artifacts,
+          }
+          Path("dist/package-artifacts.json").write_text(
+              json.dumps(payload, indent=2, sort_keys=True) + "\n",
+              encoding="utf-8",
+          )
+          PY
+      - name: Retain package hash evidence
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
+        with:
+          name: module-packages-${{ gitea.ref_name }}
+          path: dist/package-artifacts.json
+      - name: Check immutable registry state
+        shell: bash
+        env:
+          PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
+        run: |
+          set -euo pipefail
+          test -n "$PACKAGE_TOKEN"
+          python - <<'PY'
+          import hashlib
+          import json
+          import os
+          from pathlib import Path
+          import tomllib
+          from urllib.error import HTTPError
+          from urllib.parse import quote
+          from urllib.request import Request, urlopen
+
+          api_root = "https://git.add-ideas.de/api/v1/packages/GovOPlaN"
+          token = os.environ["PACKAGE_TOKEN"]
+
+          def should_publish(kind, name, version, path):
+              package_url = "/".join(
+                  (api_root, kind, quote(name, safe=""), quote(version, safe=""), "files")
+              )
+              request = Request(
+                  package_url,
+                  headers={"Accept": "application/json", "Authorization": f"token {token}"},
+              )
+              try:
+                  with urlopen(request, timeout=30) as response:
+                      files = json.load(response)
+              except HTTPError as exc:
+                  if exc.code == 404:
+                      print(f"{kind} package {name}=={version} is not published yet")
+                      return True
+                  raise
+              if not isinstance(files, list) or len(files) != 1:
+                  raise SystemExit(
+                      f"immutable {kind} package {name}=={version} has an unexpected file set"
+                  )
+              expected_sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
+              if files[0].get("sha256") != expected_sha256:
+                  raise SystemExit(
+                      f"immutable {kind} package {name}=={version} already exists with a different SHA-256"
+                  )
+              print(f"verified existing {kind} package {name}=={version} ({expected_sha256})")
+              return False
+
+          project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
+          wheels = tuple(Path("dist").glob("*.whl"))
+          if len(wheels) != 1:
+              raise SystemExit("release build must contain exactly one wheel")
+          publish_pypi = should_publish(
+              "pypi", str(project["name"]), str(project["version"]), wheels[0]
+          )
+
+          tarballs = tuple(Path("dist").glob("*.tgz"))
+          if len(tarballs) > 1:
+              raise SystemExit("release build must contain at most one npm package")
+          publish_npm = False
+          if tarballs:
+              webui = json.loads(
+                  Path(".package-webui/package.json").read_text(encoding="utf-8")
+              )
+              publish_npm = should_publish(
+                  "npm", str(webui["name"]), str(webui["version"]), tarballs[0]
+              )
+
+          with Path(os.environ["GITEA_ENV"]).open("a", encoding="utf-8") as env_file:
+              env_file.write(f"PUBLISH_PYPI={int(publish_pypi)}\n")
+              env_file.write(f"PUBLISH_NPM={int(publish_npm)}\n")
+          PY
+      - name: Publish wheel and WebUI package
+        shell: bash
+        env:
+          PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
+          PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
+        run: |
+          set -euo pipefail
+          test -n "$PACKAGE_USERNAME"
+          test -n "$PACKAGE_TOKEN"
+          if [[ "$PUBLISH_PYPI" == 1 ]]; then
+            TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
+              python -m twine upload --non-interactive \
+                --repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
+                dist/*.whl
+          else
+            echo "Exact wheel is already present; skipping immutable retry."
+          fi
+          shopt -s nullglob
+          webui_packages=(dist/*.tgz)
+          if (( ${#webui_packages[@]} )) && [[ "$PUBLISH_NPM" == 1 ]]; then
+            npmrc="$(mktemp)"
+            trap 'rm -f "$npmrc"' EXIT
+            chmod 600 "$npmrc"
+            printf '%s\n' \
+              '@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \
+              "//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \
+              > "$npmrc"
+            NPM_CONFIG_USERCONFIG="$npmrc" npm publish "./${webui_packages[0]}" \
+              --ignore-scripts --access public \
+              --registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/
+          elif (( ${#webui_packages[@]} )); then
+            echo "Exact WebUI package is already present; skipping immutable retry."
+          fi

docs/ADMIN_GUIDE.md

@@ -27,6 +27,10 @@ published by a source-owning module. Do not expose another module's ORM or an
 unbounded SQL connection as a report source. Configure an explicit schema,
 freshness policy, source fingerprint expectations, purpose, privacy,
 retention, and a row-policy provider where source access alone is not enough.
+For a Dataflow source, `source_run_ref` optionally pins one successful run that
+published an immutable Datasource materialization. Reporting passes this pin
+through rather than re-executing the pipeline. The current principal must still
+be authorized for the Dataflow definition and the exact Datasource output.
 
 On PostgreSQL installations, Reporting compiles bounded semantic filters,
 grouping, measures, calculated measures, ordering, offsets, and limits into a

docs/USER_GUIDE.md

@@ -28,6 +28,11 @@ the exact definition and output. Warnings explain freshness, inferred schema,
 or provider diagnostics. A failed quality gate records a failed execution and
 does not publish a result.
 
+A report configured against an exact published Dataflow run reads that run's
+immutable Datasource materialization. It does not rerun the flow with current
+inputs. The evidence identifies the Dataflow run and materialization, and access
+to both is checked again when the report runs.
+
 The **Effective access** explanation states when dimensions, measures, source
 rows, or actions were removed by Policy. A result with no hidden elements says
 so explicitly; catalogue visibility never grants access to protected detail.

pyproject.toml

@@ -4,15 +4,15 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "govoplan-reporting"
-version = "0.1.14"
+version = "0.1.15"
 description = "GovOPlaN governed reporting and semantic BI module."
 readme = "README.md"
 requires-python = ">=3.12"
 license = { text = "AGPL-3.0-or-later" }
 authors = [{ name = "GovOPlaN" }]
 dependencies = [
-  "govoplan-core>=0.1.14",
-  "govoplan-access>=0.1.14",
+  "govoplan-core>=0.1.15",
+  "govoplan-access>=0.1.15",
 ]
 
 [tool.setuptools.packages.find]

src/govoplan_reporting/__init__.py

@@ -1,3 +1,3 @@
 """GovOPlaN Reporting module."""
 
-__version__ = "0.1.14"
+__version__ = "0.1.15"

src/govoplan_reporting/backend/execution.py

@@ -501,6 +501,7 @@ def _read_dataset(
             request=DataflowDatasetRequest(
                 pipeline_ref=dataset.source_ref,
                 revision=dataset.source_revision or 0,
+                run_ref=dataset.source_run_ref,
                 parameters=source_parameters,
                 row_limit=2_000,
                 expected_definition_hash=dataset.definition_hash,

src/govoplan_reporting/backend/manifest.py

@@ -77,7 +77,7 @@ from govoplan_reporting.backend.search_source import create_reporting_search_sou
 
 MODULE_ID = "reporting"
 MODULE_NAME = "Reporting"
-MODULE_VERSION = "0.1.14"
+MODULE_VERSION = "0.1.15"
 
 
 def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
@@ -488,7 +488,7 @@ manifest = ModuleManifest(
                 "schedules, exports, and publication providers replace unchecked SQL in the "
                 "presentation layer. PostgreSQL executes bounded semantic plans when available. "
                 "Signed drill contexts reauthorize contributor rows, and Files/Mail publication "
-                "adapters retain idempotent evidence. Dataflow and module read models remain source owners."
+                "adapters retain idempotent evidence. A dataset may pin one successful published Dataflow run, which is read from its exact Datasource materialization after both source boundaries reauthorize the current principal. Dataflow and module read models remain source owners."
             ),
             layer="available",
             documentation_types=("admin", "user"),

src/govoplan_reporting/backend/schemas.py

@@ -101,6 +101,7 @@ class DatasetDefinition(BaseModel):
     source_kind: Literal["dataflow", "read_model", "static"]
     source_ref: str = Field(min_length=1, max_length=500)
     source_revision: int | None = Field(default=None, ge=1)
+    source_run_ref: str | None = Field(default=None, min_length=1, max_length=500)
     definition_hash: str | None = Field(default=None, min_length=1, max_length=128)
     source_parameters: dict[str, Any] = Field(default_factory=dict)
     static_rows: list[dict[str, Any]] = Field(default_factory=list, max_length=2_000)
@@ -130,6 +131,8 @@ class DatasetDefinition(BaseModel):
     def validate_source_pin(self) -> "DatasetDefinition":
         if self.source_kind == "dataflow" and self.source_revision is None:
             raise ValueError("Dataflow datasets require a pinned source revision.")
+        if self.source_run_ref is not None and self.source_kind != "dataflow":
+            raise ValueError("Only Dataflow datasets can pin a source run.")
         if self.source_kind == "static" and not self.static_rows:
             raise ValueError("Static analytical datasets require static_rows.")
         names = [item.name for item in self.fields]

tests/test_reporting_service.py

@@ -8,6 +8,10 @@ from sqlalchemy import create_engine
 from sqlalchemy.orm import Session
 
 from govoplan_core.db.base import Base
+from govoplan_core.core.dataflows import (
+    CAPABILITY_DATAFLOW_DATASET_OUTPUT,
+    DataflowDatasetResult,
+)
 from govoplan_core.core.files import (
     CAPABILITY_FILES_ARTIFACT_STORE,
     ManagedArtifactRef,
@@ -132,6 +136,34 @@ class ArtifactStore:
         )
 
 
+class DataflowOutput:
+    def __init__(self, rows: list[dict[str, object]]) -> None:
+        self.rows = tuple(dict(item) for item in rows)
+        self.last_request = None
+
+    def list_outputs(self, *_args, **_kwargs):
+        return ()
+
+    def read_output(self, _session, _principal, *, request):
+        self.last_request = request
+        return DataflowDatasetResult(
+            pipeline_ref=request.pipeline_ref,
+            revision=request.revision,
+            definition_hash=request.expected_definition_hash or "pipeline-hash",
+            rows=self.rows,
+            total_rows=len(self.rows),
+            truncated=False,
+            output_hash="d" * 64,
+            executor_version="duckdb-v1",
+            run_ref=request.run_ref,
+            source_fingerprints=(
+                {"node_id": "source", "fingerprint": "source-v1"},
+            ),
+            generated_at=NOW,
+            provenance={"immutable_run": bool(request.run_ref)},
+        )
+
+
 class ReportingServiceTests(unittest.TestCase):
     def setUp(self) -> None:
         self.engine = create_engine("sqlite+pysqlite:///:memory:")
@@ -352,6 +384,44 @@ class ReportingServiceTests(unittest.TestCase):
             )
         self.assertTrue(raised.exception.execution_id)
 
+    def test_dataflow_dataset_can_pin_an_exact_published_run(self) -> None:
+        payload = dataset_payload()
+        rows = list(payload.pop("static_rows"))
+        payload.update(
+            {
+                "source_kind": "dataflow",
+                "source_ref": "pipeline:monthly-comparison",
+                "source_revision": 4,
+                "source_run_ref": "dataflow-run:published-july",
+                "definition_hash": "pipeline-hash",
+            }
+        )
+        self._create("dataset", "dataset-1", payload)
+        self._create("semantic_model", "semantic-1", semantic_payload())
+        self._create("report", "report-1", report_payload())
+        provider = DataflowOutput(rows)
+        registry = CapabilityRegistry()
+        registry.providers[CAPABILITY_DATAFLOW_DATASET_OUTPUT] = provider
+
+        result = execute_report(
+            self.session,
+            self.principal,
+            registry=registry,
+            report_id="report-1",
+            report_revision=1,
+            parameters={},
+            query=None,
+            idempotency_key="published-dataflow-run",
+        )
+
+        self.assertEqual("succeeded", result["status"])
+        self.assertIsNotNone(provider.last_request)
+        self.assertEqual(
+            "dataflow-run:published-july",
+            provider.last_request.run_ref,
+        )
+        self.assertTrue(result["provenance"]["source"]["immutable_run"])
+
     def test_restricted_access_and_service_scope_guards(self) -> None:
         self._create_report_graph(
             report_access={

webui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@govoplan/reporting-webui",
-  "version": "0.1.14",
+  "version": "0.1.15",
   "private": true,
   "type": "module",
   "main": "src/index.ts",
@@ -17,7 +17,7 @@
     "test:interface-pattern": "node scripts/test-interface-pattern.mjs"
   },
   "peerDependencies": {
-    "@govoplan/core-webui": "^0.1.14",
+    "@govoplan/core-webui": "^0.1.15",
     "lucide-react": "^1.23.0",
     "react": ">=19.2.7 <20",
     "react-dom": ">=19.2.7 <20",