From eba35edd3c7c38adf32dfd58f394908fc0887d40 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Sat, 1 Aug 2026 17:48:39 +0200 Subject: [PATCH] feat: implement governed reporting vertical --- README.md | 26 +- docs/ADMIN_GUIDE.md | 42 + docs/OPERATIONS.md | 40 + docs/REPORTING_BOUNDARY.md | 89 +- docs/SUPERX_CAPABILITY_ASSESSMENT.md | 28 +- docs/USER_GUIDE.md | 32 + pyproject.toml | 25 + src/govoplan_reporting/__init__.py | 3 + src/govoplan_reporting/backend/__init__.py | 1 + src/govoplan_reporting/backend/acl.py | 29 + src/govoplan_reporting/backend/contracts.py | 135 +++ src/govoplan_reporting/backend/db/__init__.py | 25 + src/govoplan_reporting/backend/db/models.py | 441 ++++++++ src/govoplan_reporting/backend/definitions.py | 955 +++++++++++++++++ src/govoplan_reporting/backend/domain.py | 79 ++ src/govoplan_reporting/backend/execution.py | 976 ++++++++++++++++++ src/govoplan_reporting/backend/manifest.py | 478 +++++++++ .../backend/migrations/__init__.py | 1 + .../backend/migrations/versions/__init__.py | 1 + .../e5b2c9d4f7a1_v0114_reporting_baseline.py | 477 +++++++++ src/govoplan_reporting/backend/operations.py | 858 +++++++++++++++ .../backend/query_engine.py | 494 +++++++++ src/govoplan_reporting/backend/registry.py | 81 ++ src/govoplan_reporting/backend/router.py | 526 ++++++++++ src/govoplan_reporting/backend/schemas.py | 503 +++++++++ .../backend/search_source.py | 202 ++++ src/govoplan_reporting/py.typed | 1 + tests/test_manifest.py | 27 + tests/test_reporting_service.py | 569 ++++++++++ webui/package.json | 28 + webui/src/api/reporting.ts | 260 +++++ .../src/features/reporting/ReportingPage.tsx | 573 ++++++++++ webui/src/index.ts | 2 + webui/src/module.ts | 52 + webui/src/styles/reporting.css | 388 +++++++ 35 files changed, 8414 insertions(+), 33 deletions(-) create mode 100644 docs/ADMIN_GUIDE.md create mode 100644 docs/OPERATIONS.md create mode 100644 docs/USER_GUIDE.md create mode 100644 pyproject.toml create mode 100644 src/govoplan_reporting/__init__.py create mode 100644 src/govoplan_reporting/backend/__init__.py create mode 100644 src/govoplan_reporting/backend/acl.py create mode 100644 src/govoplan_reporting/backend/contracts.py create mode 100644 src/govoplan_reporting/backend/db/__init__.py create mode 100644 src/govoplan_reporting/backend/db/models.py create mode 100644 src/govoplan_reporting/backend/definitions.py create mode 100644 src/govoplan_reporting/backend/domain.py create mode 100644 src/govoplan_reporting/backend/execution.py create mode 100644 src/govoplan_reporting/backend/manifest.py create mode 100644 src/govoplan_reporting/backend/migrations/__init__.py create mode 100644 src/govoplan_reporting/backend/migrations/versions/__init__.py create mode 100644 src/govoplan_reporting/backend/migrations/versions/e5b2c9d4f7a1_v0114_reporting_baseline.py create mode 100644 src/govoplan_reporting/backend/operations.py create mode 100644 src/govoplan_reporting/backend/query_engine.py create mode 100644 src/govoplan_reporting/backend/registry.py create mode 100644 src/govoplan_reporting/backend/router.py create mode 100644 src/govoplan_reporting/backend/schemas.py create mode 100644 src/govoplan_reporting/backend/search_source.py create mode 100644 src/govoplan_reporting/py.typed create mode 100644 tests/test_manifest.py create mode 100644 tests/test_reporting_service.py create mode 100644 webui/package.json create mode 100644 webui/src/api/reporting.ts create mode 100644 webui/src/features/reporting/ReportingPage.tsx create mode 100644 webui/src/index.ts create mode 100644 webui/src/module.ts create mode 100644 webui/src/styles/reporting.css diff --git a/README.md b/README.md index 300fb12..2670c86 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,31 @@ scheduled outputs, and report publication/export behavior. It is intentionally separate from reusable template rendering and from generic dataflow pipelines. -This repository is currently a tag-only scaffold. It should gain package -metadata and module manifests only after the first backend or WebUI slice is -designed. +The module now provides an executable governed semantic-reporting vertical: + +- immutable, optimistic-concurrency guarded dataset, semantic-model, report, + and quality-plan revisions; +- exact source and definition pins, tenant isolation, normalized access + grants, row-policy handoff, freshness checks, and reconstructable run + provenance; +- safe dimensions, hierarchies, measures, typed calculations, filters, + detail/summary/pivot queries, and accessible chart models without executing + arbitrary report SQL; +- quality gates, saved views, interval/scheduled runs, CSV/JSON export, + provider-neutral publication targets, and import activation assessments; +- a full-height Reporting workspace for running, inspecting, saving, + scheduling, visualizing, and exporting authorized reports. + +Reporting consumes Dataflow outputs or provider-owned read models. It does not +read another module's ORM tables or take ownership of ingestion and +transformation. See [docs/REPORTING_BOUNDARY.md](docs/REPORTING_BOUNDARY.md) for the boundary decision. The behavior-level comparison with the supplied SuperX module set is recorded in [docs/SUPERX_CAPABILITY_ASSESSMENT.md](docs/SUPERX_CAPABILITY_ASSESSMENT.md). + +Operational and recovery behavior is documented in +[docs/OPERATIONS.md](docs/OPERATIONS.md), while user and administrator tasks +are covered by [docs/USER_GUIDE.md](docs/USER_GUIDE.md) and +[docs/ADMIN_GUIDE.md](docs/ADMIN_GUIDE.md). diff --git a/docs/ADMIN_GUIDE.md b/docs/ADMIN_GUIDE.md new file mode 100644 index 0000000..eb11673 --- /dev/null +++ b/docs/ADMIN_GUIDE.md @@ -0,0 +1,42 @@ +# Reporting Administration Guide + +## Definition graph + +Reporting definitions form an exact graph: + +```text +Dataset revision -> Semantic-model revision -> Report revision + -> Quality-plan revision +``` + +Create parents before children. An active child may reference only an active, +existing parent revision. Editing creates a new immutable revision and +requires the currently observed revision number. Existing runs continue to +reference the historical revisions they used. + +Datasets may bind a static fixture, a pinned Dataflow output, or a capability +published by a source-owning module. Do not expose another module's ORM or an +unbounded SQL connection as a report source. Configure an explicit schema, +freshness policy, source fingerprint expectations, purpose, privacy, +retention, and a row-policy provider where source access alone is not enough. + +## Access and publication + +Tenant-visible definitions are readable by principals with Reporting read +permission. Restricted reports use normalized account, identity, group, role, +function, assignment, organization-unit, or service-account grants. The +creator and Reporting administrators retain management access. + +Scheduled output publication requires an installed capability implementing +the Reporting publication-target contract. The target receives one immutable +execution payload and an idempotency key. It must return bounded evidence and +must not expose credentials in that evidence. + +## Import assessments + +Import assessment accepts declarative metadata only. Native datasets, +dimensions, hierarchies, measures, parameters, tables, pivots, charts, +quality assertions, and saved views map exactly. Provider-specific formatting, +dialect functions, and dashboard layouts require explicit approximation +acceptance. Raw SQL, procedures, scripts, implicit authorization, unchecked +functions, and unknown features block activation. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 0000000..f9c3217 --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,40 @@ +# Reporting Operations, Security, and Recovery + +## Runtime behavior + +Reporting API nodes are stateless. Definitions, executions, schedules, +publication evidence, and import/quality evidence live in the shared platform +database. Dataset bytes remain with their source owner; large durable output +files belong in a publication provider backed by shared object storage. + +Run and schedule idempotency keys make retries replay-safe. Schedule workers +claim bounded batches, retain success or failure state, and advance interval +time from the claimed occurrence. Providers must use the same tenant and +principal context and must enforce their source-side authorization. + +## Security controls + +- The semantic engine never evaluates Python, browser code, raw SQL, stored + procedures, or imported runtime scripts. +- Definition and execution APIs require explicit module scopes and a + tenant-bound principal. +- Restricted access is normalized into grants and rechecked before definitions + or execution history are returned. +- Row-policy providers may only reduce a dataset; returning more rows than the + source read fails closed. +- Freshness, schema, fingerprints, quality gates, definition hashes, source + provenance, actor, and output hash are retained with every run. +- CSV formula injection is neutralized before direct download. + +## Backup and restore + +Back up the shared database and every object-storage publication target as one +recovery set. Before destructive module retirement, take and verify a database +snapshot; retirement removes definitions, result evidence, schedules, and +publication records. Restore the database first, then target object storage, +then reconcile provider publications by their idempotency/evidence references. + +After restore, verify migrations, definition parent pins, schedule enablement, +execution counts and hashes, and publication target health before dispatching +due schedules. Do not claim successful recovery until an operator has run and +recorded the deployment-specific drill. diff --git a/docs/REPORTING_BOUNDARY.md b/docs/REPORTING_BOUNDARY.md index 85e5f74..ca2cc17 100644 --- a/docs/REPORTING_BOUNDARY.md +++ b/docs/REPORTING_BOUNDARY.md @@ -15,6 +15,9 @@ Reporting owns: - curated query/view inputs from modules, connectors, datasources, or configuration packages - report execution history, generated output evidence, and scheduled runs +- institutional question, goal/obligation, owner, calculation-version, + freshness/quality, and decision/action-consumption references for each + material report or indicator - report permissions and sharing rules - export targets such as file, DMS, mail, API, RSS/Atom publication, and downstream connector handoff @@ -28,13 +31,27 @@ Initial source categories: - curated SQL/query views exposed through a controlled capability - connector-staged datasets and external references - generated file/csv/xlsx inputs from files/connectors -- future `govoplan-datasources` catalog entries, if that module is created -- future `govoplan-dataflow` outputs, if that module is created +- governed `govoplan-datasources` catalogue entries and immutable + materializations +- versioned `govoplan-dataflow` outputs and run lineage Reporting should not reach into module ORM internals directly. Module-owned data must be exposed through capabilities, DTOs, events, read models, or controlled query views. +Every reproducible report definition should retain or resolve: + +- the institutional question, obligation, goal, or service measure it serves; +- owner and responsible organization/function; +- datasource/materialization and Dataflow definition/run revisions; +- source freshness, quality state, known limitations, and calculation version; +- purpose, visibility, privacy, retention, and publication decisions; +- reports, decisions, controls, projects, or actions that consumed the result. + +Reporting owns execution and presentation of the measure. Mandates, Services, +Projects, Risk Compliance, Decisions, and other domains retain ownership of +the referenced institutional concepts. + ## Output Targets - dashboard/widget view @@ -53,24 +70,60 @@ Reporting does not own: - reusable template rendering; that belongs to `govoplan-templates` - DMS lifecycle, collaborative editing, legal hold, or records management - raw file/blob storage and provider connectors -- general-purpose ingestion/transformation pipeline ownership unless a future - `govoplan-dataflow` module is justified +- ingestion, governed source identity, staging, or transformation pipeline + ownership; those belong to Connectors, Datasources, and Dataflow - cross-module search indexing; that belongs to `govoplan-search` -## Candidate Capabilities +## Capability Contracts -- `reporting.catalog` -- `reporting.run` -- `reporting.schedules` -- `reporting.exports` -- `reporting.readModels` -- `reporting.dashboardContributions` +- `reporting.registry` owns immutable definition registration and lookup. +- `reporting.runner` executes exact report graphs and returns governed result + evidence. +- `reporting.scheduler` claims due schedules and preserves run/publication + outcomes. +- `reporting.chart_renderer` renders provider-neutral visual models with an + accessible table fallback. +- `reporting.read_model:*` capabilities can expose bounded source-owned rows. +- `reporting.publication_target:*` capabilities can accept immutable result + payloads without Reporting importing the target module. -## First Implementation Slice +## Implemented Vertical -1. Define manifest metadata, permissions, and capability names. -2. Add report definition and parameter DTOs. -3. Add one read-only report source contract over module-provided summary DTOs. -4. Add export target DTOs without implementing every target. -5. Add tests that report consumers/producers use capability lookup rather than - direct imports. +The first complete vertical persists datasets, semantic models, reports, +quality plans, grants, executions, saved views, schedules, publications, +quality results, and import assessments. Each definition update creates an +immutable revision and requires optimistic concurrency. Active child +definitions require the exact pinned parent revision to be active. + +Execution resolves the report, semantic model, and dataset graph before +reading data. Static fixtures, Dataflow output, and provider-owned read models +share one bounded read contract. Source fingerprints, freshness, schema, +row-policy provenance, blocking quality plans, definition hashes, executor +version, output hash, diagnostics, and authorized rows are retained with the +execution. Failed runs also retain evidence. + +The query engine deliberately implements a typed expression and semantic +query language rather than `eval`, arbitrary SQL, stored procedures, or +runtime scripts. It supports detail, grouped summary, pivot, dimensions, +hierarchies, common aggregates, calculated measures, filters, sorting, +pagination, totals, and a provider-neutral visualization model. A saved chart +that is incompatible with an ad-hoc query degrades to its mandatory table +fallback instead of failing a valid report run. + +Direct export supports UTF-8 CSV and JSON. CSV cells that spreadsheet software +could interpret as formulas are escaped. Additional formats and delivery +destinations use an optional publication capability and preserve idempotent +evidence. Import assessments classify exact, approximated, and unsupported +semantics and block activation until every approximation is accepted and no +unsupported executable behavior remains. + +The WebUI uses the platform module loader and common controls. It exposes a +report catalogue, parameter and semantic-query controls, result visualization +and table views, history/provenance, saved views, schedules, and downloads. + +## Remaining Product Depth + +The architecture boundary is implemented. Further work is additive product +depth: richer visualization providers, drill-through navigation, packaged +domain report catalogues, XLSX/PDF formatting through optional providers, and +target-environment evidence for a maturity claim above `vertical_slice`. diff --git a/docs/SUPERX_CAPABILITY_ASSESSMENT.md b/docs/SUPERX_CAPABILITY_ASSESSMENT.md index 5cbeb80..075157a 100644 --- a/docs/SUPERX_CAPABILITY_ASSESSMENT.md +++ b/docs/SUPERX_CAPABILITY_ASSESSMENT.md @@ -42,13 +42,13 @@ The supplied modules collectively provide five relevant layers: | Scope inheritance and provenance | Available foundation | Policy/Core | | Configurable dashboard widgets | Available foundation | Dashboard | | Reusable parameterized subflows | Available foundation | Dataflow | -| BI dimensions, measures, and hierarchies | Missing | Reporting | -| Pivot/drill-down analytical view | Missing | Reporting | -| Parameterized report masks and saved views | Missing | Reporting | -| Chart renderer/provider contract | Missing | Reporting | -| Scheduled report publication/export | Partially modelled, not integrated | Reporting | -| Report-specific access explanations | Missing | Reporting/Policy | -| Data-quality test plans and evidence UI | Partial | Dataflow/Reporting | +| BI dimensions, measures, and hierarchies | Available | Reporting | +| Pivot analytical view | Available; drill-through is product depth | Reporting | +| Parameterized report masks and saved views | Available | Reporting | +| Chart renderer/provider contract | Available with tabular fallback | Reporting | +| Scheduled report publication/export | Available through provider contract | Reporting | +| Report-specific access enforcement | Available; richer explanations are product depth | Reporting/Policy | +| Data-quality test plans and evidence | Available foundation | Dataflow/Reporting | | Packaged domain report catalogues | Missing | Reporting plus domain modules | GovOPlaN already has most platform primitives below the BI layer. The largest @@ -102,9 +102,9 @@ accepts a documented approximation. Imported definitions remain ordinary versioned GovOPlaN definitions after migration; the runtime must not depend on the original SuperX installation. -## Delivery Plan +## Delivery State -### Phase 1: Semantic Core +### Phase 1: Semantic Core - complete - scaffold the Reporting module and manifest - define analytical dataset, dimension, hierarchy, measure, parameter, and @@ -112,7 +112,7 @@ the original SuperX installation. - consume one Dataflow-published dataset through a capability - enforce scope, Policy provenance, and source freshness -### Phase 2: Interactive BI +### Phase 2: Interactive BI - vertical complete - parameter panel and saved views - table, pivot, drill-down, totals, calculated measures, and export of the @@ -120,20 +120,24 @@ the original SuperX installation. - stable URL/reference contract for report state - dashboard contribution for saved reports -### Phase 3: Visualization And Publication +### Phase 3: Visualization And Publication - vertical complete - chart renderer provider contract with an initial bounded chart catalogue - scheduled runs and publication through Files, Mail, or another declared target capability - accessible tabular fallback and report-specific access explanations -### Phase 4: Quality And Domain Packs +### Phase 4: Quality And Domain Packs - foundation complete - reusable report assertions and dataset comparison plans - execution evidence and trend views - versioned domain report packages owned by the relevant domain modules - import tooling for metadata where licensing and semantic mapping permit it +Packaged domain catalogues and an importer that translates actual source +metadata remain product depth. The native contracts, quality evidence, and +blocking import assessment required to implement them are present. + ## Product Boundary Dataflow remains responsible for producing governed tabular datasets. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md new file mode 100644 index 0000000..4e7ccec --- /dev/null +++ b/docs/USER_GUIDE.md @@ -0,0 +1,32 @@ +# Reporting User Guide + +## Open and run a report + +Open **Reporting** and choose an active report from the left catalogue. The +header identifies the immutable report revision that will be used. Select a +summary, detail, or pivot shape, choose dimensions and measures, enter any +report parameters, and select **Run**. + +Only rows authorized for the current tenant and principal reach the query +engine. A successful result shows its authorized row count, visualization, +and table. When a saved chart does not match an ad-hoc query, Reporting shows +the accessible table fallback instead of changing or rejecting the query. + +## Inspect evidence + +The right panel lists previous runs and definition/source pins. Select an +earlier run to inspect its retained result. Content and output hashes identify +the exact definition and output. Warnings explain freshness, inferred schema, +or provider diagnostics. A failed quality gate records a failed execution and +does not publish a result. + +## Save and export + +Use **Save current view** to keep the current query under your account. Saved +views do not alter the report definition. CSV and JSON downloads contain only +the authorized result of the selected execution. CSV values that begin with a +spreadsheet formula marker are escaped. + +Users with scheduling permission can create an hourly, daily, weekly, or +30-day interval from the current revision, parameters, and query. Scheduled +runs continue to use those exact pins until the schedule is edited. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4aadef2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "govoplan-reporting" +version = "0.1.14" +description = "GovOPlaN governed reporting and semantic BI module." +readme = "README.md" +requires-python = ">=3.12" +license = { text = "AGPL-3.0-or-later" } +authors = [{ name = "GovOPlaN" }] +dependencies = [ + "govoplan-core>=0.1.14", + "govoplan-access>=0.1.14", +] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +govoplan_reporting = ["py.typed"] + +[project.entry-points."govoplan.modules"] +reporting = "govoplan_reporting.backend.manifest:get_manifest" diff --git a/src/govoplan_reporting/__init__.py b/src/govoplan_reporting/__init__.py new file mode 100644 index 0000000..8d01ded --- /dev/null +++ b/src/govoplan_reporting/__init__.py @@ -0,0 +1,3 @@ +"""GovOPlaN Reporting module.""" + +__version__ = "0.1.14" diff --git a/src/govoplan_reporting/backend/__init__.py b/src/govoplan_reporting/backend/__init__.py new file mode 100644 index 0000000..9104c8f --- /dev/null +++ b/src/govoplan_reporting/backend/__init__.py @@ -0,0 +1 @@ +"""Reporting backend package.""" diff --git a/src/govoplan_reporting/backend/acl.py b/src/govoplan_reporting/backend/acl.py new file mode 100644 index 0000000..dc25c27 --- /dev/null +++ b/src/govoplan_reporting/backend/acl.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from govoplan_core.auth import has_scope +from govoplan_core.core.modules import AccessDecision + + +class ReportingScopeAclProvider: + def __init__(self, resource_type: str) -> None: + self.resource_type = resource_type + + def can_read(self, principal: object, resource_id: str) -> bool: + del resource_id + return has_scope(principal, "reporting:definition:read") + + def can_write(self, principal: object, resource_id: str) -> bool: + del resource_id + return has_scope(principal, "reporting:definition:write") + + def explain(self, principal: object, resource_id: str) -> AccessDecision: + del resource_id + allowed = self.can_read(principal, "") + return AccessDecision( + allowed=allowed, + reason=None if allowed else "Missing scope: reporting:definition:read", + requirements=("reporting:definition:read",), + ) + + +__all__ = ["ReportingScopeAclProvider"] diff --git a/src/govoplan_reporting/backend/contracts.py b/src/govoplan_reporting/backend/contracts.py new file mode 100644 index 0000000..c32517e --- /dev/null +++ b/src/govoplan_reporting/backend/contracts.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import datetime +from typing import Protocol, runtime_checkable + + +CAPABILITY_REPORTING_REGISTRY = "reporting.registry" +CAPABILITY_REPORTING_RUNNER = "reporting.runner" +CAPABILITY_REPORTING_SCHEDULER = "reporting.scheduler" +CAPABILITY_REPORTING_CHART_RENDERER = "reporting.chart_renderer" + + +@dataclass(frozen=True, slots=True) +class ReportingDatasetReadRequest: + source_ref: str + source_revision: int | None + parameters: Mapping[str, object] = field(default_factory=dict) + row_limit: int = 2_000 + expected_definition_hash: str | None = None + expected_source_fingerprints: tuple[Mapping[str, object], ...] = () + + +@dataclass(frozen=True, slots=True) +class ReportingDatasetReadResult: + rows: tuple[Mapping[str, object], ...] + total_rows: int + truncated: bool + output_hash: str + executor_version: str + definition_hash: str | None = None + source_fingerprints: tuple[Mapping[str, object], ...] = () + diagnostics: tuple[Mapping[str, object], ...] = () + generated_at: datetime | None = None + provenance: Mapping[str, object] = field(default_factory=dict) + + +@runtime_checkable +class ReportingReadModelProvider(Protocol): + def read_dataset( + self, + session: object, + principal: object, + *, + request: ReportingDatasetReadRequest, + ) -> ReportingDatasetReadResult: ... + + +@dataclass(frozen=True, slots=True) +class ReportingRowPolicyRequest: + dataset_id: str + dataset_revision: int + policy_ref: str + rows: tuple[Mapping[str, object], ...] + + +@dataclass(frozen=True, slots=True) +class ReportingRowPolicyResult: + rows: tuple[Mapping[str, object], ...] + decision_ref: str + provenance: Mapping[str, object] = field(default_factory=dict) + + +@runtime_checkable +class ReportingRowPolicyProvider(Protocol): + def authorize_rows( + self, + session: object, + principal: object, + *, + request: ReportingRowPolicyRequest, + ) -> ReportingRowPolicyResult: ... + + +@dataclass(frozen=True, slots=True) +class ReportingPublicationPayload: + publication_id: str + execution_id: str + tenant_id: str + report_id: str + report_revision: int + format: str + target_ref: str | None + rows: tuple[Mapping[str, object], ...] + schema: tuple[Mapping[str, object], ...] + output_hash: str + options: Mapping[str, object] = field(default_factory=dict) + + +@runtime_checkable +class ReportingPublicationTarget(Protocol): + def publish_report( + self, + session: object, + principal: object, + *, + payload: ReportingPublicationPayload, + ) -> Mapping[str, object]: ... + + +@runtime_checkable +class ReportingChartRenderer(Protocol): + def render( + self, *, visualization: object, result: object + ) -> Mapping[str, object]: ... + + +def capability(registry: object | None, name: str) -> object | None: + if ( + registry is None + or not hasattr(registry, "has_capability") + or not hasattr(registry, "capability") + or not registry.has_capability(name) + ): + return None + return registry.capability(name) + + +__all__ = [ + "CAPABILITY_REPORTING_CHART_RENDERER", + "CAPABILITY_REPORTING_REGISTRY", + "CAPABILITY_REPORTING_RUNNER", + "CAPABILITY_REPORTING_SCHEDULER", + "ReportingChartRenderer", + "ReportingDatasetReadRequest", + "ReportingDatasetReadResult", + "ReportingPublicationPayload", + "ReportingPublicationTarget", + "ReportingReadModelProvider", + "ReportingRowPolicyProvider", + "ReportingRowPolicyRequest", + "ReportingRowPolicyResult", + "capability", +] diff --git a/src/govoplan_reporting/backend/db/__init__.py b/src/govoplan_reporting/backend/db/__init__.py new file mode 100644 index 0000000..ed66639 --- /dev/null +++ b/src/govoplan_reporting/backend/db/__init__.py @@ -0,0 +1,25 @@ +"""Reporting database models.""" + +from govoplan_reporting.backend.db.models import ( + ReportingDefinitionGrant, + ReportingDefinitionIdentity, + ReportingDefinitionRevision, + ReportingExecution, + ReportingImportAssessment, + ReportingPublication, + ReportingQualityResult, + ReportingSavedView, + ReportingSchedule, +) + +__all__ = [ + "ReportingDefinitionGrant", + "ReportingDefinitionIdentity", + "ReportingDefinitionRevision", + "ReportingExecution", + "ReportingImportAssessment", + "ReportingPublication", + "ReportingQualityResult", + "ReportingSavedView", + "ReportingSchedule", +] diff --git a/src/govoplan_reporting/backend/db/models.py b/src/govoplan_reporting/backend/db/models.py new file mode 100644 index 0000000..638232e --- /dev/null +++ b/src/govoplan_reporting/backend/db/models.py @@ -0,0 +1,441 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any +import uuid + +from sqlalchemy import ( + Boolean, + DateTime, + ForeignKey, + Index, + Integer, + JSON, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import Mapped, mapped_column + +from govoplan_core.db.base import Base, TimestampMixin + + +def new_uuid() -> str: + return str(uuid.uuid4()) + + +class ReportingDefinitionIdentity(Base, TimestampMixin): + __tablename__ = "reporting_definition_identities" + __table_args__ = ( + UniqueConstraint( + "tenant_id", + "definition_kind", + "definition_id", + name="uq_reporting_definition_identity", + ), + UniqueConstraint( + "tenant_id", + "definition_kind", + "definition_key", + name="uq_reporting_definition_key", + ), + Index( + "ix_reporting_definition_catalog", + "tenant_id", + "definition_kind", + "definition_key", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + definition_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True) + definition_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + definition_key: Mapped[str] = mapped_column(String(120), nullable=False, index=True) + created_by: Mapped[str | None] = mapped_column( + String(255), nullable=True, index=True + ) + + +class ReportingDefinitionRevision(Base, TimestampMixin): + __tablename__ = "reporting_definition_revisions" + __table_args__ = ( + UniqueConstraint( + "tenant_id", + "definition_kind", + "definition_id", + "revision", + name="uq_reporting_definition_revision", + ), + UniqueConstraint( + "tenant_id", + "idempotency_key", + name="uq_reporting_definition_idempotency", + ), + UniqueConstraint( + "tenant_id", + "event_id", + name="uq_reporting_definition_event", + ), + Index( + "ix_reporting_definition_current", + "tenant_id", + "definition_kind", + "definition_id", + "superseded_at", + ), + Index( + "ix_reporting_definition_parent", + "tenant_id", + "parent_kind", + "parent_id", + "parent_revision", + ), + Index( + "ix_reporting_definition_list", + "tenant_id", + "definition_kind", + "status", + "recorded_at", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + identity_id: Mapped[str] = mapped_column( + ForeignKey("reporting_definition_identities.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + definition_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True) + definition_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + definition_key: Mapped[str] = mapped_column(String(120), nullable=False, index=True) + revision: Mapped[int] = mapped_column(Integer, nullable=False) + previous_revision_id: Mapped[str | None] = mapped_column( + ForeignKey("reporting_definition_revisions.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + parent_kind: Mapped[str | None] = mapped_column( + String(40), nullable=True, index=True + ) + parent_id: Mapped[str | None] = mapped_column( + String(255), nullable=True, index=True + ) + parent_revision: Mapped[int | None] = mapped_column(Integer, nullable=True) + name: Mapped[str] = mapped_column(String(500), nullable=False) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + status: Mapped[str] = mapped_column(String(30), nullable=False, index=True) + visibility: Mapped[str] = mapped_column(String(30), nullable=False, index=True) + content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + change_reason: Mapped[str] = mapped_column(String(1000), nullable=False) + idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False) + request_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + event_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + recorded_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, index=True + ) + superseded_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) + payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + changed_by: Mapped[str | None] = mapped_column( + String(255), nullable=True, index=True + ) + + +class ReportingExecution(Base, TimestampMixin): + __tablename__ = "reporting_executions" + __table_args__ = ( + UniqueConstraint("tenant_id", "execution_id", name="uq_reporting_execution"), + UniqueConstraint( + "tenant_id", + "idempotency_key", + name="uq_reporting_execution_idempotency", + ), + Index( + "ix_reporting_execution_history", + "tenant_id", + "report_id", + "started_at", + ), + Index( + "ix_reporting_execution_status", + "tenant_id", + "status", + "started_at", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + report_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + report_revision: Mapped[int] = mapped_column(Integer, nullable=False) + semantic_model_id: Mapped[str] = mapped_column( + String(255), nullable=False, index=True + ) + semantic_model_revision: Mapped[int] = mapped_column(Integer, nullable=False) + dataset_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + dataset_revision: Mapped[int] = mapped_column(Integer, nullable=False) + status: Mapped[str] = mapped_column(String(30), nullable=False, index=True) + idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False) + request_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + parameters: Mapped[dict[str, Any]] = mapped_column( + JSON, default=dict, nullable=False + ) + query: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + source_fingerprints: Mapped[list[dict[str, Any]]] = mapped_column( + JSON, default=list, nullable=False + ) + definition_hashes: Mapped[dict[str, str]] = mapped_column( + JSON, default=dict, nullable=False + ) + output_hash: Mapped[str | None] = mapped_column( + String(64), nullable=True, index=True + ) + executor_version: Mapped[str | None] = mapped_column(String(255), nullable=True) + result_schema: Mapped[list[dict[str, Any]]] = mapped_column( + JSON, default=list, nullable=False + ) + result_rows: Mapped[list[dict[str, Any]]] = mapped_column( + JSON, default=list, nullable=False + ) + total_rows: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + truncated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + diagnostics: Mapped[list[dict[str, Any]]] = mapped_column( + JSON, default=list, nullable=False + ) + provenance: Mapped[dict[str, Any]] = mapped_column( + JSON, default=dict, nullable=False + ) + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, index=True + ) + finished_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) + actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True) + + +class ReportingDefinitionGrant(Base, TimestampMixin): + __tablename__ = "reporting_definition_grants" + __table_args__ = ( + UniqueConstraint( + "tenant_id", + "definition_kind", + "definition_id", + "subject_kind", + "subject_id", + name="uq_reporting_definition_grant", + ), + Index( + "ix_reporting_definition_grant_subject", + "tenant_id", + "subject_kind", + "subject_id", + "active", + ), + Index( + "ix_reporting_definition_grant_object", + "tenant_id", + "definition_kind", + "definition_id", + "active", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + definition_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True) + definition_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + subject_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True) + subject_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + permissions: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) + active: Mapped[bool] = mapped_column( + Boolean, default=True, nullable=False, index=True + ) + source_revision: Mapped[int] = mapped_column(Integer, nullable=False) + + +class ReportingSavedView(Base, TimestampMixin): + __tablename__ = "reporting_saved_views" + __table_args__ = ( + UniqueConstraint("tenant_id", "view_id", name="uq_reporting_saved_view"), + Index( + "ix_reporting_saved_view_catalog", + "tenant_id", + "report_id", + "owner_id", + "shared", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + view_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + report_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + report_revision: Mapped[int] = mapped_column(Integer, nullable=False) + owner_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True) + owner_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + name: Mapped[str] = mapped_column(String(500), nullable=False) + revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + state: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + shared: Mapped[bool] = mapped_column( + Boolean, default=False, nullable=False, index=True + ) + access: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + + +class ReportingSchedule(Base, TimestampMixin): + __tablename__ = "reporting_schedules" + __table_args__ = ( + UniqueConstraint("tenant_id", "schedule_id", name="uq_reporting_schedule"), + Index( + "ix_reporting_schedule_due", + "enabled", + "next_run_at", + "tenant_id", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + schedule_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + report_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + report_revision: Mapped[int] = mapped_column(Integer, nullable=False) + name: Mapped[str] = mapped_column(String(500), nullable=False) + revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + trigger_kind: Mapped[str] = mapped_column(String(30), nullable=False) + trigger_config: Mapped[dict[str, Any]] = mapped_column( + JSON, default=dict, nullable=False + ) + parameters: Mapped[dict[str, Any]] = mapped_column( + JSON, default=dict, nullable=False + ) + query: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + publication_target: Mapped[dict[str, Any]] = mapped_column( + JSON, default=dict, nullable=False + ) + enabled: Mapped[bool] = mapped_column( + Boolean, default=True, nullable=False, index=True + ) + next_run_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) + last_run_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + last_execution_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + created_by: Mapped[str | None] = mapped_column(String(255), nullable=True) + + +class ReportingPublication(Base, TimestampMixin): + __tablename__ = "reporting_publications" + __table_args__ = ( + UniqueConstraint( + "tenant_id", "publication_id", name="uq_reporting_publication" + ), + UniqueConstraint( + "tenant_id", + "idempotency_key", + name="uq_reporting_publication_idempotency", + ), + Index( + "ix_reporting_publication_history", + "tenant_id", + "execution_id", + "created_at", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + publication_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + target_capability: Mapped[str] = mapped_column(String(255), nullable=False) + target_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True) + format: Mapped[str] = mapped_column(String(30), nullable=False) + status: Mapped[str] = mapped_column(String(30), nullable=False, index=True) + idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False) + evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + completed_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + + +class ReportingQualityResult(Base, TimestampMixin): + __tablename__ = "reporting_quality_results" + __table_args__ = ( + UniqueConstraint("tenant_id", "result_id", name="uq_reporting_quality_result"), + Index( + "ix_reporting_quality_history", + "tenant_id", + "quality_plan_id", + "evaluated_at", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + result_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + quality_plan_id: Mapped[str] = mapped_column( + String(255), nullable=False, index=True + ) + quality_plan_revision: Mapped[int] = mapped_column(Integer, nullable=False) + dataset_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + dataset_revision: Mapped[int] = mapped_column(Integer, nullable=False) + status: Mapped[str] = mapped_column(String(30), nullable=False, index=True) + output_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + assertions: Mapped[list[dict[str, Any]]] = mapped_column( + JSON, default=list, nullable=False + ) + source_fingerprints: Mapped[list[dict[str, Any]]] = mapped_column( + JSON, default=list, nullable=False + ) + evaluated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, index=True + ) + actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True) + + +class ReportingImportAssessment(Base, TimestampMixin): + __tablename__ = "reporting_import_assessments" + __table_args__ = ( + UniqueConstraint( + "tenant_id", "assessment_id", name="uq_reporting_import_assessment" + ), + Index( + "ix_reporting_import_history", + "tenant_id", + "source_system", + "created_at", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + assessment_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + source_system: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + source_id: Mapped[str] = mapped_column(String(500), nullable=False) + source_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False) + mapping_report: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + status: Mapped[str] = mapped_column(String(30), nullable=False, index=True) + accepted_approximations: Mapped[list[str]] = mapped_column( + JSON, default=list, nullable=False + ) + assessed_by: Mapped[str | None] = mapped_column(String(255), nullable=True) + + +__all__ = [ + "ReportingDefinitionGrant", + "ReportingDefinitionIdentity", + "ReportingDefinitionRevision", + "ReportingExecution", + "ReportingImportAssessment", + "ReportingPublication", + "ReportingQualityResult", + "ReportingSavedView", + "ReportingSchedule", +] diff --git a/src/govoplan_reporting/backend/definitions.py b/src/govoplan_reporting/backend/definitions.py new file mode 100644 index 0000000..a49f0f3 --- /dev/null +++ b/src/govoplan_reporting/backend/definitions.py @@ -0,0 +1,955 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import datetime +import hashlib +import json +from typing import Any +import uuid + +from sqlalchemy import and_, exists, func, or_ +from sqlalchemy.orm import Query, Session + +from govoplan_core.core.events import ( + EventActorRef, + EventObjectRef, + EventTenantRef, + PlatformEvent, + emit_platform_event, +) +from govoplan_core.security.module_permissions import scopes_grant_compatible +from govoplan_reporting.backend.db.models import ( + ReportingDefinitionGrant, + ReportingDefinitionIdentity, + ReportingDefinitionRevision, +) +from govoplan_reporting.backend.domain import ( + ReportingDefinitionRecord, + definition_from_row, +) +from govoplan_reporting.backend.schemas import validate_definition_payload + + +READ_SCOPE = "reporting:definition:read" +WRITE_SCOPE = "reporting:definition:write" +ADMIN_SCOPE = "reporting:definition:admin" + +DEFINITION_KINDS = frozenset({"dataset", "semantic_model", "report", "quality_plan"}) +STATUS_TRANSITIONS = { + "draft": frozenset({"active", "retired"}), + "active": frozenset({"draft", "retired"}), + "retired": frozenset({"draft"}), +} +SUBJECT_KINDS = frozenset( + { + "account", + "identity", + "group", + "role", + "function", + "function_assignment", + "organization_unit", + "service_account", + } +) + + +class ReportingDefinitionError(ValueError): + pass + + +def create_definition( + session: Session, + principal: object, + *, + definition_kind: str, + definition_id: str, + definition_key: str, + name: str, + description: str | None, + status: str, + visibility: str, + recorded_at: datetime, + change_reason: str, + payload: Mapping[str, Any], + idempotency_key: str, +) -> ReportingDefinitionRecord: + _require_scope(principal, WRITE_SCOPE) + tenant_id = _principal_tenant(principal) + kind = _definition_kind(definition_kind) + clean_id = _required(definition_id, "Reporting definition identifier", 255) + clean_key = _key(definition_key, "Reporting definition key") + clean_name = _required(name, "Reporting definition name", 500) + clean_description = _optional(description, "Reporting description", 100_000) + clean_status = _status(status) + clean_visibility = _visibility(visibility) + clean_reason = _required(change_reason, "Reporting change reason", 1_000) + _aware(recorded_at, "Reporting recorded_at") + validated_payload = validate_definition_payload(kind, dict(payload)) + parent_kind, parent_id, parent_revision = _parent_reference( + kind, + validated_payload, + ) + _validate_parent( + session, + tenant_id=tenant_id, + child_status=clean_status, + parent_kind=parent_kind, + parent_id=parent_id, + parent_revision=parent_revision, + ) + request = { + "definition_kind": kind, + "definition_id": clean_id, + "definition_key": clean_key, + "name": clean_name, + "description": clean_description, + "status": clean_status, + "visibility": clean_visibility, + "recorded_at": recorded_at, + "change_reason": clean_reason, + "payload": validated_payload, + } + request_sha256 = _sha256(request) + replay = _replay( + session, + tenant_id=tenant_id, + idempotency_key=idempotency_key, + request_sha256=request_sha256, + ) + if replay is not None: + return replay + if _identity(session, tenant_id, kind, clean_id) is not None: + raise ReportingDefinitionError("Reporting definition already exists.") + duplicate = ( + session.query(ReportingDefinitionIdentity.id) + .filter( + ReportingDefinitionIdentity.tenant_id == tenant_id, + ReportingDefinitionIdentity.definition_kind == kind, + ReportingDefinitionIdentity.definition_key == clean_key, + ) + .first() + ) + if duplicate is not None: + raise ReportingDefinitionError("Reporting definition key already exists.") + identity = ReportingDefinitionIdentity( + tenant_id=tenant_id, + definition_kind=kind, + definition_id=clean_id, + definition_key=clean_key, + created_by=_actor(principal), + ) + session.add(identity) + session.flush() + return _write_revision( + session, + principal, + identity=identity, + current=None, + revision=1, + name=clean_name, + description=clean_description, + status=clean_status, + visibility=clean_visibility, + recorded_at=recorded_at, + change_reason=clean_reason, + payload=validated_payload, + parent_kind=parent_kind, + parent_id=parent_id, + parent_revision=parent_revision, + idempotency_key=idempotency_key, + request_sha256=request_sha256, + operation="created", + ) + + +def update_definition( + session: Session, + principal: object, + *, + definition_kind: str, + definition_id: str, + expected_revision: int, + recorded_at: datetime, + change_reason: str, + idempotency_key: str, + changes: Mapping[str, object], +) -> ReportingDefinitionRecord: + tenant_id = _principal_tenant(principal) + kind = _definition_kind(definition_kind) + current_row = _current_row( + session, + tenant_id=tenant_id, + definition_kind=kind, + definition_id=definition_id, + lock=True, + ) + if current_row is None: + raise LookupError("Reporting definition not found.") + if not can_write_definition( + session, + principal, + definition_kind=kind, + definition_id=definition_id, + ): + raise PermissionError("Reporting definition write access is denied.") + current = definition_from_row(current_row) + unknown = set(changes) - { + "name", + "description", + "status", + "visibility", + "payload", + } + if unknown: + raise ReportingDefinitionError( + "Unsupported Reporting definition fields: " + ", ".join(sorted(unknown)) + ) + _aware(recorded_at, "Reporting recorded_at") + clean_reason = _required(change_reason, "Reporting change reason", 1_000) + request_sha256 = _sha256( + { + "definition_kind": kind, + "definition_id": definition_id, + "expected_revision": expected_revision, + "recorded_at": recorded_at, + "change_reason": clean_reason, + "changes": changes, + } + ) + replay = _replay( + session, + tenant_id=tenant_id, + idempotency_key=idempotency_key, + request_sha256=request_sha256, + ) + if replay is not None: + return replay + if current.revision != expected_revision: + raise ReportingDefinitionError( + "Reporting definition revision conflict: the expected revision is stale." + ) + next_status = _status(str(changes.get("status", current.status))) + if ( + next_status != current.status + and next_status not in STATUS_TRANSITIONS[current.status] + ): + raise ReportingDefinitionError( + f"Cannot move Reporting definition from {current.status!r} to {next_status!r}." + ) + next_payload = validate_definition_payload( + kind, + dict(changes.get("payload", current.payload)), + ) + parent_kind, parent_id, parent_revision = _parent_reference(kind, next_payload) + _validate_parent( + session, + tenant_id=tenant_id, + child_status=next_status, + parent_kind=parent_kind, + parent_id=parent_id, + parent_revision=parent_revision, + ) + identity = _identity(session, tenant_id, kind, definition_id) + if identity is None: + raise ReportingDefinitionError("Reporting definition identity is missing.") + return _write_revision( + session, + principal, + identity=identity, + current=current_row, + revision=current.revision + 1, + name=_required( + changes.get("name", current.name), "Reporting definition name", 500 + ), + description=_optional( + changes.get("description", current.description), + "Reporting description", + 100_000, + ), + status=next_status, + visibility=_visibility(str(changes.get("visibility", current.visibility))), + recorded_at=recorded_at, + change_reason=clean_reason, + payload=next_payload, + parent_kind=parent_kind, + parent_id=parent_id, + parent_revision=parent_revision, + idempotency_key=idempotency_key, + request_sha256=request_sha256, + operation="updated" if next_status == current.status else "state_changed", + ) + + +def get_definition( + session: Session, + principal: object, + *, + definition_kind: str, + definition_id: str, + revision: int | None = None, +) -> ReportingDefinitionRecord | None: + tenant_id = _principal_tenant(principal) + kind = _definition_kind(definition_kind) + if not can_read_definition( + session, + principal, + definition_kind=kind, + definition_id=definition_id, + ): + return None + query = session.query(ReportingDefinitionRevision).filter( + ReportingDefinitionRevision.tenant_id == tenant_id, + ReportingDefinitionRevision.definition_kind == kind, + ReportingDefinitionRevision.definition_id == definition_id, + ) + if revision is None: + query = query.filter(ReportingDefinitionRevision.superseded_at.is_(None)) + else: + query = query.filter(ReportingDefinitionRevision.revision == revision) + row = query.order_by(ReportingDefinitionRevision.revision.desc()).first() + return definition_from_row(row) if row is not None else None + + +def list_definitions( + session: Session, + principal: object, + *, + definition_kinds: Sequence[str] | None = None, + statuses: Sequence[str] | None = None, + query: str = "", + offset: int = 0, + limit: int = 100, +) -> tuple[tuple[ReportingDefinitionRecord, ...], int]: + _require_scope(principal, READ_SCOPE) + if offset < 0 or not 1 <= limit <= 200: + raise ReportingDefinitionError( + "Reporting list offset must be non-negative and limit between 1 and 200." + ) + kinds = tuple(dict.fromkeys(definition_kinds or DEFINITION_KINDS)) + if any(item not in DEFINITION_KINDS for item in kinds): + raise ReportingDefinitionError("Unsupported Reporting definition kind filter.") + statement = session.query(ReportingDefinitionRevision).filter( + ReportingDefinitionRevision.tenant_id == _principal_tenant(principal), + ReportingDefinitionRevision.superseded_at.is_(None), + ReportingDefinitionRevision.definition_kind.in_(kinds), + ) + statement = _filter_accessible(statement, principal) + if statuses: + statement = statement.filter( + ReportingDefinitionRevision.status.in_(tuple(dict.fromkeys(statuses))) + ) + clean_query = query.strip().casefold() + if clean_query: + pattern = f"%{clean_query}%" + statement = statement.filter( + or_( + func.lower(ReportingDefinitionRevision.name).like(pattern), + func.lower(ReportingDefinitionRevision.definition_key).like(pattern), + func.lower(ReportingDefinitionRevision.description).like(pattern), + ) + ) + total = int(statement.with_entities(func.count()).scalar() or 0) + rows = ( + statement.order_by( + ReportingDefinitionRevision.definition_kind.asc(), + ReportingDefinitionRevision.name.asc(), + ) + .offset(offset) + .limit(limit) + .all() + ) + return tuple(definition_from_row(row) for row in rows), total + + +def definition_history( + session: Session, + principal: object, + *, + definition_kind: str, + definition_id: str, + limit: int = 100, +) -> tuple[ReportingDefinitionRecord, ...]: + if not can_read_definition( + session, + principal, + definition_kind=definition_kind, + definition_id=definition_id, + ): + return () + rows = ( + session.query(ReportingDefinitionRevision) + .filter( + ReportingDefinitionRevision.tenant_id == _principal_tenant(principal), + ReportingDefinitionRevision.definition_kind == definition_kind, + ReportingDefinitionRevision.definition_id == definition_id, + ) + .order_by(ReportingDefinitionRevision.revision.desc()) + .limit(max(1, min(limit, 200))) + .all() + ) + return tuple(definition_from_row(row) for row in rows) + + +def can_read_definition( + session: Session, + principal: object, + *, + definition_kind: str, + definition_id: str, +) -> bool: + if not _has_scope(principal, READ_SCOPE): + return False + return _can_access( + session, + principal, + definition_kind=definition_kind, + definition_id=definition_id, + permission="read", + ) + + +def can_write_definition( + session: Session, + principal: object, + *, + definition_kind: str, + definition_id: str, +) -> bool: + if not _has_scope(principal, WRITE_SCOPE): + return False + return _can_access( + session, + principal, + definition_kind=definition_kind, + definition_id=definition_id, + permission="write", + ) + + +def _write_revision( + session: Session, + principal: object, + *, + identity: ReportingDefinitionIdentity, + current: ReportingDefinitionRevision | None, + revision: int, + name: str, + description: str | None, + status: str, + visibility: str, + recorded_at: datetime, + change_reason: str, + payload: dict[str, Any], + parent_kind: str | None, + parent_id: str | None, + parent_revision: int | None, + idempotency_key: str, + request_sha256: str, + operation: str, +) -> ReportingDefinitionRecord: + if current is not None: + current.superseded_at = recorded_at + content_hash = _sha256( + { + "name": name, + "description": description, + "status": status, + "visibility": visibility, + "parent_kind": parent_kind, + "parent_id": parent_id, + "parent_revision": parent_revision, + "payload": payload, + } + ) + event_id = str(uuid.uuid4()) + row = ReportingDefinitionRevision( + tenant_id=identity.tenant_id, + identity_id=identity.id, + definition_kind=identity.definition_kind, + definition_id=identity.definition_id, + definition_key=identity.definition_key, + revision=revision, + previous_revision_id=current.id if current else None, + parent_kind=parent_kind, + parent_id=parent_id, + parent_revision=parent_revision, + name=name, + description=description, + status=status, + visibility=visibility, + content_hash=content_hash, + change_reason=change_reason, + idempotency_key=_required(idempotency_key, "Reporting idempotency key", 255), + request_sha256=request_sha256, + event_id=event_id, + recorded_at=recorded_at, + payload=payload, + changed_by=_actor(principal), + ) + session.add(row) + session.flush() + _sync_grants(session, row) + event_type = f"reporting.{identity.definition_kind}.{operation}" + emit_platform_event( + session, + PlatformEvent( + event_id=event_id, + type=event_type, + module_id="reporting", + payload={ + "definition_kind": identity.definition_kind, + "definition_key": identity.definition_key, + "revision": revision, + "status": status, + "content_hash": content_hash, + "change_reason": change_reason, + }, + occurred_at=recorded_at, + actor=EventActorRef(type="account", id=_actor(principal)), + tenant=EventTenantRef(id=identity.tenant_id), + resource=EventObjectRef( + type=f"reporting_{identity.definition_kind}", + id=identity.definition_id, + label=name, + ), + classification="restricted" if visibility == "restricted" else "internal", + ), + ) + return definition_from_row(row) + + +def _sync_grants(session: Session, row: ReportingDefinitionRevision) -> None: + access_policy = row.payload.get("access_policy") + subjects = ( + access_policy.get("subjects", []) if isinstance(access_policy, Mapping) else [] + ) + desired: dict[tuple[str, str], list[str]] = {} + if not isinstance(subjects, list): + raise ReportingDefinitionError("Report access_policy subjects must be a list.") + for subject in subjects: + if not isinstance(subject, Mapping): + raise ReportingDefinitionError("Report access subjects must be objects.") + kind = str(subject.get("kind") or "") + subject_id = str(subject.get("id") or "").strip() + if kind not in SUBJECT_KINDS or not subject_id: + raise ReportingDefinitionError("Report access subject is invalid.") + permissions = tuple( + dict.fromkeys(str(item) for item in subject.get("permissions", ["read"])) + ) + if not permissions or set(permissions) - {"read", "write", "publish", "admin"}: + raise ReportingDefinitionError("Report access permissions are invalid.") + desired[(kind, subject_id)] = list(permissions) + rows = ( + session.query(ReportingDefinitionGrant) + .filter( + ReportingDefinitionGrant.tenant_id == row.tenant_id, + ReportingDefinitionGrant.definition_kind == row.definition_kind, + ReportingDefinitionGrant.definition_id == row.definition_id, + ) + .all() + ) + existing = {(item.subject_kind, item.subject_id): item for item in rows} + for key, item in existing.items(): + if key not in desired: + item.active = False + item.source_revision = row.revision + for key, permissions in desired.items(): + item = existing.get(key) + if item is None: + session.add( + ReportingDefinitionGrant( + tenant_id=row.tenant_id, + definition_kind=row.definition_kind, + definition_id=row.definition_id, + subject_kind=key[0], + subject_id=key[1], + permissions=permissions, + active=True, + source_revision=row.revision, + ) + ) + else: + item.permissions = permissions + item.active = True + item.source_revision = row.revision + session.flush() + + +def _validate_parent( + session: Session, + *, + tenant_id: str, + child_status: str, + parent_kind: str | None, + parent_id: str | None, + parent_revision: int | None, +) -> None: + if parent_kind is None: + return + row = ( + session.query(ReportingDefinitionRevision) + .filter( + ReportingDefinitionRevision.tenant_id == tenant_id, + ReportingDefinitionRevision.definition_kind == parent_kind, + ReportingDefinitionRevision.definition_id == parent_id, + ReportingDefinitionRevision.revision == parent_revision, + ) + .one_or_none() + ) + if row is None: + raise ReportingDefinitionError( + f"Reporting definition references a missing {parent_kind} revision." + ) + if child_status == "active" and row.status != "active": + raise ReportingDefinitionError( + f"An active Reporting definition requires an active {parent_kind} revision." + ) + + +def _parent_reference( + definition_kind: str, + payload: Mapping[str, object], +) -> tuple[str | None, str | None, int | None]: + if definition_kind == "semantic_model": + return "dataset", str(payload["dataset_id"]), int(payload["dataset_revision"]) + if definition_kind == "report": + return ( + "semantic_model", + str(payload["semantic_model_id"]), + int(payload["semantic_model_revision"]), + ) + if definition_kind == "quality_plan": + return "dataset", str(payload["dataset_id"]), int(payload["dataset_revision"]) + return None, None, None + + +def _replay( + session: Session, + *, + tenant_id: str, + idempotency_key: str, + request_sha256: str, +) -> ReportingDefinitionRecord | None: + key = _required(idempotency_key, "Reporting idempotency key", 255) + row = ( + session.query(ReportingDefinitionRevision) + .filter( + ReportingDefinitionRevision.tenant_id == tenant_id, + ReportingDefinitionRevision.idempotency_key == key, + ) + .one_or_none() + ) + if row is None: + return None + if row.request_sha256 != request_sha256: + raise ReportingDefinitionError( + "Reporting idempotency conflict: the key belongs to another request." + ) + return definition_from_row(row) + + +def _can_access( + session: Session, + principal: object, + *, + definition_kind: str, + definition_id: str, + permission: str, +) -> bool: + tenant_id = _principal_tenant(principal) + row = _current_row( + session, + tenant_id=tenant_id, + definition_kind=definition_kind, + definition_id=definition_id, + lock=False, + ) + if row is None: + return False + if _has_scope(principal, ADMIN_SCOPE): + return True + identity = _identity(session, tenant_id, definition_kind, definition_id) + actor_ids = _actor_ids(principal) + if identity is not None and identity.created_by in actor_ids: + return True + if permission == "read" and row.visibility == "tenant": + return True + subjects = _principal_subjects(principal) + if not subjects: + return False + clauses = [ + and_( + ReportingDefinitionGrant.subject_kind == kind, + ReportingDefinitionGrant.subject_id == subject_id, + ) + for kind, subject_id in subjects + ] + grants = ( + session.query(ReportingDefinitionGrant) + .filter( + ReportingDefinitionGrant.tenant_id == tenant_id, + ReportingDefinitionGrant.definition_kind == definition_kind, + ReportingDefinitionGrant.definition_id == definition_id, + ReportingDefinitionGrant.active.is_(True), + or_(*clauses), + ) + .all() + ) + return any( + permission == "read" + or permission in set(item.permissions or ()) + or "admin" in set(item.permissions or ()) + for item in grants + ) + + +def _require_scope(principal: object, scope: str) -> None: + if not _has_scope(principal, scope): + raise PermissionError(f"Missing scope: {scope}") + + +def _filter_accessible(query: Query, principal: object) -> Query: + if _has_scope(principal, ADMIN_SCOPE): + return query + conditions = [ReportingDefinitionRevision.visibility == "tenant"] + actor_ids = _actor_ids(principal) + if actor_ids: + conditions.append( + exists() + .where( + ReportingDefinitionIdentity.id + == ReportingDefinitionRevision.identity_id + ) + .where(ReportingDefinitionIdentity.created_by.in_(actor_ids)) + ) + subjects = _principal_subjects(principal) + if subjects: + conditions.append( + exists() + .where( + ReportingDefinitionGrant.tenant_id + == ReportingDefinitionRevision.tenant_id + ) + .where( + ReportingDefinitionGrant.definition_kind + == ReportingDefinitionRevision.definition_kind + ) + .where( + ReportingDefinitionGrant.definition_id + == ReportingDefinitionRevision.definition_id + ) + .where(ReportingDefinitionGrant.active.is_(True)) + .where( + or_( + *( + and_( + ReportingDefinitionGrant.subject_kind == kind, + ReportingDefinitionGrant.subject_id == subject_id, + ) + for kind, subject_id in subjects + ) + ) + ) + ) + return query.filter(or_(*conditions)) + + +def _current_row( + session: Session, + *, + tenant_id: str, + definition_kind: str, + definition_id: str, + lock: bool, +) -> ReportingDefinitionRevision | None: + query = session.query(ReportingDefinitionRevision).filter( + ReportingDefinitionRevision.tenant_id == tenant_id, + ReportingDefinitionRevision.definition_kind == definition_kind, + ReportingDefinitionRevision.definition_id == definition_id, + ReportingDefinitionRevision.superseded_at.is_(None), + ) + if lock: + query = query.with_for_update() + return query.one_or_none() + + +def _identity( + session: Session, + tenant_id: str, + definition_kind: str, + definition_id: str, +) -> ReportingDefinitionIdentity | None: + return ( + session.query(ReportingDefinitionIdentity) + .filter( + ReportingDefinitionIdentity.tenant_id == tenant_id, + ReportingDefinitionIdentity.definition_kind == definition_kind, + ReportingDefinitionIdentity.definition_id == definition_id, + ) + .one_or_none() + ) + + +def _principal_subjects(principal: object) -> tuple[tuple[str, str], ...]: + values: list[tuple[str, str]] = [] + for kind, attribute in ( + ("account", "account_id"), + ("identity", "identity_id"), + ("function_assignment", "acting_assignment_id"), + ("service_account", "service_account_id"), + ): + value = getattr(principal, attribute, None) + if str(value or "").strip(): + values.append((kind, str(value))) + for kind, attribute in ( + ("group", "group_ids"), + ("role", "role_ids"), + ("function_assignment", "function_assignment_ids"), + ): + values.extend( + (kind, str(value)) + for value in getattr(principal, attribute, ()) or () + if str(value or "").strip() + ) + return tuple(dict.fromkeys(values)) + + +def _actor(principal: object) -> str | None: + for value in ( + getattr(principal, "account_id", None), + getattr(principal, "identity_id", None), + getattr(principal, "membership_id", None), + ): + if str(value or "").strip(): + return str(value) + return None + + +def _actor_ids(principal: object) -> tuple[str, ...]: + user = getattr(principal, "user", None) + return tuple( + dict.fromkeys( + str(value) + for value in ( + getattr(principal, "account_id", None), + getattr(principal, "identity_id", None), + getattr(principal, "membership_id", None), + getattr(user, "id", None), + ) + if str(value or "").strip() + ) + ) + + +def _principal_tenant(principal: object) -> str: + tenant_id = str(getattr(principal, "tenant_id", "") or "").strip() + if not tenant_id: + raise ReportingDefinitionError( + "Reporting operations require a tenant-bound principal." + ) + return tenant_id + + +def _has_scope(principal: object, scope: str) -> bool: + method = getattr(principal, "has", None) + if callable(method): + return bool(method(scope)) + return scopes_grant_compatible( + frozenset(getattr(principal, "scopes", ()) or ()), + scope, + ) + + +def _definition_kind(value: str) -> str: + if value not in DEFINITION_KINDS: + raise ReportingDefinitionError( + f"Unsupported Reporting definition kind: {value!r}." + ) + return value + + +def _status(value: str) -> str: + if value not in STATUS_TRANSITIONS: + raise ReportingDefinitionError(f"Unsupported Reporting status: {value!r}.") + return value + + +def _visibility(value: str) -> str: + if value not in {"tenant", "restricted"}: + raise ReportingDefinitionError(f"Unsupported Reporting visibility: {value!r}.") + return value + + +def _key(value: object, label: str) -> str: + result = _required(value, label, 120).casefold() + if any( + character not in "abcdefghijklmnopqrstuvwxyz0123456789._-" + for character in result + ): + raise ReportingDefinitionError( + f"{label} may contain only letters, digits, dot, underscore, and hyphen." + ) + return result + + +def _required(value: object, label: str, maximum: int) -> str: + result = str(value or "").strip() + if not result: + raise ReportingDefinitionError(f"{label} is required.") + if len(result) > maximum: + raise ReportingDefinitionError(f"{label} is limited to {maximum} characters.") + return result + + +def _optional(value: object, label: str, maximum: int) -> str | None: + result = str(value or "").strip() + if not result: + return None + if len(result) > maximum: + raise ReportingDefinitionError(f"{label} is limited to {maximum} characters.") + return result + + +def _aware(value: datetime, label: str) -> None: + if value.tzinfo is None or value.utcoffset() is None: + raise ReportingDefinitionError(f"{label} must include a timezone.") + + +def _sha256(value: object) -> str: + return hashlib.sha256( + json.dumps( + _json_value(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + ).hexdigest() + + +def _json_value(value: object) -> Any: + if isinstance(value, datetime): + return value.isoformat() + if hasattr(value, "model_dump"): + return _json_value(value.model_dump(mode="json")) + if isinstance(value, Mapping): + return {str(key): _json_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + return value + + +__all__ = [ + "ADMIN_SCOPE", + "DEFINITION_KINDS", + "READ_SCOPE", + "ReportingDefinitionError", + "WRITE_SCOPE", + "can_read_definition", + "can_write_definition", + "create_definition", + "definition_history", + "get_definition", + "list_definitions", + "update_definition", +] diff --git a/src/govoplan_reporting/backend/domain.py b/src/govoplan_reporting/backend/domain.py new file mode 100644 index 0000000..511fdd0 --- /dev/null +++ b/src/govoplan_reporting/backend/domain.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + +from govoplan_reporting.backend.db.models import ReportingDefinitionRevision + + +@dataclass(frozen=True, slots=True) +class ReportingDefinitionRecord: + tenant_id: str + definition_kind: str + definition_id: str + definition_key: str + revision: int + name: str + description: str | None + status: str + visibility: str + content_hash: str + recorded_at: datetime + change_reason: str + payload: dict[str, Any] + parent_kind: str | None = None + parent_id: str | None = None + parent_revision: int | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "tenant_id": self.tenant_id, + "definition_kind": self.definition_kind, + "definition_id": self.definition_id, + "definition_key": self.definition_key, + "revision": self.revision, + "name": self.name, + "description": self.description, + "status": self.status, + "visibility": self.visibility, + "content_hash": self.content_hash, + "parent_kind": self.parent_kind, + "parent_id": self.parent_id, + "parent_revision": self.parent_revision, + "recorded_at": _datetime_text(self.recorded_at), + "change_reason": self.change_reason, + "payload": dict(self.payload), + } + + +def definition_from_row( + row: ReportingDefinitionRevision, +) -> ReportingDefinitionRecord: + return ReportingDefinitionRecord( + tenant_id=row.tenant_id, + definition_kind=row.definition_kind, + definition_id=row.definition_id, + definition_key=row.definition_key, + revision=row.revision, + name=row.name, + description=row.description, + status=row.status, + visibility=row.visibility, + content_hash=row.content_hash, + parent_kind=row.parent_kind, + parent_id=row.parent_id, + parent_revision=row.parent_revision, + recorded_at=row.recorded_at, + change_reason=row.change_reason, + payload=dict(row.payload or {}), + ) + + +def _datetime_text(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=UTC) + return value.isoformat() + + +__all__ = ["ReportingDefinitionRecord", "definition_from_row"] diff --git a/src/govoplan_reporting/backend/execution.py b/src/govoplan_reporting/backend/execution.py new file mode 100644 index 0000000..0a0b3a6 --- /dev/null +++ b/src/govoplan_reporting/backend/execution.py @@ -0,0 +1,976 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import UTC, date, datetime +import hashlib +import json +from typing import Any +import uuid + +from sqlalchemy.orm import Session + +from govoplan_core.core.dataflows import ( + DataflowDatasetRequest, + dataflow_dataset_output, +) +from govoplan_core.core.events import ( + EventActorRef, + EventObjectRef, + EventTenantRef, + PlatformEvent, + emit_platform_event, +) +from govoplan_core.security.module_permissions import scopes_grant_compatible +from govoplan_core.security.time import utc_now +from govoplan_reporting.backend.contracts import ( + CAPABILITY_REPORTING_CHART_RENDERER, + ReportingChartRenderer, + ReportingDatasetReadRequest, + ReportingDatasetReadResult, + ReportingReadModelProvider, + ReportingRowPolicyProvider, + ReportingRowPolicyRequest, + capability, +) +from govoplan_reporting.backend.db.models import ( + ReportingExecution, + ReportingQualityResult, +) +from govoplan_reporting.backend.definitions import get_definition, list_definitions +from govoplan_reporting.backend.query_engine import ( + QUERY_ENGINE_VERSION, + DefaultChartRenderer, + QueryResult, + execute_semantic_query, +) +from govoplan_reporting.backend.schemas import ( + DatasetDefinition, + QualityPlanDefinition, + ReportDefinition, + ReportQuery, + SemanticModelDefinition, +) + + +RUN_SCOPE = "reporting:report:run" +QUALITY_SCOPE = "reporting:quality:run" + + +class ReportingExecutionError(ValueError): + pass + + +class ReportingExecutionFailure(ReportingExecutionError): + def __init__(self, message: str, execution_id: str) -> None: + super().__init__(message) + self.execution_id = execution_id + + +class SqlReportingRunner: + def __init__(self, registry: object | None) -> None: + self.registry = registry + + def execute( + self, + session: object, + principal: object, + *, + report_id: str, + report_revision: int | None = None, + parameters: Mapping[str, object] | None = None, + query: ReportQuery | None = None, + idempotency_key: str, + ) -> Mapping[str, object]: + return execute_report( + _session(session), + principal, + registry=self.registry, + report_id=report_id, + report_revision=report_revision, + parameters=parameters or {}, + query=query, + idempotency_key=idempotency_key, + ) + + def get_execution( + self, + session: object, + principal: object, + *, + execution_id: str, + ) -> Mapping[str, object] | None: + return get_execution(_session(session), principal, execution_id=execution_id) + + +def execute_report( + session: Session, + principal: object, + *, + registry: object | None, + report_id: str, + report_revision: int | None, + parameters: Mapping[str, object], + query: ReportQuery | None, + idempotency_key: str, +) -> dict[str, object]: + _require_scope(principal, RUN_SCOPE) + report_record = get_definition( + session, + principal, + definition_kind="report", + definition_id=report_id, + revision=report_revision, + ) + if report_record is None: + raise LookupError("Reporting report definition not found.") + if report_record.status != "active": + raise ReportingExecutionError("Only active report definitions can run.") + report = ReportDefinition.model_validate(report_record.payload) + semantic_record = get_definition( + session, + principal, + definition_kind="semantic_model", + definition_id=report.semantic_model_id, + revision=report.semantic_model_revision, + ) + if semantic_record is None or semantic_record.status != "active": + raise ReportingExecutionError( + "The report's pinned semantic model is unavailable or inactive." + ) + semantic = SemanticModelDefinition.model_validate(semantic_record.payload) + dataset_record = get_definition( + session, + principal, + definition_kind="dataset", + definition_id=semantic.dataset_id, + revision=semantic.dataset_revision, + ) + if dataset_record is None or dataset_record.status != "active": + raise ReportingExecutionError( + "The report's pinned analytical dataset is unavailable or inactive." + ) + dataset = DatasetDefinition.model_validate(dataset_record.payload) + bound_parameters = _bind_parameters(report, parameters) + effective_query = query or report.default_query + clean_idempotency_key = _required( + idempotency_key, + "Reporting execution idempotency key", + 255, + ) + request_sha256 = _sha256( + { + "report_id": report_id, + "report_revision": report_record.revision, + "semantic_model_id": semantic_record.definition_id, + "semantic_model_revision": semantic_record.revision, + "dataset_id": dataset_record.definition_id, + "dataset_revision": dataset_record.revision, + "parameters": bound_parameters, + "query": effective_query.model_dump(mode="json"), + } + ) + replay = _execution_replay( + session, + tenant_id=_tenant(principal), + idempotency_key=clean_idempotency_key, + request_sha256=request_sha256, + ) + if replay is not None: + return _execution_payload(replay, report=report, registry=registry) + started_at = utc_now() + execution = ReportingExecution( + tenant_id=_tenant(principal), + execution_id=str(uuid.uuid4()), + report_id=report_record.definition_id, + report_revision=report_record.revision, + semantic_model_id=semantic_record.definition_id, + semantic_model_revision=semantic_record.revision, + dataset_id=dataset_record.definition_id, + dataset_revision=dataset_record.revision, + status="running", + idempotency_key=clean_idempotency_key, + request_sha256=request_sha256, + parameters=_json_value(bound_parameters), + query=effective_query.model_dump(mode="json"), + definition_hashes={ + "report": report_record.content_hash, + "semantic_model": semantic_record.content_hash, + "dataset": dataset_record.content_hash, + }, + started_at=started_at, + actor_id=_actor(principal), + ) + session.add(execution) + session.flush() + try: + source = _read_dataset( + session, + principal, + registry=registry, + dataset=dataset, + parameters=bound_parameters, + ) + diagnostics = list(source.diagnostics) + diagnostics.extend(_freshness_diagnostics(dataset, source, now=started_at)) + normalized_rows = tuple(_json_value(dict(row)) for row in source.rows) + diagnostics.extend(_validate_schema(dataset, normalized_rows)) + authorized_rows, policy_provenance = _apply_row_policy( + session, + principal, + registry=registry, + dataset_id=dataset_record.definition_id, + dataset_revision=dataset_record.revision, + dataset=dataset, + rows=normalized_rows, + ) + quality_results = _evaluate_blocking_quality_plans( + session, + principal, + dataset_id=dataset_record.definition_id, + dataset_revision=dataset_record.revision, + rows=authorized_rows, + output_hash=source.output_hash, + source_fingerprints=source.source_fingerprints, + ) + result = execute_semantic_query(authorized_rows, semantic, effective_query) + output_hash = _sha256( + { + "rows": result.rows, + "schema": result.schema, + "query": effective_query.model_dump(mode="json"), + "source_output_hash": source.output_hash, + } + ) + execution.status = "succeeded" + execution.source_fingerprints = _json_value(source.source_fingerprints) + execution.output_hash = output_hash + execution.executor_version = f"{QUERY_ENGINE_VERSION}+{source.executor_version}" + execution.result_schema = list(result.schema) + execution.result_rows = list(result.rows) + execution.total_rows = result.total_rows + execution.truncated = result.truncated or source.truncated + execution.diagnostics = _json_value(diagnostics) + execution.provenance = { + "source": _json_value(source.provenance), + "source_output_hash": source.output_hash, + "row_policy": _json_value(policy_provenance), + "quality_result_ids": [item.result_id for item in quality_results], + "authorized_source_rows": len(authorized_rows), + "report_content_hash": report_record.content_hash, + "semantic_model_content_hash": semantic_record.content_hash, + "dataset_content_hash": dataset_record.content_hash, + } + execution.finished_at = utc_now() + session.flush() + _emit_execution_event(session, execution, report_record.name) + return _execution_payload(execution, report=report, registry=registry) + except Exception as exc: + execution.status = "failed" + execution.finished_at = utc_now() + execution.diagnostics = [ + { + "severity": "error", + "code": "report_execution_failed", + "message": str(exc), + } + ] + session.flush() + _emit_execution_event(session, execution, report_record.name) + raise ReportingExecutionFailure(str(exc), execution.execution_id) from exc + + +def get_execution( + session: Session, + principal: object, + *, + execution_id: str, +) -> dict[str, object] | None: + row = ( + session.query(ReportingExecution) + .filter( + ReportingExecution.tenant_id == _tenant(principal), + ReportingExecution.execution_id == execution_id, + ) + .one_or_none() + ) + if row is None: + return None + report_record = get_definition( + session, + principal, + definition_kind="report", + definition_id=row.report_id, + revision=row.report_revision, + ) + if report_record is None: + return None + return _execution_payload( + row, + report=ReportDefinition.model_validate(report_record.payload), + registry=None, + ) + + +def list_executions( + session: Session, + principal: object, + *, + report_id: str, + limit: int = 100, +) -> tuple[dict[str, object], ...]: + if ( + get_definition( + session, + principal, + definition_kind="report", + definition_id=report_id, + ) + is None + ): + return () + rows = ( + session.query(ReportingExecution) + .filter( + ReportingExecution.tenant_id == _tenant(principal), + ReportingExecution.report_id == report_id, + ) + .order_by(ReportingExecution.started_at.desc()) + .limit(max(1, min(limit, 200))) + .all() + ) + return tuple(_execution_payload(row, report=None, registry=None) for row in rows) + + +def _read_dataset( + session: Session, + principal: object, + *, + registry: object | None, + dataset: DatasetDefinition, + parameters: Mapping[str, object], +) -> ReportingDatasetReadResult: + source_parameters = {**dataset.source_parameters, **parameters} + if dataset.source_kind == "static": + rows = tuple(dict(item) for item in dataset.static_rows) + return ReportingDatasetReadResult( + rows=rows, + total_rows=len(rows), + truncated=False, + output_hash=_sha256(rows), + executor_version="reporting-static-v1", + definition_hash=dataset.definition_hash, + generated_at=utc_now(), + provenance={"source_kind": "static", "source_ref": dataset.source_ref}, + ) + if dataset.source_kind == "dataflow": + provider = dataflow_dataset_output(registry) + if provider is None: + raise ReportingExecutionError( + "The Dataflow dataset provider is not enabled." + ) + result = provider.read_output( + session, + principal, + request=DataflowDatasetRequest( + pipeline_ref=dataset.source_ref, + revision=dataset.source_revision or 0, + parameters=source_parameters, + row_limit=2_000, + expected_definition_hash=dataset.definition_hash, + expected_source_fingerprints=tuple( + dataset.expected_source_fingerprints + ), + ), + ) + return ReportingDatasetReadResult( + rows=result.rows, + total_rows=result.total_rows, + truncated=result.truncated, + output_hash=result.output_hash, + executor_version=result.executor_version, + definition_hash=result.definition_hash, + source_fingerprints=result.source_fingerprints, + diagnostics=result.diagnostics, + generated_at=result.generated_at, + provenance=result.provenance, + ) + provider = capability(registry, dataset.source_ref) + if not isinstance(provider, ReportingReadModelProvider): + raise ReportingExecutionError( + f"Reporting read-model provider {dataset.source_ref!r} is unavailable." + ) + return provider.read_dataset( + session, + principal, + request=ReportingDatasetReadRequest( + source_ref=dataset.source_ref, + source_revision=dataset.source_revision, + parameters=source_parameters, + expected_definition_hash=dataset.definition_hash, + expected_source_fingerprints=tuple(dataset.expected_source_fingerprints), + ), + ) + + +def _apply_row_policy( + session: Session, + principal: object, + *, + registry: object | None, + dataset_id: str, + dataset_revision: int, + dataset: DatasetDefinition, + rows: tuple[Mapping[str, object], ...], +) -> tuple[tuple[Mapping[str, object], ...], Mapping[str, object]]: + if dataset.row_policy_ref is None: + return rows, {"mode": "dataset_access_only"} + provider = capability(registry, dataset.row_policy_ref) + if not isinstance(provider, ReportingRowPolicyProvider): + raise ReportingExecutionError( + f"Required row-policy provider {dataset.row_policy_ref!r} is unavailable." + ) + result = provider.authorize_rows( + session, + principal, + request=ReportingRowPolicyRequest( + dataset_id=dataset_id, + dataset_revision=dataset_revision, + policy_ref=dataset.row_policy_ref, + rows=rows, + ), + ) + if len(result.rows) > len(rows): + raise ReportingExecutionError( + "A row-policy provider returned more rows than it received." + ) + return result.rows, { + "mode": "provider", + "provider": dataset.row_policy_ref, + "decision_ref": result.decision_ref, + **dict(result.provenance), + } + + +def _freshness_diagnostics( + dataset: DatasetDefinition, + source: ReportingDatasetReadResult, + *, + now: datetime, +) -> tuple[dict[str, object], ...]: + policy = dataset.freshness + if ( + policy.require_source_fingerprints + and not source.source_fingerprints + and dataset.source_kind != "static" + ): + raise ReportingExecutionError( + "The analytical dataset requires source fingerprints, but the provider returned none." + ) + if policy.max_age_seconds is None: + return () + generated_at = source.generated_at + if generated_at is None: + message = "The dataset provider did not report a generation timestamp." + if policy.stale_action == "block": + raise ReportingExecutionError(message) + return ( + {"severity": "warning", "code": "freshness_unknown", "message": message}, + ) + if generated_at.tzinfo is None: + generated_at = generated_at.replace(tzinfo=UTC) + age_seconds = max(0, int((now - generated_at).total_seconds())) + if age_seconds <= policy.max_age_seconds: + return () + message = ( + f"Dataset age {age_seconds}s exceeds the configured " + f"{policy.max_age_seconds}s freshness limit." + ) + if policy.stale_action == "block": + raise ReportingExecutionError(message) + if policy.stale_action == "warn": + return ({"severity": "warning", "code": "dataset_stale", "message": message},) + return () + + +def _validate_schema( + dataset: DatasetDefinition, + rows: Sequence[Mapping[str, object]], +) -> tuple[dict[str, object], ...]: + if not dataset.fields: + return ( + { + "severity": "info", + "code": "schema_inferred", + "message": "Dataset schema is inferred because no explicit fields are pinned.", + }, + ) + errors: list[str] = [] + for field in dataset.fields: + for index, row in enumerate(rows): + value = row.get(field.name) + if value is None: + if not field.nullable: + errors.append(f"row {index + 1}: {field.name} is null") + continue + if not _matches_type(value, field.type): + errors.append( + f"row {index + 1}: {field.name} does not match {field.type}" + ) + if len(errors) >= 20: + break + if len(errors) >= 20: + break + if errors: + raise ReportingExecutionError( + "Analytical dataset schema validation failed: " + "; ".join(errors) + ) + return () + + +def _evaluate_blocking_quality_plans( + session: Session, + principal: object, + *, + dataset_id: str, + dataset_revision: int, + rows: tuple[Mapping[str, object], ...], + output_hash: str, + source_fingerprints: Sequence[Mapping[str, object]], +) -> tuple[ReportingQualityResult, ...]: + plans, _total = list_definitions( + session, + principal, + definition_kinds=("quality_plan",), + statuses=("active",), + limit=200, + ) + relevant = tuple( + plan + for plan in plans + if plan.parent_id == dataset_id and plan.parent_revision == dataset_revision + ) + results: list[ReportingQualityResult] = [] + for plan_record in relevant: + plan = QualityPlanDefinition.model_validate(plan_record.payload) + result = _record_quality_result( + session, + principal, + plan_id=plan_record.definition_id, + plan_revision=plan_record.revision, + dataset_id=dataset_id, + dataset_revision=dataset_revision, + plan=plan, + rows=rows, + output_hash=output_hash, + source_fingerprints=source_fingerprints, + ) + results.append(result) + if plan.block_report_execution and result.status == "failed": + raise ReportingExecutionError( + f"Quality plan {plan_record.name!r} blocked report execution." + ) + return tuple(results) + + +def run_quality_plan( + session: Session, + principal: object, + *, + registry: object | None, + quality_plan_id: str, + quality_plan_revision: int | None, + parameters: Mapping[str, object], +) -> dict[str, object]: + _require_scope(principal, QUALITY_SCOPE) + record = get_definition( + session, + principal, + definition_kind="quality_plan", + definition_id=quality_plan_id, + revision=quality_plan_revision, + ) + if record is None or record.status != "active": + raise LookupError("Active Reporting quality plan not found.") + plan = QualityPlanDefinition.model_validate(record.payload) + dataset_record = get_definition( + session, + principal, + definition_kind="dataset", + definition_id=plan.dataset_id, + revision=plan.dataset_revision, + ) + if dataset_record is None or dataset_record.status != "active": + raise ReportingExecutionError("Quality plan dataset is unavailable.") + dataset = DatasetDefinition.model_validate(dataset_record.payload) + source = _read_dataset( + session, + principal, + registry=registry, + dataset=dataset, + parameters=parameters, + ) + rows = tuple(_json_value(dict(item)) for item in source.rows) + result = _record_quality_result( + session, + principal, + plan_id=record.definition_id, + plan_revision=record.revision, + dataset_id=dataset_record.definition_id, + dataset_revision=dataset_record.revision, + plan=plan, + rows=rows, + output_hash=source.output_hash, + source_fingerprints=source.source_fingerprints, + ) + return _quality_payload(result) + + +def _record_quality_result( + session: Session, + principal: object, + *, + plan_id: str, + plan_revision: int, + dataset_id: str, + dataset_revision: int, + plan: QualityPlanDefinition, + rows: Sequence[Mapping[str, object]], + output_hash: str, + source_fingerprints: Sequence[Mapping[str, object]], +) -> ReportingQualityResult: + assertions = [_evaluate_assertion(item, rows) for item in plan.assertions] + failed = any( + not item["passed"] and item["severity"] in {"error", "blocker"} + for item in assertions + ) + warning = any(not item["passed"] for item in assertions) + row = ReportingQualityResult( + tenant_id=_tenant(principal), + result_id=str(uuid.uuid4()), + quality_plan_id=plan_id, + quality_plan_revision=plan_revision, + dataset_id=dataset_id, + dataset_revision=dataset_revision, + status="failed" if failed else "warning" if warning else "passed", + output_hash=output_hash, + assertions=assertions, + source_fingerprints=_json_value(source_fingerprints), + evaluated_at=utc_now(), + actor_id=_actor(principal), + ) + session.add(row) + session.flush() + return row + + +def _evaluate_assertion( + assertion, rows: Sequence[Mapping[str, object]] +) -> dict[str, object]: + field = assertion.field + config = assertion.config + failures = 0 + if assertion.kind == "not_null": + failures = sum(item.get(field or "") is None for item in rows) + elif assertion.kind == "unique": + values = [item.get(field or "") for item in rows] + failures = len(values) - len({_stable_value(item) for item in values}) + elif assertion.kind == "range": + minimum = config.get("minimum") + maximum = config.get("maximum") + failures = sum( + value is not None + and ( + (minimum is not None and value < minimum) + or (maximum is not None and value > maximum) + ) + for value in (item.get(field or "") for item in rows) + ) + elif assertion.kind == "accepted_values": + accepted = {_stable_value(item) for item in config.get("values", [])} + failures = sum( + _stable_value(item.get(field or "")) not in accepted for item in rows + ) + elif assertion.kind == "row_count": + minimum = int(config.get("minimum", 0)) + maximum = int(config.get("maximum", 2_000_000_000)) + failures = 0 if minimum <= len(rows) <= maximum else 1 + elif assertion.kind == "comparison": + other_field = str(config.get("other_field") or "") + operator = str(config.get("operator") or "eq") + if not field or not other_field or operator not in {"eq", "ne"}: + raise ReportingExecutionError("Comparison quality assertion is invalid.") + failures = sum( + (item.get(field) == item.get(other_field)) != (operator == "eq") + for item in rows + ) + else: + raise ReportingExecutionError( + f"Unsupported quality assertion: {assertion.kind!r}." + ) + return { + "key": assertion.key, + "kind": assertion.kind, + "severity": assertion.severity, + "passed": failures == 0, + "failure_count": failures, + "row_count": len(rows), + } + + +def _execution_payload( + row: ReportingExecution, + *, + report: ReportDefinition | None, + registry: object | None, +) -> dict[str, object]: + payload: dict[str, object] = { + "execution_id": row.execution_id, + "report_id": row.report_id, + "report_revision": row.report_revision, + "semantic_model_id": row.semantic_model_id, + "semantic_model_revision": row.semantic_model_revision, + "dataset_id": row.dataset_id, + "dataset_revision": row.dataset_revision, + "status": row.status, + "parameters": dict(row.parameters or {}), + "query": dict(row.query or {}), + "definition_hashes": dict(row.definition_hashes or {}), + "source_fingerprints": list(row.source_fingerprints or []), + "output_hash": row.output_hash, + "executor_version": row.executor_version, + "schema": list(row.result_schema or []), + "rows": list(row.result_rows or []), + "total_rows": row.total_rows, + "truncated": row.truncated, + "diagnostics": list(row.diagnostics or []), + "provenance": dict(row.provenance or {}), + "started_at": _datetime_text(row.started_at), + "finished_at": _datetime_text(row.finished_at), + "actor_id": row.actor_id, + } + if row.status == "succeeded" and report is not None: + result = QueryResult( + rows=tuple(dict(item) for item in row.result_rows or []), + total_rows=row.total_rows, + schema=tuple(dict(item) for item in row.result_schema or []), + truncated=row.truncated, + ) + renderer = capability(registry, CAPABILITY_REPORTING_CHART_RENDERER) + if not isinstance(renderer, ReportingChartRenderer): + renderer = DefaultChartRenderer() + payload["visualization"] = dict( + renderer.render(visualization=report.visualization, result=result) + ) + return payload + + +def _quality_payload(row: ReportingQualityResult) -> dict[str, object]: + return { + "result_id": row.result_id, + "quality_plan_id": row.quality_plan_id, + "quality_plan_revision": row.quality_plan_revision, + "dataset_id": row.dataset_id, + "dataset_revision": row.dataset_revision, + "status": row.status, + "output_hash": row.output_hash, + "assertions": list(row.assertions or []), + "source_fingerprints": list(row.source_fingerprints or []), + "evaluated_at": _datetime_text(row.evaluated_at), + } + + +def _execution_replay( + session: Session, + *, + tenant_id: str, + idempotency_key: str, + request_sha256: str, +) -> ReportingExecution | None: + row = ( + session.query(ReportingExecution) + .filter( + ReportingExecution.tenant_id == tenant_id, + ReportingExecution.idempotency_key == idempotency_key, + ) + .one_or_none() + ) + if row is not None and row.request_sha256 != request_sha256: + raise ReportingExecutionError( + "Reporting execution idempotency conflict: the key belongs to another request." + ) + return row + + +def _bind_parameters( + report: ReportDefinition, + supplied: Mapping[str, object], +) -> dict[str, object]: + definitions = {item.key: item for item in report.parameters} + unknown = set(supplied) - set(definitions) + if unknown: + raise ReportingExecutionError( + "Unknown report parameters: " + ", ".join(sorted(unknown)) + ) + result: dict[str, object] = {} + for key, definition in definitions.items(): + value = supplied.get(key, definition.default) + if value is None and definition.required: + raise ReportingExecutionError(f"Report parameter {key!r} is required.") + if definition.allowed_values and value not in definition.allowed_values: + raise ReportingExecutionError( + f"Report parameter {key!r} is outside its allowed values." + ) + if value is not None: + result[key] = value + return result + + +def _emit_execution_event( + session: Session, + execution: ReportingExecution, + report_name: str, +) -> None: + emit_platform_event( + session, + PlatformEvent( + type=f"reporting.execution.{execution.status}", + module_id="reporting", + payload={ + "execution_id": execution.execution_id, + "report_id": execution.report_id, + "report_revision": execution.report_revision, + "output_hash": execution.output_hash, + "total_rows": execution.total_rows, + "truncated": execution.truncated, + }, + occurred_at=execution.finished_at or execution.started_at, + actor=EventActorRef(type="account", id=execution.actor_id), + tenant=EventTenantRef(id=execution.tenant_id), + resource=EventObjectRef( + type="report_execution", + id=execution.execution_id, + label=report_name, + ), + classification="internal", + ), + ) + + +def _matches_type(value: object, field_type: str) -> bool: + if field_type == "string": + return isinstance(value, str) + if field_type == "integer": + return isinstance(value, int) and not isinstance(value, bool) + if field_type == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + if field_type == "boolean": + return isinstance(value, bool) + if field_type in {"date", "datetime"}: + if isinstance(value, (date, datetime)): + return True + if isinstance(value, str): + try: + datetime.fromisoformat(value.replace("Z", "+00:00")) + return True + except ValueError: + return False + return False + if field_type == "json": + return isinstance(value, (dict, list, tuple)) + return False + + +def _stable_value(value: object) -> str: + return json.dumps(_json_value(value), sort_keys=True, separators=(",", ":")) + + +def _required(value: object, label: str, maximum: int) -> str: + result = str(value or "").strip() + if not result: + raise ReportingExecutionError(f"{label} is required.") + if len(result) > maximum: + raise ReportingExecutionError(f"{label} is limited to {maximum} characters.") + return result + + +def _require_scope(principal: object, scope: str) -> None: + method = getattr(principal, "has", None) + allowed = ( + bool(method(scope)) + if callable(method) + else scopes_grant_compatible( + frozenset(getattr(principal, "scopes", ()) or ()), scope + ) + ) + if not allowed: + raise PermissionError(f"Missing scope: {scope}") + + +def _tenant(principal: object) -> str: + tenant_id = str(getattr(principal, "tenant_id", "") or "").strip() + if not tenant_id: + raise ReportingExecutionError( + "Reporting execution requires a tenant-bound principal." + ) + return tenant_id + + +def _actor(principal: object) -> str | None: + for value in ( + getattr(principal, "account_id", None), + getattr(principal, "identity_id", None), + getattr(principal, "membership_id", None), + ): + if str(value or "").strip(): + return str(value) + return None + + +def _session(value: object) -> Session: + if not isinstance(value, Session): + raise TypeError("Reporting execution requires a SQLAlchemy session.") + return value + + +def _sha256(value: object) -> str: + return hashlib.sha256( + json.dumps( + _json_value(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + ).hexdigest() + + +def _json_value(value: object) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + if hasattr(value, "model_dump"): + return _json_value(value.model_dump(mode="json")) + if isinstance(value, Mapping): + return {str(key): _json_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + return value + + +def _datetime_text(value: datetime | None) -> str | None: + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=UTC) + return value.isoformat() + + +__all__ = [ + "QUALITY_SCOPE", + "RUN_SCOPE", + "ReportingExecutionError", + "ReportingExecutionFailure", + "SqlReportingRunner", + "execute_report", + "get_execution", + "list_executions", + "run_quality_plan", +] diff --git a/src/govoplan_reporting/backend/manifest.py b/src/govoplan_reporting/backend/manifest.py new file mode 100644 index 0000000..ad49fc3 --- /dev/null +++ b/src/govoplan_reporting/backend/manifest.py @@ -0,0 +1,478 @@ +from __future__ import annotations + +from pathlib import Path + +from govoplan_core.core.access import ( + CAPABILITY_AUTH_PERMISSION_EVALUATOR, + CAPABILITY_AUTH_PRINCIPAL_RESOLVER, +) +from govoplan_core.core.dataflows import CAPABILITY_DATAFLOW_DATASET_OUTPUT +from govoplan_core.core.module_guards import ( + drop_table_retirement_provider, + persistent_table_uninstall_guard, +) +from govoplan_core.core.modules import ( + CapabilityDocumentation, + DocumentationLink, + DocumentationTopic, + FrontendModule, + FrontendRoute, + MigrationSpec, + ModuleContext, + ModuleInterfaceProvider, + ModuleInterfaceRequirement, + ModuleManifest, + NavItem, + PermissionDefinition, + RoleTemplate, +) +from govoplan_core.core.provider_governance import ( + ModuleArchitectureDeclaration, + ModuleArchitectureDocumentation, + ModuleMaturityEvidence, +) +from govoplan_core.core.search import SearchSourceProviderRegistration +from govoplan_core.core.views import ViewSurface +from govoplan_core.db.base import Base +from govoplan_reporting.backend.acl import ReportingScopeAclProvider +from govoplan_reporting.backend.contracts import ( + CAPABILITY_REPORTING_CHART_RENDERER, + CAPABILITY_REPORTING_REGISTRY, + CAPABILITY_REPORTING_RUNNER, + CAPABILITY_REPORTING_SCHEDULER, +) +from govoplan_reporting.backend.db import models as reporting_models +from govoplan_reporting.backend.definitions import ( + ADMIN_SCOPE, + READ_SCOPE, + WRITE_SCOPE, +) +from govoplan_reporting.backend.execution import ( + QUALITY_SCOPE, + RUN_SCOPE, + SqlReportingRunner, +) +from govoplan_reporting.backend.operations import ( + IMPORT_SCOPE, + PUBLISH_SCOPE, + SCHEDULE_SCOPE, + SqlReportingScheduler, +) +from govoplan_reporting.backend.query_engine import DefaultChartRenderer +from govoplan_reporting.backend.registry import SqlReportingRegistry +from govoplan_reporting.backend.search_source import create_reporting_search_source + + +MODULE_ID = "reporting" +MODULE_NAME = "Reporting" +MODULE_VERSION = "0.1.14" + + +def _permission(scope: str, label: str, description: str) -> PermissionDefinition: + module_id, resource, action = scope.split(":", 2) + return PermissionDefinition( + scope=scope, + label=label, + description=description, + category=MODULE_NAME, + level="tenant", + module_id=module_id, + resource=resource, + action=action, + ) + + +PERMISSIONS = ( + _permission( + READ_SCOPE, + "View reporting definitions", + "Read accessible datasets, semantic models, reports, and quality plans.", + ), + _permission( + WRITE_SCOPE, + "Manage reporting definitions", + "Create immutable revisions of Reporting definitions.", + ), + _permission( + ADMIN_SCOPE, + "Administer reporting", + "Manage restricted definitions and Reporting governance.", + ), + _permission( + RUN_SCOPE, + "Run reports", + "Execute accessible report revisions and export their authorized result.", + ), + _permission( + PUBLISH_SCOPE, + "Publish reports", + "Send successful report results to configured publication providers.", + ), + _permission( + SCHEDULE_SCOPE, + "Schedule reports", + "Create schedules and dispatch due report runs.", + ), + _permission( + QUALITY_SCOPE, + "Run report quality plans", + "Evaluate dataset quality plans and inspect evidence.", + ), + _permission( + IMPORT_SCOPE, + "Assess report imports", + "Assess external BI metadata and accept bounded approximations.", + ), +) + +ROLE_TEMPLATES = ( + RoleTemplate( + slug="reporting_analyst", + name="Reporting analyst", + description="Define semantic reports, run them, and save analytical views.", + permissions=(READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, QUALITY_SCOPE), + ), + RoleTemplate( + slug="reporting_publisher", + name="Reporting publisher", + description="Run, schedule, export, and publish accessible reports.", + permissions=(READ_SCOPE, RUN_SCOPE, PUBLISH_SCOPE, SCHEDULE_SCOPE), + ), + RoleTemplate( + slug="reporting_administrator", + name="Reporting administrator", + description="Administer definitions, imports, quality, schedules, and publications.", + permissions=tuple(item.scope for item in PERMISSIONS), + ), +) + + +def _router(context: ModuleContext): + from govoplan_reporting.backend.router import create_router + + return create_router(context.registry) + + +def _registry(context: ModuleContext) -> SqlReportingRegistry: + del context + return SqlReportingRegistry() + + +def _runner(context: ModuleContext) -> SqlReportingRunner: + return SqlReportingRunner(context.registry) + + +def _scheduler(context: ModuleContext) -> SqlReportingScheduler: + return SqlReportingScheduler(context.registry) + + +def _chart_renderer(context: ModuleContext) -> DefaultChartRenderer: + del context + return DefaultChartRenderer() + + +def _tenant_summary(session, tenant_id: str) -> dict[str, int]: + definitions = ( + session.query(reporting_models.ReportingDefinitionRevision) + .filter( + reporting_models.ReportingDefinitionRevision.tenant_id == tenant_id, + reporting_models.ReportingDefinitionRevision.superseded_at.is_(None), + ) + .count() + ) + reports = ( + session.query(reporting_models.ReportingDefinitionRevision) + .filter( + reporting_models.ReportingDefinitionRevision.tenant_id == tenant_id, + reporting_models.ReportingDefinitionRevision.definition_kind == "report", + reporting_models.ReportingDefinitionRevision.superseded_at.is_(None), + ) + .count() + ) + executions = ( + session.query(reporting_models.ReportingExecution) + .filter(reporting_models.ReportingExecution.tenant_id == tenant_id) + .count() + ) + schedules = ( + session.query(reporting_models.ReportingSchedule) + .filter( + reporting_models.ReportingSchedule.tenant_id == tenant_id, + reporting_models.ReportingSchedule.enabled.is_(True), + ) + .count() + ) + return { + "reporting_definitions": definitions, + "reports": reports, + "report_executions": executions, + "active_report_schedules": schedules, + } + + +manifest = ModuleManifest( + id=MODULE_ID, + name=MODULE_NAME, + version=MODULE_VERSION, + dependencies=("access",), + optional_dependencies=( + "dataflow", + "datasources", + "connectors", + "dashboard", + "files", + "mail", + "templates", + "workflow_engine", + "policy", + "search", + "notifications", + ), + required_capabilities=( + CAPABILITY_AUTH_PRINCIPAL_RESOLVER, + CAPABILITY_AUTH_PERMISSION_EVALUATOR, + ), + optional_capabilities=(CAPABILITY_DATAFLOW_DATASET_OUTPUT,), + permissions=PERMISSIONS, + role_templates=ROLE_TEMPLATES, + route_factory=_router, + nav_items=( + NavItem( + path="/reporting", + label="Reporting", + icon="clipboard-pen-line", + required_any=(READ_SCOPE,), + order=74, + ), + ), + frontend=FrontendModule( + module_id=MODULE_ID, + package_name="@govoplan/reporting-webui", + routes=( + FrontendRoute( + path="/reporting", + component="ReportingPage", + required_any=(READ_SCOPE,), + order=74, + ), + ), + nav_items=( + NavItem( + path="/reporting", + label="Reporting", + icon="clipboard-pen-line", + required_any=(READ_SCOPE,), + order=74, + ), + ), + view_surfaces=( + ViewSurface( + id="reporting.navigation", + module_id=MODULE_ID, + kind="navigation", + label="Reporting navigation", + order=10, + ), + ViewSurface( + id="reporting.workspace", + module_id=MODULE_ID, + kind="route", + label="Reporting workspace", + order=20, + ), + ViewSurface( + id="reporting.parameters", + module_id=MODULE_ID, + kind="section", + label="Report parameters and filters", + parent_id="reporting.workspace", + order=30, + ), + ViewSurface( + id="reporting.results", + module_id=MODULE_ID, + kind="section", + label="Authorized report results", + parent_id="reporting.workspace", + order=40, + ), + ), + ), + provides_interfaces=( + ModuleInterfaceProvider(name="reporting.registry", version="0.1.0"), + ModuleInterfaceProvider(name="reporting.runner", version="0.1.0"), + ModuleInterfaceProvider(name="reporting.scheduler", version="0.1.0"), + ModuleInterfaceProvider(name="reporting.chart_renderer", version="0.1.0"), + ), + requires_interfaces=( + ModuleInterfaceRequirement( + name="dataflow.dataset_output", + version_min="0.1.0", + version_max_exclusive="0.2.0", + optional=True, + ), + ), + capability_factories={ + CAPABILITY_REPORTING_REGISTRY: _registry, + CAPABILITY_REPORTING_RUNNER: _runner, + CAPABILITY_REPORTING_SCHEDULER: _scheduler, + CAPABILITY_REPORTING_CHART_RENDERER: _chart_renderer, + }, + capability_documentation={ + CAPABILITY_REPORTING_REGISTRY: CapabilityDocumentation( + label="Reporting definition registry", + summary="Stores versioned datasets, semantic models, reports, and quality plans.", + contract_version="0.1.0", + ), + CAPABILITY_REPORTING_RUNNER: CapabilityDocumentation( + label="Governed report runner", + summary="Executes a pinned report graph over an authorized provider-owned dataset.", + contract_version="0.1.0", + ), + CAPABILITY_REPORTING_SCHEDULER: CapabilityDocumentation( + label="Report schedule dispatcher", + summary="Claims due report schedules and records run/publication evidence.", + contract_version="0.1.0", + ), + CAPABILITY_REPORTING_CHART_RENDERER: CapabilityDocumentation( + label="Report chart renderer", + summary="Builds provider-neutral chart models with an accessible tabular fallback.", + contract_version="0.1.0", + ), + }, + search_sources=( + SearchSourceProviderRegistration( + id="reporting.reports", + factory=create_reporting_search_source, + ), + ), + migration_spec=MigrationSpec( + module_id=MODULE_ID, + metadata=Base.metadata, + script_location=str(Path(__file__).with_name("migrations") / "versions"), + retirement_supported=True, + retirement_provider=drop_table_retirement_provider( + reporting_models.ReportingImportAssessment, + reporting_models.ReportingQualityResult, + reporting_models.ReportingPublication, + reporting_models.ReportingSchedule, + reporting_models.ReportingSavedView, + reporting_models.ReportingDefinitionGrant, + reporting_models.ReportingExecution, + reporting_models.ReportingDefinitionRevision, + reporting_models.ReportingDefinitionIdentity, + label="Reporting", + ), + retirement_notes=( + "Destructive retirement requires a database snapshot and removes " + "Reporting definitions, results, quality evidence, schedules, and publications." + ), + ), + uninstall_guard_providers=( + persistent_table_uninstall_guard( + reporting_models.ReportingDefinitionIdentity, + reporting_models.ReportingDefinitionRevision, + reporting_models.ReportingDefinitionGrant, + reporting_models.ReportingExecution, + reporting_models.ReportingSavedView, + reporting_models.ReportingSchedule, + reporting_models.ReportingPublication, + reporting_models.ReportingQualityResult, + reporting_models.ReportingImportAssessment, + label="Reporting", + ), + ), + resource_acl_providers=( + ReportingScopeAclProvider("analytical_dataset"), + ReportingScopeAclProvider("semantic_model"), + ReportingScopeAclProvider("report"), + ReportingScopeAclProvider("report_execution"), + ), + tenant_summary_providers=(_tenant_summary,), + documentation=( + DocumentationTopic( + id="reporting.governed-bi", + title="Governed reporting and semantic BI", + summary="Build reproducible reports over provider-owned datasets without bypassing module or row-level access.", + body=( + "Reporting pins dataset, semantic-model, and report revisions. Runs retain " + "definition hashes, source fingerprints, policy provenance, quality evidence, " + "authorized result rows, diagnostics, and output hashes. Safe dimensions, " + "aggregations, typed expressions, filters, pivots, saved views, chart models, " + "schedules, exports, and publication providers replace unchecked SQL in the " + "presentation layer. Dataflow and module read models remain source owners." + ), + layer="available", + documentation_types=("admin", "user"), + audience=("user", "operator", "module_admin", "product_owner"), + links=( + DocumentationLink( + label="Reporting module boundary", + href="govoplan-reporting/docs/REPORTING_BOUNDARY.md", + kind="repository", + ), + DocumentationLink( + label="SuperX capability assessment", + href="govoplan-reporting/docs/SUPERX_CAPABILITY_ASSESSMENT.md", + kind="repository", + ), + DocumentationLink( + label="Reporting user guide", + href="govoplan-reporting/docs/USER_GUIDE.md", + kind="repository", + ), + DocumentationLink( + label="Reporting administration guide", + href="govoplan-reporting/docs/ADMIN_GUIDE.md", + kind="repository", + ), + ), + ), + ), + architecture=ModuleArchitectureDeclaration( + layer="data_reporting_integration", + kind="domain", + maturity="vertical_slice", + evidence=( + ModuleMaturityEvidence( + kind="test", + reference="tests/test_reporting_service.py", + summary="Proves revision pinning, safe semantic execution, quality gates, access, replay, exports, and import blocking.", + ), + ModuleMaturityEvidence( + kind="documentation", + reference="docs/REPORTING_BOUNDARY.md", + summary="Defines governed analytical source, semantic, execution, and publication ownership.", + ), + ), + known_limits=( + "Dataflow is the first live dataset adapter; additional module read models use the provider-neutral contract.", + "Direct browser export supports CSV and JSON; XLSX, PDF, Files, Mail, and DMS delivery require an optional publication provider.", + "Import assessment produces blocking diagnostics but does not execute source SQL or automatically activate generated definitions.", + "The initial chart provider emits a renderer-neutral model and accessible table; richer visual renderers remain replaceable adapters.", + ), + owned_concepts=( + "analytical dataset binding", + "semantic dimension hierarchy and measure", + "report definition and saved view", + "report execution and publication evidence", + "report quality plan and import assessment", + ), + non_owned_concepts=( + "raw datasource ingestion", + "data transformation pipeline", + "source module authorization", + "template document rendering", + "file or DMS storage", + ), + documentation=ModuleArchitectureDocumentation( + operations=("docs/OPERATIONS.md",), + recovery=("docs/OPERATIONS.md",), + security=("docs/OPERATIONS.md",), + ), + ), +) + + +def get_manifest() -> ModuleManifest: + return manifest diff --git a/src/govoplan_reporting/backend/migrations/__init__.py b/src/govoplan_reporting/backend/migrations/__init__.py new file mode 100644 index 0000000..8969202 --- /dev/null +++ b/src/govoplan_reporting/backend/migrations/__init__.py @@ -0,0 +1 @@ +"""Reporting Alembic revisions.""" diff --git a/src/govoplan_reporting/backend/migrations/versions/__init__.py b/src/govoplan_reporting/backend/migrations/versions/__init__.py new file mode 100644 index 0000000..fe0e57f --- /dev/null +++ b/src/govoplan_reporting/backend/migrations/versions/__init__.py @@ -0,0 +1 @@ +"""Reporting migration versions.""" diff --git a/src/govoplan_reporting/backend/migrations/versions/e5b2c9d4f7a1_v0114_reporting_baseline.py b/src/govoplan_reporting/backend/migrations/versions/e5b2c9d4f7a1_v0114_reporting_baseline.py new file mode 100644 index 0000000..ee5d33e --- /dev/null +++ b/src/govoplan_reporting/backend/migrations/versions/e5b2c9d4f7a1_v0114_reporting_baseline.py @@ -0,0 +1,477 @@ +"""v0.1.14 governed Reporting baseline. + +Revision ID: e5b2c9d4f7a1 +Revises: None +""" + +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "e5b2c9d4f7a1" +down_revision = None +branch_labels = None +depends_on = "4f2a9c8e7b6d" + + +def upgrade() -> None: + op.create_table( + "reporting_definition_identities", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("definition_kind", sa.String(length=40), nullable=False), + sa.Column("definition_id", sa.String(length=255), nullable=False), + sa.Column("definition_key", sa.String(length=120), nullable=False), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_definition_identities")), + sa.UniqueConstraint( + "tenant_id", + "definition_kind", + "definition_id", + name="uq_reporting_definition_identity", + ), + sa.UniqueConstraint( + "tenant_id", + "definition_kind", + "definition_key", + name="uq_reporting_definition_key", + ), + ) + _indexes( + "reporting_definition_identities", + "tenant_id", + "definition_kind", + "definition_id", + "definition_key", + "created_by", + ) + op.create_index( + "ix_reporting_definition_catalog", + "reporting_definition_identities", + ["tenant_id", "definition_kind", "definition_key"], + unique=False, + ) + + op.create_table( + "reporting_definition_revisions", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("identity_id", sa.String(length=36), nullable=False), + sa.Column("definition_kind", sa.String(length=40), nullable=False), + sa.Column("definition_id", sa.String(length=255), nullable=False), + sa.Column("definition_key", sa.String(length=120), nullable=False), + sa.Column("revision", sa.Integer(), nullable=False), + sa.Column("previous_revision_id", sa.String(length=36), nullable=True), + sa.Column("parent_kind", sa.String(length=40), nullable=True), + sa.Column("parent_id", sa.String(length=255), nullable=True), + sa.Column("parent_revision", sa.Integer(), nullable=True), + sa.Column("name", sa.String(length=500), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("status", sa.String(length=30), nullable=False), + sa.Column("visibility", sa.String(length=30), nullable=False), + sa.Column("content_hash", sa.String(length=64), nullable=False), + sa.Column("change_reason", sa.String(length=1000), nullable=False), + sa.Column("idempotency_key", sa.String(length=255), nullable=False), + sa.Column("request_sha256", sa.String(length=64), nullable=False), + sa.Column("event_id", sa.String(length=36), nullable=False), + sa.Column("recorded_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("superseded_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("payload", sa.JSON(), nullable=False), + sa.Column("changed_by", sa.String(length=255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["identity_id"], + ["reporting_definition_identities.id"], + name=op.f( + "fk_reporting_definition_revisions_identity_id_reporting_definition_identities" + ), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["previous_revision_id"], + ["reporting_definition_revisions.id"], + name=op.f( + "fk_reporting_definition_revisions_previous_revision_id_reporting_definition_revisions" + ), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_definition_revisions")), + sa.UniqueConstraint( + "tenant_id", + "definition_kind", + "definition_id", + "revision", + name="uq_reporting_definition_revision", + ), + sa.UniqueConstraint( + "tenant_id", + "idempotency_key", + name="uq_reporting_definition_idempotency", + ), + sa.UniqueConstraint( + "tenant_id", + "event_id", + name="uq_reporting_definition_event", + ), + ) + _indexes( + "reporting_definition_revisions", + "tenant_id", + "identity_id", + "definition_kind", + "definition_id", + "definition_key", + "previous_revision_id", + "parent_kind", + "parent_id", + "status", + "visibility", + "content_hash", + "event_id", + "recorded_at", + "superseded_at", + "changed_by", + ) + op.create_index( + "ix_reporting_definition_current", + "reporting_definition_revisions", + ["tenant_id", "definition_kind", "definition_id", "superseded_at"], + unique=False, + ) + op.create_index( + "ix_reporting_definition_parent", + "reporting_definition_revisions", + ["tenant_id", "parent_kind", "parent_id", "parent_revision"], + unique=False, + ) + op.create_index( + "ix_reporting_definition_list", + "reporting_definition_revisions", + ["tenant_id", "definition_kind", "status", "recorded_at"], + unique=False, + ) + + op.create_table( + "reporting_definition_grants", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("definition_kind", sa.String(length=40), nullable=False), + sa.Column("definition_id", sa.String(length=255), nullable=False), + sa.Column("subject_kind", sa.String(length=40), nullable=False), + sa.Column("subject_id", sa.String(length=255), nullable=False), + sa.Column("permissions", sa.JSON(), nullable=False), + sa.Column("active", sa.Boolean(), nullable=False), + sa.Column("source_revision", sa.Integer(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_definition_grants")), + sa.UniqueConstraint( + "tenant_id", + "definition_kind", + "definition_id", + "subject_kind", + "subject_id", + name="uq_reporting_definition_grant", + ), + ) + _indexes( + "reporting_definition_grants", + "tenant_id", + "definition_kind", + "definition_id", + "subject_kind", + "subject_id", + "active", + ) + op.create_index( + "ix_reporting_definition_grant_subject", + "reporting_definition_grants", + ["tenant_id", "subject_kind", "subject_id", "active"], + unique=False, + ) + op.create_index( + "ix_reporting_definition_grant_object", + "reporting_definition_grants", + ["tenant_id", "definition_kind", "definition_id", "active"], + unique=False, + ) + + op.create_table( + "reporting_executions", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("execution_id", sa.String(length=36), nullable=False), + sa.Column("report_id", sa.String(length=255), nullable=False), + sa.Column("report_revision", sa.Integer(), nullable=False), + sa.Column("semantic_model_id", sa.String(length=255), nullable=False), + sa.Column("semantic_model_revision", sa.Integer(), nullable=False), + sa.Column("dataset_id", sa.String(length=255), nullable=False), + sa.Column("dataset_revision", sa.Integer(), nullable=False), + sa.Column("status", sa.String(length=30), nullable=False), + sa.Column("idempotency_key", sa.String(length=255), nullable=False), + sa.Column("request_sha256", sa.String(length=64), nullable=False), + sa.Column("parameters", sa.JSON(), nullable=False), + sa.Column("query", sa.JSON(), nullable=False), + sa.Column("source_fingerprints", sa.JSON(), nullable=False), + sa.Column("definition_hashes", sa.JSON(), nullable=False), + sa.Column("output_hash", sa.String(length=64), nullable=True), + sa.Column("executor_version", sa.String(length=255), nullable=True), + sa.Column("result_schema", sa.JSON(), nullable=False), + sa.Column("result_rows", sa.JSON(), nullable=False), + sa.Column("total_rows", sa.Integer(), nullable=False), + sa.Column("truncated", sa.Boolean(), nullable=False), + sa.Column("diagnostics", sa.JSON(), nullable=False), + sa.Column("provenance", sa.JSON(), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("actor_id", sa.String(length=255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_executions")), + sa.UniqueConstraint("tenant_id", "execution_id", name="uq_reporting_execution"), + sa.UniqueConstraint( + "tenant_id", + "idempotency_key", + name="uq_reporting_execution_idempotency", + ), + ) + _indexes( + "reporting_executions", + "tenant_id", + "execution_id", + "report_id", + "semantic_model_id", + "dataset_id", + "status", + "output_hash", + "started_at", + "finished_at", + "actor_id", + ) + op.create_index( + "ix_reporting_execution_history", + "reporting_executions", + ["tenant_id", "report_id", "started_at"], + unique=False, + ) + op.create_index( + "ix_reporting_execution_status", + "reporting_executions", + ["tenant_id", "status", "started_at"], + unique=False, + ) + + op.create_table( + "reporting_saved_views", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("view_id", sa.String(length=36), nullable=False), + sa.Column("report_id", sa.String(length=255), nullable=False), + sa.Column("report_revision", sa.Integer(), nullable=False), + sa.Column("owner_kind", sa.String(length=40), nullable=False), + sa.Column("owner_id", sa.String(length=255), nullable=False), + sa.Column("name", sa.String(length=500), nullable=False), + sa.Column("revision", sa.Integer(), nullable=False), + sa.Column("state", sa.JSON(), nullable=False), + sa.Column("shared", sa.Boolean(), nullable=False), + sa.Column("access", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_saved_views")), + sa.UniqueConstraint("tenant_id", "view_id", name="uq_reporting_saved_view"), + ) + _indexes( + "reporting_saved_views", + "tenant_id", + "view_id", + "report_id", + "owner_kind", + "owner_id", + "shared", + ) + op.create_index( + "ix_reporting_saved_view_catalog", + "reporting_saved_views", + ["tenant_id", "report_id", "owner_id", "shared"], + unique=False, + ) + + op.create_table( + "reporting_schedules", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("schedule_id", sa.String(length=36), nullable=False), + sa.Column("report_id", sa.String(length=255), nullable=False), + sa.Column("report_revision", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=500), nullable=False), + sa.Column("revision", sa.Integer(), nullable=False), + sa.Column("trigger_kind", sa.String(length=30), nullable=False), + sa.Column("trigger_config", sa.JSON(), nullable=False), + sa.Column("parameters", sa.JSON(), nullable=False), + sa.Column("query", sa.JSON(), nullable=False), + sa.Column("publication_target", sa.JSON(), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False), + sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_run_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_execution_id", sa.String(length=36), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_schedules")), + sa.UniqueConstraint("tenant_id", "schedule_id", name="uq_reporting_schedule"), + ) + _indexes( + "reporting_schedules", + "tenant_id", + "schedule_id", + "report_id", + "enabled", + "next_run_at", + ) + op.create_index( + "ix_reporting_schedule_due", + "reporting_schedules", + ["enabled", "next_run_at", "tenant_id"], + unique=False, + ) + + op.create_table( + "reporting_publications", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("publication_id", sa.String(length=36), nullable=False), + sa.Column("execution_id", sa.String(length=36), nullable=False), + sa.Column("target_capability", sa.String(length=255), nullable=False), + sa.Column("target_ref", sa.String(length=1000), nullable=True), + sa.Column("format", sa.String(length=30), nullable=False), + sa.Column("status", sa.String(length=30), nullable=False), + sa.Column("idempotency_key", sa.String(length=255), nullable=False), + sa.Column("evidence", sa.JSON(), nullable=False), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_publications")), + sa.UniqueConstraint( + "tenant_id", "publication_id", name="uq_reporting_publication" + ), + sa.UniqueConstraint( + "tenant_id", + "idempotency_key", + name="uq_reporting_publication_idempotency", + ), + ) + _indexes( + "reporting_publications", + "tenant_id", + "publication_id", + "execution_id", + "status", + ) + op.create_index( + "ix_reporting_publication_history", + "reporting_publications", + ["tenant_id", "execution_id", "created_at"], + unique=False, + ) + + op.create_table( + "reporting_quality_results", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("result_id", sa.String(length=36), nullable=False), + sa.Column("quality_plan_id", sa.String(length=255), nullable=False), + sa.Column("quality_plan_revision", sa.Integer(), nullable=False), + sa.Column("dataset_id", sa.String(length=255), nullable=False), + sa.Column("dataset_revision", sa.Integer(), nullable=False), + sa.Column("status", sa.String(length=30), nullable=False), + sa.Column("output_hash", sa.String(length=64), nullable=False), + sa.Column("assertions", sa.JSON(), nullable=False), + sa.Column("source_fingerprints", sa.JSON(), nullable=False), + sa.Column("evaluated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("actor_id", sa.String(length=255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_quality_results")), + sa.UniqueConstraint( + "tenant_id", "result_id", name="uq_reporting_quality_result" + ), + ) + _indexes( + "reporting_quality_results", + "tenant_id", + "result_id", + "quality_plan_id", + "dataset_id", + "status", + "output_hash", + "evaluated_at", + ) + op.create_index( + "ix_reporting_quality_history", + "reporting_quality_results", + ["tenant_id", "quality_plan_id", "evaluated_at"], + unique=False, + ) + + op.create_table( + "reporting_import_assessments", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("assessment_id", sa.String(length=36), nullable=False), + sa.Column("source_system", sa.String(length=255), nullable=False), + sa.Column("source_id", sa.String(length=500), nullable=False), + sa.Column("source_fingerprint", sa.String(length=64), nullable=False), + sa.Column("mapping_report", sa.JSON(), nullable=False), + sa.Column("status", sa.String(length=30), nullable=False), + sa.Column("accepted_approximations", sa.JSON(), nullable=False), + sa.Column("assessed_by", sa.String(length=255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_import_assessments")), + sa.UniqueConstraint( + "tenant_id", + "assessment_id", + name="uq_reporting_import_assessment", + ), + ) + _indexes( + "reporting_import_assessments", + "tenant_id", + "assessment_id", + "source_system", + "status", + ) + op.create_index( + "ix_reporting_import_history", + "reporting_import_assessments", + ["tenant_id", "source_system", "created_at"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_table("reporting_import_assessments") + op.drop_table("reporting_quality_results") + op.drop_table("reporting_publications") + op.drop_table("reporting_schedules") + op.drop_table("reporting_saved_views") + op.drop_table("reporting_definition_grants") + op.drop_table("reporting_executions") + op.drop_table("reporting_definition_revisions") + op.drop_table("reporting_definition_identities") + + +def _indexes(table: str, *columns: str) -> None: + for column in columns: + op.create_index( + op.f(f"ix_{table}_{column}"), + table, + [column], + unique=False, + ) diff --git a/src/govoplan_reporting/backend/operations.py b/src/govoplan_reporting/backend/operations.py new file mode 100644 index 0000000..1930ea7 --- /dev/null +++ b/src/govoplan_reporting/backend/operations.py @@ -0,0 +1,858 @@ +from __future__ import annotations + +from collections.abc import Mapping +import csv +from datetime import UTC, datetime, timedelta +from io import StringIO +import hashlib +import json +from typing import Any +import uuid + +from sqlalchemy import or_ +from sqlalchemy.orm import Session + +from govoplan_core.core.events import ( + EventActorRef, + EventObjectRef, + EventTenantRef, + PlatformEvent, + emit_platform_event, +) +from govoplan_core.security.module_permissions import scopes_grant_compatible +from govoplan_core.security.time import utc_now +from govoplan_reporting.backend.contracts import ( + ReportingPublicationPayload, + ReportingPublicationTarget, + capability, +) +from govoplan_reporting.backend.db.models import ( + ReportingImportAssessment, + ReportingPublication, + ReportingSavedView, + ReportingSchedule, +) +from govoplan_reporting.backend.definitions import ADMIN_SCOPE, get_definition +from govoplan_reporting.backend.execution import ( + ReportingExecutionFailure, + execute_report, + get_execution, +) +from govoplan_reporting.backend.schemas import ReportQuery + + +PUBLISH_SCOPE = "reporting:report:publish" +SCHEDULE_SCOPE = "reporting:schedule:write" +IMPORT_SCOPE = "reporting:import:assess" + + +class ReportingOperationError(ValueError): + pass + + +class SqlReportingScheduler: + def __init__(self, registry: object | None) -> None: + self.registry = registry + + def dispatch_due( + self, + session: object, + principal: object, + *, + now: datetime | None = None, + limit: int = 20, + ) -> Mapping[str, object]: + return dispatch_due_schedules( + _session(session), + principal, + registry=self.registry, + now=now, + limit=limit, + ) + + +def upsert_saved_view( + session: Session, + principal: object, + *, + view_id: str, + report_id: str, + report_revision: int, + name: str, + state: Mapping[str, object], + shared: bool, + access: Mapping[str, object], + expected_revision: int | None, +) -> dict[str, object]: + report = get_definition( + session, + principal, + definition_kind="report", + definition_id=report_id, + revision=report_revision, + ) + if report is None: + raise LookupError("Reporting report definition not found.") + _validate_saved_view_state(state) + owner_id = _actor(principal) + if owner_id is None: + raise ReportingOperationError("Saved views require an account owner.") + row = ( + session.query(ReportingSavedView) + .filter( + ReportingSavedView.tenant_id == _tenant(principal), + ReportingSavedView.view_id == view_id, + ) + .with_for_update() + .one_or_none() + ) + if row is None: + if expected_revision is not None: + raise ReportingOperationError( + "Saved-view revision conflict: no view exists." + ) + row = ReportingSavedView( + tenant_id=_tenant(principal), + view_id=_required(view_id, "Saved-view identifier", 36), + report_id=report_id, + report_revision=report_revision, + owner_kind="account", + owner_id=owner_id, + name=_required(name, "Saved-view name", 500), + revision=1, + state=_json_value(state), + shared=shared, + access=_json_value(access), + ) + session.add(row) + else: + if row.owner_id != owner_id and not _has_scope(principal, ADMIN_SCOPE): + raise PermissionError("Only the saved-view owner or an admin may edit it.") + if expected_revision != row.revision: + raise ReportingOperationError( + "Saved-view revision conflict: the expected revision is stale." + ) + row.report_id = report_id + row.report_revision = report_revision + row.name = _required(name, "Saved-view name", 500) + row.state = _json_value(state) + row.shared = shared + row.access = _json_value(access) + row.revision += 1 + session.flush() + return _saved_view_payload(row) + + +def list_saved_views( + session: Session, + principal: object, + *, + report_id: str, +) -> tuple[dict[str, object], ...]: + if ( + get_definition( + session, + principal, + definition_kind="report", + definition_id=report_id, + ) + is None + ): + return () + owner_id = _actor(principal) + rows = ( + session.query(ReportingSavedView) + .filter( + ReportingSavedView.tenant_id == _tenant(principal), + ReportingSavedView.report_id == report_id, + or_( + ReportingSavedView.shared.is_(True), + ReportingSavedView.owner_id == owner_id, + ), + ) + .order_by(ReportingSavedView.name.asc()) + .all() + ) + return tuple(_saved_view_payload(row) for row in rows) + + +def delete_saved_view( + session: Session, + principal: object, + *, + view_id: str, +) -> bool: + row = ( + session.query(ReportingSavedView) + .filter( + ReportingSavedView.tenant_id == _tenant(principal), + ReportingSavedView.view_id == view_id, + ) + .with_for_update() + .one_or_none() + ) + if row is None: + return False + if row.owner_id != _actor(principal) and not _has_scope(principal, ADMIN_SCOPE): + raise PermissionError("Only the saved-view owner or an admin may delete it.") + session.delete(row) + session.flush() + return True + + +def upsert_schedule( + session: Session, + principal: object, + *, + schedule_id: str, + report_id: str, + report_revision: int, + name: str, + trigger_kind: str, + trigger_config: Mapping[str, object], + parameters: Mapping[str, object], + query: ReportQuery, + publication_target: Mapping[str, object], + enabled: bool, + next_run_at: datetime | None, + expected_revision: int | None, +) -> dict[str, object]: + _require_scope(principal, SCHEDULE_SCOPE) + report = get_definition( + session, + principal, + definition_kind="report", + definition_id=report_id, + revision=report_revision, + ) + if report is None or report.status != "active": + raise ReportingOperationError("Schedules require an active report revision.") + normalized_next = _validate_trigger( + trigger_kind, + trigger_config, + next_run_at=next_run_at, + ) + row = ( + session.query(ReportingSchedule) + .filter( + ReportingSchedule.tenant_id == _tenant(principal), + ReportingSchedule.schedule_id == schedule_id, + ) + .with_for_update() + .one_or_none() + ) + if row is None: + if expected_revision is not None: + raise ReportingOperationError( + "Reporting schedule revision conflict: no schedule exists." + ) + row = ReportingSchedule( + tenant_id=_tenant(principal), + schedule_id=_required(schedule_id, "Reporting schedule identifier", 36), + report_id=report_id, + report_revision=report_revision, + name=_required(name, "Reporting schedule name", 500), + revision=1, + trigger_kind=trigger_kind, + trigger_config=_json_value(trigger_config), + parameters=_json_value(parameters), + query=query.model_dump(mode="json"), + publication_target=_json_value(publication_target), + enabled=enabled, + next_run_at=normalized_next, + created_by=_actor(principal), + ) + session.add(row) + else: + if expected_revision != row.revision: + raise ReportingOperationError( + "Reporting schedule revision conflict: the expected revision is stale." + ) + row.report_id = report_id + row.report_revision = report_revision + row.name = _required(name, "Reporting schedule name", 500) + row.trigger_kind = trigger_kind + row.trigger_config = _json_value(trigger_config) + row.parameters = _json_value(parameters) + row.query = query.model_dump(mode="json") + row.publication_target = _json_value(publication_target) + row.enabled = enabled + row.next_run_at = normalized_next + row.revision += 1 + session.flush() + return _schedule_payload(row) + + +def list_schedules( + session: Session, + principal: object, + *, + report_id: str | None = None, +) -> tuple[dict[str, object], ...]: + statement = session.query(ReportingSchedule).filter( + ReportingSchedule.tenant_id == _tenant(principal) + ) + if report_id: + if ( + get_definition( + session, + principal, + definition_kind="report", + definition_id=report_id, + ) + is None + ): + return () + statement = statement.filter(ReportingSchedule.report_id == report_id) + return tuple( + _schedule_payload(row) + for row in statement.order_by(ReportingSchedule.name.asc()).all() + ) + + +def dispatch_due_schedules( + session: Session, + principal: object, + *, + registry: object | None, + now: datetime | None, + limit: int, +) -> dict[str, object]: + _require_scope(principal, SCHEDULE_SCOPE) + current = now or utc_now() + _aware(current, "Reporting scheduler time") + rows = ( + session.query(ReportingSchedule) + .filter( + ReportingSchedule.enabled.is_(True), + ReportingSchedule.next_run_at.is_not(None), + ReportingSchedule.next_run_at <= current, + ) + .order_by(ReportingSchedule.next_run_at.asc()) + .limit(max(1, min(limit, 100))) + .with_for_update(skip_locked=True) + .all() + ) + succeeded = 0 + failed = 0 + execution_ids: list[str] = [] + for row in rows: + scheduled_for = row.next_run_at or current + try: + execution = execute_report( + session, + principal, + registry=registry, + report_id=row.report_id, + report_revision=row.report_revision, + parameters=dict(row.parameters or {}), + query=ReportQuery.model_validate(row.query or {}), + idempotency_key=f"schedule:{row.schedule_id}:{scheduled_for.isoformat()}", + ) + execution_id = str(execution["execution_id"]) + execution_ids.append(execution_id) + target = dict(row.publication_target or {}) + if target: + publish_execution( + session, + principal, + registry=registry, + execution_id=execution_id, + target_capability=str(target.get("target_capability") or ""), + target_ref=( + str(target["target_ref"]) + if target.get("target_ref") is not None + else None + ), + format=str(target.get("format") or "csv"), + idempotency_key=f"schedule-publication:{row.schedule_id}:{scheduled_for.isoformat()}", + options=dict(target.get("options") or {}), + ) + succeeded += 1 + row.last_execution_id = execution_id + except (ReportingExecutionFailure, ReportingOperationError, LookupError): + failed += 1 + row.last_run_at = current + _advance_schedule(row, scheduled_for) + session.flush() + return { + "claimed": len(rows), + "succeeded": succeeded, + "failed": failed, + "execution_ids": execution_ids, + } + + +def publish_execution( + session: Session, + principal: object, + *, + registry: object | None, + execution_id: str, + target_capability: str, + target_ref: str | None, + format: str, + idempotency_key: str, + options: Mapping[str, object], +) -> dict[str, object]: + _require_scope(principal, PUBLISH_SCOPE) + execution_payload = get_execution(session, principal, execution_id=execution_id) + if execution_payload is None: + raise LookupError("Reporting execution not found.") + if execution_payload["status"] != "succeeded": + raise ReportingOperationError("Only successful report executions can publish.") + clean_capability = _required( + target_capability, + "Reporting publication target capability", + 255, + ) + clean_format = str(format).casefold() + if clean_format not in {"json", "csv", "xlsx", "html", "pdf"}: + raise ReportingOperationError("Unsupported Reporting publication format.") + clean_key = _required(idempotency_key, "Reporting publication idempotency key", 255) + existing = ( + session.query(ReportingPublication) + .filter( + ReportingPublication.tenant_id == _tenant(principal), + ReportingPublication.idempotency_key == clean_key, + ) + .one_or_none() + ) + if existing is not None: + if ( + existing.execution_id != execution_id + or existing.target_capability != clean_capability + or existing.target_ref != target_ref + or existing.format != clean_format + ): + raise ReportingOperationError("Reporting publication idempotency conflict.") + return _publication_payload(existing) + provider = capability(registry, clean_capability) + if not isinstance(provider, ReportingPublicationTarget): + raise ReportingOperationError( + f"Reporting publication provider {clean_capability!r} is unavailable." + ) + publication = ReportingPublication( + tenant_id=_tenant(principal), + publication_id=str(uuid.uuid4()), + execution_id=execution_id, + target_capability=clean_capability, + target_ref=target_ref, + format=clean_format, + status="running", + idempotency_key=clean_key, + ) + session.add(publication) + session.flush() + try: + evidence = provider.publish_report( + session, + principal, + payload=ReportingPublicationPayload( + publication_id=publication.publication_id, + execution_id=execution_id, + tenant_id=_tenant(principal), + report_id=str(execution_payload["report_id"]), + report_revision=int(execution_payload["report_revision"]), + format=clean_format, + target_ref=target_ref, + rows=tuple(execution_payload["rows"]), # type: ignore[arg-type] + schema=tuple(execution_payload["schema"]), # type: ignore[arg-type] + output_hash=str(execution_payload["output_hash"]), + options=dict(options), + ), + ) + publication.status = "succeeded" + publication.evidence = _json_value(evidence) + publication.completed_at = utc_now() + except Exception as exc: + publication.status = "failed" + publication.error = str(exc) + publication.completed_at = utc_now() + session.flush() + raise ReportingOperationError(str(exc)) from exc + session.flush() + emit_platform_event( + session, + PlatformEvent( + type=f"reporting.publication.{publication.status}", + module_id="reporting", + payload={ + "publication_id": publication.publication_id, + "execution_id": publication.execution_id, + "target_capability": publication.target_capability, + "target_ref": publication.target_ref, + "format": publication.format, + "evidence": dict(publication.evidence or {}), + }, + actor=EventActorRef(type="account", id=_actor(principal)), + tenant=EventTenantRef(id=publication.tenant_id), + resource=EventObjectRef( + type="report_publication", + id=publication.publication_id, + ), + classification="internal", + ), + ) + return _publication_payload(publication) + + +def export_execution( + session: Session, + principal: object, + *, + execution_id: str, + format: str, +) -> tuple[bytes, str, str]: + payload = get_execution(session, principal, execution_id=execution_id) + if payload is None: + raise LookupError("Reporting execution not found.") + if payload["status"] != "succeeded": + raise ReportingOperationError("Only successful executions can be exported.") + rows = tuple(payload["rows"]) # type: ignore[arg-type] + if format == "json": + content = json.dumps( + {"schema": payload["schema"], "rows": rows}, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + return content, "application/json", f"report-{execution_id}.json" + if format != "csv": + raise ReportingOperationError("Direct export supports CSV or JSON.") + fields = tuple( + dict.fromkeys( + str(key) for row in rows if isinstance(row, Mapping) for key in row + ) + ) + stream = StringIO(newline="") + writer = csv.DictWriter(stream, fieldnames=fields, extrasaction="ignore") + writer.writeheader() + for row in rows: + if isinstance(row, Mapping): + writer.writerow({key: _safe_csv_cell(row.get(key)) for key in fields}) + return ( + stream.getvalue().encode("utf-8-sig"), + "text/csv; charset=utf-8", + f"report-{execution_id}.csv", + ) + + +def assess_import( + session: Session, + principal: object, + *, + source_system: str, + source_id: str, + metadata: Mapping[str, object], + accepted_approximations: list[str], +) -> dict[str, object]: + _require_scope(principal, IMPORT_SCOPE) + raw_features = metadata.get("features", []) + if not isinstance(raw_features, list): + raise ReportingOperationError("Import metadata features must be a list.") + features = tuple(dict.fromkeys(str(item) for item in raw_features)) + exact_features = { + "dataset", + "dimension", + "hierarchy", + "measure", + "parameter", + "table", + "pivot", + "chart", + "quality_assertion", + "saved_view", + } + approximated_features = { + "provider_specific_format", + "dialect_function", + "dashboard_layout", + } + unsupported_features = { + "raw_sql", + "stored_procedure", + "runtime_script", + "implicit_authorization", + "unchecked_custom_function", + } + exact = sorted(set(features) & exact_features) + approximated = sorted(set(features) & approximated_features) + unsupported = sorted( + (set(features) & unsupported_features) + | ( + set(features) + - exact_features + - approximated_features + - unsupported_features + ) + ) + accepted = sorted(set(accepted_approximations) & set(approximated)) + pending = sorted(set(approximated) - set(accepted)) + status = "ready" if not unsupported and not pending else "blocked" + mapping_report = { + "contract_version": "1", + "source_system": source_system, + "source_id": source_id, + "exact": exact, + "approximated": approximated, + "accepted_approximations": accepted, + "pending_approximations": pending, + "unsupported": unsupported, + "manual_bindings": list(metadata.get("manual_bindings", [])), + "provider_assumptions": list(metadata.get("provider_assumptions", [])), + "activation_allowed": status == "ready", + } + row = ReportingImportAssessment( + tenant_id=_tenant(principal), + assessment_id=str(uuid.uuid4()), + source_system=_required(source_system, "Import source system", 255), + source_id=_required(source_id, "Import source identifier", 500), + source_fingerprint=_sha256(metadata), + mapping_report=mapping_report, + status=status, + accepted_approximations=accepted, + assessed_by=_actor(principal), + ) + session.add(row) + session.flush() + return _assessment_payload(row) + + +def list_import_assessments( + session: Session, + principal: object, + *, + limit: int = 100, +) -> tuple[dict[str, object], ...]: + _require_scope(principal, IMPORT_SCOPE) + rows = ( + session.query(ReportingImportAssessment) + .filter(ReportingImportAssessment.tenant_id == _tenant(principal)) + .order_by(ReportingImportAssessment.created_at.desc()) + .limit(max(1, min(limit, 200))) + .all() + ) + return tuple(_assessment_payload(row) for row in rows) + + +def _validate_saved_view_state(state: Mapping[str, object]) -> None: + query = state.get("query") + if query is not None: + if not isinstance(query, Mapping): + raise ReportingOperationError("Saved-view query must be an object.") + ReportQuery.model_validate(query) + if len(state) > 100: + raise ReportingOperationError("Saved-view state is limited to 100 entries.") + + +def _validate_trigger( + trigger_kind: str, + trigger_config: Mapping[str, object], + *, + next_run_at: datetime | None, +) -> datetime | None: + if trigger_kind not in {"scheduled", "interval"}: + raise ReportingOperationError("Unsupported Reporting schedule trigger.") + if next_run_at is not None: + _aware(next_run_at, "Reporting next_run_at") + if trigger_kind == "scheduled": + if next_run_at is None: + raise ReportingOperationError("Scheduled reports require next_run_at.") + return next_run_at + seconds = int(trigger_config.get("seconds", 0)) + if not 60 <= seconds <= 31_536_000: + raise ReportingOperationError( + "Reporting intervals must be between 60 seconds and one year." + ) + return next_run_at or utc_now() + timedelta(seconds=seconds) + + +def _advance_schedule(row: ReportingSchedule, scheduled_for: datetime) -> None: + if row.trigger_kind == "scheduled": + row.enabled = False + row.next_run_at = None + return + if scheduled_for.tzinfo is None or scheduled_for.utcoffset() is None: + scheduled_for = scheduled_for.replace(tzinfo=UTC) + seconds = int((row.trigger_config or {}).get("seconds", 0)) + next_run = scheduled_for + timedelta(seconds=seconds) + now = utc_now() + while next_run <= now: + next_run += timedelta(seconds=seconds) + row.next_run_at = next_run + + +def _saved_view_payload(row: ReportingSavedView) -> dict[str, object]: + return { + "view_id": row.view_id, + "report_id": row.report_id, + "report_revision": row.report_revision, + "owner_kind": row.owner_kind, + "owner_id": row.owner_id, + "name": row.name, + "revision": row.revision, + "state": dict(row.state or {}), + "shared": row.shared, + "access": dict(row.access or {}), + "updated_at": _datetime_text(row.updated_at), + } + + +def _schedule_payload(row: ReportingSchedule) -> dict[str, object]: + return { + "schedule_id": row.schedule_id, + "report_id": row.report_id, + "report_revision": row.report_revision, + "name": row.name, + "revision": row.revision, + "trigger_kind": row.trigger_kind, + "trigger_config": dict(row.trigger_config or {}), + "parameters": dict(row.parameters or {}), + "query": dict(row.query or {}), + "publication_target": dict(row.publication_target or {}), + "enabled": row.enabled, + "next_run_at": _datetime_text(row.next_run_at), + "last_run_at": _datetime_text(row.last_run_at), + "last_execution_id": row.last_execution_id, + } + + +def _publication_payload(row: ReportingPublication) -> dict[str, object]: + return { + "publication_id": row.publication_id, + "execution_id": row.execution_id, + "target_capability": row.target_capability, + "target_ref": row.target_ref, + "format": row.format, + "status": row.status, + "evidence": dict(row.evidence or {}), + "error": row.error, + "completed_at": _datetime_text(row.completed_at), + } + + +def _assessment_payload(row: ReportingImportAssessment) -> dict[str, object]: + return { + "assessment_id": row.assessment_id, + "source_system": row.source_system, + "source_id": row.source_id, + "source_fingerprint": row.source_fingerprint, + "mapping_report": dict(row.mapping_report or {}), + "status": row.status, + "accepted_approximations": list(row.accepted_approximations or []), + "assessed_by": row.assessed_by, + "created_at": _datetime_text(row.created_at), + } + + +def _safe_csv_cell(value: object) -> object: + if isinstance(value, (dict, list, tuple)): + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + if isinstance(value, str) and value.startswith(("=", "+", "-", "@")): + return f"'{value}" + return value + + +def _has_scope(principal: object, scope: str) -> bool: + method = getattr(principal, "has", None) + if callable(method): + return bool(method(scope)) + return scopes_grant_compatible( + frozenset(getattr(principal, "scopes", ()) or ()), + scope, + ) + + +def _require_scope(principal: object, scope: str) -> None: + if not _has_scope(principal, scope): + raise PermissionError(f"Missing scope: {scope}") + + +def _tenant(principal: object) -> str: + tenant_id = str(getattr(principal, "tenant_id", "") or "").strip() + if not tenant_id: + raise ReportingOperationError( + "Reporting operations require a tenant-bound principal." + ) + return tenant_id + + +def _actor(principal: object) -> str | None: + for value in ( + getattr(principal, "account_id", None), + getattr(principal, "identity_id", None), + getattr(principal, "membership_id", None), + ): + if str(value or "").strip(): + return str(value) + return None + + +def _session(value: object) -> Session: + if not isinstance(value, Session): + raise TypeError("Reporting operations require a SQLAlchemy session.") + return value + + +def _required(value: object, label: str, maximum: int) -> str: + result = str(value or "").strip() + if not result: + raise ReportingOperationError(f"{label} is required.") + if len(result) > maximum: + raise ReportingOperationError(f"{label} is limited to {maximum} characters.") + return result + + +def _aware(value: datetime, label: str) -> None: + if value.tzinfo is None or value.utcoffset() is None: + raise ReportingOperationError(f"{label} must include a timezone.") + + +def _sha256(value: object) -> str: + return hashlib.sha256( + json.dumps( + _json_value(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + ).hexdigest() + + +def _json_value(value: object) -> Any: + if isinstance(value, datetime): + return value.isoformat() + if hasattr(value, "model_dump"): + return _json_value(value.model_dump(mode="json")) + if isinstance(value, Mapping): + return {str(key): _json_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + return value + + +def _datetime_text(value: datetime | None) -> str | None: + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=UTC) + return value.isoformat() + + +__all__ = [ + "IMPORT_SCOPE", + "PUBLISH_SCOPE", + "SCHEDULE_SCOPE", + "ReportingOperationError", + "SqlReportingScheduler", + "assess_import", + "delete_saved_view", + "dispatch_due_schedules", + "export_execution", + "list_import_assessments", + "list_saved_views", + "list_schedules", + "publish_execution", + "upsert_saved_view", + "upsert_schedule", +] diff --git a/src/govoplan_reporting/backend/query_engine.py b/src/govoplan_reporting/backend/query_engine.py new file mode 100644 index 0000000..69b1bc3 --- /dev/null +++ b/src/govoplan_reporting/backend/query_engine.py @@ -0,0 +1,494 @@ +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal +from typing import Any + +from govoplan_reporting.backend.schemas import ( + FilterClause, + MeasureDefinition, + ReportQuery, + SemanticModelDefinition, + TypedExpression, + VisualizationDefinition, +) + + +QUERY_ENGINE_VERSION = "reporting-query-v1" + + +class ReportingQueryError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class QueryResult: + rows: tuple[dict[str, Any], ...] + total_rows: int + schema: tuple[dict[str, Any], ...] + truncated: bool + diagnostics: tuple[dict[str, Any], ...] = () + + +def execute_semantic_query( + rows: Sequence[Mapping[str, object]], + semantic_model: SemanticModelDefinition, + query: ReportQuery, +) -> QueryResult: + dimensions = {item.key: item for item in semantic_model.dimensions} + measures = {item.key: item for item in semantic_model.measures} + selected_dimensions = tuple(query.dimensions or semantic_model.default_dimensions) + selected_measures = tuple(query.measures or semantic_model.default_measures) + if query.mode == "pivot" and query.pivot is not None: + selected_dimensions = tuple( + dict.fromkeys((*query.pivot.rows, *query.pivot.columns)) + ) + selected_measures = tuple(query.pivot.measures or selected_measures) + _require_known(selected_dimensions, dimensions, "dimensions") + _require_known(selected_measures, measures, "measures") + _require_known( + tuple(item.dimension for item in query.filters), + dimensions, + "filter dimensions", + ) + filtered = tuple( + dict(row) + for row in rows + if all( + _matches_filter(row.get(dimensions[item.dimension].field), item) + for item in query.filters + ) + ) + if query.mode == "detail": + result_rows = _detail_rows(filtered, selected_dimensions, dimensions) + else: + result_rows = _summary_rows( + filtered, + selected_dimensions, + selected_measures, + dimensions, + measures, + ) + if query.mode == "pivot" and query.pivot is not None: + result_rows = _pivot_rows( + result_rows, + row_dimensions=tuple(query.pivot.rows), + column_dimensions=tuple(query.pivot.columns), + measures=selected_measures, + include_totals=query.pivot.include_totals, + ) + sorted_rows = _sort_rows(result_rows, query) + total = len(sorted_rows) + selected = sorted_rows[query.offset : query.offset + query.limit] + return QueryResult( + rows=tuple(selected), + total_rows=total, + schema=_infer_schema(selected or sorted_rows[:1]), + truncated=query.offset + len(selected) < total, + ) + + +class DefaultChartRenderer: + """Build a provider-neutral chart model with a mandatory table fallback.""" + + def render( + self, + *, + visualization: VisualizationDefinition, + result: QueryResult, + ) -> Mapping[str, object]: + category = visualization.category_dimension + measure_keys = tuple(visualization.measures) + missing: set[str] = set() + if visualization.kind not in {"table", "pivot", "metric"}: + if not category or not measure_keys: + missing = {"chart category and measure configuration"} + else: + available = {str(item.get("name")) for item in result.schema} + missing = {category, *measure_keys} - available + if missing and not visualization.tabular_fallback: + raise ReportingQueryError( + "Chart references unavailable result fields: " + + ", ".join(sorted(missing)) + ) + return { + "contract_version": "1", + "kind": "table" if missing else visualization.kind, + "requested_kind": visualization.kind, + "category": category, + "series": visualization.series_dimension, + "measures": list(measure_keys), + "options": dict(visualization.options), + "fallback_reason": ( + "The selected query does not expose the fields required by the " + "saved visualization. Showing its accessible table fallback." + if missing + else None + ), + "rows": list(result.rows), + "schema": list(result.schema), + "tabular_fallback": ( + { + "rows": list(result.rows), + "schema": list(result.schema), + } + if visualization.tabular_fallback or missing + else None + ), + } + + +def _detail_rows( + rows: Sequence[Mapping[str, object]], + selected_dimensions: Sequence[str], + dimensions: Mapping[str, object], +) -> list[dict[str, Any]]: + if not selected_dimensions: + return [ + {str(key): _json_value(value) for key, value in row.items()} for row in rows + ] + return [ + { + key: _json_value(row.get(getattr(dimensions[key], "field"))) + for key in selected_dimensions + } + for row in rows + ] + + +def _summary_rows( + rows: Sequence[Mapping[str, object]], + selected_dimensions: Sequence[str], + selected_measures: Sequence[str], + dimensions: Mapping[str, object], + measures: Mapping[str, MeasureDefinition], +) -> list[dict[str, Any]]: + grouped: dict[tuple[object, ...], list[Mapping[str, object]]] = defaultdict(list) + if selected_dimensions: + for row in rows: + key = tuple( + row.get(getattr(dimensions[item], "field")) + for item in selected_dimensions + ) + grouped[key].append(row) + else: + grouped[()] = list(rows) + result: list[dict[str, Any]] = [] + for group_key, group_rows in grouped.items(): + item: dict[str, Any] = { + dimension: _json_value(value) + for dimension, value in zip(selected_dimensions, group_key, strict=True) + } + pending: list[MeasureDefinition] = [] + for key in selected_measures: + measure = measures[key] + if measure.aggregation == "calculated": + pending.append(measure) + else: + item[key] = _aggregate(group_rows, measure) + for measure in pending: + item[measure.key] = _evaluate_expression( + measure.expression, + row=None, + measures=item, + ) + result.append(item) + return result + + +def _aggregate( + rows: Sequence[Mapping[str, object]], + measure: MeasureDefinition, +) -> object: + if measure.aggregation == "count": + if measure.field is None: + return len(rows) + return sum(row.get(measure.field) is not None for row in rows) + values = [ + row.get(measure.field or "") + for row in rows + if row.get(measure.field or "") is not None + ] + if measure.aggregation == "count_distinct": + return len({_hashable(value) for value in values}) + if not values: + return None + if measure.aggregation == "sum": + return _numeric_result(sum(_number(value) for value in values)) + if measure.aggregation == "average": + return _numeric_result(sum(_number(value) for value in values) / len(values)) + if measure.aggregation == "minimum": + return _json_value(min(values)) + if measure.aggregation == "maximum": + return _json_value(max(values)) + raise ReportingQueryError( + f"Unsupported measure aggregation: {measure.aggregation!r}." + ) + + +def _evaluate_expression( + expression: TypedExpression | None, + *, + row: Mapping[str, object] | None, + measures: Mapping[str, object], +) -> object: + if expression is None: + return None + if expression.op == "literal": + return expression.value + if expression.op == "field": + return row.get(expression.ref or "") if row is not None else None + if expression.op == "measure": + return measures.get(expression.ref or "") + values = [ + _evaluate_expression(item, row=row, measures=measures) + for item in expression.args + ] + if expression.op == "add": + return _numeric_result(sum(_number(item) for item in values)) + if expression.op == "subtract": + _arity(values, 2, expression.op) + return _numeric_result(_number(values[0]) - _number(values[1])) + if expression.op == "multiply": + total = Decimal(1) + for value in values: + total *= _number(value) + return _numeric_result(total) + if expression.op == "divide": + _arity(values, 2, expression.op) + divisor = _number(values[1]) + return None if divisor == 0 else _numeric_result(_number(values[0]) / divisor) + if expression.op == "coalesce": + return next((item for item in values if item is not None), None) + if expression.op == "case": + if len(values) < 3: + raise ReportingQueryError( + "Case expressions require condition, value, and default." + ) + pairs = values[:-1] + for index in range(0, len(pairs) - 1, 2): + if bool(pairs[index]): + return pairs[index + 1] + return values[-1] + if expression.op in {"eq", "ne", "gt", "gte", "lt", "lte"}: + _arity(values, 2, expression.op) + return _compare(values[0], values[1], expression.op) + if expression.op == "and": + return all(bool(item) for item in values) + if expression.op == "or": + return any(bool(item) for item in values) + if expression.op == "not": + _arity(values, 1, expression.op) + return not bool(values[0]) + raise ReportingQueryError(f"Unsupported expression operator: {expression.op!r}.") + + +def _pivot_rows( + rows: Sequence[Mapping[str, object]], + *, + row_dimensions: Sequence[str], + column_dimensions: Sequence[str], + measures: Sequence[str], + include_totals: bool, +) -> list[dict[str, Any]]: + grouped: dict[tuple[object, ...], dict[str, Any]] = {} + totals: dict[tuple[object, ...], dict[str, Decimal]] = defaultdict( + lambda: defaultdict(Decimal) + ) + for source in rows: + row_key = tuple(source.get(item) for item in row_dimensions) + target = grouped.setdefault( + row_key, + { + item: _json_value(value) + for item, value in zip(row_dimensions, row_key, strict=True) + }, + ) + column_key = ( + " / ".join( + str(source.get(item) if source.get(item) is not None else "(blank)") + for item in column_dimensions + ) + or "value" + ) + for measure in measures: + field_name = f"{column_key}.{measure}" + value = source.get(measure) + target[field_name] = _json_value(value) + if include_totals and value is not None: + try: + totals[row_key][measure] += _number(value) + except ReportingQueryError: + pass + if include_totals: + for row_key, target in grouped.items(): + for measure in measures: + if measure in totals[row_key]: + target[f"total.{measure}"] = _numeric_result( + totals[row_key][measure] + ) + return list(grouped.values()) + + +def _matches_filter(value: object, clause: FilterClause) -> bool: + expected = clause.value + if clause.operator == "is_null": + return value is None + if clause.operator == "not_null": + return value is not None + if clause.operator == "eq": + return value == expected + if clause.operator == "ne": + return value != expected + if clause.operator in {"in", "not_in"}: + if not isinstance(expected, (list, tuple, set, frozenset)): + raise ReportingQueryError("Set filters require a list value.") + result = value in expected + return result if clause.operator == "in" else not result + if clause.operator == "contains": + return str(expected or "").casefold() in str(value or "").casefold() + if clause.operator == "starts_with": + return str(value or "").casefold().startswith(str(expected or "").casefold()) + if clause.operator == "between": + if not isinstance(expected, (list, tuple)) or len(expected) != 2: + raise ReportingQueryError("Between filters require two values.") + return value is not None and expected[0] <= value <= expected[1] + if clause.operator in {"gt", "gte", "lt", "lte"}: + if value is None or expected is None: + return False + return bool(_compare(value, expected, clause.operator)) + raise ReportingQueryError(f"Unsupported filter operator: {clause.operator!r}.") + + +def _sort_rows( + rows: Sequence[dict[str, Any]], + query: ReportQuery, +) -> list[dict[str, Any]]: + result = list(rows) + for clause in reversed(query.sort): + result.sort( + key=lambda item: _sort_key(item.get(clause.key)), + reverse=clause.direction == "desc", + ) + return result + + +def _infer_schema(rows: Sequence[Mapping[str, object]]) -> tuple[dict[str, Any], ...]: + names = tuple(dict.fromkeys(str(key) for row in rows for key in row)) + return tuple( + { + "name": name, + "type": _value_type( + next((row.get(name) for row in rows if row.get(name) is not None), None) + ), + "nullable": any(row.get(name) is None for row in rows), + } + for name in names + ) + + +def _value_type(value: object) -> str: + if value is None: + return "string" + if isinstance(value, bool): + return "boolean" + if isinstance(value, int): + return "integer" + if isinstance(value, (float, Decimal)): + return "number" + if isinstance(value, datetime): + return "datetime" + if isinstance(value, date): + return "date" + if isinstance(value, (dict, list, tuple)): + return "json" + return "string" + + +def _require_known( + keys: Sequence[str], + available: Mapping[str, object], + label: str, +) -> None: + unknown = set(keys) - set(available) + if unknown: + raise ReportingQueryError( + f"Report query references unknown {label}: " + ", ".join(sorted(unknown)) + ) + + +def _compare(left: object, right: object, operator: str) -> bool: + if operator == "eq": + return left == right + if operator == "ne": + return left != right + try: + if operator == "gt": + return left > right # type: ignore[operator] + if operator == "gte": + return left >= right # type: ignore[operator] + if operator == "lt": + return left < right # type: ignore[operator] + if operator == "lte": + return left <= right # type: ignore[operator] + except TypeError as exc: + raise ReportingQueryError("Report comparison values are incompatible.") from exc + raise ReportingQueryError(f"Unsupported comparison operator: {operator!r}.") + + +def _arity(values: Sequence[object], count: int, operator: str) -> None: + if len(values) != count: + raise ReportingQueryError( + f"Expression {operator} requires exactly {count} arguments." + ) + + +def _number(value: object) -> Decimal: + if value is None or isinstance(value, bool): + raise ReportingQueryError("A numeric report expression received a non-number.") + try: + return Decimal(str(value)) + except Exception as exc: + raise ReportingQueryError("A numeric report expression is invalid.") from exc + + +def _numeric_result(value: Decimal) -> int | float: + if value == value.to_integral_value(): + return int(value) + return float(value) + + +def _sort_key(value: object) -> tuple[bool, str, str]: + return value is None, type(value).__name__, str(value).casefold() + + +def _hashable(value: object) -> object: + if isinstance(value, Mapping): + return tuple(sorted((str(key), _hashable(item)) for key, item in value.items())) + if isinstance(value, (list, tuple)): + return tuple(_hashable(item) for item in value) + return value + + +def _json_value(value: object) -> Any: + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, Decimal): + return _numeric_result(value) + if isinstance(value, Mapping): + return {str(key): _json_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + return value + + +__all__ = [ + "QUERY_ENGINE_VERSION", + "DefaultChartRenderer", + "QueryResult", + "ReportingQueryError", + "execute_semantic_query", +] diff --git a/src/govoplan_reporting/backend/registry.py b/src/govoplan_reporting/backend/registry.py new file mode 100644 index 0000000..a26562d --- /dev/null +++ b/src/govoplan_reporting/backend/registry.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence + +from govoplan_reporting.backend.definitions import ( + create_definition, + get_definition, + list_definitions, + update_definition, +) + + +class SqlReportingRegistry: + def get_definition( + self, + session: object, + principal: object, + *, + definition_kind: str, + definition_id: str, + revision: int | None = None, + ): + return get_definition( + session, # type: ignore[arg-type] + principal, + definition_kind=definition_kind, + definition_id=definition_id, + revision=revision, + ) + + def list_definitions( + self, + session: object, + principal: object, + *, + definition_kinds: Sequence[str] | None = None, + limit: int = 100, + ): + records, _total = list_definitions( + session, # type: ignore[arg-type] + principal, + definition_kinds=definition_kinds, + limit=limit, + ) + return records + + def create_definition( + self, + session: object, + principal: object, + **values, + ): + return create_definition(session, principal, **values) # type: ignore[arg-type] + + def update_definition( + self, + session: object, + principal: object, + *, + definition_kind: str, + definition_id: str, + expected_revision: int, + recorded_at, + change_reason: str, + idempotency_key: str, + changes: Mapping[str, object], + ): + return update_definition( + session, # type: ignore[arg-type] + principal, + definition_kind=definition_kind, + definition_id=definition_id, + expected_revision=expected_revision, + recorded_at=recorded_at, + change_reason=change_reason, + idempotency_key=idempotency_key, + changes=changes, + ) + + +__all__ = ["SqlReportingRegistry"] diff --git a/src/govoplan_reporting/backend/router.py b/src/govoplan_reporting/backend/router.py new file mode 100644 index 0000000..e5f8aec --- /dev/null +++ b/src/govoplan_reporting/backend/router.py @@ -0,0 +1,526 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Query, Response, status +from sqlalchemy.orm import Session + +from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope +from govoplan_core.core.dataflows import dataflow_dataset_output +from govoplan_core.db.session import get_session +from govoplan_reporting.backend.definitions import ( + READ_SCOPE, + WRITE_SCOPE, + ReportingDefinitionError, + create_definition, + definition_history, + get_definition, + list_definitions, + update_definition, +) +from govoplan_reporting.backend.execution import ( + QUALITY_SCOPE, + RUN_SCOPE, + ReportingExecutionError, + ReportingExecutionFailure, + execute_report, + get_execution, + list_executions, + run_quality_plan, +) +from govoplan_reporting.backend.operations import ( + IMPORT_SCOPE, + PUBLISH_SCOPE, + SCHEDULE_SCOPE, + ReportingOperationError, + assess_import, + delete_saved_view, + dispatch_due_schedules, + export_execution, + list_import_assessments, + list_saved_views, + list_schedules, + publish_execution, + upsert_saved_view, + upsert_schedule, +) +from govoplan_reporting.backend.query_engine import ReportingQueryError +from govoplan_reporting.backend.schemas import ( + DefinitionUpdateRequest, + DefinitionWriteRequest, + ImportAssessmentRequest, + PublicationRequest, + QualityRunRequest, + ReportExecutionRequest, + SavedViewWriteRequest, + ScheduleWriteRequest, +) + + +def create_router(registry: object | None) -> APIRouter: + router = APIRouter(prefix="/reporting", tags=["reporting"]) + + @router.get("/definitions") + def api_list_definitions( + definition_kind: list[str] | None = Query(default=None), + definition_status: list[str] | None = Query(default=None, alias="status"), + query: str = "", + offset: int = Query(default=0, ge=0), + limit: int = Query(default=100, ge=1, le=200), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, READ_SCOPE) + try: + records, total = list_definitions( + session, + principal, + definition_kinds=definition_kind, + statuses=definition_status, + query=query, + offset=offset, + limit=limit, + ) + except ReportingDefinitionError as exc: + raise _error(exc) from exc + return { + "definitions": [item.to_dict() for item in records], + "total": total, + "offset": offset, + "limit": limit, + } + + @router.post( + "/definitions", + status_code=status.HTTP_201_CREATED, + ) + def api_create_definition( + payload: DefinitionWriteRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, Any]: + _require(principal, WRITE_SCOPE) + try: + record = create_definition( + session, + principal, + **payload.model_dump(), + ) + session.commit() + except (ReportingDefinitionError, ValueError) as exc: + session.rollback() + raise _error(exc) from exc + return record.to_dict() + + @router.get("/definitions/{definition_kind}/{definition_id}") + def api_get_definition( + definition_kind: str, + definition_id: str, + revision: int | None = Query(default=None, ge=1), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, Any]: + _require(principal, READ_SCOPE) + try: + record = get_definition( + session, + principal, + definition_kind=definition_kind, + definition_id=definition_id, + revision=revision, + ) + except ReportingDefinitionError as exc: + raise _error(exc) from exc + if record is None: + raise HTTPException( + status_code=404, detail="Reporting definition not found" + ) + return record.to_dict() + + @router.patch("/definitions/{definition_kind}/{definition_id}") + def api_update_definition( + definition_kind: str, + definition_id: str, + payload: DefinitionUpdateRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, Any]: + _require(principal, WRITE_SCOPE) + changes = payload.model_dump( + exclude={ + "expected_revision", + "recorded_at", + "change_reason", + "idempotency_key", + }, + exclude_unset=True, + ) + try: + record = update_definition( + session, + principal, + definition_kind=definition_kind, + definition_id=definition_id, + expected_revision=payload.expected_revision, + recorded_at=payload.recorded_at, + change_reason=payload.change_reason, + idempotency_key=payload.idempotency_key, + changes=changes, + ) + session.commit() + except ( + ReportingDefinitionError, + PermissionError, + LookupError, + ValueError, + ) as exc: + session.rollback() + raise _error(exc) from exc + return record.to_dict() + + @router.get("/definitions/{definition_kind}/{definition_id}/history") + def api_definition_history( + definition_kind: str, + definition_id: str, + limit: int = Query(default=100, ge=1, le=200), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, READ_SCOPE) + records = definition_history( + session, + principal, + definition_kind=definition_kind, + definition_id=definition_id, + limit=limit, + ) + if not records: + raise HTTPException( + status_code=404, detail="Reporting definition not found" + ) + return {"revisions": [item.to_dict() for item in records]} + + @router.get("/sources/dataflow") + def api_dataflow_sources( + query: str = "", + limit: int = Query(default=100, ge=1, le=200), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, WRITE_SCOPE) + provider = dataflow_dataset_output(registry) + if provider is None: + return { + "available": False, + "reason": "The Dataflow dataset provider is not enabled.", + "sources": [], + } + sources = provider.list_outputs( + session, + principal, + query=query, + limit=limit, + ) + return { + "available": True, + "reason": None, + "sources": [ + { + "pipeline_ref": item.pipeline_ref, + "name": item.name, + "description": item.description, + "revision": item.revision, + "definition_hash": item.definition_hash, + "status": item.status, + "updated_at": item.updated_at.isoformat() + if item.updated_at + else None, + "parameters": dict(item.parameters), + "provenance": dict(item.provenance), + } + for item in sources + ], + } + + @router.post("/reports/{report_id}/executions") + def api_execute_report( + report_id: str, + payload: ReportExecutionRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, RUN_SCOPE) + try: + result = execute_report( + session, + principal, + registry=registry, + report_id=report_id, + report_revision=payload.report_revision, + parameters=payload.parameters, + query=payload.query, + idempotency_key=payload.idempotency_key, + ) + session.commit() + except ReportingExecutionFailure as exc: + session.commit() + raise HTTPException( + status_code=422, + detail={"message": str(exc), "execution_id": exc.execution_id}, + ) from exc + except ( + ReportingExecutionError, + ReportingQueryError, + PermissionError, + LookupError, + ValueError, + ) as exc: + session.rollback() + raise _error(exc) from exc + return result + + @router.get("/reports/{report_id}/executions") + def api_list_executions( + report_id: str, + limit: int = Query(default=100, ge=1, le=200), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, RUN_SCOPE) + return { + "executions": list( + list_executions(session, principal, report_id=report_id, limit=limit) + ) + } + + @router.get("/executions/{execution_id}") + def api_get_execution( + execution_id: str, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, RUN_SCOPE) + result = get_execution(session, principal, execution_id=execution_id) + if result is None: + raise HTTPException(status_code=404, detail="Reporting execution not found") + return result + + @router.get("/executions/{execution_id}/export") + def api_export_execution( + execution_id: str, + format: str = Query(default="csv", pattern="^(csv|json)$"), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> Response: + _require(principal, RUN_SCOPE) + try: + content, media_type, filename = export_execution( + session, + principal, + execution_id=execution_id, + format=format, + ) + except (ReportingOperationError, LookupError) as exc: + raise _error(exc) from exc + return Response( + content=content, + media_type=media_type, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + @router.post("/executions/{execution_id}/publications") + def api_publish_execution( + execution_id: str, + payload: PublicationRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, PUBLISH_SCOPE) + try: + result = publish_execution( + session, + principal, + registry=registry, + execution_id=execution_id, + **payload.model_dump(), + ) + session.commit() + except (ReportingOperationError, PermissionError, LookupError) as exc: + session.commit() + raise _error(exc) from exc + return result + + @router.get("/reports/{report_id}/saved-views") + def api_list_saved_views( + report_id: str, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, READ_SCOPE) + return { + "views": list(list_saved_views(session, principal, report_id=report_id)) + } + + @router.put("/reports/{report_id}/saved-views/{view_id}") + def api_upsert_saved_view( + report_id: str, + view_id: str, + payload: SavedViewWriteRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, READ_SCOPE) + if view_id != payload.view_id: + raise HTTPException(status_code=400, detail="Saved-view identifiers differ") + try: + result = upsert_saved_view( + session, + principal, + report_id=report_id, + **payload.model_dump(), + ) + session.commit() + except (ReportingOperationError, PermissionError, LookupError) as exc: + session.rollback() + raise _error(exc) from exc + return result + + @router.delete("/saved-views/{view_id}", status_code=204) + def api_delete_saved_view( + view_id: str, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> Response: + _require(principal, READ_SCOPE) + try: + deleted = delete_saved_view(session, principal, view_id=view_id) + session.commit() + except PermissionError as exc: + session.rollback() + raise _error(exc) from exc + if not deleted: + raise HTTPException(status_code=404, detail="Saved view not found") + return Response(status_code=204) + + @router.get("/schedules") + def api_list_schedules( + report_id: str | None = None, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, SCHEDULE_SCOPE) + return { + "schedules": list(list_schedules(session, principal, report_id=report_id)) + } + + @router.put("/schedules/{schedule_id}") + def api_upsert_schedule( + schedule_id: str, + payload: ScheduleWriteRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, SCHEDULE_SCOPE) + if schedule_id != payload.schedule_id: + raise HTTPException(status_code=400, detail="Schedule identifiers differ") + try: + result = upsert_schedule(session, principal, **payload.model_dump()) + session.commit() + except (ReportingOperationError, PermissionError, LookupError) as exc: + session.rollback() + raise _error(exc) from exc + return result + + @router.post("/schedules/dispatch") + def api_dispatch_schedules( + limit: int = Query(default=20, ge=1, le=100), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, SCHEDULE_SCOPE) + result = dispatch_due_schedules( + session, + principal, + registry=registry, + now=None, + limit=limit, + ) + session.commit() + return result + + @router.post("/quality-plans/{quality_plan_id}/runs") + def api_run_quality_plan( + quality_plan_id: str, + payload: QualityRunRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, QUALITY_SCOPE) + try: + result = run_quality_plan( + session, + principal, + registry=registry, + quality_plan_id=quality_plan_id, + quality_plan_revision=payload.quality_plan_revision, + parameters=payload.parameters, + ) + session.commit() + except (ReportingExecutionError, PermissionError, LookupError) as exc: + session.rollback() + raise _error(exc) from exc + return result + + @router.post("/imports/assessments", status_code=201) + def api_assess_import( + payload: ImportAssessmentRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, IMPORT_SCOPE) + try: + result = assess_import(session, principal, **payload.model_dump()) + session.commit() + except (ReportingOperationError, PermissionError) as exc: + session.rollback() + raise _error(exc) from exc + return result + + @router.get("/imports/assessments") + def api_list_import_assessments( + limit: int = Query(default=100, ge=1, le=200), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), + ) -> dict[str, object]: + _require(principal, IMPORT_SCOPE) + return { + "assessments": list( + list_import_assessments(session, principal, limit=limit) + ) + } + + return router + + +def _require(principal: ApiPrincipal, scope: str) -> None: + if not has_scope(principal, scope): + raise HTTPException(status_code=403, detail=f"Missing scope: {scope}") + + +def _error(exc: Exception) -> HTTPException: + message = str(exc) + lowered = message.casefold() + if isinstance(exc, LookupError): + code = 404 + elif isinstance(exc, PermissionError): + code = 403 + elif any(word in lowered for word in ("conflict", "already", "stale")): + code = 409 + elif "unavailable" in lowered or "not enabled" in lowered: + code = 503 + else: + code = 400 + return HTTPException(status_code=code, detail=message) + + +__all__ = ["create_router"] diff --git a/src/govoplan_reporting/backend/schemas.py b/src/govoplan_reporting/backend/schemas.py new file mode 100644 index 0000000..ec211b7 --- /dev/null +++ b/src/govoplan_reporting/backend/schemas.py @@ -0,0 +1,503 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +DefinitionKind = Literal[ + "dataset", + "semantic_model", + "report", + "quality_plan", +] +FieldType = Literal[ + "string", + "integer", + "number", + "boolean", + "date", + "datetime", + "json", +] + + +class ReportingReference(BaseModel): + model_config = ConfigDict(extra="forbid") + + owner_module: str = Field(min_length=1, max_length=100) + resource_type: str = Field(min_length=1, max_length=100) + resource_id: str = Field(min_length=1, max_length=255) + revision: str | None = Field(default=None, max_length=255) + relationship: str = Field(default="related", min_length=1, max_length=80) + label: str | None = Field(default=None, max_length=500) + + +class DatasetField(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1, max_length=255) + type: FieldType = "string" + label: str | None = Field(default=None, max_length=500) + nullable: bool = True + description: str | None = Field(default=None, max_length=4_000) + classification: Literal[ + "public", + "internal", + "confidential", + "restricted", + ] = "internal" + + +class FreshnessPolicy(BaseModel): + model_config = ConfigDict(extra="forbid") + + max_age_seconds: int | None = Field(default=None, ge=1, le=31_536_000) + stale_action: Literal["allow", "warn", "block"] = "warn" + require_source_fingerprints: bool = True + + +class DatasetDefinition(BaseModel): + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + source_kind: Literal["dataflow", "read_model", "static"] + source_ref: str = Field(min_length=1, max_length=500) + source_revision: int | None = Field(default=None, ge=1) + definition_hash: str | None = Field(default=None, min_length=1, max_length=128) + source_parameters: dict[str, Any] = Field(default_factory=dict) + static_rows: list[dict[str, Any]] = Field(default_factory=list, max_length=2_000) + expected_source_fingerprints: list[dict[str, Any]] = Field( + default_factory=list, + max_length=100, + ) + fields: list[DatasetField] = Field( + default_factory=list, + max_length=500, + validation_alias="schema", + serialization_alias="schema", + ) + freshness: FreshnessPolicy = Field(default_factory=FreshnessPolicy) + row_policy_ref: str | None = Field(default=None, max_length=500) + purpose: str = Field(min_length=1, max_length=4_000) + privacy: dict[str, Any] = Field(default_factory=dict) + retention: dict[str, Any] = Field(default_factory=dict) + policy_provenance: dict[str, Any] = Field(default_factory=dict) + institutional_references: list[ReportingReference] = Field( + default_factory=list, + max_length=200, + ) + + @model_validator(mode="after") + def validate_source_pin(self) -> "DatasetDefinition": + if self.source_kind == "dataflow" and self.source_revision is None: + raise ValueError("Dataflow datasets require a pinned source revision.") + if self.source_kind == "static" and not self.static_rows: + raise ValueError("Static analytical datasets require static_rows.") + names = [item.name for item in self.fields] + if len(names) != len(set(names)): + raise ValueError("Dataset schema field names must be unique.") + return self + + +class DimensionDefinition(BaseModel): + model_config = ConfigDict(extra="forbid") + + key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$") + field: str = Field(min_length=1, max_length=255) + label: str = Field(min_length=1, max_length=500) + type: FieldType = "string" + format: str | None = Field(default=None, max_length=255) + default_sort: Literal["asc", "desc"] | None = None + + +class HierarchyLevel(BaseModel): + model_config = ConfigDict(extra="forbid") + + dimension: str = Field(min_length=1, max_length=120) + label: str | None = Field(default=None, max_length=500) + + +class HierarchyDefinition(BaseModel): + model_config = ConfigDict(extra="forbid") + + key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$") + label: str = Field(min_length=1, max_length=500) + levels: list[HierarchyLevel] = Field(min_length=1, max_length=20) + + +class TypedExpression(BaseModel): + model_config = ConfigDict(extra="forbid") + + op: Literal[ + "literal", + "field", + "measure", + "add", + "subtract", + "multiply", + "divide", + "coalesce", + "case", + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "and", + "or", + "not", + ] + value: Any = None + ref: str | None = Field(default=None, max_length=255) + args: list["TypedExpression"] = Field(default_factory=list, max_length=50) + result_type: FieldType | None = None + + @model_validator(mode="after") + def validate_shape(self) -> "TypedExpression": + if self.op in {"field", "measure"} and not self.ref: + raise ValueError(f"Expression {self.op} requires ref.") + if self.op == "literal" and self.args: + raise ValueError("Literal expressions cannot have arguments.") + if self.op not in {"literal", "field", "measure"} and not self.args: + raise ValueError(f"Expression {self.op} requires arguments.") + return self + + +class MeasureDefinition(BaseModel): + model_config = ConfigDict(extra="forbid") + + key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$") + label: str = Field(min_length=1, max_length=500) + aggregation: Literal[ + "sum", + "count", + "count_distinct", + "average", + "minimum", + "maximum", + "calculated", + ] + field: str | None = Field(default=None, max_length=255) + expression: TypedExpression | None = None + format: str | None = Field(default=None, max_length=255) + + @model_validator(mode="after") + def validate_measure(self) -> "MeasureDefinition": + if self.aggregation == "calculated" and self.expression is None: + raise ValueError("Calculated measures require an expression.") + if self.aggregation not in {"count", "calculated"} and not self.field: + raise ValueError(f"{self.aggregation} measures require a field.") + return self + + +class SemanticModelDefinition(BaseModel): + model_config = ConfigDict(extra="forbid") + + dataset_id: str = Field(min_length=1, max_length=255) + dataset_revision: int = Field(ge=1) + dimensions: list[DimensionDefinition] = Field(default_factory=list, max_length=300) + hierarchies: list[HierarchyDefinition] = Field(default_factory=list, max_length=100) + measures: list[MeasureDefinition] = Field(default_factory=list, max_length=300) + default_dimensions: list[str] = Field(default_factory=list, max_length=50) + default_measures: list[str] = Field(default_factory=list, max_length=50) + metadata: dict[str, Any] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_semantics(self) -> "SemanticModelDefinition": + dimensions = {item.key for item in self.dimensions} + measures = {item.key for item in self.measures} + if len(dimensions) != len(self.dimensions): + raise ValueError("Semantic dimensions require unique keys.") + if len(measures) != len(self.measures): + raise ValueError("Semantic measures require unique keys.") + unknown_dimensions = set(self.default_dimensions) - dimensions + unknown_measures = set(self.default_measures) - measures + hierarchy_dimensions = { + level.dimension + for hierarchy in self.hierarchies + for level in hierarchy.levels + } + unknown_dimensions |= hierarchy_dimensions - dimensions + if unknown_dimensions or unknown_measures: + raise ValueError( + "Semantic model references unknown dimensions or measures: " + + ", ".join(sorted(unknown_dimensions | unknown_measures)) + ) + return self + + +class ParameterDefinition(BaseModel): + model_config = ConfigDict(extra="forbid") + + key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$") + label: str = Field(min_length=1, max_length=500) + type: FieldType = "string" + required: bool = False + default: Any = None + allowed_values: list[Any] = Field(default_factory=list, max_length=500) + + +class FilterClause(BaseModel): + model_config = ConfigDict(extra="forbid") + + dimension: str = Field(min_length=1, max_length=120) + operator: Literal[ + "eq", + "ne", + "in", + "not_in", + "contains", + "starts_with", + "gt", + "gte", + "lt", + "lte", + "between", + "is_null", + "not_null", + ] = "eq" + value: Any = None + + +class SortClause(BaseModel): + model_config = ConfigDict(extra="forbid") + + key: str = Field(min_length=1, max_length=120) + direction: Literal["asc", "desc"] = "asc" + + +class PivotDefinition(BaseModel): + model_config = ConfigDict(extra="forbid") + + rows: list[str] = Field(default_factory=list, max_length=10) + columns: list[str] = Field(default_factory=list, max_length=5) + measures: list[str] = Field(default_factory=list, max_length=20) + include_totals: bool = True + + +class ReportQuery(BaseModel): + model_config = ConfigDict(extra="forbid") + + mode: Literal["summary", "detail", "pivot"] = "summary" + dimensions: list[str] = Field(default_factory=list, max_length=50) + measures: list[str] = Field(default_factory=list, max_length=50) + filters: list[FilterClause] = Field(default_factory=list, max_length=100) + sort: list[SortClause] = Field(default_factory=list, max_length=20) + pivot: PivotDefinition | None = None + offset: int = Field(default=0, ge=0) + limit: int = Field(default=200, ge=1, le=2_000) + + @model_validator(mode="after") + def validate_pivot(self) -> "ReportQuery": + if self.mode == "pivot" and self.pivot is None: + raise ValueError("Pivot mode requires a pivot definition.") + return self + + +class VisualizationDefinition(BaseModel): + model_config = ConfigDict(extra="forbid") + + kind: Literal[ + "table", + "pivot", + "bar", + "line", + "area", + "column", + "pie", + "metric", + ] = "table" + category_dimension: str | None = Field(default=None, max_length=120) + series_dimension: str | None = Field(default=None, max_length=120) + measures: list[str] = Field(default_factory=list, max_length=20) + options: dict[str, Any] = Field(default_factory=dict) + tabular_fallback: bool = True + + +class ReportDefinition(BaseModel): + model_config = ConfigDict(extra="forbid") + + semantic_model_id: str = Field(min_length=1, max_length=255) + semantic_model_revision: int = Field(ge=1) + parameters: list[ParameterDefinition] = Field(default_factory=list, max_length=100) + default_query: ReportQuery = Field(default_factory=ReportQuery) + visualization: VisualizationDefinition = Field( + default_factory=VisualizationDefinition + ) + layout: dict[str, Any] = Field(default_factory=dict) + access_policy: dict[str, Any] = Field(default_factory=dict) + publication_defaults: dict[str, Any] = Field(default_factory=dict) + institutional_references: list[ReportingReference] = Field( + default_factory=list, + max_length=200, + ) + + +class QualityAssertion(BaseModel): + model_config = ConfigDict(extra="forbid") + + key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$") + kind: Literal[ + "not_null", + "unique", + "range", + "accepted_values", + "row_count", + "comparison", + ] + field: str | None = Field(default=None, max_length=255) + severity: Literal["info", "warning", "error", "blocker"] = "error" + config: dict[str, Any] = Field(default_factory=dict) + + +class QualityPlanDefinition(BaseModel): + model_config = ConfigDict(extra="forbid") + + dataset_id: str = Field(min_length=1, max_length=255) + dataset_revision: int = Field(ge=1) + assertions: list[QualityAssertion] = Field(min_length=1, max_length=200) + block_report_execution: bool = True + + +DEFINITION_PAYLOAD_TYPES = { + "dataset": DatasetDefinition, + "semantic_model": SemanticModelDefinition, + "report": ReportDefinition, + "quality_plan": QualityPlanDefinition, +} + + +def validate_definition_payload( + kind: str, + payload: dict[str, Any], +) -> dict[str, Any]: + model = DEFINITION_PAYLOAD_TYPES.get(kind) + if model is None: + raise ValueError(f"Unsupported Reporting definition kind: {kind!r}.") + return model.model_validate(payload).model_dump(mode="json", by_alias=True) + + +class DefinitionWriteRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + definition_kind: DefinitionKind + definition_id: str = Field(min_length=1, max_length=255) + definition_key: str = Field( + min_length=1, + max_length=120, + pattern=r"^[a-z0-9._-]+$", + ) + name: str = Field(min_length=1, max_length=500) + description: str | None = Field(default=None, max_length=100_000) + status: Literal["draft", "active", "retired"] = "draft" + visibility: Literal["tenant", "restricted"] = "tenant" + recorded_at: datetime + change_reason: str = Field(min_length=1, max_length=1_000) + payload: dict[str, Any] + idempotency_key: str = Field(min_length=1, max_length=255) + + +class DefinitionUpdateRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + expected_revision: int = Field(ge=1) + recorded_at: datetime + change_reason: str = Field(min_length=1, max_length=1_000) + idempotency_key: str = Field(min_length=1, max_length=255) + name: str | None = Field(default=None, min_length=1, max_length=500) + description: str | None = Field(default=None, max_length=100_000) + status: Literal["draft", "active", "retired"] | None = None + visibility: Literal["tenant", "restricted"] | None = None + payload: dict[str, Any] | None = None + + +class ReportExecutionRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + report_revision: int | None = Field(default=None, ge=1) + parameters: dict[str, Any] = Field(default_factory=dict) + query: ReportQuery | None = None + idempotency_key: str = Field(min_length=1, max_length=255) + + +class SavedViewWriteRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + view_id: str = Field(min_length=1, max_length=36) + report_revision: int = Field(ge=1) + name: str = Field(min_length=1, max_length=500) + state: dict[str, Any] + shared: bool = False + access: dict[str, Any] = Field(default_factory=dict) + expected_revision: int | None = Field(default=None, ge=1) + + +class ScheduleWriteRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + schedule_id: str = Field(min_length=1, max_length=36) + report_id: str = Field(min_length=1, max_length=255) + report_revision: int = Field(ge=1) + name: str = Field(min_length=1, max_length=500) + trigger_kind: Literal["scheduled", "interval"] + trigger_config: dict[str, Any] + parameters: dict[str, Any] = Field(default_factory=dict) + query: ReportQuery = Field(default_factory=ReportQuery) + publication_target: dict[str, Any] = Field(default_factory=dict) + enabled: bool = True + next_run_at: datetime | None = None + expected_revision: int | None = Field(default=None, ge=1) + + +class PublicationRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + target_capability: str = Field(min_length=1, max_length=255) + target_ref: str | None = Field(default=None, max_length=1_000) + format: Literal["json", "csv", "xlsx", "html", "pdf"] = "csv" + idempotency_key: str = Field(min_length=1, max_length=255) + options: dict[str, Any] = Field(default_factory=dict) + + +class QualityRunRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + quality_plan_revision: int | None = Field(default=None, ge=1) + parameters: dict[str, Any] = Field(default_factory=dict) + + +class ImportAssessmentRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + source_system: str = Field(min_length=1, max_length=255) + source_id: str = Field(min_length=1, max_length=500) + metadata: dict[str, Any] + accepted_approximations: list[str] = Field(default_factory=list, max_length=500) + + +TypedExpression.model_rebuild() + + +__all__ = [ + "DatasetDefinition", + "DefinitionUpdateRequest", + "DefinitionWriteRequest", + "DimensionDefinition", + "FilterClause", + "ImportAssessmentRequest", + "MeasureDefinition", + "PublicationRequest", + "QualityPlanDefinition", + "QualityRunRequest", + "ReportDefinition", + "ReportExecutionRequest", + "ReportQuery", + "SavedViewWriteRequest", + "ScheduleWriteRequest", + "SemanticModelDefinition", + "TypedExpression", + "VisualizationDefinition", + "validate_definition_payload", +] diff --git a/src/govoplan_reporting/backend/search_source.py b/src/govoplan_reporting/backend/search_source.py new file mode 100644 index 0000000..7d3652a --- /dev/null +++ b/src/govoplan_reporting/backend/search_source.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Mapping, Sequence +from urllib.parse import quote + +from sqlalchemy import func +from sqlalchemy.orm import Session + +from govoplan_core.core.modules import ModuleContext +from govoplan_core.core.search import ( + SearchAuthorizationRequest, + SearchBackfillPage, + SearchBackfillRequest, + SearchDocument, + SearchResourceType, +) +from govoplan_reporting.backend.db.models import ( + ReportingDefinitionGrant, + ReportingDefinitionIdentity, + ReportingDefinitionRevision, +) +from govoplan_reporting.backend.definitions import can_read_definition + + +PROVIDER_ID = "reporting.reports" +RESOURCE_TYPE = "report" + + +class ReportingSearchSource: + def resource_types(self) -> Sequence[SearchResourceType]: + return ( + SearchResourceType( + provider_id=PROVIDER_ID, + module_id="reporting", + resource_type=RESOURCE_TYPE, + label="Reports", + requires_authorization_recheck=True, + ), + ) + + def backfill( + self, + session: object, + *, + request: SearchBackfillRequest, + ) -> SearchBackfillPage: + if request.provider_id != PROVIDER_ID or request.resource_type != RESOURCE_TYPE: + raise ValueError("Unsupported Reporting search source.") + db = _session(session) + query = db.query(ReportingDefinitionRevision).filter( + ReportingDefinitionRevision.tenant_id == request.tenant_id, + ReportingDefinitionRevision.definition_kind == "report", + ReportingDefinitionRevision.status != "retired", + ReportingDefinitionRevision.superseded_at.is_(None), + ) + if request.cursor: + query = query.filter(ReportingDefinitionRevision.id > request.cursor) + rows = ( + query.order_by(ReportingDefinitionRevision.id.asc()) + .limit(request.limit + 1) + .all() + ) + has_more = len(rows) > request.limit + selected = rows[: request.limit] + tokens = _acl_tokens(db, selected) + high_watermark = ( + db.query(func.max(ReportingDefinitionRevision.updated_at)) + .filter( + ReportingDefinitionRevision.tenant_id == request.tenant_id, + ReportingDefinitionRevision.definition_kind == "report", + ReportingDefinitionRevision.superseded_at.is_(None), + ) + .scalar() + ) + return SearchBackfillPage( + documents=tuple( + _document(row, tokens[row.definition_id]) for row in selected + ), + next_cursor=selected[-1].id if has_more and selected else None, + complete=not has_more, + high_watermark=( + high_watermark.isoformat() if high_watermark is not None else None + ), + ) + + def authorize( + self, + session: object, + principal: object, + *, + requests: Sequence[SearchAuthorizationRequest], + ) -> Mapping[str, bool]: + db = _session(session) + tenant_id = str(getattr(principal, "tenant_id", "") or "") + decisions = {request.reference.key: False for request in requests} + for request in requests: + reference = request.reference + if ( + reference.tenant_id != tenant_id + or reference.module_id != "reporting" + or reference.resource_type != RESOURCE_TYPE + ): + continue + decisions[reference.key] = can_read_definition( + db, + principal, + definition_kind="report", + definition_id=reference.resource_id, + ) + return decisions + + +def create_reporting_search_source( + context: ModuleContext, +) -> ReportingSearchSource: + del context + return ReportingSearchSource() + + +def _document( + row: ReportingDefinitionRevision, + tokens: tuple[str, ...], +) -> SearchDocument: + restricted_tokens = tuple( + dict.fromkeys((*tokens, "scope:reporting:definition:admin")) + ) + return SearchDocument( + tenant_id=row.tenant_id, + module_id="reporting", + provider_id=PROVIDER_ID, + resource_type=RESOURCE_TYPE, + resource_id=row.definition_id, + title=row.name, + url=f"/reporting?reportId={quote(row.definition_id, safe='')}", + summary=row.description, + body=row.description, + keywords=(row.definition_key, row.status), + visibility=row.visibility, + acl_tokens=restricted_tokens if row.visibility == "restricted" else (), + source_revision=str(row.revision), + source_updated_at=row.updated_at or row.recorded_at, + metadata={ + "definition_key": row.definition_key, + "status": row.status, + "content_hash": row.content_hash, + }, + requires_authorization_recheck=True, + ) + + +def _acl_tokens( + session: Session, + rows: Sequence[ReportingDefinitionRevision], +) -> Mapping[str, tuple[str, ...]]: + result: dict[str, list[str]] = defaultdict(list) + if not rows: + return result + identity_ids = {row.identity_id for row in rows} + for identity in ( + session.query(ReportingDefinitionIdentity) + .filter(ReportingDefinitionIdentity.id.in_(identity_ids)) + .all() + ): + if identity.created_by: + result[identity.definition_id].append(f"account:{identity.created_by}") + definition_ids = {row.definition_id for row in rows} + grants = ( + session.query(ReportingDefinitionGrant) + .filter( + ReportingDefinitionGrant.tenant_id == rows[0].tenant_id, + ReportingDefinitionGrant.definition_kind == "report", + ReportingDefinitionGrant.definition_id.in_(definition_ids), + ReportingDefinitionGrant.active.is_(True), + ) + .all() + ) + for grant in grants: + prefix = ( + "function" + if grant.subject_kind == "function_assignment" + else grant.subject_kind + ) + result[grant.definition_id].append(f"{prefix}:{grant.subject_id}") + return { + definition_id: tuple(dict.fromkeys(values)) + for definition_id, values in result.items() + } + + +def _session(value: object) -> Session: + if not isinstance(value, Session): + raise TypeError("Reporting search requires a SQLAlchemy session.") + return value + + +__all__ = [ + "PROVIDER_ID", + "RESOURCE_TYPE", + "ReportingSearchSource", + "create_reporting_search_source", +] diff --git a/src/govoplan_reporting/py.typed b/src/govoplan_reporting/py.typed new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/govoplan_reporting/py.typed @@ -0,0 +1 @@ + diff --git a/tests/test_manifest.py b/tests/test_manifest.py new file mode 100644 index 0000000..2fdf948 --- /dev/null +++ b/tests/test_manifest.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import unittest + +from govoplan_reporting.backend.manifest import get_manifest + + +class ReportingManifestTests(unittest.TestCase): + def test_manifest_exposes_the_governed_reporting_vertical(self) -> None: + manifest = get_manifest() + + self.assertEqual("reporting", manifest.id) + self.assertEqual("vertical_slice", manifest.architecture.maturity) + self.assertEqual("@govoplan/reporting-webui", manifest.frontend.package_name) + self.assertIsNotNone(manifest.route_factory) + self.assertIsNotNone(manifest.migration_spec) + self.assertEqual(4, len(manifest.provides_interfaces)) + self.assertEqual(1, len(manifest.search_sources)) + self.assertIn("dataflow", manifest.optional_dependencies) + self.assertIn( + "report execution and publication evidence", + manifest.architecture.owned_concepts, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_reporting_service.py b/tests/test_reporting_service.py new file mode 100644 index 0000000..8386abe --- /dev/null +++ b/tests/test_reporting_service.py @@ -0,0 +1,569 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from types import SimpleNamespace +import unittest + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from govoplan_core.db.base import Base +from govoplan_core.security.module_permissions import scopes_grant_compatible +from govoplan_reporting.backend.definitions import ( + ADMIN_SCOPE, + READ_SCOPE, + WRITE_SCOPE, + ReportingDefinitionError, + create_definition, + definition_history, + get_definition, + list_definitions, + update_definition, +) +from govoplan_reporting.backend.execution import ( + QUALITY_SCOPE, + RUN_SCOPE, + ReportingExecutionFailure, + execute_report, + run_quality_plan, +) +from govoplan_reporting.backend.operations import ( + IMPORT_SCOPE, + PUBLISH_SCOPE, + SCHEDULE_SCOPE, + ReportingOperationError, + assess_import, + dispatch_due_schedules, + export_execution, + upsert_saved_view, + upsert_schedule, +) +from govoplan_reporting.backend.schemas import ReportQuery + + +NOW = datetime(2026, 8, 1, 10, 0, tzinfo=UTC) +ALL_SCOPES = ( + READ_SCOPE, + WRITE_SCOPE, + ADMIN_SCOPE, + RUN_SCOPE, + QUALITY_SCOPE, + PUBLISH_SCOPE, + SCHEDULE_SCOPE, + IMPORT_SCOPE, +) + + +class Principal: + def __init__( + self, + tenant_id: str = "tenant-1", + account_id: str = "analyst-1", + *, + scopes: tuple[str, ...] = ALL_SCOPES, + group_ids: tuple[str, ...] = (), + ) -> None: + self.tenant_id = tenant_id + self.account_id = account_id + self.identity_id = f"identity-{account_id}" + self.membership_id = f"membership-{account_id}" + self.group_ids = frozenset(group_ids) + self.role_ids = frozenset() + self.function_assignment_ids = frozenset() + self.acting_assignment_id = None + self.service_account_id = None + self.scopes = frozenset(scopes) + self.user = SimpleNamespace(id=f"user-{account_id}") + + def has(self, scope: str) -> bool: + return scopes_grant_compatible(self.scopes, scope) + + +class ReportingServiceTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(self.engine) + self.session = Session(self.engine, expire_on_commit=False) + self.principal = Principal() + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def test_definition_graph_is_pinned_versioned_and_occ_guarded(self) -> None: + dataset, semantic, report = self._create_report_graph() + self.assertEqual( + ("dataset", dataset.definition_id, 1), + ( + semantic.parent_kind, + semantic.parent_id, + semantic.parent_revision, + ), + ) + replay = self._create( + "report", + "report-1", + report_payload(), + idempotency_key="create-report", + ) + self.assertEqual(report.content_hash, replay.content_hash) + + updated = update_definition( + self.session, + self.principal, + definition_kind="report", + definition_id="report-1", + expected_revision=1, + recorded_at=NOW.replace(hour=11), + change_reason="Clarify the institutional question.", + idempotency_key="update-report", + changes={"description": "Regional workload and value."}, + ) + self.assertEqual(2, updated.revision) + self.assertEqual( + [2, 1], + [ + item.revision + for item in definition_history( + self.session, + self.principal, + definition_kind="report", + definition_id="report-1", + ) + ], + ) + with self.assertRaisesRegex(ReportingDefinitionError, "stale"): + update_definition( + self.session, + self.principal, + definition_kind="report", + definition_id="report-1", + expected_revision=1, + recorded_at=NOW.replace(hour=12), + change_reason="Stale edit.", + idempotency_key="stale-report", + changes={"name": "Stale report"}, + ) + + def test_active_children_require_exact_active_parent_revision(self) -> None: + self._create("dataset", "dataset-draft", dataset_payload(), status="draft") + payload = semantic_payload(dataset_id="dataset-draft") + with self.assertRaisesRegex(ReportingDefinitionError, "active dataset"): + self._create("semantic_model", "semantic-invalid", payload) + payload["dataset_id"] = "missing" + with self.assertRaisesRegex(ReportingDefinitionError, "missing dataset"): + self._create("semantic_model", "semantic-missing", payload) + + def test_semantic_execution_replay_pivot_saved_view_and_schedule(self) -> None: + self._create_report_graph() + result = execute_report( + self.session, + self.principal, + registry=None, + report_id="report-1", + report_revision=1, + parameters={}, + query=None, + idempotency_key="run-report-1", + ) + self.assertEqual("succeeded", result["status"]) + self.assertEqual( + [ + {"region": "North", "amount": 150, "cases": 15, "value_per_case": 10}, + {"region": "South", "amount": 40, "cases": 8, "value_per_case": 5}, + ], + result["rows"], + ) + self.assertEqual( + result["execution_id"], + execute_report( + self.session, + self.principal, + registry=None, + report_id="report-1", + report_revision=1, + parameters={}, + query=None, + idempotency_key="run-report-1", + )["execution_id"], + ) + self.assertEqual("bar", result["visualization"]["kind"]) + self.assertIsNotNone(result["visualization"]["tabular_fallback"]) + + pivot = execute_report( + self.session, + self.principal, + registry=None, + report_id="report-1", + report_revision=1, + parameters={}, + query=ReportQuery.model_validate( + { + "mode": "pivot", + "pivot": { + "rows": ["region"], + "columns": ["category"], + "measures": ["amount"], + }, + } + ), + idempotency_key="run-report-pivot", + ) + self.assertEqual(2, pivot["total_rows"]) + self.assertEqual(100, pivot["rows"][0]["A.amount"]) + + view = upsert_saved_view( + self.session, + self.principal, + view_id="view-1", + report_id="report-1", + report_revision=1, + name="Regional pivot", + state={"query": pivot["query"]}, + shared=True, + access={}, + expected_revision=None, + ) + self.assertEqual(1, view["revision"]) + schedule = upsert_schedule( + self.session, + self.principal, + schedule_id="schedule-1", + report_id="report-1", + report_revision=1, + name="Daily regional report", + trigger_kind="interval", + trigger_config={"seconds": 86_400}, + parameters={}, + query=ReportQuery(), + publication_target={}, + enabled=True, + next_run_at=NOW, + expected_revision=None, + ) + self.assertEqual(1, schedule["revision"]) + dispatched = dispatch_due_schedules( + self.session, + self.principal, + registry=None, + now=NOW, + limit=10, + ) + self.assertEqual( + (1, 1, 0), + ( + dispatched["claimed"], + dispatched["succeeded"], + dispatched["failed"], + ), + ) + + def test_quality_gate_blocks_and_preserves_failure_evidence(self) -> None: + self._create_report_graph() + self._create( + "quality_plan", + "quality-1", + { + "dataset_id": "dataset-1", + "dataset_revision": 1, + "block_report_execution": True, + "assertions": [ + { + "key": "region-unique", + "kind": "unique", + "field": "region", + "severity": "blocker", + } + ], + }, + ) + quality = run_quality_plan( + self.session, + self.principal, + registry=None, + quality_plan_id="quality-1", + quality_plan_revision=1, + parameters={}, + ) + self.assertEqual("failed", quality["status"]) + with self.assertRaises(ReportingExecutionFailure) as raised: + execute_report( + self.session, + self.principal, + registry=None, + report_id="report-1", + report_revision=1, + parameters={}, + query=None, + idempotency_key="quality-blocked-run", + ) + self.assertTrue(raised.exception.execution_id) + + def test_restricted_access_and_service_scope_guards(self) -> None: + self._create_report_graph( + report_access={ + "subjects": [ + { + "kind": "group", + "id": "group-auditors", + "permissions": ["read"], + } + ] + }, + report_visibility="restricted", + ) + regular_scopes = (READ_SCOPE, WRITE_SCOPE, RUN_SCOPE) + outsider = Principal(account_id="outsider", scopes=regular_scopes) + auditor = Principal( + account_id="auditor", + scopes=regular_scopes, + group_ids=("group-auditors",), + ) + self.assertIsNone( + get_definition( + self.session, + outsider, + definition_kind="report", + definition_id="report-1", + ) + ) + self.assertIsNotNone( + get_definition( + self.session, + auditor, + definition_kind="report", + definition_id="report-1", + ) + ) + self.assertEqual( + 0, + list_definitions( + self.session, + outsider, + definition_kinds=("report",), + )[1], + ) + without_permissions = Principal(account_id="none", scopes=()) + with self.assertRaises(PermissionError): + list_definitions(self.session, without_permissions) + with self.assertRaises(PermissionError): + self._create( + "dataset", + "unauthorized", + dataset_payload(), + principal=without_permissions, + ) + + def test_safe_export_and_import_activation_assessment(self) -> None: + self._create_report_graph() + detail = execute_report( + self.session, + self.principal, + registry=None, + report_id="report-1", + report_revision=1, + parameters={}, + query=ReportQuery(mode="detail", dimensions=["note"]), + idempotency_key="detail-export", + ) + content, content_type, _filename = export_execution( + self.session, + self.principal, + execution_id=str(detail["execution_id"]), + format="csv", + ) + self.assertEqual("text/csv; charset=utf-8", content_type) + self.assertIn("'=cmd", content.decode("utf-8-sig")) + + blocked = assess_import( + self.session, + self.principal, + source_system="SuperX", + source_id="legacy-report", + metadata={"features": ["dataset", "raw_sql", "dashboard_layout"]}, + accepted_approximations=[], + ) + self.assertEqual("blocked", blocked["status"]) + self.assertFalse(blocked["mapping_report"]["activation_allowed"]) + ready = assess_import( + self.session, + self.principal, + source_system="SuperX", + source_id="mapped-report", + metadata={"features": ["dataset", "dashboard_layout"]}, + accepted_approximations=["dashboard_layout"], + ) + self.assertEqual("ready", ready["status"]) + with self.assertRaisesRegex(ReportingOperationError, "must be a list"): + assess_import( + self.session, + self.principal, + source_system="invalid", + source_id="invalid", + metadata={"features": "raw_sql"}, + accepted_approximations=[], + ) + + def _create_report_graph( + self, + *, + report_access: dict[str, object] | None = None, + report_visibility: str = "tenant", + ): + dataset = self._create("dataset", "dataset-1", dataset_payload()) + semantic = self._create( + "semantic_model", + "semantic-1", + semantic_payload(), + ) + report = self._create( + "report", + "report-1", + report_payload(access_policy=report_access), + visibility=report_visibility, + idempotency_key="create-report", + ) + return dataset, semantic, report + + def _create( + self, + kind: str, + definition_id: str, + payload: dict[str, object], + *, + status: str = "active", + visibility: str = "tenant", + idempotency_key: str | None = None, + principal: Principal | None = None, + ): + return create_definition( + self.session, + principal or self.principal, + definition_kind=kind, + definition_id=definition_id, + definition_key=definition_id, + name=definition_id.replace("-", " ").title(), + description="Governed reporting test definition.", + status=status, + visibility=visibility, + recorded_at=NOW, + change_reason="Initial governed baseline.", + payload=payload, + idempotency_key=idempotency_key or f"create-{definition_id}", + ) + + +def dataset_payload() -> dict[str, object]: + return { + "source_kind": "static", + "source_ref": "fixture.regional-workload", + "purpose": "Verify governed institutional reporting.", + "schema": [ + {"name": "region", "type": "string", "nullable": False}, + {"name": "category", "type": "string", "nullable": False}, + {"name": "amount", "type": "number", "nullable": False}, + {"name": "cases", "type": "integer", "nullable": False}, + {"name": "note", "type": "string", "nullable": False}, + ], + "static_rows": [ + { + "region": "North", + "category": "A", + "amount": 100, + "cases": 10, + "note": "=cmd", + }, + { + "region": "North", + "category": "B", + "amount": 50, + "cases": 5, + "note": "ordinary", + }, + { + "region": "South", + "category": "A", + "amount": 40, + "cases": 8, + "note": "ordinary", + }, + ], + } + + +def semantic_payload(*, dataset_id: str = "dataset-1") -> dict[str, object]: + return { + "dataset_id": dataset_id, + "dataset_revision": 1, + "dimensions": [ + {"key": "region", "field": "region", "label": "Region"}, + {"key": "category", "field": "category", "label": "Category"}, + {"key": "note", "field": "note", "label": "Note"}, + ], + "hierarchies": [ + { + "key": "regional-category", + "label": "Region and category", + "levels": [{"dimension": "region"}, {"dimension": "category"}], + } + ], + "measures": [ + { + "key": "amount", + "label": "Amount", + "aggregation": "sum", + "field": "amount", + }, + {"key": "cases", "label": "Cases", "aggregation": "sum", "field": "cases"}, + { + "key": "value_per_case", + "label": "Value per case", + "aggregation": "calculated", + "expression": { + "op": "divide", + "args": [ + {"op": "measure", "ref": "amount"}, + {"op": "measure", "ref": "cases"}, + ], + }, + }, + ], + "default_dimensions": ["region"], + "default_measures": ["amount", "cases", "value_per_case"], + } + + +def report_payload( + *, + access_policy: dict[str, object] | None = None, +) -> dict[str, object]: + return { + "semantic_model_id": "semantic-1", + "semantic_model_revision": 1, + "default_query": { + "mode": "summary", + "dimensions": ["region"], + "measures": ["amount", "cases", "value_per_case"], + "sort": [{"key": "region", "direction": "asc"}], + }, + "visualization": { + "kind": "bar", + "category_dimension": "region", + "measures": ["amount"], + "tabular_fallback": True, + }, + "access_policy": access_policy or {}, + "institutional_references": [ + { + "owner_module": "projects", + "resource_type": "outcome", + "resource_id": "faster-decisions", + "revision": "1", + "relationship": "measures", + } + ], + } + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/package.json b/webui/package.json new file mode 100644 index 0000000..2f9b4b3 --- /dev/null +++ b/webui/package.json @@ -0,0 +1,28 @@ +{ + "name": "@govoplan/reporting-webui", + "version": "0.1.14", + "private": true, + "type": "module", + "main": "src/index.ts", + "module": "src/index.ts", + "types": "src/index.ts", + "exports": { + ".": { + "types": "./src/index.ts", + "import": "./src/index.ts" + }, + "./styles/reporting.css": "./src/styles/reporting.css" + }, + "peerDependencies": { + "@govoplan/core-webui": "^0.1.14", + "lucide-react": "^1.23.0", + "react": ">=19.2.7 <20", + "react-dom": ">=19.2.7 <20", + "react-router": ">=8.3.0 <9" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } +} diff --git a/webui/src/api/reporting.ts b/webui/src/api/reporting.ts new file mode 100644 index 0000000..7349ef0 --- /dev/null +++ b/webui/src/api/reporting.ts @@ -0,0 +1,260 @@ +import { + apiFetch, + apiPath, + apiUrl, + authHeaders, + type ApiSettings +} from "@govoplan/core-webui"; + + +export type ReportingDefinitionKind = "dataset" | "semantic_model" | "report" | "quality_plan"; +export type ReportingQueryMode = "summary" | "detail" | "pivot"; + +export type ReportingQuery = { + mode: ReportingQueryMode; + dimensions: string[]; + measures: string[]; + filters: Array>; + sort: Array<{ key: string; direction: "asc" | "desc" }>; + pivot?: { + rows: string[]; + columns: string[]; + measures: string[]; + include_totals: boolean; + } | null; + offset: number; + limit: number; +}; + +export type ReportingDefinition = { + tenant_id: string; + definition_kind: ReportingDefinitionKind; + definition_id: string; + definition_key: string; + revision: number; + name: string; + description?: string | null; + status: "draft" | "active" | "retired"; + visibility: "tenant" | "restricted"; + content_hash: string; + parent_kind?: ReportingDefinitionKind | null; + parent_id?: string | null; + parent_revision?: number | null; + recorded_at: string; + change_reason: string; + payload: Record; +}; + +export type SemanticModelPayload = { + dataset_id: string; + dataset_revision: number; + dimensions: Array<{ key: string; field: string; label: string }>; + measures: Array<{ key: string; label: string; aggregation: string }>; + default_dimensions: string[]; + default_measures: string[]; +}; + +export type ReportPayload = { + semantic_model_id: string; + semantic_model_revision: number; + parameters: Array<{ + key: string; + label: string; + type: string; + required: boolean; + default?: unknown; + allowed_values: unknown[]; + }>; + default_query: ReportingQuery; + visualization: { + kind: string; + category_dimension?: string | null; + series_dimension?: string | null; + measures: string[]; + tabular_fallback: boolean; + }; + institutional_references: Array>; +}; + +export type ReportExecution = { + execution_id: string; + report_id: string; + report_revision: number; + semantic_model_id: string; + semantic_model_revision: number; + dataset_id: string; + dataset_revision: number; + status: "running" | "succeeded" | "failed"; + parameters: Record; + query: ReportingQuery; + definition_hashes: Record; + source_fingerprints: Array>; + output_hash?: string | null; + executor_version?: string | null; + schema: Array<{ name: string; type: string }>; + rows: Array>; + total_rows: number; + truncated: boolean; + diagnostics: Array<{ severity?: string; code?: string; message?: string }>; + provenance: Record; + started_at: string; + finished_at?: string | null; + visualization?: { + kind: string; + requested_kind?: string; + category?: string | null; + measures?: string[]; + fallback_reason?: string | null; + }; +}; + +export type ReportingSavedView = { + view_id: string; + report_id: string; + report_revision: number; + name: string; + revision: number; + state: { query?: ReportingQuery }; + shared: boolean; + updated_at: string; +}; + +export function listDefinitions( + settings: ApiSettings, + options: { kinds?: ReportingDefinitionKind[]; status?: string[]; query?: string; limit?: number }, + signal?: AbortSignal +): Promise<{ definitions: ReportingDefinition[]; total: number }> { + return apiFetch(settings, apiPath("/api/v1/reporting/definitions", { + definition_kind: options.kinds, + status: options.status, + query: options.query, + limit: options.limit ?? 200 + }), { signal }); +} + +export function getDefinition( + settings: ApiSettings, + kind: ReportingDefinitionKind, + id: string, + revision?: number, + signal?: AbortSignal +): Promise { + return apiFetch(settings, apiPath( + `/api/v1/reporting/definitions/${encodeURIComponent(kind)}/${encodeURIComponent(id)}`, + { revision } + ), { signal }); +} + +export function runReport( + settings: ApiSettings, + report: ReportingDefinition, + query: ReportingQuery, + parameters: Record +): Promise { + return apiFetch(settings, `/api/v1/reporting/reports/${encodeURIComponent(report.definition_id)}/executions`, { + method: "POST", + body: JSON.stringify({ + report_revision: report.revision, + parameters, + query, + idempotency_key: crypto.randomUUID() + }) + }); +} + +export function listExecutions( + settings: ApiSettings, + reportId: string, + signal?: AbortSignal +): Promise<{ executions: ReportExecution[] }> { + return apiFetch(settings, `/api/v1/reporting/reports/${encodeURIComponent(reportId)}/executions?limit=30`, { signal }); +} + +export function listSavedViews( + settings: ApiSettings, + reportId: string, + signal?: AbortSignal +): Promise<{ views: ReportingSavedView[] }> { + return apiFetch(settings, `/api/v1/reporting/reports/${encodeURIComponent(reportId)}/saved-views`, { signal }); +} + +export function saveView( + settings: ApiSettings, + report: ReportingDefinition, + name: string, + query: ReportingQuery +): Promise { + const viewId = crypto.randomUUID(); + return apiFetch(settings, `/api/v1/reporting/reports/${encodeURIComponent(report.definition_id)}/saved-views/${viewId}`, { + method: "PUT", + body: JSON.stringify({ + view_id: viewId, + report_revision: report.revision, + name, + state: { query }, + shared: false, + access: {}, + expected_revision: null + }) + }); +} + +export function createIntervalSchedule( + settings: ApiSettings, + report: ReportingDefinition, + name: string, + seconds: number, + query: ReportingQuery, + parameters: Record +): Promise> { + const scheduleId = crypto.randomUUID(); + return apiFetch(settings, `/api/v1/reporting/schedules/${scheduleId}`, { + method: "PUT", + body: JSON.stringify({ + schedule_id: scheduleId, + report_id: report.definition_id, + report_revision: report.revision, + name, + trigger_kind: "interval", + trigger_config: { seconds }, + parameters, + query, + publication_target: {}, + enabled: true, + next_run_at: null, + expected_revision: null + }) + }); +} + +export async function downloadExecution( + settings: ApiSettings, + executionId: string, + format: "csv" | "json" +): Promise { + const response = await fetch(apiUrl(settings, apiPath( + `/api/v1/reporting/executions/${encodeURIComponent(executionId)}/export`, + { format } + )), { + headers: authHeaders(settings), + credentials: "include" + }); + if (!response.ok) throw new Error(await response.text() || "The report export failed."); + const blob = await response.blob(); + const disposition = response.headers.get("content-disposition") ?? ""; + const filename = disposition.match(/filename="?([^";]+)"?/i)?.[1] ?? `report.${format}`; + const href = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = href; + link.download = filename; + link.click(); + URL.revokeObjectURL(href); +} + +export function reportPayload(definition: ReportingDefinition): ReportPayload { + return definition.payload as unknown as ReportPayload; +} + +export function semanticPayload(definition: ReportingDefinition): SemanticModelPayload { + return definition.payload as unknown as SemanticModelPayload; +} diff --git a/webui/src/features/reporting/ReportingPage.tsx b/webui/src/features/reporting/ReportingPage.tsx new file mode 100644 index 0000000..daf0992 --- /dev/null +++ b/webui/src/features/reporting/ReportingPage.tsx @@ -0,0 +1,573 @@ +import { + BarChart3, + CalendarClock, + Download, + FileJson, + History, + Play, + RefreshCw, + Save, + Search, + SlidersHorizontal, + Table2 +} from "lucide-react"; +import { + useEffect, + useMemo, + useState, + type FormEvent +} from "react"; +import { + Button, + DataGrid, + Dialog, + DismissibleAlert, + IconButton, + LoadingIndicator, + PageScrollViewport, + SegmentedControl, + StatusBadge, + hasScope, + type DataGridColumn, + type PlatformRouteContext +} from "@govoplan/core-webui"; +import { + createIntervalSchedule, + downloadExecution, + getDefinition, + listDefinitions, + listExecutions, + listSavedViews, + reportPayload, + runReport, + saveView, + semanticPayload, + type ReportExecution, + type ReportingDefinition, + type ReportingQuery, + type ReportingQueryMode, + type ReportingSavedView, + type SemanticModelPayload +} from "../../api/reporting"; + + +type OutputMode = "visual" | "table"; + +export default function ReportingPage({ settings, auth }: PlatformRouteContext) { + const [search, setSearch] = useState(""); + const [submittedSearch, setSubmittedSearch] = useState(""); + const [reports, setReports] = useState([]); + const [selectedId, setSelectedId] = useState(""); + const [semantic, setSemantic] = useState(null); + const [query, setQuery] = useState(emptyQuery()); + const [parameters, setParameters] = useState>({}); + const [execution, setExecution] = useState(null); + const [history, setHistory] = useState([]); + const [savedViews, setSavedViews] = useState([]); + const [outputMode, setOutputMode] = useState("visual"); + const [loading, setLoading] = useState(true); + const [running, setRunning] = useState(false); + const [error, setError] = useState(""); + const [saveDialogOpen, setSaveDialogOpen] = useState(false); + const [scheduleDialogOpen, setScheduleDialogOpen] = useState(false); + const canRun = hasScope(auth, "reporting:report:run"); + const canSchedule = hasScope(auth, "reporting:schedule:write"); + + const selected = useMemo( + () => reports.find((item) => item.definition_id === selectedId) ?? null, + [reports, selectedId] + ); + const report = selected ? reportPayload(selected) : null; + + function reload(signal?: AbortSignal) { + setLoading(true); + setError(""); + return listDefinitions(settings, { + kinds: ["report"], + status: ["active"], + query: submittedSearch, + limit: 200 + }, signal). + then((result) => { + setReports(result.definitions); + setSelectedId((current) => result.definitions.some((item) => item.definition_id === current) ? + current : result.definitions[0]?.definition_id ?? ""); + }). + catch((reason) => { + if ((reason as Error).name !== "AbortError") setError(message(reason)); + }). + finally(() => setLoading(false)); + } + + useEffect(() => { + const controller = new AbortController(); + void reload(controller.signal); + return () => controller.abort(); + }, [settings, submittedSearch]); + + useEffect(() => { + if (!selected || !report) { + setSemantic(null); + setExecution(null); + setHistory([]); + setSavedViews([]); + return; + } + const controller = new AbortController(); + setQuery(normalizeQuery(report.default_query)); + setParameters(defaultParameters(report.parameters)); + setExecution(null); + Promise.all([ + getDefinition(settings, "semantic_model", report.semantic_model_id, report.semantic_model_revision, controller.signal), + canRun ? listExecutions(settings, selected.definition_id, controller.signal) : Promise.resolve({ executions: [] }), + listSavedViews(settings, selected.definition_id, controller.signal) + ]). + then(([semanticDefinition, executions, views]) => { + setSemantic(semanticPayload(semanticDefinition)); + setHistory(executions.executions); + setSavedViews(views.views); + setExecution(executions.executions.find((item) => item.status === "succeeded") ?? null); + }). + catch((reason) => { + if ((reason as Error).name !== "AbortError") setError(message(reason)); + }); + return () => controller.abort(); + }, [settings, selectedId]); + + function submitSearch(event: FormEvent) { + event.preventDefault(); + setSubmittedSearch(search.trim()); + } + + async function execute() { + if (!selected) return; + setRunning(true); + setError(""); + try { + const result = await runReport(settings, selected, query, parameters); + setExecution(result); + setHistory((current) => [result, ...current.filter((item) => item.execution_id !== result.execution_id)]); + setOutputMode(result.visualization?.kind === "table" ? "table" : "visual"); + } catch (reason) { + setError(message(reason)); + } finally { + setRunning(false); + } + } + + function applySavedView(view: ReportingSavedView) { + if (view.state.query) setQuery(normalizeQuery(view.state.query)); + } + + return ( +
+
+
+
+
+ {error && + + {error} + + } +
+ + {loading && } + {!loading && reports.length === 0 &&
No active reports are available.
} +
+ {reports.map((item) => + + )} +
+ +
+ {selected && report ? + <> +
+
+ {selected.definition_key} · revision {selected.revision} +

{selected.name}

+
+
+ {canSchedule && + } variant="ghost" onClick={() => setScheduleDialogOpen(true)} /> + } + } variant="ghost" onClick={() => setSaveDialogOpen(true)} /> + +
+
+ +
+ Visual }, + { id: "table", label: <> Table } + ]} + /> + {execution && + <> + {execution.total_rows} rows{execution.truncated ? " (truncated)" : ""} + } variant="ghost" onClick={() => void downloadExecution(settings, execution.execution_id, "csv").catch((reason) => setError(message(reason)))} /> + } variant="ghost" onClick={() => void downloadExecution(settings, execution.execution_id, "json").catch((reason) => setError(message(reason)))} /> + + } +
+
+ {!execution &&
Run the report or select a previous execution.
} + {execution && outputMode === "visual" && } + {execution && outputMode === "table" && } +
+ : +
Select a report.
+ } +
+ + + +
+
+ setSaveDialogOpen(false)} + onSave={async (name) => { + if (!selected) return; + const saved = await saveView(settings, selected, name, query); + setSavedViews((current) => [...current, saved].sort((a, b) => a.name.localeCompare(b.name))); + setSaveDialogOpen(false); + }} + /> + setScheduleDialogOpen(false)} + onSave={async (name, seconds) => { + if (!selected) return; + await createIntervalSchedule(settings, selected, name, seconds, query, parameters); + setScheduleDialogOpen(false); + }} + /> +
+ ); +} + +function QueryControls({ query, semantic, parameters, parameterValues, onQueryChange, onParametersChange }: { + query: ReportingQuery; + semantic: SemanticModelPayload | null; + parameters: Array<{ key: string; label: string; type: string; required: boolean; default?: unknown; allowed_values: unknown[] }>; + parameterValues: Record; + onQueryChange: (query: ReportingQuery) => void; + onParametersChange: (parameters: Record) => void; +}) { + const dimensions = semantic?.dimensions ?? []; + const measures = semantic?.measures ?? []; + + function setMode(mode: ReportingQueryMode) { + const row = query.dimensions[0] ?? dimensions[0]?.key; + const column = query.dimensions[1] ?? dimensions[1]?.key; + onQueryChange({ + ...query, + mode, + pivot: mode === "pivot" ? { + rows: row ? [row] : [], + columns: column ? [column] : [], + measures: query.measures.length ? query.measures : measures.slice(0, 1).map((item) => item.key), + include_totals: true + } : null + }); + } + + return ( +
+
+
+
+ Dimensions +
+ {dimensions.map((item) => + + )} +
+
+ {query.mode !== "detail" && +
+ Measures +
+ {measures.map((item) => + + )} +
+
+ } + {parameters.map((parameter) => + + )} +
+ ); +} + +function ReportTable({ execution }: { execution: ReportExecution }) { + const [page, setPage] = useState(0); + useEffect(() => setPage(0), [execution.execution_id]); + const columns = useMemo>[]>(() => + execution.schema.map((field) => ({ + id: field.name, + header: humanize(field.name), + width: "1fr", + minWidth: 120, + resizable: true, + sortable: true, + filterable: true, + filterType: field.type === "integer" || field.type === "number" ? field.type : "text", + value: (row) => row[field.name], + render: (row) => formatValue(row[field.name]) + })), [execution]); + return ( + `${execution.execution_id}:${index}`} + initialFit="container" + resizeBehavior="cover" + emptyText="The report returned no rows." + pagination={{ page, pageSize: 50, onPageChange: setPage }} + /> + ); +} + +function ReportVisual({ execution }: { execution: ReportExecution }) { + const visual = execution.visualization; + if (!visual || visual.kind === "table" || !visual.category || !visual.measures?.length) { + return ( +
+ {visual?.fallback_reason && {visual.fallback_reason}} + +
+ ); + } + const measure = visual.measures[0]; + const values = execution.rows.map((row) => Number(row[measure] ?? 0)); + const maximum = Math.max(...values.map((value) => Math.abs(value)), 1); + return ( +
+ {execution.rows.map((row, index) => +
+ {formatValue(row[visual.category ?? ""])} +
+ {formatValue(row[measure])} +
+ )} +
+
+ ); +} + +function Inspector({ selected, execution, history, savedViews, onSelectExecution, onApplySavedView }: { + selected: ReportingDefinition | null; + execution: ReportExecution | null; + history: ReportExecution[]; + savedViews: ReportingSavedView[]; + onSelectExecution: (execution: ReportExecution) => void; + onApplySavedView: (view: ReportingSavedView) => void; +}) { + return ( +
+
+

Runs

+ {history.length === 0 &&

No recorded runs.

} + {history.map((item) => + + )} +
+
+

Saved views

+ {savedViews.length === 0 &&

No saved views.

} + {savedViews.map((view) => + + )} +
+ {selected && +
+

Definition

+
+
Report revision
{selected.revision}
+
Content hash
{shortHash(selected.content_hash)}
+ {execution && <>
Dataset
{execution.dataset_id} r{execution.dataset_revision}
} + {execution?.output_hash && <>
Output hash
{shortHash(execution.output_hash)}
} + {execution?.executor_version && <>
Executor
{execution.executor_version}
} +
+ {execution?.diagnostics.map((item, index) => + + {item.message ?? item.code ?? "Execution diagnostic"} + + )} +
+ } +
+ ); +} + +function SaveViewDialog({ open, onClose, onSave }: { open: boolean; onClose: () => void; onSave: (name: string) => Promise }) { + const [name, setName] = useState(""); + const [saving, setSaving] = useState(false); + return ( + + + + }> + + + ); +} + +function ScheduleDialog({ open, onClose, onSave }: { open: boolean; onClose: () => void; onSave: (name: string, seconds: number) => Promise }) { + const [name, setName] = useState(""); + const [interval, setIntervalValue] = useState("86400"); + const [saving, setSaving] = useState(false); + return ( + + + + }> +
+ + +
+ + ); +} + +function emptyQuery(): ReportingQuery { + return { mode: "summary", dimensions: [], measures: [], filters: [], sort: [], pivot: null, offset: 0, limit: 200 }; +} + +function normalizeQuery(value: ReportingQuery): ReportingQuery { + return { ...emptyQuery(), ...value, dimensions: [...(value.dimensions ?? [])], measures: [...(value.measures ?? [])], filters: [...(value.filters ?? [])], sort: [...(value.sort ?? [])] }; +} + +function defaultParameters(parameters: Array<{ key: string; default?: unknown }>): Record { + return Object.fromEntries(parameters.filter((item) => item.default !== null && item.default !== undefined).map((item) => [item.key, item.default])); +} + +function toggle(values: string[], value: string): string[] { + return values.includes(value) ? values.filter((item) => item !== value) : [...values, value]; +} + +function typedValue(value: string, type: string): unknown { + if (value === "") return null; + if (type === "integer") return Number.parseInt(value, 10); + if (type === "number") return Number(value); + return value; +} + +function formatValue(value: unknown): string { + if (value === null || value === undefined) return "—"; + if (typeof value === "number") return new Intl.NumberFormat().format(value); + if (typeof value === "object") return JSON.stringify(value); + return String(value); +} + +function formatDateTime(value: string): string { + const parsed = new Date(value); + return Number.isNaN(parsed.valueOf()) ? value : new Intl.DateTimeFormat(undefined, { dateStyle: "medium", timeStyle: "short" }).format(parsed); +} + +function humanize(value: string): string { + return value.replaceAll("_", " ").replace(/\b\w/g, (letter) => letter.toUpperCase()); +} + +function shortHash(value: string): string { + return `${value.slice(0, 10)}…${value.slice(-6)}`; +} + +function message(reason: unknown): string { + return reason instanceof Error ? reason.message : "Reporting could not complete the request."; +} diff --git a/webui/src/index.ts b/webui/src/index.ts new file mode 100644 index 0000000..8b0f76e --- /dev/null +++ b/webui/src/index.ts @@ -0,0 +1,2 @@ +export { default, reportingModule } from "./module"; +export * from "./api/reporting"; diff --git a/webui/src/module.ts b/webui/src/module.ts new file mode 100644 index 0000000..74001cc --- /dev/null +++ b/webui/src/module.ts @@ -0,0 +1,52 @@ +import { createElement, lazy } from "react"; +import type { PlatformWebModule } from "@govoplan/core-webui"; +import "./styles/reporting.css"; + + +const ReportingPage = lazy(() => import("./features/reporting/ReportingPage")); + +export const reportingModule: PlatformWebModule = { + id: "reporting", + label: "Reporting", + version: "0.1.14", + optionalDependencies: [ + "dataflow", + "datasources", + "connectors", + "dashboard", + "files", + "mail", + "templates", + "workflow_engine", + "policy", + "search", + "notifications" + ], + routes: [ + { + path: "/reporting", + anyOf: ["reporting:definition:read"], + order: 74, + surfaceId: "reporting.workspace", + render: (context) => createElement(ReportingPage, context) + } + ], + navItems: [ + { + to: "/reporting", + label: "Reporting", + iconName: "clipboard-pen-line", + anyOf: ["reporting:definition:read"], + order: 74, + surfaceId: "reporting.navigation" + } + ], + viewSurfaces: [ + { id: "reporting.navigation", moduleId: "reporting", kind: "navigation", label: "Reporting navigation", order: 10 }, + { id: "reporting.workspace", moduleId: "reporting", kind: "route", label: "Reporting workspace", order: 20 }, + { id: "reporting.parameters", moduleId: "reporting", kind: "section", label: "Report parameters and filters", parentId: "reporting.workspace", order: 30 }, + { id: "reporting.results", moduleId: "reporting", kind: "section", label: "Authorized report results", parentId: "reporting.workspace", order: 40 } + ] +}; + +export default reportingModule; diff --git a/webui/src/styles/reporting.css b/webui/src/styles/reporting.css new file mode 100644 index 0000000..fc1edd2 --- /dev/null +++ b/webui/src/styles/reporting.css @@ -0,0 +1,388 @@ +.reporting-page, +.reporting-shell { + height: 100%; + min-height: 0; + overflow: hidden; +} + +.reporting-shell { + display: flex; + flex-direction: column; + background: var(--surface); +} + +.reporting-toolbar, +.reporting-result-header, +.reporting-output-toolbar { + display: flex; + align-items: center; + gap: 10px; + border-bottom: 1px solid var(--border); + background: var(--surface-raised); +} + +.reporting-toolbar { + min-height: 56px; + padding: 9px 14px; +} + +.reporting-search { + display: flex; + align-items: center; + gap: 8px; + width: min(440px, 46vw); +} + +.reporting-search input { + min-width: 140px; + flex: 1; +} + +.reporting-count { + margin-left: auto; + color: var(--text-soft); + font-size: 0.8rem; +} + +.reporting-workspace { + display: grid; + min-height: 0; + flex: 1; + grid-template-columns: minmax(250px, 20%) minmax(460px, 1fr) minmax(260px, 20%); +} + +.reporting-catalogue, +.reporting-inspector { + min-height: 0; + padding: 10px; + background: var(--surface-subtle, var(--surface)); +} + +.reporting-catalogue { + border-right: 1px solid var(--border); +} + +.reporting-inspector { + border-left: 1px solid var(--border); +} + +.reporting-report-list, +.reporting-inspector-content section { + overflow: hidden; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--surface-raised); +} + +.reporting-report-row, +.reporting-inspector-content section > button { + display: grid; + align-items: center; + width: 100%; + border: 0; + border-bottom: 1px solid var(--border); + background: transparent; + color: inherit; + text-align: left; + cursor: pointer; +} + +.reporting-report-row { + grid-template-columns: 22px minmax(0, 1fr) auto; + gap: 8px; + min-height: 58px; + padding: 8px 10px; +} + +.reporting-report-row:last-child, +.reporting-inspector-content section > button:last-child { + border-bottom: 0; +} + +.reporting-report-row:hover, +.reporting-report-row:focus-visible, +.reporting-report-row.is-selected, +.reporting-inspector-content section > button:hover, +.reporting-inspector-content section > button.is-selected { + background: var(--hover-bg); +} + +.reporting-report-row.is-selected { + box-shadow: inset 3px 0 0 var(--accent); +} + +.reporting-report-row > span:nth-child(2) { + display: flex; + min-width: 0; + flex-direction: column; + gap: 2px; +} + +.reporting-report-row strong, +.reporting-report-row small { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.reporting-report-row small, +.reporting-inspector-content small, +.reporting-inspector-content p { + color: var(--text-soft); + font-size: 0.76rem; +} + +.reporting-result-region { + display: flex; + min-width: 0; + min-height: 0; + flex-direction: column; +} + +.reporting-result-header { + min-height: 62px; + padding: 8px 14px; +} + +.reporting-title { + min-width: 0; +} + +.reporting-title span { + color: var(--text-soft); + font-size: 0.75rem; +} + +.reporting-title h1 { + overflow: hidden; + margin: 2px 0 0; + font-size: 1.05rem; + letter-spacing: 0; + text-overflow: ellipsis; + white-space: nowrap; +} + +.reporting-run-actions { + display: flex; + align-items: center; + gap: 6px; + margin-left: auto; +} + +.reporting-query-controls { + display: flex; + align-items: center; + gap: 14px; + min-height: 66px; + padding: 8px 14px; + overflow-x: auto; + border-bottom: 1px solid var(--border); + background: var(--surface-subtle, var(--surface)); +} + +.reporting-query-heading { + display: flex; + align-items: center; + gap: 7px; +} + +.reporting-query-controls fieldset { + min-width: 170px; + margin: 0; + padding: 4px 8px 7px; + border: 1px solid var(--border); + border-radius: 6px; +} + +.reporting-query-controls legend, +.reporting-parameter > span, +.reporting-dialog-field > span { + color: var(--text-soft); + font-size: 0.72rem; +} + +.reporting-option-list { + display: flex; + gap: 9px; + max-width: 340px; + overflow-x: auto; +} + +.reporting-option-list label { + display: flex; + align-items: center; + gap: 4px; + white-space: nowrap; +} + +.reporting-parameter, +.reporting-dialog-field { + display: flex; + min-width: 150px; + flex-direction: column; + gap: 4px; +} + +.reporting-output-toolbar { + min-height: 48px; + padding: 6px 12px; +} + +.reporting-output-toolbar > span { + margin-left: auto; + color: var(--text-soft); + font-size: 0.78rem; +} + +.reporting-output { + min-height: 0; + flex: 1; + overflow: auto; + padding: 10px; +} + +.reporting-output .data-grid-shell { + min-width: 100%; +} + +.reporting-empty { + padding: 38px 12px; + color: var(--text-soft); + text-align: center; +} + +.reporting-inspector-content { + display: flex; + flex-direction: column; + gap: 10px; +} + +.reporting-inspector-content section h2 { + display: flex; + align-items: center; + gap: 6px; + margin: 0; + padding: 9px 10px; + border-bottom: 1px solid var(--border); + font-size: 0.82rem; + letter-spacing: 0; +} + +.reporting-inspector-content section > p { + margin: 0; + padding: 12px 10px; +} + +.reporting-inspector-content section > button { + grid-template-columns: minmax(0, 1fr) auto; + gap: 5px 8px; + padding: 8px 10px; +} + +.reporting-inspector-content section > button small { + grid-column: 1 / -1; +} + +.reporting-provenance { + padding-bottom: 8px; +} + +.reporting-provenance dl { + display: grid; + grid-template-columns: minmax(90px, auto) minmax(0, 1fr); + gap: 6px 8px; + margin: 0; + padding: 10px; + font-size: 0.76rem; +} + +.reporting-provenance dt { + color: var(--text-soft); +} + +.reporting-provenance dd { + min-width: 0; + margin: 0; + overflow-wrap: anywhere; +} + +.reporting-provenance .alert { + margin: 8px; +} + +.reporting-bar-chart { + display: flex; + min-width: 420px; + flex-direction: column; + gap: 8px; + padding: 10px; +} + +.reporting-bar-row { + display: grid; + grid-template-columns: minmax(100px, 22%) minmax(180px, 1fr) minmax(80px, auto); + align-items: center; + gap: 9px; + font-size: 0.8rem; +} + +.reporting-bar-row > div { + height: 22px; + overflow: hidden; + border: 1px solid var(--border); + border-radius: 4px; + background: var(--surface-subtle, var(--surface)); +} + +.reporting-bar-row i { + display: block; + height: 100%; + background: var(--accent); +} + +.reporting-chart-table { + margin-top: 14px; + padding-top: 12px; + border-top: 1px solid var(--border); +} + +.reporting-dialog-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 12px; +} + +@media (max-width: 1100px) { + .reporting-workspace { + grid-template-columns: minmax(220px, 28%) minmax(0, 1fr); + } + + .reporting-inspector { + display: none; + } +} + +@media (max-width: 760px) { + .reporting-workspace { + grid-template-columns: 1fr; + } + + .reporting-catalogue { + max-height: 30vh; + border-right: 0; + border-bottom: 1px solid var(--border); + } + + .reporting-result-header, + .reporting-toolbar { + flex-wrap: wrap; + } + + .reporting-search { + width: 100%; + } + + .reporting-dialog-grid { + grid-template-columns: 1fr; + } +}