feat: implement governed reporting vertical

This commit is contained in:
2026-08-01 17:48:39 +02:00
parent 720959536b
commit eba35edd3c
35 changed files with 8414 additions and 33 deletions
+3
View File
@@ -0,0 +1,3 @@
"""GovOPlaN Reporting module."""
__version__ = "0.1.14"
@@ -0,0 +1 @@
"""Reporting backend package."""
+29
View File
@@ -0,0 +1,29 @@
from __future__ import annotations
from govoplan_core.auth import has_scope
from govoplan_core.core.modules import AccessDecision
class ReportingScopeAclProvider:
def __init__(self, resource_type: str) -> None:
self.resource_type = resource_type
def can_read(self, principal: object, resource_id: str) -> bool:
del resource_id
return has_scope(principal, "reporting:definition:read")
def can_write(self, principal: object, resource_id: str) -> bool:
del resource_id
return has_scope(principal, "reporting:definition:write")
def explain(self, principal: object, resource_id: str) -> AccessDecision:
del resource_id
allowed = self.can_read(principal, "")
return AccessDecision(
allowed=allowed,
reason=None if allowed else "Missing scope: reporting:definition:read",
requirements=("reporting:definition:read",),
)
__all__ = ["ReportingScopeAclProvider"]
+135
View File
@@ -0,0 +1,135 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime
from typing import Protocol, runtime_checkable
CAPABILITY_REPORTING_REGISTRY = "reporting.registry"
CAPABILITY_REPORTING_RUNNER = "reporting.runner"
CAPABILITY_REPORTING_SCHEDULER = "reporting.scheduler"
CAPABILITY_REPORTING_CHART_RENDERER = "reporting.chart_renderer"
@dataclass(frozen=True, slots=True)
class ReportingDatasetReadRequest:
source_ref: str
source_revision: int | None
parameters: Mapping[str, object] = field(default_factory=dict)
row_limit: int = 2_000
expected_definition_hash: str | None = None
expected_source_fingerprints: tuple[Mapping[str, object], ...] = ()
@dataclass(frozen=True, slots=True)
class ReportingDatasetReadResult:
rows: tuple[Mapping[str, object], ...]
total_rows: int
truncated: bool
output_hash: str
executor_version: str
definition_hash: str | None = None
source_fingerprints: tuple[Mapping[str, object], ...] = ()
diagnostics: tuple[Mapping[str, object], ...] = ()
generated_at: datetime | None = None
provenance: Mapping[str, object] = field(default_factory=dict)
@runtime_checkable
class ReportingReadModelProvider(Protocol):
def read_dataset(
self,
session: object,
principal: object,
*,
request: ReportingDatasetReadRequest,
) -> ReportingDatasetReadResult: ...
@dataclass(frozen=True, slots=True)
class ReportingRowPolicyRequest:
dataset_id: str
dataset_revision: int
policy_ref: str
rows: tuple[Mapping[str, object], ...]
@dataclass(frozen=True, slots=True)
class ReportingRowPolicyResult:
rows: tuple[Mapping[str, object], ...]
decision_ref: str
provenance: Mapping[str, object] = field(default_factory=dict)
@runtime_checkable
class ReportingRowPolicyProvider(Protocol):
def authorize_rows(
self,
session: object,
principal: object,
*,
request: ReportingRowPolicyRequest,
) -> ReportingRowPolicyResult: ...
@dataclass(frozen=True, slots=True)
class ReportingPublicationPayload:
publication_id: str
execution_id: str
tenant_id: str
report_id: str
report_revision: int
format: str
target_ref: str | None
rows: tuple[Mapping[str, object], ...]
schema: tuple[Mapping[str, object], ...]
output_hash: str
options: Mapping[str, object] = field(default_factory=dict)
@runtime_checkable
class ReportingPublicationTarget(Protocol):
def publish_report(
self,
session: object,
principal: object,
*,
payload: ReportingPublicationPayload,
) -> Mapping[str, object]: ...
@runtime_checkable
class ReportingChartRenderer(Protocol):
def render(
self, *, visualization: object, result: object
) -> Mapping[str, object]: ...
def capability(registry: object | None, name: str) -> object | None:
if (
registry is None
or not hasattr(registry, "has_capability")
or not hasattr(registry, "capability")
or not registry.has_capability(name)
):
return None
return registry.capability(name)
__all__ = [
"CAPABILITY_REPORTING_CHART_RENDERER",
"CAPABILITY_REPORTING_REGISTRY",
"CAPABILITY_REPORTING_RUNNER",
"CAPABILITY_REPORTING_SCHEDULER",
"ReportingChartRenderer",
"ReportingDatasetReadRequest",
"ReportingDatasetReadResult",
"ReportingPublicationPayload",
"ReportingPublicationTarget",
"ReportingReadModelProvider",
"ReportingRowPolicyProvider",
"ReportingRowPolicyRequest",
"ReportingRowPolicyResult",
"capability",
]
@@ -0,0 +1,25 @@
"""Reporting database models."""
from govoplan_reporting.backend.db.models import (
ReportingDefinitionGrant,
ReportingDefinitionIdentity,
ReportingDefinitionRevision,
ReportingExecution,
ReportingImportAssessment,
ReportingPublication,
ReportingQualityResult,
ReportingSavedView,
ReportingSchedule,
)
__all__ = [
"ReportingDefinitionGrant",
"ReportingDefinitionIdentity",
"ReportingDefinitionRevision",
"ReportingExecution",
"ReportingImportAssessment",
"ReportingPublication",
"ReportingQualityResult",
"ReportingSavedView",
"ReportingSchedule",
]
+441
View File
@@ -0,0 +1,441 @@
from __future__ import annotations
from datetime import datetime
from typing import Any
import uuid
from sqlalchemy import (
Boolean,
DateTime,
ForeignKey,
Index,
Integer,
JSON,
String,
Text,
UniqueConstraint,
)
from sqlalchemy.orm import Mapped, mapped_column
from govoplan_core.db.base import Base, TimestampMixin
def new_uuid() -> str:
return str(uuid.uuid4())
class ReportingDefinitionIdentity(Base, TimestampMixin):
__tablename__ = "reporting_definition_identities"
__table_args__ = (
UniqueConstraint(
"tenant_id",
"definition_kind",
"definition_id",
name="uq_reporting_definition_identity",
),
UniqueConstraint(
"tenant_id",
"definition_kind",
"definition_key",
name="uq_reporting_definition_key",
),
Index(
"ix_reporting_definition_catalog",
"tenant_id",
"definition_kind",
"definition_key",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
definition_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
definition_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
definition_key: Mapped[str] = mapped_column(String(120), nullable=False, index=True)
created_by: Mapped[str | None] = mapped_column(
String(255), nullable=True, index=True
)
class ReportingDefinitionRevision(Base, TimestampMixin):
__tablename__ = "reporting_definition_revisions"
__table_args__ = (
UniqueConstraint(
"tenant_id",
"definition_kind",
"definition_id",
"revision",
name="uq_reporting_definition_revision",
),
UniqueConstraint(
"tenant_id",
"idempotency_key",
name="uq_reporting_definition_idempotency",
),
UniqueConstraint(
"tenant_id",
"event_id",
name="uq_reporting_definition_event",
),
Index(
"ix_reporting_definition_current",
"tenant_id",
"definition_kind",
"definition_id",
"superseded_at",
),
Index(
"ix_reporting_definition_parent",
"tenant_id",
"parent_kind",
"parent_id",
"parent_revision",
),
Index(
"ix_reporting_definition_list",
"tenant_id",
"definition_kind",
"status",
"recorded_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
identity_id: Mapped[str] = mapped_column(
ForeignKey("reporting_definition_identities.id", ondelete="RESTRICT"),
nullable=False,
index=True,
)
definition_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
definition_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
definition_key: Mapped[str] = mapped_column(String(120), nullable=False, index=True)
revision: Mapped[int] = mapped_column(Integer, nullable=False)
previous_revision_id: Mapped[str | None] = mapped_column(
ForeignKey("reporting_definition_revisions.id", ondelete="RESTRICT"),
nullable=True,
index=True,
)
parent_kind: Mapped[str | None] = mapped_column(
String(40), nullable=True, index=True
)
parent_id: Mapped[str | None] = mapped_column(
String(255), nullable=True, index=True
)
parent_revision: Mapped[int | None] = mapped_column(Integer, nullable=True)
name: Mapped[str] = mapped_column(String(500), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
visibility: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
change_reason: Mapped[str] = mapped_column(String(1000), nullable=False)
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
request_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
event_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
recorded_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True
)
superseded_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True, index=True
)
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
changed_by: Mapped[str | None] = mapped_column(
String(255), nullable=True, index=True
)
class ReportingExecution(Base, TimestampMixin):
__tablename__ = "reporting_executions"
__table_args__ = (
UniqueConstraint("tenant_id", "execution_id", name="uq_reporting_execution"),
UniqueConstraint(
"tenant_id",
"idempotency_key",
name="uq_reporting_execution_idempotency",
),
Index(
"ix_reporting_execution_history",
"tenant_id",
"report_id",
"started_at",
),
Index(
"ix_reporting_execution_status",
"tenant_id",
"status",
"started_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
report_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
report_revision: Mapped[int] = mapped_column(Integer, nullable=False)
semantic_model_id: Mapped[str] = mapped_column(
String(255), nullable=False, index=True
)
semantic_model_revision: Mapped[int] = mapped_column(Integer, nullable=False)
dataset_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
dataset_revision: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
request_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
parameters: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
query: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
source_fingerprints: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
definition_hashes: Mapped[dict[str, str]] = mapped_column(
JSON, default=dict, nullable=False
)
output_hash: Mapped[str | None] = mapped_column(
String(64), nullable=True, index=True
)
executor_version: Mapped[str | None] = mapped_column(String(255), nullable=True)
result_schema: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
result_rows: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
total_rows: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
truncated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
provenance: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
started_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True
)
finished_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True, index=True
)
actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
class ReportingDefinitionGrant(Base, TimestampMixin):
__tablename__ = "reporting_definition_grants"
__table_args__ = (
UniqueConstraint(
"tenant_id",
"definition_kind",
"definition_id",
"subject_kind",
"subject_id",
name="uq_reporting_definition_grant",
),
Index(
"ix_reporting_definition_grant_subject",
"tenant_id",
"subject_kind",
"subject_id",
"active",
),
Index(
"ix_reporting_definition_grant_object",
"tenant_id",
"definition_kind",
"definition_id",
"active",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
definition_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
definition_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
subject_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
subject_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
permissions: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
active: Mapped[bool] = mapped_column(
Boolean, default=True, nullable=False, index=True
)
source_revision: Mapped[int] = mapped_column(Integer, nullable=False)
class ReportingSavedView(Base, TimestampMixin):
__tablename__ = "reporting_saved_views"
__table_args__ = (
UniqueConstraint("tenant_id", "view_id", name="uq_reporting_saved_view"),
Index(
"ix_reporting_saved_view_catalog",
"tenant_id",
"report_id",
"owner_id",
"shared",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
view_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
report_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
report_revision: Mapped[int] = mapped_column(Integer, nullable=False)
owner_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
owner_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
name: Mapped[str] = mapped_column(String(500), nullable=False)
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
state: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
shared: Mapped[bool] = mapped_column(
Boolean, default=False, nullable=False, index=True
)
access: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
class ReportingSchedule(Base, TimestampMixin):
__tablename__ = "reporting_schedules"
__table_args__ = (
UniqueConstraint("tenant_id", "schedule_id", name="uq_reporting_schedule"),
Index(
"ix_reporting_schedule_due",
"enabled",
"next_run_at",
"tenant_id",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
schedule_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
report_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
report_revision: Mapped[int] = mapped_column(Integer, nullable=False)
name: Mapped[str] = mapped_column(String(500), nullable=False)
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
trigger_kind: Mapped[str] = mapped_column(String(30), nullable=False)
trigger_config: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
parameters: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
query: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
publication_target: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
enabled: Mapped[bool] = mapped_column(
Boolean, default=True, nullable=False, index=True
)
next_run_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True, index=True
)
last_run_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
last_execution_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
class ReportingPublication(Base, TimestampMixin):
__tablename__ = "reporting_publications"
__table_args__ = (
UniqueConstraint(
"tenant_id", "publication_id", name="uq_reporting_publication"
),
UniqueConstraint(
"tenant_id",
"idempotency_key",
name="uq_reporting_publication_idempotency",
),
Index(
"ix_reporting_publication_history",
"tenant_id",
"execution_id",
"created_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
publication_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
target_capability: Mapped[str] = mapped_column(String(255), nullable=False)
target_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
format: Mapped[str] = mapped_column(String(30), nullable=False)
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
class ReportingQualityResult(Base, TimestampMixin):
__tablename__ = "reporting_quality_results"
__table_args__ = (
UniqueConstraint("tenant_id", "result_id", name="uq_reporting_quality_result"),
Index(
"ix_reporting_quality_history",
"tenant_id",
"quality_plan_id",
"evaluated_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
result_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
quality_plan_id: Mapped[str] = mapped_column(
String(255), nullable=False, index=True
)
quality_plan_revision: Mapped[int] = mapped_column(Integer, nullable=False)
dataset_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
dataset_revision: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
output_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
assertions: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
source_fingerprints: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
evaluated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True
)
actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
class ReportingImportAssessment(Base, TimestampMixin):
__tablename__ = "reporting_import_assessments"
__table_args__ = (
UniqueConstraint(
"tenant_id", "assessment_id", name="uq_reporting_import_assessment"
),
Index(
"ix_reporting_import_history",
"tenant_id",
"source_system",
"created_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
assessment_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
source_system: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
source_id: Mapped[str] = mapped_column(String(500), nullable=False)
source_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
mapping_report: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
accepted_approximations: Mapped[list[str]] = mapped_column(
JSON, default=list, nullable=False
)
assessed_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
__all__ = [
"ReportingDefinitionGrant",
"ReportingDefinitionIdentity",
"ReportingDefinitionRevision",
"ReportingExecution",
"ReportingImportAssessment",
"ReportingPublication",
"ReportingQualityResult",
"ReportingSavedView",
"ReportingSchedule",
]
@@ -0,0 +1,955 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from datetime import datetime
import hashlib
import json
from typing import Any
import uuid
from sqlalchemy import and_, exists, func, or_
from sqlalchemy.orm import Query, Session
from govoplan_core.core.events import (
EventActorRef,
EventObjectRef,
EventTenantRef,
PlatformEvent,
emit_platform_event,
)
from govoplan_core.security.module_permissions import scopes_grant_compatible
from govoplan_reporting.backend.db.models import (
ReportingDefinitionGrant,
ReportingDefinitionIdentity,
ReportingDefinitionRevision,
)
from govoplan_reporting.backend.domain import (
ReportingDefinitionRecord,
definition_from_row,
)
from govoplan_reporting.backend.schemas import validate_definition_payload
READ_SCOPE = "reporting:definition:read"
WRITE_SCOPE = "reporting:definition:write"
ADMIN_SCOPE = "reporting:definition:admin"
DEFINITION_KINDS = frozenset({"dataset", "semantic_model", "report", "quality_plan"})
STATUS_TRANSITIONS = {
"draft": frozenset({"active", "retired"}),
"active": frozenset({"draft", "retired"}),
"retired": frozenset({"draft"}),
}
SUBJECT_KINDS = frozenset(
{
"account",
"identity",
"group",
"role",
"function",
"function_assignment",
"organization_unit",
"service_account",
}
)
class ReportingDefinitionError(ValueError):
pass
def create_definition(
session: Session,
principal: object,
*,
definition_kind: str,
definition_id: str,
definition_key: str,
name: str,
description: str | None,
status: str,
visibility: str,
recorded_at: datetime,
change_reason: str,
payload: Mapping[str, Any],
idempotency_key: str,
) -> ReportingDefinitionRecord:
_require_scope(principal, WRITE_SCOPE)
tenant_id = _principal_tenant(principal)
kind = _definition_kind(definition_kind)
clean_id = _required(definition_id, "Reporting definition identifier", 255)
clean_key = _key(definition_key, "Reporting definition key")
clean_name = _required(name, "Reporting definition name", 500)
clean_description = _optional(description, "Reporting description", 100_000)
clean_status = _status(status)
clean_visibility = _visibility(visibility)
clean_reason = _required(change_reason, "Reporting change reason", 1_000)
_aware(recorded_at, "Reporting recorded_at")
validated_payload = validate_definition_payload(kind, dict(payload))
parent_kind, parent_id, parent_revision = _parent_reference(
kind,
validated_payload,
)
_validate_parent(
session,
tenant_id=tenant_id,
child_status=clean_status,
parent_kind=parent_kind,
parent_id=parent_id,
parent_revision=parent_revision,
)
request = {
"definition_kind": kind,
"definition_id": clean_id,
"definition_key": clean_key,
"name": clean_name,
"description": clean_description,
"status": clean_status,
"visibility": clean_visibility,
"recorded_at": recorded_at,
"change_reason": clean_reason,
"payload": validated_payload,
}
request_sha256 = _sha256(request)
replay = _replay(
session,
tenant_id=tenant_id,
idempotency_key=idempotency_key,
request_sha256=request_sha256,
)
if replay is not None:
return replay
if _identity(session, tenant_id, kind, clean_id) is not None:
raise ReportingDefinitionError("Reporting definition already exists.")
duplicate = (
session.query(ReportingDefinitionIdentity.id)
.filter(
ReportingDefinitionIdentity.tenant_id == tenant_id,
ReportingDefinitionIdentity.definition_kind == kind,
ReportingDefinitionIdentity.definition_key == clean_key,
)
.first()
)
if duplicate is not None:
raise ReportingDefinitionError("Reporting definition key already exists.")
identity = ReportingDefinitionIdentity(
tenant_id=tenant_id,
definition_kind=kind,
definition_id=clean_id,
definition_key=clean_key,
created_by=_actor(principal),
)
session.add(identity)
session.flush()
return _write_revision(
session,
principal,
identity=identity,
current=None,
revision=1,
name=clean_name,
description=clean_description,
status=clean_status,
visibility=clean_visibility,
recorded_at=recorded_at,
change_reason=clean_reason,
payload=validated_payload,
parent_kind=parent_kind,
parent_id=parent_id,
parent_revision=parent_revision,
idempotency_key=idempotency_key,
request_sha256=request_sha256,
operation="created",
)
def update_definition(
session: Session,
principal: object,
*,
definition_kind: str,
definition_id: str,
expected_revision: int,
recorded_at: datetime,
change_reason: str,
idempotency_key: str,
changes: Mapping[str, object],
) -> ReportingDefinitionRecord:
tenant_id = _principal_tenant(principal)
kind = _definition_kind(definition_kind)
current_row = _current_row(
session,
tenant_id=tenant_id,
definition_kind=kind,
definition_id=definition_id,
lock=True,
)
if current_row is None:
raise LookupError("Reporting definition not found.")
if not can_write_definition(
session,
principal,
definition_kind=kind,
definition_id=definition_id,
):
raise PermissionError("Reporting definition write access is denied.")
current = definition_from_row(current_row)
unknown = set(changes) - {
"name",
"description",
"status",
"visibility",
"payload",
}
if unknown:
raise ReportingDefinitionError(
"Unsupported Reporting definition fields: " + ", ".join(sorted(unknown))
)
_aware(recorded_at, "Reporting recorded_at")
clean_reason = _required(change_reason, "Reporting change reason", 1_000)
request_sha256 = _sha256(
{
"definition_kind": kind,
"definition_id": definition_id,
"expected_revision": expected_revision,
"recorded_at": recorded_at,
"change_reason": clean_reason,
"changes": changes,
}
)
replay = _replay(
session,
tenant_id=tenant_id,
idempotency_key=idempotency_key,
request_sha256=request_sha256,
)
if replay is not None:
return replay
if current.revision != expected_revision:
raise ReportingDefinitionError(
"Reporting definition revision conflict: the expected revision is stale."
)
next_status = _status(str(changes.get("status", current.status)))
if (
next_status != current.status
and next_status not in STATUS_TRANSITIONS[current.status]
):
raise ReportingDefinitionError(
f"Cannot move Reporting definition from {current.status!r} to {next_status!r}."
)
next_payload = validate_definition_payload(
kind,
dict(changes.get("payload", current.payload)),
)
parent_kind, parent_id, parent_revision = _parent_reference(kind, next_payload)
_validate_parent(
session,
tenant_id=tenant_id,
child_status=next_status,
parent_kind=parent_kind,
parent_id=parent_id,
parent_revision=parent_revision,
)
identity = _identity(session, tenant_id, kind, definition_id)
if identity is None:
raise ReportingDefinitionError("Reporting definition identity is missing.")
return _write_revision(
session,
principal,
identity=identity,
current=current_row,
revision=current.revision + 1,
name=_required(
changes.get("name", current.name), "Reporting definition name", 500
),
description=_optional(
changes.get("description", current.description),
"Reporting description",
100_000,
),
status=next_status,
visibility=_visibility(str(changes.get("visibility", current.visibility))),
recorded_at=recorded_at,
change_reason=clean_reason,
payload=next_payload,
parent_kind=parent_kind,
parent_id=parent_id,
parent_revision=parent_revision,
idempotency_key=idempotency_key,
request_sha256=request_sha256,
operation="updated" if next_status == current.status else "state_changed",
)
def get_definition(
session: Session,
principal: object,
*,
definition_kind: str,
definition_id: str,
revision: int | None = None,
) -> ReportingDefinitionRecord | None:
tenant_id = _principal_tenant(principal)
kind = _definition_kind(definition_kind)
if not can_read_definition(
session,
principal,
definition_kind=kind,
definition_id=definition_id,
):
return None
query = session.query(ReportingDefinitionRevision).filter(
ReportingDefinitionRevision.tenant_id == tenant_id,
ReportingDefinitionRevision.definition_kind == kind,
ReportingDefinitionRevision.definition_id == definition_id,
)
if revision is None:
query = query.filter(ReportingDefinitionRevision.superseded_at.is_(None))
else:
query = query.filter(ReportingDefinitionRevision.revision == revision)
row = query.order_by(ReportingDefinitionRevision.revision.desc()).first()
return definition_from_row(row) if row is not None else None
def list_definitions(
session: Session,
principal: object,
*,
definition_kinds: Sequence[str] | None = None,
statuses: Sequence[str] | None = None,
query: str = "",
offset: int = 0,
limit: int = 100,
) -> tuple[tuple[ReportingDefinitionRecord, ...], int]:
_require_scope(principal, READ_SCOPE)
if offset < 0 or not 1 <= limit <= 200:
raise ReportingDefinitionError(
"Reporting list offset must be non-negative and limit between 1 and 200."
)
kinds = tuple(dict.fromkeys(definition_kinds or DEFINITION_KINDS))
if any(item not in DEFINITION_KINDS for item in kinds):
raise ReportingDefinitionError("Unsupported Reporting definition kind filter.")
statement = session.query(ReportingDefinitionRevision).filter(
ReportingDefinitionRevision.tenant_id == _principal_tenant(principal),
ReportingDefinitionRevision.superseded_at.is_(None),
ReportingDefinitionRevision.definition_kind.in_(kinds),
)
statement = _filter_accessible(statement, principal)
if statuses:
statement = statement.filter(
ReportingDefinitionRevision.status.in_(tuple(dict.fromkeys(statuses)))
)
clean_query = query.strip().casefold()
if clean_query:
pattern = f"%{clean_query}%"
statement = statement.filter(
or_(
func.lower(ReportingDefinitionRevision.name).like(pattern),
func.lower(ReportingDefinitionRevision.definition_key).like(pattern),
func.lower(ReportingDefinitionRevision.description).like(pattern),
)
)
total = int(statement.with_entities(func.count()).scalar() or 0)
rows = (
statement.order_by(
ReportingDefinitionRevision.definition_kind.asc(),
ReportingDefinitionRevision.name.asc(),
)
.offset(offset)
.limit(limit)
.all()
)
return tuple(definition_from_row(row) for row in rows), total
def definition_history(
session: Session,
principal: object,
*,
definition_kind: str,
definition_id: str,
limit: int = 100,
) -> tuple[ReportingDefinitionRecord, ...]:
if not can_read_definition(
session,
principal,
definition_kind=definition_kind,
definition_id=definition_id,
):
return ()
rows = (
session.query(ReportingDefinitionRevision)
.filter(
ReportingDefinitionRevision.tenant_id == _principal_tenant(principal),
ReportingDefinitionRevision.definition_kind == definition_kind,
ReportingDefinitionRevision.definition_id == definition_id,
)
.order_by(ReportingDefinitionRevision.revision.desc())
.limit(max(1, min(limit, 200)))
.all()
)
return tuple(definition_from_row(row) for row in rows)
def can_read_definition(
session: Session,
principal: object,
*,
definition_kind: str,
definition_id: str,
) -> bool:
if not _has_scope(principal, READ_SCOPE):
return False
return _can_access(
session,
principal,
definition_kind=definition_kind,
definition_id=definition_id,
permission="read",
)
def can_write_definition(
session: Session,
principal: object,
*,
definition_kind: str,
definition_id: str,
) -> bool:
if not _has_scope(principal, WRITE_SCOPE):
return False
return _can_access(
session,
principal,
definition_kind=definition_kind,
definition_id=definition_id,
permission="write",
)
def _write_revision(
session: Session,
principal: object,
*,
identity: ReportingDefinitionIdentity,
current: ReportingDefinitionRevision | None,
revision: int,
name: str,
description: str | None,
status: str,
visibility: str,
recorded_at: datetime,
change_reason: str,
payload: dict[str, Any],
parent_kind: str | None,
parent_id: str | None,
parent_revision: int | None,
idempotency_key: str,
request_sha256: str,
operation: str,
) -> ReportingDefinitionRecord:
if current is not None:
current.superseded_at = recorded_at
content_hash = _sha256(
{
"name": name,
"description": description,
"status": status,
"visibility": visibility,
"parent_kind": parent_kind,
"parent_id": parent_id,
"parent_revision": parent_revision,
"payload": payload,
}
)
event_id = str(uuid.uuid4())
row = ReportingDefinitionRevision(
tenant_id=identity.tenant_id,
identity_id=identity.id,
definition_kind=identity.definition_kind,
definition_id=identity.definition_id,
definition_key=identity.definition_key,
revision=revision,
previous_revision_id=current.id if current else None,
parent_kind=parent_kind,
parent_id=parent_id,
parent_revision=parent_revision,
name=name,
description=description,
status=status,
visibility=visibility,
content_hash=content_hash,
change_reason=change_reason,
idempotency_key=_required(idempotency_key, "Reporting idempotency key", 255),
request_sha256=request_sha256,
event_id=event_id,
recorded_at=recorded_at,
payload=payload,
changed_by=_actor(principal),
)
session.add(row)
session.flush()
_sync_grants(session, row)
event_type = f"reporting.{identity.definition_kind}.{operation}"
emit_platform_event(
session,
PlatformEvent(
event_id=event_id,
type=event_type,
module_id="reporting",
payload={
"definition_kind": identity.definition_kind,
"definition_key": identity.definition_key,
"revision": revision,
"status": status,
"content_hash": content_hash,
"change_reason": change_reason,
},
occurred_at=recorded_at,
actor=EventActorRef(type="account", id=_actor(principal)),
tenant=EventTenantRef(id=identity.tenant_id),
resource=EventObjectRef(
type=f"reporting_{identity.definition_kind}",
id=identity.definition_id,
label=name,
),
classification="restricted" if visibility == "restricted" else "internal",
),
)
return definition_from_row(row)
def _sync_grants(session: Session, row: ReportingDefinitionRevision) -> None:
access_policy = row.payload.get("access_policy")
subjects = (
access_policy.get("subjects", []) if isinstance(access_policy, Mapping) else []
)
desired: dict[tuple[str, str], list[str]] = {}
if not isinstance(subjects, list):
raise ReportingDefinitionError("Report access_policy subjects must be a list.")
for subject in subjects:
if not isinstance(subject, Mapping):
raise ReportingDefinitionError("Report access subjects must be objects.")
kind = str(subject.get("kind") or "")
subject_id = str(subject.get("id") or "").strip()
if kind not in SUBJECT_KINDS or not subject_id:
raise ReportingDefinitionError("Report access subject is invalid.")
permissions = tuple(
dict.fromkeys(str(item) for item in subject.get("permissions", ["read"]))
)
if not permissions or set(permissions) - {"read", "write", "publish", "admin"}:
raise ReportingDefinitionError("Report access permissions are invalid.")
desired[(kind, subject_id)] = list(permissions)
rows = (
session.query(ReportingDefinitionGrant)
.filter(
ReportingDefinitionGrant.tenant_id == row.tenant_id,
ReportingDefinitionGrant.definition_kind == row.definition_kind,
ReportingDefinitionGrant.definition_id == row.definition_id,
)
.all()
)
existing = {(item.subject_kind, item.subject_id): item for item in rows}
for key, item in existing.items():
if key not in desired:
item.active = False
item.source_revision = row.revision
for key, permissions in desired.items():
item = existing.get(key)
if item is None:
session.add(
ReportingDefinitionGrant(
tenant_id=row.tenant_id,
definition_kind=row.definition_kind,
definition_id=row.definition_id,
subject_kind=key[0],
subject_id=key[1],
permissions=permissions,
active=True,
source_revision=row.revision,
)
)
else:
item.permissions = permissions
item.active = True
item.source_revision = row.revision
session.flush()
def _validate_parent(
session: Session,
*,
tenant_id: str,
child_status: str,
parent_kind: str | None,
parent_id: str | None,
parent_revision: int | None,
) -> None:
if parent_kind is None:
return
row = (
session.query(ReportingDefinitionRevision)
.filter(
ReportingDefinitionRevision.tenant_id == tenant_id,
ReportingDefinitionRevision.definition_kind == parent_kind,
ReportingDefinitionRevision.definition_id == parent_id,
ReportingDefinitionRevision.revision == parent_revision,
)
.one_or_none()
)
if row is None:
raise ReportingDefinitionError(
f"Reporting definition references a missing {parent_kind} revision."
)
if child_status == "active" and row.status != "active":
raise ReportingDefinitionError(
f"An active Reporting definition requires an active {parent_kind} revision."
)
def _parent_reference(
definition_kind: str,
payload: Mapping[str, object],
) -> tuple[str | None, str | None, int | None]:
if definition_kind == "semantic_model":
return "dataset", str(payload["dataset_id"]), int(payload["dataset_revision"])
if definition_kind == "report":
return (
"semantic_model",
str(payload["semantic_model_id"]),
int(payload["semantic_model_revision"]),
)
if definition_kind == "quality_plan":
return "dataset", str(payload["dataset_id"]), int(payload["dataset_revision"])
return None, None, None
def _replay(
session: Session,
*,
tenant_id: str,
idempotency_key: str,
request_sha256: str,
) -> ReportingDefinitionRecord | None:
key = _required(idempotency_key, "Reporting idempotency key", 255)
row = (
session.query(ReportingDefinitionRevision)
.filter(
ReportingDefinitionRevision.tenant_id == tenant_id,
ReportingDefinitionRevision.idempotency_key == key,
)
.one_or_none()
)
if row is None:
return None
if row.request_sha256 != request_sha256:
raise ReportingDefinitionError(
"Reporting idempotency conflict: the key belongs to another request."
)
return definition_from_row(row)
def _can_access(
session: Session,
principal: object,
*,
definition_kind: str,
definition_id: str,
permission: str,
) -> bool:
tenant_id = _principal_tenant(principal)
row = _current_row(
session,
tenant_id=tenant_id,
definition_kind=definition_kind,
definition_id=definition_id,
lock=False,
)
if row is None:
return False
if _has_scope(principal, ADMIN_SCOPE):
return True
identity = _identity(session, tenant_id, definition_kind, definition_id)
actor_ids = _actor_ids(principal)
if identity is not None and identity.created_by in actor_ids:
return True
if permission == "read" and row.visibility == "tenant":
return True
subjects = _principal_subjects(principal)
if not subjects:
return False
clauses = [
and_(
ReportingDefinitionGrant.subject_kind == kind,
ReportingDefinitionGrant.subject_id == subject_id,
)
for kind, subject_id in subjects
]
grants = (
session.query(ReportingDefinitionGrant)
.filter(
ReportingDefinitionGrant.tenant_id == tenant_id,
ReportingDefinitionGrant.definition_kind == definition_kind,
ReportingDefinitionGrant.definition_id == definition_id,
ReportingDefinitionGrant.active.is_(True),
or_(*clauses),
)
.all()
)
return any(
permission == "read"
or permission in set(item.permissions or ())
or "admin" in set(item.permissions or ())
for item in grants
)
def _require_scope(principal: object, scope: str) -> None:
if not _has_scope(principal, scope):
raise PermissionError(f"Missing scope: {scope}")
def _filter_accessible(query: Query, principal: object) -> Query:
if _has_scope(principal, ADMIN_SCOPE):
return query
conditions = [ReportingDefinitionRevision.visibility == "tenant"]
actor_ids = _actor_ids(principal)
if actor_ids:
conditions.append(
exists()
.where(
ReportingDefinitionIdentity.id
== ReportingDefinitionRevision.identity_id
)
.where(ReportingDefinitionIdentity.created_by.in_(actor_ids))
)
subjects = _principal_subjects(principal)
if subjects:
conditions.append(
exists()
.where(
ReportingDefinitionGrant.tenant_id
== ReportingDefinitionRevision.tenant_id
)
.where(
ReportingDefinitionGrant.definition_kind
== ReportingDefinitionRevision.definition_kind
)
.where(
ReportingDefinitionGrant.definition_id
== ReportingDefinitionRevision.definition_id
)
.where(ReportingDefinitionGrant.active.is_(True))
.where(
or_(
*(
and_(
ReportingDefinitionGrant.subject_kind == kind,
ReportingDefinitionGrant.subject_id == subject_id,
)
for kind, subject_id in subjects
)
)
)
)
return query.filter(or_(*conditions))
def _current_row(
session: Session,
*,
tenant_id: str,
definition_kind: str,
definition_id: str,
lock: bool,
) -> ReportingDefinitionRevision | None:
query = session.query(ReportingDefinitionRevision).filter(
ReportingDefinitionRevision.tenant_id == tenant_id,
ReportingDefinitionRevision.definition_kind == definition_kind,
ReportingDefinitionRevision.definition_id == definition_id,
ReportingDefinitionRevision.superseded_at.is_(None),
)
if lock:
query = query.with_for_update()
return query.one_or_none()
def _identity(
session: Session,
tenant_id: str,
definition_kind: str,
definition_id: str,
) -> ReportingDefinitionIdentity | None:
return (
session.query(ReportingDefinitionIdentity)
.filter(
ReportingDefinitionIdentity.tenant_id == tenant_id,
ReportingDefinitionIdentity.definition_kind == definition_kind,
ReportingDefinitionIdentity.definition_id == definition_id,
)
.one_or_none()
)
def _principal_subjects(principal: object) -> tuple[tuple[str, str], ...]:
values: list[tuple[str, str]] = []
for kind, attribute in (
("account", "account_id"),
("identity", "identity_id"),
("function_assignment", "acting_assignment_id"),
("service_account", "service_account_id"),
):
value = getattr(principal, attribute, None)
if str(value or "").strip():
values.append((kind, str(value)))
for kind, attribute in (
("group", "group_ids"),
("role", "role_ids"),
("function_assignment", "function_assignment_ids"),
):
values.extend(
(kind, str(value))
for value in getattr(principal, attribute, ()) or ()
if str(value or "").strip()
)
return tuple(dict.fromkeys(values))
def _actor(principal: object) -> str | None:
for value in (
getattr(principal, "account_id", None),
getattr(principal, "identity_id", None),
getattr(principal, "membership_id", None),
):
if str(value or "").strip():
return str(value)
return None
def _actor_ids(principal: object) -> tuple[str, ...]:
user = getattr(principal, "user", None)
return tuple(
dict.fromkeys(
str(value)
for value in (
getattr(principal, "account_id", None),
getattr(principal, "identity_id", None),
getattr(principal, "membership_id", None),
getattr(user, "id", None),
)
if str(value or "").strip()
)
)
def _principal_tenant(principal: object) -> str:
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
if not tenant_id:
raise ReportingDefinitionError(
"Reporting operations require a tenant-bound principal."
)
return tenant_id
def _has_scope(principal: object, scope: str) -> bool:
method = getattr(principal, "has", None)
if callable(method):
return bool(method(scope))
return scopes_grant_compatible(
frozenset(getattr(principal, "scopes", ()) or ()),
scope,
)
def _definition_kind(value: str) -> str:
if value not in DEFINITION_KINDS:
raise ReportingDefinitionError(
f"Unsupported Reporting definition kind: {value!r}."
)
return value
def _status(value: str) -> str:
if value not in STATUS_TRANSITIONS:
raise ReportingDefinitionError(f"Unsupported Reporting status: {value!r}.")
return value
def _visibility(value: str) -> str:
if value not in {"tenant", "restricted"}:
raise ReportingDefinitionError(f"Unsupported Reporting visibility: {value!r}.")
return value
def _key(value: object, label: str) -> str:
result = _required(value, label, 120).casefold()
if any(
character not in "abcdefghijklmnopqrstuvwxyz0123456789._-"
for character in result
):
raise ReportingDefinitionError(
f"{label} may contain only letters, digits, dot, underscore, and hyphen."
)
return result
def _required(value: object, label: str, maximum: int) -> str:
result = str(value or "").strip()
if not result:
raise ReportingDefinitionError(f"{label} is required.")
if len(result) > maximum:
raise ReportingDefinitionError(f"{label} is limited to {maximum} characters.")
return result
def _optional(value: object, label: str, maximum: int) -> str | None:
result = str(value or "").strip()
if not result:
return None
if len(result) > maximum:
raise ReportingDefinitionError(f"{label} is limited to {maximum} characters.")
return result
def _aware(value: datetime, label: str) -> None:
if value.tzinfo is None or value.utcoffset() is None:
raise ReportingDefinitionError(f"{label} must include a timezone.")
def _sha256(value: object) -> str:
return hashlib.sha256(
json.dumps(
_json_value(value),
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
).encode("utf-8")
).hexdigest()
def _json_value(value: object) -> Any:
if isinstance(value, datetime):
return value.isoformat()
if hasattr(value, "model_dump"):
return _json_value(value.model_dump(mode="json"))
if isinstance(value, Mapping):
return {str(key): _json_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_value(item) for item in value]
return value
__all__ = [
"ADMIN_SCOPE",
"DEFINITION_KINDS",
"READ_SCOPE",
"ReportingDefinitionError",
"WRITE_SCOPE",
"can_read_definition",
"can_write_definition",
"create_definition",
"definition_history",
"get_definition",
"list_definitions",
"update_definition",
]
+79
View File
@@ -0,0 +1,79 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any
from govoplan_reporting.backend.db.models import ReportingDefinitionRevision
@dataclass(frozen=True, slots=True)
class ReportingDefinitionRecord:
tenant_id: str
definition_kind: str
definition_id: str
definition_key: str
revision: int
name: str
description: str | None
status: str
visibility: str
content_hash: str
recorded_at: datetime
change_reason: str
payload: dict[str, Any]
parent_kind: str | None = None
parent_id: str | None = None
parent_revision: int | None = None
def to_dict(self) -> dict[str, Any]:
return {
"tenant_id": self.tenant_id,
"definition_kind": self.definition_kind,
"definition_id": self.definition_id,
"definition_key": self.definition_key,
"revision": self.revision,
"name": self.name,
"description": self.description,
"status": self.status,
"visibility": self.visibility,
"content_hash": self.content_hash,
"parent_kind": self.parent_kind,
"parent_id": self.parent_id,
"parent_revision": self.parent_revision,
"recorded_at": _datetime_text(self.recorded_at),
"change_reason": self.change_reason,
"payload": dict(self.payload),
}
def definition_from_row(
row: ReportingDefinitionRevision,
) -> ReportingDefinitionRecord:
return ReportingDefinitionRecord(
tenant_id=row.tenant_id,
definition_kind=row.definition_kind,
definition_id=row.definition_id,
definition_key=row.definition_key,
revision=row.revision,
name=row.name,
description=row.description,
status=row.status,
visibility=row.visibility,
content_hash=row.content_hash,
parent_kind=row.parent_kind,
parent_id=row.parent_id,
parent_revision=row.parent_revision,
recorded_at=row.recorded_at,
change_reason=row.change_reason,
payload=dict(row.payload or {}),
)
def _datetime_text(value: datetime) -> str:
if value.tzinfo is None:
value = value.replace(tzinfo=UTC)
return value.isoformat()
__all__ = ["ReportingDefinitionRecord", "definition_from_row"]
+976
View File
@@ -0,0 +1,976 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from datetime import UTC, date, datetime
import hashlib
import json
from typing import Any
import uuid
from sqlalchemy.orm import Session
from govoplan_core.core.dataflows import (
DataflowDatasetRequest,
dataflow_dataset_output,
)
from govoplan_core.core.events import (
EventActorRef,
EventObjectRef,
EventTenantRef,
PlatformEvent,
emit_platform_event,
)
from govoplan_core.security.module_permissions import scopes_grant_compatible
from govoplan_core.security.time import utc_now
from govoplan_reporting.backend.contracts import (
CAPABILITY_REPORTING_CHART_RENDERER,
ReportingChartRenderer,
ReportingDatasetReadRequest,
ReportingDatasetReadResult,
ReportingReadModelProvider,
ReportingRowPolicyProvider,
ReportingRowPolicyRequest,
capability,
)
from govoplan_reporting.backend.db.models import (
ReportingExecution,
ReportingQualityResult,
)
from govoplan_reporting.backend.definitions import get_definition, list_definitions
from govoplan_reporting.backend.query_engine import (
QUERY_ENGINE_VERSION,
DefaultChartRenderer,
QueryResult,
execute_semantic_query,
)
from govoplan_reporting.backend.schemas import (
DatasetDefinition,
QualityPlanDefinition,
ReportDefinition,
ReportQuery,
SemanticModelDefinition,
)
RUN_SCOPE = "reporting:report:run"
QUALITY_SCOPE = "reporting:quality:run"
class ReportingExecutionError(ValueError):
pass
class ReportingExecutionFailure(ReportingExecutionError):
def __init__(self, message: str, execution_id: str) -> None:
super().__init__(message)
self.execution_id = execution_id
class SqlReportingRunner:
def __init__(self, registry: object | None) -> None:
self.registry = registry
def execute(
self,
session: object,
principal: object,
*,
report_id: str,
report_revision: int | None = None,
parameters: Mapping[str, object] | None = None,
query: ReportQuery | None = None,
idempotency_key: str,
) -> Mapping[str, object]:
return execute_report(
_session(session),
principal,
registry=self.registry,
report_id=report_id,
report_revision=report_revision,
parameters=parameters or {},
query=query,
idempotency_key=idempotency_key,
)
def get_execution(
self,
session: object,
principal: object,
*,
execution_id: str,
) -> Mapping[str, object] | None:
return get_execution(_session(session), principal, execution_id=execution_id)
def execute_report(
session: Session,
principal: object,
*,
registry: object | None,
report_id: str,
report_revision: int | None,
parameters: Mapping[str, object],
query: ReportQuery | None,
idempotency_key: str,
) -> dict[str, object]:
_require_scope(principal, RUN_SCOPE)
report_record = get_definition(
session,
principal,
definition_kind="report",
definition_id=report_id,
revision=report_revision,
)
if report_record is None:
raise LookupError("Reporting report definition not found.")
if report_record.status != "active":
raise ReportingExecutionError("Only active report definitions can run.")
report = ReportDefinition.model_validate(report_record.payload)
semantic_record = get_definition(
session,
principal,
definition_kind="semantic_model",
definition_id=report.semantic_model_id,
revision=report.semantic_model_revision,
)
if semantic_record is None or semantic_record.status != "active":
raise ReportingExecutionError(
"The report's pinned semantic model is unavailable or inactive."
)
semantic = SemanticModelDefinition.model_validate(semantic_record.payload)
dataset_record = get_definition(
session,
principal,
definition_kind="dataset",
definition_id=semantic.dataset_id,
revision=semantic.dataset_revision,
)
if dataset_record is None or dataset_record.status != "active":
raise ReportingExecutionError(
"The report's pinned analytical dataset is unavailable or inactive."
)
dataset = DatasetDefinition.model_validate(dataset_record.payload)
bound_parameters = _bind_parameters(report, parameters)
effective_query = query or report.default_query
clean_idempotency_key = _required(
idempotency_key,
"Reporting execution idempotency key",
255,
)
request_sha256 = _sha256(
{
"report_id": report_id,
"report_revision": report_record.revision,
"semantic_model_id": semantic_record.definition_id,
"semantic_model_revision": semantic_record.revision,
"dataset_id": dataset_record.definition_id,
"dataset_revision": dataset_record.revision,
"parameters": bound_parameters,
"query": effective_query.model_dump(mode="json"),
}
)
replay = _execution_replay(
session,
tenant_id=_tenant(principal),
idempotency_key=clean_idempotency_key,
request_sha256=request_sha256,
)
if replay is not None:
return _execution_payload(replay, report=report, registry=registry)
started_at = utc_now()
execution = ReportingExecution(
tenant_id=_tenant(principal),
execution_id=str(uuid.uuid4()),
report_id=report_record.definition_id,
report_revision=report_record.revision,
semantic_model_id=semantic_record.definition_id,
semantic_model_revision=semantic_record.revision,
dataset_id=dataset_record.definition_id,
dataset_revision=dataset_record.revision,
status="running",
idempotency_key=clean_idempotency_key,
request_sha256=request_sha256,
parameters=_json_value(bound_parameters),
query=effective_query.model_dump(mode="json"),
definition_hashes={
"report": report_record.content_hash,
"semantic_model": semantic_record.content_hash,
"dataset": dataset_record.content_hash,
},
started_at=started_at,
actor_id=_actor(principal),
)
session.add(execution)
session.flush()
try:
source = _read_dataset(
session,
principal,
registry=registry,
dataset=dataset,
parameters=bound_parameters,
)
diagnostics = list(source.diagnostics)
diagnostics.extend(_freshness_diagnostics(dataset, source, now=started_at))
normalized_rows = tuple(_json_value(dict(row)) for row in source.rows)
diagnostics.extend(_validate_schema(dataset, normalized_rows))
authorized_rows, policy_provenance = _apply_row_policy(
session,
principal,
registry=registry,
dataset_id=dataset_record.definition_id,
dataset_revision=dataset_record.revision,
dataset=dataset,
rows=normalized_rows,
)
quality_results = _evaluate_blocking_quality_plans(
session,
principal,
dataset_id=dataset_record.definition_id,
dataset_revision=dataset_record.revision,
rows=authorized_rows,
output_hash=source.output_hash,
source_fingerprints=source.source_fingerprints,
)
result = execute_semantic_query(authorized_rows, semantic, effective_query)
output_hash = _sha256(
{
"rows": result.rows,
"schema": result.schema,
"query": effective_query.model_dump(mode="json"),
"source_output_hash": source.output_hash,
}
)
execution.status = "succeeded"
execution.source_fingerprints = _json_value(source.source_fingerprints)
execution.output_hash = output_hash
execution.executor_version = f"{QUERY_ENGINE_VERSION}+{source.executor_version}"
execution.result_schema = list(result.schema)
execution.result_rows = list(result.rows)
execution.total_rows = result.total_rows
execution.truncated = result.truncated or source.truncated
execution.diagnostics = _json_value(diagnostics)
execution.provenance = {
"source": _json_value(source.provenance),
"source_output_hash": source.output_hash,
"row_policy": _json_value(policy_provenance),
"quality_result_ids": [item.result_id for item in quality_results],
"authorized_source_rows": len(authorized_rows),
"report_content_hash": report_record.content_hash,
"semantic_model_content_hash": semantic_record.content_hash,
"dataset_content_hash": dataset_record.content_hash,
}
execution.finished_at = utc_now()
session.flush()
_emit_execution_event(session, execution, report_record.name)
return _execution_payload(execution, report=report, registry=registry)
except Exception as exc:
execution.status = "failed"
execution.finished_at = utc_now()
execution.diagnostics = [
{
"severity": "error",
"code": "report_execution_failed",
"message": str(exc),
}
]
session.flush()
_emit_execution_event(session, execution, report_record.name)
raise ReportingExecutionFailure(str(exc), execution.execution_id) from exc
def get_execution(
session: Session,
principal: object,
*,
execution_id: str,
) -> dict[str, object] | None:
row = (
session.query(ReportingExecution)
.filter(
ReportingExecution.tenant_id == _tenant(principal),
ReportingExecution.execution_id == execution_id,
)
.one_or_none()
)
if row is None:
return None
report_record = get_definition(
session,
principal,
definition_kind="report",
definition_id=row.report_id,
revision=row.report_revision,
)
if report_record is None:
return None
return _execution_payload(
row,
report=ReportDefinition.model_validate(report_record.payload),
registry=None,
)
def list_executions(
session: Session,
principal: object,
*,
report_id: str,
limit: int = 100,
) -> tuple[dict[str, object], ...]:
if (
get_definition(
session,
principal,
definition_kind="report",
definition_id=report_id,
)
is None
):
return ()
rows = (
session.query(ReportingExecution)
.filter(
ReportingExecution.tenant_id == _tenant(principal),
ReportingExecution.report_id == report_id,
)
.order_by(ReportingExecution.started_at.desc())
.limit(max(1, min(limit, 200)))
.all()
)
return tuple(_execution_payload(row, report=None, registry=None) for row in rows)
def _read_dataset(
session: Session,
principal: object,
*,
registry: object | None,
dataset: DatasetDefinition,
parameters: Mapping[str, object],
) -> ReportingDatasetReadResult:
source_parameters = {**dataset.source_parameters, **parameters}
if dataset.source_kind == "static":
rows = tuple(dict(item) for item in dataset.static_rows)
return ReportingDatasetReadResult(
rows=rows,
total_rows=len(rows),
truncated=False,
output_hash=_sha256(rows),
executor_version="reporting-static-v1",
definition_hash=dataset.definition_hash,
generated_at=utc_now(),
provenance={"source_kind": "static", "source_ref": dataset.source_ref},
)
if dataset.source_kind == "dataflow":
provider = dataflow_dataset_output(registry)
if provider is None:
raise ReportingExecutionError(
"The Dataflow dataset provider is not enabled."
)
result = provider.read_output(
session,
principal,
request=DataflowDatasetRequest(
pipeline_ref=dataset.source_ref,
revision=dataset.source_revision or 0,
parameters=source_parameters,
row_limit=2_000,
expected_definition_hash=dataset.definition_hash,
expected_source_fingerprints=tuple(
dataset.expected_source_fingerprints
),
),
)
return ReportingDatasetReadResult(
rows=result.rows,
total_rows=result.total_rows,
truncated=result.truncated,
output_hash=result.output_hash,
executor_version=result.executor_version,
definition_hash=result.definition_hash,
source_fingerprints=result.source_fingerprints,
diagnostics=result.diagnostics,
generated_at=result.generated_at,
provenance=result.provenance,
)
provider = capability(registry, dataset.source_ref)
if not isinstance(provider, ReportingReadModelProvider):
raise ReportingExecutionError(
f"Reporting read-model provider {dataset.source_ref!r} is unavailable."
)
return provider.read_dataset(
session,
principal,
request=ReportingDatasetReadRequest(
source_ref=dataset.source_ref,
source_revision=dataset.source_revision,
parameters=source_parameters,
expected_definition_hash=dataset.definition_hash,
expected_source_fingerprints=tuple(dataset.expected_source_fingerprints),
),
)
def _apply_row_policy(
session: Session,
principal: object,
*,
registry: object | None,
dataset_id: str,
dataset_revision: int,
dataset: DatasetDefinition,
rows: tuple[Mapping[str, object], ...],
) -> tuple[tuple[Mapping[str, object], ...], Mapping[str, object]]:
if dataset.row_policy_ref is None:
return rows, {"mode": "dataset_access_only"}
provider = capability(registry, dataset.row_policy_ref)
if not isinstance(provider, ReportingRowPolicyProvider):
raise ReportingExecutionError(
f"Required row-policy provider {dataset.row_policy_ref!r} is unavailable."
)
result = provider.authorize_rows(
session,
principal,
request=ReportingRowPolicyRequest(
dataset_id=dataset_id,
dataset_revision=dataset_revision,
policy_ref=dataset.row_policy_ref,
rows=rows,
),
)
if len(result.rows) > len(rows):
raise ReportingExecutionError(
"A row-policy provider returned more rows than it received."
)
return result.rows, {
"mode": "provider",
"provider": dataset.row_policy_ref,
"decision_ref": result.decision_ref,
**dict(result.provenance),
}
def _freshness_diagnostics(
dataset: DatasetDefinition,
source: ReportingDatasetReadResult,
*,
now: datetime,
) -> tuple[dict[str, object], ...]:
policy = dataset.freshness
if (
policy.require_source_fingerprints
and not source.source_fingerprints
and dataset.source_kind != "static"
):
raise ReportingExecutionError(
"The analytical dataset requires source fingerprints, but the provider returned none."
)
if policy.max_age_seconds is None:
return ()
generated_at = source.generated_at
if generated_at is None:
message = "The dataset provider did not report a generation timestamp."
if policy.stale_action == "block":
raise ReportingExecutionError(message)
return (
{"severity": "warning", "code": "freshness_unknown", "message": message},
)
if generated_at.tzinfo is None:
generated_at = generated_at.replace(tzinfo=UTC)
age_seconds = max(0, int((now - generated_at).total_seconds()))
if age_seconds <= policy.max_age_seconds:
return ()
message = (
f"Dataset age {age_seconds}s exceeds the configured "
f"{policy.max_age_seconds}s freshness limit."
)
if policy.stale_action == "block":
raise ReportingExecutionError(message)
if policy.stale_action == "warn":
return ({"severity": "warning", "code": "dataset_stale", "message": message},)
return ()
def _validate_schema(
dataset: DatasetDefinition,
rows: Sequence[Mapping[str, object]],
) -> tuple[dict[str, object], ...]:
if not dataset.fields:
return (
{
"severity": "info",
"code": "schema_inferred",
"message": "Dataset schema is inferred because no explicit fields are pinned.",
},
)
errors: list[str] = []
for field in dataset.fields:
for index, row in enumerate(rows):
value = row.get(field.name)
if value is None:
if not field.nullable:
errors.append(f"row {index + 1}: {field.name} is null")
continue
if not _matches_type(value, field.type):
errors.append(
f"row {index + 1}: {field.name} does not match {field.type}"
)
if len(errors) >= 20:
break
if len(errors) >= 20:
break
if errors:
raise ReportingExecutionError(
"Analytical dataset schema validation failed: " + "; ".join(errors)
)
return ()
def _evaluate_blocking_quality_plans(
session: Session,
principal: object,
*,
dataset_id: str,
dataset_revision: int,
rows: tuple[Mapping[str, object], ...],
output_hash: str,
source_fingerprints: Sequence[Mapping[str, object]],
) -> tuple[ReportingQualityResult, ...]:
plans, _total = list_definitions(
session,
principal,
definition_kinds=("quality_plan",),
statuses=("active",),
limit=200,
)
relevant = tuple(
plan
for plan in plans
if plan.parent_id == dataset_id and plan.parent_revision == dataset_revision
)
results: list[ReportingQualityResult] = []
for plan_record in relevant:
plan = QualityPlanDefinition.model_validate(plan_record.payload)
result = _record_quality_result(
session,
principal,
plan_id=plan_record.definition_id,
plan_revision=plan_record.revision,
dataset_id=dataset_id,
dataset_revision=dataset_revision,
plan=plan,
rows=rows,
output_hash=output_hash,
source_fingerprints=source_fingerprints,
)
results.append(result)
if plan.block_report_execution and result.status == "failed":
raise ReportingExecutionError(
f"Quality plan {plan_record.name!r} blocked report execution."
)
return tuple(results)
def run_quality_plan(
session: Session,
principal: object,
*,
registry: object | None,
quality_plan_id: str,
quality_plan_revision: int | None,
parameters: Mapping[str, object],
) -> dict[str, object]:
_require_scope(principal, QUALITY_SCOPE)
record = get_definition(
session,
principal,
definition_kind="quality_plan",
definition_id=quality_plan_id,
revision=quality_plan_revision,
)
if record is None or record.status != "active":
raise LookupError("Active Reporting quality plan not found.")
plan = QualityPlanDefinition.model_validate(record.payload)
dataset_record = get_definition(
session,
principal,
definition_kind="dataset",
definition_id=plan.dataset_id,
revision=plan.dataset_revision,
)
if dataset_record is None or dataset_record.status != "active":
raise ReportingExecutionError("Quality plan dataset is unavailable.")
dataset = DatasetDefinition.model_validate(dataset_record.payload)
source = _read_dataset(
session,
principal,
registry=registry,
dataset=dataset,
parameters=parameters,
)
rows = tuple(_json_value(dict(item)) for item in source.rows)
result = _record_quality_result(
session,
principal,
plan_id=record.definition_id,
plan_revision=record.revision,
dataset_id=dataset_record.definition_id,
dataset_revision=dataset_record.revision,
plan=plan,
rows=rows,
output_hash=source.output_hash,
source_fingerprints=source.source_fingerprints,
)
return _quality_payload(result)
def _record_quality_result(
session: Session,
principal: object,
*,
plan_id: str,
plan_revision: int,
dataset_id: str,
dataset_revision: int,
plan: QualityPlanDefinition,
rows: Sequence[Mapping[str, object]],
output_hash: str,
source_fingerprints: Sequence[Mapping[str, object]],
) -> ReportingQualityResult:
assertions = [_evaluate_assertion(item, rows) for item in plan.assertions]
failed = any(
not item["passed"] and item["severity"] in {"error", "blocker"}
for item in assertions
)
warning = any(not item["passed"] for item in assertions)
row = ReportingQualityResult(
tenant_id=_tenant(principal),
result_id=str(uuid.uuid4()),
quality_plan_id=plan_id,
quality_plan_revision=plan_revision,
dataset_id=dataset_id,
dataset_revision=dataset_revision,
status="failed" if failed else "warning" if warning else "passed",
output_hash=output_hash,
assertions=assertions,
source_fingerprints=_json_value(source_fingerprints),
evaluated_at=utc_now(),
actor_id=_actor(principal),
)
session.add(row)
session.flush()
return row
def _evaluate_assertion(
assertion, rows: Sequence[Mapping[str, object]]
) -> dict[str, object]:
field = assertion.field
config = assertion.config
failures = 0
if assertion.kind == "not_null":
failures = sum(item.get(field or "") is None for item in rows)
elif assertion.kind == "unique":
values = [item.get(field or "") for item in rows]
failures = len(values) - len({_stable_value(item) for item in values})
elif assertion.kind == "range":
minimum = config.get("minimum")
maximum = config.get("maximum")
failures = sum(
value is not None
and (
(minimum is not None and value < minimum)
or (maximum is not None and value > maximum)
)
for value in (item.get(field or "") for item in rows)
)
elif assertion.kind == "accepted_values":
accepted = {_stable_value(item) for item in config.get("values", [])}
failures = sum(
_stable_value(item.get(field or "")) not in accepted for item in rows
)
elif assertion.kind == "row_count":
minimum = int(config.get("minimum", 0))
maximum = int(config.get("maximum", 2_000_000_000))
failures = 0 if minimum <= len(rows) <= maximum else 1
elif assertion.kind == "comparison":
other_field = str(config.get("other_field") or "")
operator = str(config.get("operator") or "eq")
if not field or not other_field or operator not in {"eq", "ne"}:
raise ReportingExecutionError("Comparison quality assertion is invalid.")
failures = sum(
(item.get(field) == item.get(other_field)) != (operator == "eq")
for item in rows
)
else:
raise ReportingExecutionError(
f"Unsupported quality assertion: {assertion.kind!r}."
)
return {
"key": assertion.key,
"kind": assertion.kind,
"severity": assertion.severity,
"passed": failures == 0,
"failure_count": failures,
"row_count": len(rows),
}
def _execution_payload(
row: ReportingExecution,
*,
report: ReportDefinition | None,
registry: object | None,
) -> dict[str, object]:
payload: dict[str, object] = {
"execution_id": row.execution_id,
"report_id": row.report_id,
"report_revision": row.report_revision,
"semantic_model_id": row.semantic_model_id,
"semantic_model_revision": row.semantic_model_revision,
"dataset_id": row.dataset_id,
"dataset_revision": row.dataset_revision,
"status": row.status,
"parameters": dict(row.parameters or {}),
"query": dict(row.query or {}),
"definition_hashes": dict(row.definition_hashes or {}),
"source_fingerprints": list(row.source_fingerprints or []),
"output_hash": row.output_hash,
"executor_version": row.executor_version,
"schema": list(row.result_schema or []),
"rows": list(row.result_rows or []),
"total_rows": row.total_rows,
"truncated": row.truncated,
"diagnostics": list(row.diagnostics or []),
"provenance": dict(row.provenance or {}),
"started_at": _datetime_text(row.started_at),
"finished_at": _datetime_text(row.finished_at),
"actor_id": row.actor_id,
}
if row.status == "succeeded" and report is not None:
result = QueryResult(
rows=tuple(dict(item) for item in row.result_rows or []),
total_rows=row.total_rows,
schema=tuple(dict(item) for item in row.result_schema or []),
truncated=row.truncated,
)
renderer = capability(registry, CAPABILITY_REPORTING_CHART_RENDERER)
if not isinstance(renderer, ReportingChartRenderer):
renderer = DefaultChartRenderer()
payload["visualization"] = dict(
renderer.render(visualization=report.visualization, result=result)
)
return payload
def _quality_payload(row: ReportingQualityResult) -> dict[str, object]:
return {
"result_id": row.result_id,
"quality_plan_id": row.quality_plan_id,
"quality_plan_revision": row.quality_plan_revision,
"dataset_id": row.dataset_id,
"dataset_revision": row.dataset_revision,
"status": row.status,
"output_hash": row.output_hash,
"assertions": list(row.assertions or []),
"source_fingerprints": list(row.source_fingerprints or []),
"evaluated_at": _datetime_text(row.evaluated_at),
}
def _execution_replay(
session: Session,
*,
tenant_id: str,
idempotency_key: str,
request_sha256: str,
) -> ReportingExecution | None:
row = (
session.query(ReportingExecution)
.filter(
ReportingExecution.tenant_id == tenant_id,
ReportingExecution.idempotency_key == idempotency_key,
)
.one_or_none()
)
if row is not None and row.request_sha256 != request_sha256:
raise ReportingExecutionError(
"Reporting execution idempotency conflict: the key belongs to another request."
)
return row
def _bind_parameters(
report: ReportDefinition,
supplied: Mapping[str, object],
) -> dict[str, object]:
definitions = {item.key: item for item in report.parameters}
unknown = set(supplied) - set(definitions)
if unknown:
raise ReportingExecutionError(
"Unknown report parameters: " + ", ".join(sorted(unknown))
)
result: dict[str, object] = {}
for key, definition in definitions.items():
value = supplied.get(key, definition.default)
if value is None and definition.required:
raise ReportingExecutionError(f"Report parameter {key!r} is required.")
if definition.allowed_values and value not in definition.allowed_values:
raise ReportingExecutionError(
f"Report parameter {key!r} is outside its allowed values."
)
if value is not None:
result[key] = value
return result
def _emit_execution_event(
session: Session,
execution: ReportingExecution,
report_name: str,
) -> None:
emit_platform_event(
session,
PlatformEvent(
type=f"reporting.execution.{execution.status}",
module_id="reporting",
payload={
"execution_id": execution.execution_id,
"report_id": execution.report_id,
"report_revision": execution.report_revision,
"output_hash": execution.output_hash,
"total_rows": execution.total_rows,
"truncated": execution.truncated,
},
occurred_at=execution.finished_at or execution.started_at,
actor=EventActorRef(type="account", id=execution.actor_id),
tenant=EventTenantRef(id=execution.tenant_id),
resource=EventObjectRef(
type="report_execution",
id=execution.execution_id,
label=report_name,
),
classification="internal",
),
)
def _matches_type(value: object, field_type: str) -> bool:
if field_type == "string":
return isinstance(value, str)
if field_type == "integer":
return isinstance(value, int) and not isinstance(value, bool)
if field_type == "number":
return isinstance(value, (int, float)) and not isinstance(value, bool)
if field_type == "boolean":
return isinstance(value, bool)
if field_type in {"date", "datetime"}:
if isinstance(value, (date, datetime)):
return True
if isinstance(value, str):
try:
datetime.fromisoformat(value.replace("Z", "+00:00"))
return True
except ValueError:
return False
return False
if field_type == "json":
return isinstance(value, (dict, list, tuple))
return False
def _stable_value(value: object) -> str:
return json.dumps(_json_value(value), sort_keys=True, separators=(",", ":"))
def _required(value: object, label: str, maximum: int) -> str:
result = str(value or "").strip()
if not result:
raise ReportingExecutionError(f"{label} is required.")
if len(result) > maximum:
raise ReportingExecutionError(f"{label} is limited to {maximum} characters.")
return result
def _require_scope(principal: object, scope: str) -> None:
method = getattr(principal, "has", None)
allowed = (
bool(method(scope))
if callable(method)
else scopes_grant_compatible(
frozenset(getattr(principal, "scopes", ()) or ()), scope
)
)
if not allowed:
raise PermissionError(f"Missing scope: {scope}")
def _tenant(principal: object) -> str:
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
if not tenant_id:
raise ReportingExecutionError(
"Reporting execution requires a tenant-bound principal."
)
return tenant_id
def _actor(principal: object) -> str | None:
for value in (
getattr(principal, "account_id", None),
getattr(principal, "identity_id", None),
getattr(principal, "membership_id", None),
):
if str(value or "").strip():
return str(value)
return None
def _session(value: object) -> Session:
if not isinstance(value, Session):
raise TypeError("Reporting execution requires a SQLAlchemy session.")
return value
def _sha256(value: object) -> str:
return hashlib.sha256(
json.dumps(
_json_value(value),
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
).encode("utf-8")
).hexdigest()
def _json_value(value: object) -> Any:
if isinstance(value, (date, datetime)):
return value.isoformat()
if hasattr(value, "model_dump"):
return _json_value(value.model_dump(mode="json"))
if isinstance(value, Mapping):
return {str(key): _json_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_value(item) for item in value]
return value
def _datetime_text(value: datetime | None) -> str | None:
if value is None:
return None
if value.tzinfo is None:
value = value.replace(tzinfo=UTC)
return value.isoformat()
__all__ = [
"QUALITY_SCOPE",
"RUN_SCOPE",
"ReportingExecutionError",
"ReportingExecutionFailure",
"SqlReportingRunner",
"execute_report",
"get_execution",
"list_executions",
"run_quality_plan",
]
+478
View File
@@ -0,0 +1,478 @@
from __future__ import annotations
from pathlib import Path
from govoplan_core.core.access import (
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
)
from govoplan_core.core.dataflows import CAPABILITY_DATAFLOW_DATASET_OUTPUT
from govoplan_core.core.module_guards import (
drop_table_retirement_provider,
persistent_table_uninstall_guard,
)
from govoplan_core.core.modules import (
CapabilityDocumentation,
DocumentationLink,
DocumentationTopic,
FrontendModule,
FrontendRoute,
MigrationSpec,
ModuleContext,
ModuleInterfaceProvider,
ModuleInterfaceRequirement,
ModuleManifest,
NavItem,
PermissionDefinition,
RoleTemplate,
)
from govoplan_core.core.provider_governance import (
ModuleArchitectureDeclaration,
ModuleArchitectureDocumentation,
ModuleMaturityEvidence,
)
from govoplan_core.core.search import SearchSourceProviderRegistration
from govoplan_core.core.views import ViewSurface
from govoplan_core.db.base import Base
from govoplan_reporting.backend.acl import ReportingScopeAclProvider
from govoplan_reporting.backend.contracts import (
CAPABILITY_REPORTING_CHART_RENDERER,
CAPABILITY_REPORTING_REGISTRY,
CAPABILITY_REPORTING_RUNNER,
CAPABILITY_REPORTING_SCHEDULER,
)
from govoplan_reporting.backend.db import models as reporting_models
from govoplan_reporting.backend.definitions import (
ADMIN_SCOPE,
READ_SCOPE,
WRITE_SCOPE,
)
from govoplan_reporting.backend.execution import (
QUALITY_SCOPE,
RUN_SCOPE,
SqlReportingRunner,
)
from govoplan_reporting.backend.operations import (
IMPORT_SCOPE,
PUBLISH_SCOPE,
SCHEDULE_SCOPE,
SqlReportingScheduler,
)
from govoplan_reporting.backend.query_engine import DefaultChartRenderer
from govoplan_reporting.backend.registry import SqlReportingRegistry
from govoplan_reporting.backend.search_source import create_reporting_search_source
MODULE_ID = "reporting"
MODULE_NAME = "Reporting"
MODULE_VERSION = "0.1.14"
def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
module_id, resource, action = scope.split(":", 2)
return PermissionDefinition(
scope=scope,
label=label,
description=description,
category=MODULE_NAME,
level="tenant",
module_id=module_id,
resource=resource,
action=action,
)
PERMISSIONS = (
_permission(
READ_SCOPE,
"View reporting definitions",
"Read accessible datasets, semantic models, reports, and quality plans.",
),
_permission(
WRITE_SCOPE,
"Manage reporting definitions",
"Create immutable revisions of Reporting definitions.",
),
_permission(
ADMIN_SCOPE,
"Administer reporting",
"Manage restricted definitions and Reporting governance.",
),
_permission(
RUN_SCOPE,
"Run reports",
"Execute accessible report revisions and export their authorized result.",
),
_permission(
PUBLISH_SCOPE,
"Publish reports",
"Send successful report results to configured publication providers.",
),
_permission(
SCHEDULE_SCOPE,
"Schedule reports",
"Create schedules and dispatch due report runs.",
),
_permission(
QUALITY_SCOPE,
"Run report quality plans",
"Evaluate dataset quality plans and inspect evidence.",
),
_permission(
IMPORT_SCOPE,
"Assess report imports",
"Assess external BI metadata and accept bounded approximations.",
),
)
ROLE_TEMPLATES = (
RoleTemplate(
slug="reporting_analyst",
name="Reporting analyst",
description="Define semantic reports, run them, and save analytical views.",
permissions=(READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, QUALITY_SCOPE),
),
RoleTemplate(
slug="reporting_publisher",
name="Reporting publisher",
description="Run, schedule, export, and publish accessible reports.",
permissions=(READ_SCOPE, RUN_SCOPE, PUBLISH_SCOPE, SCHEDULE_SCOPE),
),
RoleTemplate(
slug="reporting_administrator",
name="Reporting administrator",
description="Administer definitions, imports, quality, schedules, and publications.",
permissions=tuple(item.scope for item in PERMISSIONS),
),
)
def _router(context: ModuleContext):
from govoplan_reporting.backend.router import create_router
return create_router(context.registry)
def _registry(context: ModuleContext) -> SqlReportingRegistry:
del context
return SqlReportingRegistry()
def _runner(context: ModuleContext) -> SqlReportingRunner:
return SqlReportingRunner(context.registry)
def _scheduler(context: ModuleContext) -> SqlReportingScheduler:
return SqlReportingScheduler(context.registry)
def _chart_renderer(context: ModuleContext) -> DefaultChartRenderer:
del context
return DefaultChartRenderer()
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
definitions = (
session.query(reporting_models.ReportingDefinitionRevision)
.filter(
reporting_models.ReportingDefinitionRevision.tenant_id == tenant_id,
reporting_models.ReportingDefinitionRevision.superseded_at.is_(None),
)
.count()
)
reports = (
session.query(reporting_models.ReportingDefinitionRevision)
.filter(
reporting_models.ReportingDefinitionRevision.tenant_id == tenant_id,
reporting_models.ReportingDefinitionRevision.definition_kind == "report",
reporting_models.ReportingDefinitionRevision.superseded_at.is_(None),
)
.count()
)
executions = (
session.query(reporting_models.ReportingExecution)
.filter(reporting_models.ReportingExecution.tenant_id == tenant_id)
.count()
)
schedules = (
session.query(reporting_models.ReportingSchedule)
.filter(
reporting_models.ReportingSchedule.tenant_id == tenant_id,
reporting_models.ReportingSchedule.enabled.is_(True),
)
.count()
)
return {
"reporting_definitions": definitions,
"reports": reports,
"report_executions": executions,
"active_report_schedules": schedules,
}
manifest = ModuleManifest(
id=MODULE_ID,
name=MODULE_NAME,
version=MODULE_VERSION,
dependencies=("access",),
optional_dependencies=(
"dataflow",
"datasources",
"connectors",
"dashboard",
"files",
"mail",
"templates",
"workflow_engine",
"policy",
"search",
"notifications",
),
required_capabilities=(
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
),
optional_capabilities=(CAPABILITY_DATAFLOW_DATASET_OUTPUT,),
permissions=PERMISSIONS,
role_templates=ROLE_TEMPLATES,
route_factory=_router,
nav_items=(
NavItem(
path="/reporting",
label="Reporting",
icon="clipboard-pen-line",
required_any=(READ_SCOPE,),
order=74,
),
),
frontend=FrontendModule(
module_id=MODULE_ID,
package_name="@govoplan/reporting-webui",
routes=(
FrontendRoute(
path="/reporting",
component="ReportingPage",
required_any=(READ_SCOPE,),
order=74,
),
),
nav_items=(
NavItem(
path="/reporting",
label="Reporting",
icon="clipboard-pen-line",
required_any=(READ_SCOPE,),
order=74,
),
),
view_surfaces=(
ViewSurface(
id="reporting.navigation",
module_id=MODULE_ID,
kind="navigation",
label="Reporting navigation",
order=10,
),
ViewSurface(
id="reporting.workspace",
module_id=MODULE_ID,
kind="route",
label="Reporting workspace",
order=20,
),
ViewSurface(
id="reporting.parameters",
module_id=MODULE_ID,
kind="section",
label="Report parameters and filters",
parent_id="reporting.workspace",
order=30,
),
ViewSurface(
id="reporting.results",
module_id=MODULE_ID,
kind="section",
label="Authorized report results",
parent_id="reporting.workspace",
order=40,
),
),
),
provides_interfaces=(
ModuleInterfaceProvider(name="reporting.registry", version="0.1.0"),
ModuleInterfaceProvider(name="reporting.runner", version="0.1.0"),
ModuleInterfaceProvider(name="reporting.scheduler", version="0.1.0"),
ModuleInterfaceProvider(name="reporting.chart_renderer", version="0.1.0"),
),
requires_interfaces=(
ModuleInterfaceRequirement(
name="dataflow.dataset_output",
version_min="0.1.0",
version_max_exclusive="0.2.0",
optional=True,
),
),
capability_factories={
CAPABILITY_REPORTING_REGISTRY: _registry,
CAPABILITY_REPORTING_RUNNER: _runner,
CAPABILITY_REPORTING_SCHEDULER: _scheduler,
CAPABILITY_REPORTING_CHART_RENDERER: _chart_renderer,
},
capability_documentation={
CAPABILITY_REPORTING_REGISTRY: CapabilityDocumentation(
label="Reporting definition registry",
summary="Stores versioned datasets, semantic models, reports, and quality plans.",
contract_version="0.1.0",
),
CAPABILITY_REPORTING_RUNNER: CapabilityDocumentation(
label="Governed report runner",
summary="Executes a pinned report graph over an authorized provider-owned dataset.",
contract_version="0.1.0",
),
CAPABILITY_REPORTING_SCHEDULER: CapabilityDocumentation(
label="Report schedule dispatcher",
summary="Claims due report schedules and records run/publication evidence.",
contract_version="0.1.0",
),
CAPABILITY_REPORTING_CHART_RENDERER: CapabilityDocumentation(
label="Report chart renderer",
summary="Builds provider-neutral chart models with an accessible tabular fallback.",
contract_version="0.1.0",
),
},
search_sources=(
SearchSourceProviderRegistration(
id="reporting.reports",
factory=create_reporting_search_source,
),
),
migration_spec=MigrationSpec(
module_id=MODULE_ID,
metadata=Base.metadata,
script_location=str(Path(__file__).with_name("migrations") / "versions"),
retirement_supported=True,
retirement_provider=drop_table_retirement_provider(
reporting_models.ReportingImportAssessment,
reporting_models.ReportingQualityResult,
reporting_models.ReportingPublication,
reporting_models.ReportingSchedule,
reporting_models.ReportingSavedView,
reporting_models.ReportingDefinitionGrant,
reporting_models.ReportingExecution,
reporting_models.ReportingDefinitionRevision,
reporting_models.ReportingDefinitionIdentity,
label="Reporting",
),
retirement_notes=(
"Destructive retirement requires a database snapshot and removes "
"Reporting definitions, results, quality evidence, schedules, and publications."
),
),
uninstall_guard_providers=(
persistent_table_uninstall_guard(
reporting_models.ReportingDefinitionIdentity,
reporting_models.ReportingDefinitionRevision,
reporting_models.ReportingDefinitionGrant,
reporting_models.ReportingExecution,
reporting_models.ReportingSavedView,
reporting_models.ReportingSchedule,
reporting_models.ReportingPublication,
reporting_models.ReportingQualityResult,
reporting_models.ReportingImportAssessment,
label="Reporting",
),
),
resource_acl_providers=(
ReportingScopeAclProvider("analytical_dataset"),
ReportingScopeAclProvider("semantic_model"),
ReportingScopeAclProvider("report"),
ReportingScopeAclProvider("report_execution"),
),
tenant_summary_providers=(_tenant_summary,),
documentation=(
DocumentationTopic(
id="reporting.governed-bi",
title="Governed reporting and semantic BI",
summary="Build reproducible reports over provider-owned datasets without bypassing module or row-level access.",
body=(
"Reporting pins dataset, semantic-model, and report revisions. Runs retain "
"definition hashes, source fingerprints, policy provenance, quality evidence, "
"authorized result rows, diagnostics, and output hashes. Safe dimensions, "
"aggregations, typed expressions, filters, pivots, saved views, chart models, "
"schedules, exports, and publication providers replace unchecked SQL in the "
"presentation layer. Dataflow and module read models remain source owners."
),
layer="available",
documentation_types=("admin", "user"),
audience=("user", "operator", "module_admin", "product_owner"),
links=(
DocumentationLink(
label="Reporting module boundary",
href="govoplan-reporting/docs/REPORTING_BOUNDARY.md",
kind="repository",
),
DocumentationLink(
label="SuperX capability assessment",
href="govoplan-reporting/docs/SUPERX_CAPABILITY_ASSESSMENT.md",
kind="repository",
),
DocumentationLink(
label="Reporting user guide",
href="govoplan-reporting/docs/USER_GUIDE.md",
kind="repository",
),
DocumentationLink(
label="Reporting administration guide",
href="govoplan-reporting/docs/ADMIN_GUIDE.md",
kind="repository",
),
),
),
),
architecture=ModuleArchitectureDeclaration(
layer="data_reporting_integration",
kind="domain",
maturity="vertical_slice",
evidence=(
ModuleMaturityEvidence(
kind="test",
reference="tests/test_reporting_service.py",
summary="Proves revision pinning, safe semantic execution, quality gates, access, replay, exports, and import blocking.",
),
ModuleMaturityEvidence(
kind="documentation",
reference="docs/REPORTING_BOUNDARY.md",
summary="Defines governed analytical source, semantic, execution, and publication ownership.",
),
),
known_limits=(
"Dataflow is the first live dataset adapter; additional module read models use the provider-neutral contract.",
"Direct browser export supports CSV and JSON; XLSX, PDF, Files, Mail, and DMS delivery require an optional publication provider.",
"Import assessment produces blocking diagnostics but does not execute source SQL or automatically activate generated definitions.",
"The initial chart provider emits a renderer-neutral model and accessible table; richer visual renderers remain replaceable adapters.",
),
owned_concepts=(
"analytical dataset binding",
"semantic dimension hierarchy and measure",
"report definition and saved view",
"report execution and publication evidence",
"report quality plan and import assessment",
),
non_owned_concepts=(
"raw datasource ingestion",
"data transformation pipeline",
"source module authorization",
"template document rendering",
"file or DMS storage",
),
documentation=ModuleArchitectureDocumentation(
operations=("docs/OPERATIONS.md",),
recovery=("docs/OPERATIONS.md",),
security=("docs/OPERATIONS.md",),
),
),
)
def get_manifest() -> ModuleManifest:
return manifest
@@ -0,0 +1 @@
"""Reporting Alembic revisions."""
@@ -0,0 +1 @@
"""Reporting migration versions."""
@@ -0,0 +1,477 @@
"""v0.1.14 governed Reporting baseline.
Revision ID: e5b2c9d4f7a1
Revises: None
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "e5b2c9d4f7a1"
down_revision = None
branch_labels = None
depends_on = "4f2a9c8e7b6d"
def upgrade() -> None:
op.create_table(
"reporting_definition_identities",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("definition_kind", sa.String(length=40), nullable=False),
sa.Column("definition_id", sa.String(length=255), nullable=False),
sa.Column("definition_key", sa.String(length=120), nullable=False),
sa.Column("created_by", sa.String(length=255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_definition_identities")),
sa.UniqueConstraint(
"tenant_id",
"definition_kind",
"definition_id",
name="uq_reporting_definition_identity",
),
sa.UniqueConstraint(
"tenant_id",
"definition_kind",
"definition_key",
name="uq_reporting_definition_key",
),
)
_indexes(
"reporting_definition_identities",
"tenant_id",
"definition_kind",
"definition_id",
"definition_key",
"created_by",
)
op.create_index(
"ix_reporting_definition_catalog",
"reporting_definition_identities",
["tenant_id", "definition_kind", "definition_key"],
unique=False,
)
op.create_table(
"reporting_definition_revisions",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("identity_id", sa.String(length=36), nullable=False),
sa.Column("definition_kind", sa.String(length=40), nullable=False),
sa.Column("definition_id", sa.String(length=255), nullable=False),
sa.Column("definition_key", sa.String(length=120), nullable=False),
sa.Column("revision", sa.Integer(), nullable=False),
sa.Column("previous_revision_id", sa.String(length=36), nullable=True),
sa.Column("parent_kind", sa.String(length=40), nullable=True),
sa.Column("parent_id", sa.String(length=255), nullable=True),
sa.Column("parent_revision", sa.Integer(), nullable=True),
sa.Column("name", sa.String(length=500), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("status", sa.String(length=30), nullable=False),
sa.Column("visibility", sa.String(length=30), nullable=False),
sa.Column("content_hash", sa.String(length=64), nullable=False),
sa.Column("change_reason", sa.String(length=1000), nullable=False),
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
sa.Column("request_sha256", sa.String(length=64), nullable=False),
sa.Column("event_id", sa.String(length=36), nullable=False),
sa.Column("recorded_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("superseded_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("payload", sa.JSON(), nullable=False),
sa.Column("changed_by", sa.String(length=255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["identity_id"],
["reporting_definition_identities.id"],
name=op.f(
"fk_reporting_definition_revisions_identity_id_reporting_definition_identities"
),
ondelete="RESTRICT",
),
sa.ForeignKeyConstraint(
["previous_revision_id"],
["reporting_definition_revisions.id"],
name=op.f(
"fk_reporting_definition_revisions_previous_revision_id_reporting_definition_revisions"
),
ondelete="RESTRICT",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_definition_revisions")),
sa.UniqueConstraint(
"tenant_id",
"definition_kind",
"definition_id",
"revision",
name="uq_reporting_definition_revision",
),
sa.UniqueConstraint(
"tenant_id",
"idempotency_key",
name="uq_reporting_definition_idempotency",
),
sa.UniqueConstraint(
"tenant_id",
"event_id",
name="uq_reporting_definition_event",
),
)
_indexes(
"reporting_definition_revisions",
"tenant_id",
"identity_id",
"definition_kind",
"definition_id",
"definition_key",
"previous_revision_id",
"parent_kind",
"parent_id",
"status",
"visibility",
"content_hash",
"event_id",
"recorded_at",
"superseded_at",
"changed_by",
)
op.create_index(
"ix_reporting_definition_current",
"reporting_definition_revisions",
["tenant_id", "definition_kind", "definition_id", "superseded_at"],
unique=False,
)
op.create_index(
"ix_reporting_definition_parent",
"reporting_definition_revisions",
["tenant_id", "parent_kind", "parent_id", "parent_revision"],
unique=False,
)
op.create_index(
"ix_reporting_definition_list",
"reporting_definition_revisions",
["tenant_id", "definition_kind", "status", "recorded_at"],
unique=False,
)
op.create_table(
"reporting_definition_grants",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("definition_kind", sa.String(length=40), nullable=False),
sa.Column("definition_id", sa.String(length=255), nullable=False),
sa.Column("subject_kind", sa.String(length=40), nullable=False),
sa.Column("subject_id", sa.String(length=255), nullable=False),
sa.Column("permissions", sa.JSON(), nullable=False),
sa.Column("active", sa.Boolean(), nullable=False),
sa.Column("source_revision", sa.Integer(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_definition_grants")),
sa.UniqueConstraint(
"tenant_id",
"definition_kind",
"definition_id",
"subject_kind",
"subject_id",
name="uq_reporting_definition_grant",
),
)
_indexes(
"reporting_definition_grants",
"tenant_id",
"definition_kind",
"definition_id",
"subject_kind",
"subject_id",
"active",
)
op.create_index(
"ix_reporting_definition_grant_subject",
"reporting_definition_grants",
["tenant_id", "subject_kind", "subject_id", "active"],
unique=False,
)
op.create_index(
"ix_reporting_definition_grant_object",
"reporting_definition_grants",
["tenant_id", "definition_kind", "definition_id", "active"],
unique=False,
)
op.create_table(
"reporting_executions",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("execution_id", sa.String(length=36), nullable=False),
sa.Column("report_id", sa.String(length=255), nullable=False),
sa.Column("report_revision", sa.Integer(), nullable=False),
sa.Column("semantic_model_id", sa.String(length=255), nullable=False),
sa.Column("semantic_model_revision", sa.Integer(), nullable=False),
sa.Column("dataset_id", sa.String(length=255), nullable=False),
sa.Column("dataset_revision", sa.Integer(), nullable=False),
sa.Column("status", sa.String(length=30), nullable=False),
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
sa.Column("request_sha256", sa.String(length=64), nullable=False),
sa.Column("parameters", sa.JSON(), nullable=False),
sa.Column("query", sa.JSON(), nullable=False),
sa.Column("source_fingerprints", sa.JSON(), nullable=False),
sa.Column("definition_hashes", sa.JSON(), nullable=False),
sa.Column("output_hash", sa.String(length=64), nullable=True),
sa.Column("executor_version", sa.String(length=255), nullable=True),
sa.Column("result_schema", sa.JSON(), nullable=False),
sa.Column("result_rows", sa.JSON(), nullable=False),
sa.Column("total_rows", sa.Integer(), nullable=False),
sa.Column("truncated", sa.Boolean(), nullable=False),
sa.Column("diagnostics", sa.JSON(), nullable=False),
sa.Column("provenance", sa.JSON(), nullable=False),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("actor_id", sa.String(length=255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_executions")),
sa.UniqueConstraint("tenant_id", "execution_id", name="uq_reporting_execution"),
sa.UniqueConstraint(
"tenant_id",
"idempotency_key",
name="uq_reporting_execution_idempotency",
),
)
_indexes(
"reporting_executions",
"tenant_id",
"execution_id",
"report_id",
"semantic_model_id",
"dataset_id",
"status",
"output_hash",
"started_at",
"finished_at",
"actor_id",
)
op.create_index(
"ix_reporting_execution_history",
"reporting_executions",
["tenant_id", "report_id", "started_at"],
unique=False,
)
op.create_index(
"ix_reporting_execution_status",
"reporting_executions",
["tenant_id", "status", "started_at"],
unique=False,
)
op.create_table(
"reporting_saved_views",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("view_id", sa.String(length=36), nullable=False),
sa.Column("report_id", sa.String(length=255), nullable=False),
sa.Column("report_revision", sa.Integer(), nullable=False),
sa.Column("owner_kind", sa.String(length=40), nullable=False),
sa.Column("owner_id", sa.String(length=255), nullable=False),
sa.Column("name", sa.String(length=500), nullable=False),
sa.Column("revision", sa.Integer(), nullable=False),
sa.Column("state", sa.JSON(), nullable=False),
sa.Column("shared", sa.Boolean(), nullable=False),
sa.Column("access", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_saved_views")),
sa.UniqueConstraint("tenant_id", "view_id", name="uq_reporting_saved_view"),
)
_indexes(
"reporting_saved_views",
"tenant_id",
"view_id",
"report_id",
"owner_kind",
"owner_id",
"shared",
)
op.create_index(
"ix_reporting_saved_view_catalog",
"reporting_saved_views",
["tenant_id", "report_id", "owner_id", "shared"],
unique=False,
)
op.create_table(
"reporting_schedules",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("schedule_id", sa.String(length=36), nullable=False),
sa.Column("report_id", sa.String(length=255), nullable=False),
sa.Column("report_revision", sa.Integer(), nullable=False),
sa.Column("name", sa.String(length=500), nullable=False),
sa.Column("revision", sa.Integer(), nullable=False),
sa.Column("trigger_kind", sa.String(length=30), nullable=False),
sa.Column("trigger_config", sa.JSON(), nullable=False),
sa.Column("parameters", sa.JSON(), nullable=False),
sa.Column("query", sa.JSON(), nullable=False),
sa.Column("publication_target", sa.JSON(), nullable=False),
sa.Column("enabled", sa.Boolean(), nullable=False),
sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_run_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_execution_id", sa.String(length=36), nullable=True),
sa.Column("created_by", sa.String(length=255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_schedules")),
sa.UniqueConstraint("tenant_id", "schedule_id", name="uq_reporting_schedule"),
)
_indexes(
"reporting_schedules",
"tenant_id",
"schedule_id",
"report_id",
"enabled",
"next_run_at",
)
op.create_index(
"ix_reporting_schedule_due",
"reporting_schedules",
["enabled", "next_run_at", "tenant_id"],
unique=False,
)
op.create_table(
"reporting_publications",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("publication_id", sa.String(length=36), nullable=False),
sa.Column("execution_id", sa.String(length=36), nullable=False),
sa.Column("target_capability", sa.String(length=255), nullable=False),
sa.Column("target_ref", sa.String(length=1000), nullable=True),
sa.Column("format", sa.String(length=30), nullable=False),
sa.Column("status", sa.String(length=30), nullable=False),
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
sa.Column("evidence", sa.JSON(), nullable=False),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_publications")),
sa.UniqueConstraint(
"tenant_id", "publication_id", name="uq_reporting_publication"
),
sa.UniqueConstraint(
"tenant_id",
"idempotency_key",
name="uq_reporting_publication_idempotency",
),
)
_indexes(
"reporting_publications",
"tenant_id",
"publication_id",
"execution_id",
"status",
)
op.create_index(
"ix_reporting_publication_history",
"reporting_publications",
["tenant_id", "execution_id", "created_at"],
unique=False,
)
op.create_table(
"reporting_quality_results",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("result_id", sa.String(length=36), nullable=False),
sa.Column("quality_plan_id", sa.String(length=255), nullable=False),
sa.Column("quality_plan_revision", sa.Integer(), nullable=False),
sa.Column("dataset_id", sa.String(length=255), nullable=False),
sa.Column("dataset_revision", sa.Integer(), nullable=False),
sa.Column("status", sa.String(length=30), nullable=False),
sa.Column("output_hash", sa.String(length=64), nullable=False),
sa.Column("assertions", sa.JSON(), nullable=False),
sa.Column("source_fingerprints", sa.JSON(), nullable=False),
sa.Column("evaluated_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("actor_id", sa.String(length=255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_quality_results")),
sa.UniqueConstraint(
"tenant_id", "result_id", name="uq_reporting_quality_result"
),
)
_indexes(
"reporting_quality_results",
"tenant_id",
"result_id",
"quality_plan_id",
"dataset_id",
"status",
"output_hash",
"evaluated_at",
)
op.create_index(
"ix_reporting_quality_history",
"reporting_quality_results",
["tenant_id", "quality_plan_id", "evaluated_at"],
unique=False,
)
op.create_table(
"reporting_import_assessments",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("assessment_id", sa.String(length=36), nullable=False),
sa.Column("source_system", sa.String(length=255), nullable=False),
sa.Column("source_id", sa.String(length=500), nullable=False),
sa.Column("source_fingerprint", sa.String(length=64), nullable=False),
sa.Column("mapping_report", sa.JSON(), nullable=False),
sa.Column("status", sa.String(length=30), nullable=False),
sa.Column("accepted_approximations", sa.JSON(), nullable=False),
sa.Column("assessed_by", sa.String(length=255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_import_assessments")),
sa.UniqueConstraint(
"tenant_id",
"assessment_id",
name="uq_reporting_import_assessment",
),
)
_indexes(
"reporting_import_assessments",
"tenant_id",
"assessment_id",
"source_system",
"status",
)
op.create_index(
"ix_reporting_import_history",
"reporting_import_assessments",
["tenant_id", "source_system", "created_at"],
unique=False,
)
def downgrade() -> None:
op.drop_table("reporting_import_assessments")
op.drop_table("reporting_quality_results")
op.drop_table("reporting_publications")
op.drop_table("reporting_schedules")
op.drop_table("reporting_saved_views")
op.drop_table("reporting_definition_grants")
op.drop_table("reporting_executions")
op.drop_table("reporting_definition_revisions")
op.drop_table("reporting_definition_identities")
def _indexes(table: str, *columns: str) -> None:
for column in columns:
op.create_index(
op.f(f"ix_{table}_{column}"),
table,
[column],
unique=False,
)
@@ -0,0 +1,858 @@
from __future__ import annotations
from collections.abc import Mapping
import csv
from datetime import UTC, datetime, timedelta
from io import StringIO
import hashlib
import json
from typing import Any
import uuid
from sqlalchemy import or_
from sqlalchemy.orm import Session
from govoplan_core.core.events import (
EventActorRef,
EventObjectRef,
EventTenantRef,
PlatformEvent,
emit_platform_event,
)
from govoplan_core.security.module_permissions import scopes_grant_compatible
from govoplan_core.security.time import utc_now
from govoplan_reporting.backend.contracts import (
ReportingPublicationPayload,
ReportingPublicationTarget,
capability,
)
from govoplan_reporting.backend.db.models import (
ReportingImportAssessment,
ReportingPublication,
ReportingSavedView,
ReportingSchedule,
)
from govoplan_reporting.backend.definitions import ADMIN_SCOPE, get_definition
from govoplan_reporting.backend.execution import (
ReportingExecutionFailure,
execute_report,
get_execution,
)
from govoplan_reporting.backend.schemas import ReportQuery
PUBLISH_SCOPE = "reporting:report:publish"
SCHEDULE_SCOPE = "reporting:schedule:write"
IMPORT_SCOPE = "reporting:import:assess"
class ReportingOperationError(ValueError):
pass
class SqlReportingScheduler:
def __init__(self, registry: object | None) -> None:
self.registry = registry
def dispatch_due(
self,
session: object,
principal: object,
*,
now: datetime | None = None,
limit: int = 20,
) -> Mapping[str, object]:
return dispatch_due_schedules(
_session(session),
principal,
registry=self.registry,
now=now,
limit=limit,
)
def upsert_saved_view(
session: Session,
principal: object,
*,
view_id: str,
report_id: str,
report_revision: int,
name: str,
state: Mapping[str, object],
shared: bool,
access: Mapping[str, object],
expected_revision: int | None,
) -> dict[str, object]:
report = get_definition(
session,
principal,
definition_kind="report",
definition_id=report_id,
revision=report_revision,
)
if report is None:
raise LookupError("Reporting report definition not found.")
_validate_saved_view_state(state)
owner_id = _actor(principal)
if owner_id is None:
raise ReportingOperationError("Saved views require an account owner.")
row = (
session.query(ReportingSavedView)
.filter(
ReportingSavedView.tenant_id == _tenant(principal),
ReportingSavedView.view_id == view_id,
)
.with_for_update()
.one_or_none()
)
if row is None:
if expected_revision is not None:
raise ReportingOperationError(
"Saved-view revision conflict: no view exists."
)
row = ReportingSavedView(
tenant_id=_tenant(principal),
view_id=_required(view_id, "Saved-view identifier", 36),
report_id=report_id,
report_revision=report_revision,
owner_kind="account",
owner_id=owner_id,
name=_required(name, "Saved-view name", 500),
revision=1,
state=_json_value(state),
shared=shared,
access=_json_value(access),
)
session.add(row)
else:
if row.owner_id != owner_id and not _has_scope(principal, ADMIN_SCOPE):
raise PermissionError("Only the saved-view owner or an admin may edit it.")
if expected_revision != row.revision:
raise ReportingOperationError(
"Saved-view revision conflict: the expected revision is stale."
)
row.report_id = report_id
row.report_revision = report_revision
row.name = _required(name, "Saved-view name", 500)
row.state = _json_value(state)
row.shared = shared
row.access = _json_value(access)
row.revision += 1
session.flush()
return _saved_view_payload(row)
def list_saved_views(
session: Session,
principal: object,
*,
report_id: str,
) -> tuple[dict[str, object], ...]:
if (
get_definition(
session,
principal,
definition_kind="report",
definition_id=report_id,
)
is None
):
return ()
owner_id = _actor(principal)
rows = (
session.query(ReportingSavedView)
.filter(
ReportingSavedView.tenant_id == _tenant(principal),
ReportingSavedView.report_id == report_id,
or_(
ReportingSavedView.shared.is_(True),
ReportingSavedView.owner_id == owner_id,
),
)
.order_by(ReportingSavedView.name.asc())
.all()
)
return tuple(_saved_view_payload(row) for row in rows)
def delete_saved_view(
session: Session,
principal: object,
*,
view_id: str,
) -> bool:
row = (
session.query(ReportingSavedView)
.filter(
ReportingSavedView.tenant_id == _tenant(principal),
ReportingSavedView.view_id == view_id,
)
.with_for_update()
.one_or_none()
)
if row is None:
return False
if row.owner_id != _actor(principal) and not _has_scope(principal, ADMIN_SCOPE):
raise PermissionError("Only the saved-view owner or an admin may delete it.")
session.delete(row)
session.flush()
return True
def upsert_schedule(
session: Session,
principal: object,
*,
schedule_id: str,
report_id: str,
report_revision: int,
name: str,
trigger_kind: str,
trigger_config: Mapping[str, object],
parameters: Mapping[str, object],
query: ReportQuery,
publication_target: Mapping[str, object],
enabled: bool,
next_run_at: datetime | None,
expected_revision: int | None,
) -> dict[str, object]:
_require_scope(principal, SCHEDULE_SCOPE)
report = get_definition(
session,
principal,
definition_kind="report",
definition_id=report_id,
revision=report_revision,
)
if report is None or report.status != "active":
raise ReportingOperationError("Schedules require an active report revision.")
normalized_next = _validate_trigger(
trigger_kind,
trigger_config,
next_run_at=next_run_at,
)
row = (
session.query(ReportingSchedule)
.filter(
ReportingSchedule.tenant_id == _tenant(principal),
ReportingSchedule.schedule_id == schedule_id,
)
.with_for_update()
.one_or_none()
)
if row is None:
if expected_revision is not None:
raise ReportingOperationError(
"Reporting schedule revision conflict: no schedule exists."
)
row = ReportingSchedule(
tenant_id=_tenant(principal),
schedule_id=_required(schedule_id, "Reporting schedule identifier", 36),
report_id=report_id,
report_revision=report_revision,
name=_required(name, "Reporting schedule name", 500),
revision=1,
trigger_kind=trigger_kind,
trigger_config=_json_value(trigger_config),
parameters=_json_value(parameters),
query=query.model_dump(mode="json"),
publication_target=_json_value(publication_target),
enabled=enabled,
next_run_at=normalized_next,
created_by=_actor(principal),
)
session.add(row)
else:
if expected_revision != row.revision:
raise ReportingOperationError(
"Reporting schedule revision conflict: the expected revision is stale."
)
row.report_id = report_id
row.report_revision = report_revision
row.name = _required(name, "Reporting schedule name", 500)
row.trigger_kind = trigger_kind
row.trigger_config = _json_value(trigger_config)
row.parameters = _json_value(parameters)
row.query = query.model_dump(mode="json")
row.publication_target = _json_value(publication_target)
row.enabled = enabled
row.next_run_at = normalized_next
row.revision += 1
session.flush()
return _schedule_payload(row)
def list_schedules(
session: Session,
principal: object,
*,
report_id: str | None = None,
) -> tuple[dict[str, object], ...]:
statement = session.query(ReportingSchedule).filter(
ReportingSchedule.tenant_id == _tenant(principal)
)
if report_id:
if (
get_definition(
session,
principal,
definition_kind="report",
definition_id=report_id,
)
is None
):
return ()
statement = statement.filter(ReportingSchedule.report_id == report_id)
return tuple(
_schedule_payload(row)
for row in statement.order_by(ReportingSchedule.name.asc()).all()
)
def dispatch_due_schedules(
session: Session,
principal: object,
*,
registry: object | None,
now: datetime | None,
limit: int,
) -> dict[str, object]:
_require_scope(principal, SCHEDULE_SCOPE)
current = now or utc_now()
_aware(current, "Reporting scheduler time")
rows = (
session.query(ReportingSchedule)
.filter(
ReportingSchedule.enabled.is_(True),
ReportingSchedule.next_run_at.is_not(None),
ReportingSchedule.next_run_at <= current,
)
.order_by(ReportingSchedule.next_run_at.asc())
.limit(max(1, min(limit, 100)))
.with_for_update(skip_locked=True)
.all()
)
succeeded = 0
failed = 0
execution_ids: list[str] = []
for row in rows:
scheduled_for = row.next_run_at or current
try:
execution = execute_report(
session,
principal,
registry=registry,
report_id=row.report_id,
report_revision=row.report_revision,
parameters=dict(row.parameters or {}),
query=ReportQuery.model_validate(row.query or {}),
idempotency_key=f"schedule:{row.schedule_id}:{scheduled_for.isoformat()}",
)
execution_id = str(execution["execution_id"])
execution_ids.append(execution_id)
target = dict(row.publication_target or {})
if target:
publish_execution(
session,
principal,
registry=registry,
execution_id=execution_id,
target_capability=str(target.get("target_capability") or ""),
target_ref=(
str(target["target_ref"])
if target.get("target_ref") is not None
else None
),
format=str(target.get("format") or "csv"),
idempotency_key=f"schedule-publication:{row.schedule_id}:{scheduled_for.isoformat()}",
options=dict(target.get("options") or {}),
)
succeeded += 1
row.last_execution_id = execution_id
except (ReportingExecutionFailure, ReportingOperationError, LookupError):
failed += 1
row.last_run_at = current
_advance_schedule(row, scheduled_for)
session.flush()
return {
"claimed": len(rows),
"succeeded": succeeded,
"failed": failed,
"execution_ids": execution_ids,
}
def publish_execution(
session: Session,
principal: object,
*,
registry: object | None,
execution_id: str,
target_capability: str,
target_ref: str | None,
format: str,
idempotency_key: str,
options: Mapping[str, object],
) -> dict[str, object]:
_require_scope(principal, PUBLISH_SCOPE)
execution_payload = get_execution(session, principal, execution_id=execution_id)
if execution_payload is None:
raise LookupError("Reporting execution not found.")
if execution_payload["status"] != "succeeded":
raise ReportingOperationError("Only successful report executions can publish.")
clean_capability = _required(
target_capability,
"Reporting publication target capability",
255,
)
clean_format = str(format).casefold()
if clean_format not in {"json", "csv", "xlsx", "html", "pdf"}:
raise ReportingOperationError("Unsupported Reporting publication format.")
clean_key = _required(idempotency_key, "Reporting publication idempotency key", 255)
existing = (
session.query(ReportingPublication)
.filter(
ReportingPublication.tenant_id == _tenant(principal),
ReportingPublication.idempotency_key == clean_key,
)
.one_or_none()
)
if existing is not None:
if (
existing.execution_id != execution_id
or existing.target_capability != clean_capability
or existing.target_ref != target_ref
or existing.format != clean_format
):
raise ReportingOperationError("Reporting publication idempotency conflict.")
return _publication_payload(existing)
provider = capability(registry, clean_capability)
if not isinstance(provider, ReportingPublicationTarget):
raise ReportingOperationError(
f"Reporting publication provider {clean_capability!r} is unavailable."
)
publication = ReportingPublication(
tenant_id=_tenant(principal),
publication_id=str(uuid.uuid4()),
execution_id=execution_id,
target_capability=clean_capability,
target_ref=target_ref,
format=clean_format,
status="running",
idempotency_key=clean_key,
)
session.add(publication)
session.flush()
try:
evidence = provider.publish_report(
session,
principal,
payload=ReportingPublicationPayload(
publication_id=publication.publication_id,
execution_id=execution_id,
tenant_id=_tenant(principal),
report_id=str(execution_payload["report_id"]),
report_revision=int(execution_payload["report_revision"]),
format=clean_format,
target_ref=target_ref,
rows=tuple(execution_payload["rows"]), # type: ignore[arg-type]
schema=tuple(execution_payload["schema"]), # type: ignore[arg-type]
output_hash=str(execution_payload["output_hash"]),
options=dict(options),
),
)
publication.status = "succeeded"
publication.evidence = _json_value(evidence)
publication.completed_at = utc_now()
except Exception as exc:
publication.status = "failed"
publication.error = str(exc)
publication.completed_at = utc_now()
session.flush()
raise ReportingOperationError(str(exc)) from exc
session.flush()
emit_platform_event(
session,
PlatformEvent(
type=f"reporting.publication.{publication.status}",
module_id="reporting",
payload={
"publication_id": publication.publication_id,
"execution_id": publication.execution_id,
"target_capability": publication.target_capability,
"target_ref": publication.target_ref,
"format": publication.format,
"evidence": dict(publication.evidence or {}),
},
actor=EventActorRef(type="account", id=_actor(principal)),
tenant=EventTenantRef(id=publication.tenant_id),
resource=EventObjectRef(
type="report_publication",
id=publication.publication_id,
),
classification="internal",
),
)
return _publication_payload(publication)
def export_execution(
session: Session,
principal: object,
*,
execution_id: str,
format: str,
) -> tuple[bytes, str, str]:
payload = get_execution(session, principal, execution_id=execution_id)
if payload is None:
raise LookupError("Reporting execution not found.")
if payload["status"] != "succeeded":
raise ReportingOperationError("Only successful executions can be exported.")
rows = tuple(payload["rows"]) # type: ignore[arg-type]
if format == "json":
content = json.dumps(
{"schema": payload["schema"], "rows": rows},
ensure_ascii=False,
separators=(",", ":"),
).encode("utf-8")
return content, "application/json", f"report-{execution_id}.json"
if format != "csv":
raise ReportingOperationError("Direct export supports CSV or JSON.")
fields = tuple(
dict.fromkeys(
str(key) for row in rows if isinstance(row, Mapping) for key in row
)
)
stream = StringIO(newline="")
writer = csv.DictWriter(stream, fieldnames=fields, extrasaction="ignore")
writer.writeheader()
for row in rows:
if isinstance(row, Mapping):
writer.writerow({key: _safe_csv_cell(row.get(key)) for key in fields})
return (
stream.getvalue().encode("utf-8-sig"),
"text/csv; charset=utf-8",
f"report-{execution_id}.csv",
)
def assess_import(
session: Session,
principal: object,
*,
source_system: str,
source_id: str,
metadata: Mapping[str, object],
accepted_approximations: list[str],
) -> dict[str, object]:
_require_scope(principal, IMPORT_SCOPE)
raw_features = metadata.get("features", [])
if not isinstance(raw_features, list):
raise ReportingOperationError("Import metadata features must be a list.")
features = tuple(dict.fromkeys(str(item) for item in raw_features))
exact_features = {
"dataset",
"dimension",
"hierarchy",
"measure",
"parameter",
"table",
"pivot",
"chart",
"quality_assertion",
"saved_view",
}
approximated_features = {
"provider_specific_format",
"dialect_function",
"dashboard_layout",
}
unsupported_features = {
"raw_sql",
"stored_procedure",
"runtime_script",
"implicit_authorization",
"unchecked_custom_function",
}
exact = sorted(set(features) & exact_features)
approximated = sorted(set(features) & approximated_features)
unsupported = sorted(
(set(features) & unsupported_features)
| (
set(features)
- exact_features
- approximated_features
- unsupported_features
)
)
accepted = sorted(set(accepted_approximations) & set(approximated))
pending = sorted(set(approximated) - set(accepted))
status = "ready" if not unsupported and not pending else "blocked"
mapping_report = {
"contract_version": "1",
"source_system": source_system,
"source_id": source_id,
"exact": exact,
"approximated": approximated,
"accepted_approximations": accepted,
"pending_approximations": pending,
"unsupported": unsupported,
"manual_bindings": list(metadata.get("manual_bindings", [])),
"provider_assumptions": list(metadata.get("provider_assumptions", [])),
"activation_allowed": status == "ready",
}
row = ReportingImportAssessment(
tenant_id=_tenant(principal),
assessment_id=str(uuid.uuid4()),
source_system=_required(source_system, "Import source system", 255),
source_id=_required(source_id, "Import source identifier", 500),
source_fingerprint=_sha256(metadata),
mapping_report=mapping_report,
status=status,
accepted_approximations=accepted,
assessed_by=_actor(principal),
)
session.add(row)
session.flush()
return _assessment_payload(row)
def list_import_assessments(
session: Session,
principal: object,
*,
limit: int = 100,
) -> tuple[dict[str, object], ...]:
_require_scope(principal, IMPORT_SCOPE)
rows = (
session.query(ReportingImportAssessment)
.filter(ReportingImportAssessment.tenant_id == _tenant(principal))
.order_by(ReportingImportAssessment.created_at.desc())
.limit(max(1, min(limit, 200)))
.all()
)
return tuple(_assessment_payload(row) for row in rows)
def _validate_saved_view_state(state: Mapping[str, object]) -> None:
query = state.get("query")
if query is not None:
if not isinstance(query, Mapping):
raise ReportingOperationError("Saved-view query must be an object.")
ReportQuery.model_validate(query)
if len(state) > 100:
raise ReportingOperationError("Saved-view state is limited to 100 entries.")
def _validate_trigger(
trigger_kind: str,
trigger_config: Mapping[str, object],
*,
next_run_at: datetime | None,
) -> datetime | None:
if trigger_kind not in {"scheduled", "interval"}:
raise ReportingOperationError("Unsupported Reporting schedule trigger.")
if next_run_at is not None:
_aware(next_run_at, "Reporting next_run_at")
if trigger_kind == "scheduled":
if next_run_at is None:
raise ReportingOperationError("Scheduled reports require next_run_at.")
return next_run_at
seconds = int(trigger_config.get("seconds", 0))
if not 60 <= seconds <= 31_536_000:
raise ReportingOperationError(
"Reporting intervals must be between 60 seconds and one year."
)
return next_run_at or utc_now() + timedelta(seconds=seconds)
def _advance_schedule(row: ReportingSchedule, scheduled_for: datetime) -> None:
if row.trigger_kind == "scheduled":
row.enabled = False
row.next_run_at = None
return
if scheduled_for.tzinfo is None or scheduled_for.utcoffset() is None:
scheduled_for = scheduled_for.replace(tzinfo=UTC)
seconds = int((row.trigger_config or {}).get("seconds", 0))
next_run = scheduled_for + timedelta(seconds=seconds)
now = utc_now()
while next_run <= now:
next_run += timedelta(seconds=seconds)
row.next_run_at = next_run
def _saved_view_payload(row: ReportingSavedView) -> dict[str, object]:
return {
"view_id": row.view_id,
"report_id": row.report_id,
"report_revision": row.report_revision,
"owner_kind": row.owner_kind,
"owner_id": row.owner_id,
"name": row.name,
"revision": row.revision,
"state": dict(row.state or {}),
"shared": row.shared,
"access": dict(row.access or {}),
"updated_at": _datetime_text(row.updated_at),
}
def _schedule_payload(row: ReportingSchedule) -> dict[str, object]:
return {
"schedule_id": row.schedule_id,
"report_id": row.report_id,
"report_revision": row.report_revision,
"name": row.name,
"revision": row.revision,
"trigger_kind": row.trigger_kind,
"trigger_config": dict(row.trigger_config or {}),
"parameters": dict(row.parameters or {}),
"query": dict(row.query or {}),
"publication_target": dict(row.publication_target or {}),
"enabled": row.enabled,
"next_run_at": _datetime_text(row.next_run_at),
"last_run_at": _datetime_text(row.last_run_at),
"last_execution_id": row.last_execution_id,
}
def _publication_payload(row: ReportingPublication) -> dict[str, object]:
return {
"publication_id": row.publication_id,
"execution_id": row.execution_id,
"target_capability": row.target_capability,
"target_ref": row.target_ref,
"format": row.format,
"status": row.status,
"evidence": dict(row.evidence or {}),
"error": row.error,
"completed_at": _datetime_text(row.completed_at),
}
def _assessment_payload(row: ReportingImportAssessment) -> dict[str, object]:
return {
"assessment_id": row.assessment_id,
"source_system": row.source_system,
"source_id": row.source_id,
"source_fingerprint": row.source_fingerprint,
"mapping_report": dict(row.mapping_report or {}),
"status": row.status,
"accepted_approximations": list(row.accepted_approximations or []),
"assessed_by": row.assessed_by,
"created_at": _datetime_text(row.created_at),
}
def _safe_csv_cell(value: object) -> object:
if isinstance(value, (dict, list, tuple)):
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
if isinstance(value, str) and value.startswith(("=", "+", "-", "@")):
return f"'{value}"
return value
def _has_scope(principal: object, scope: str) -> bool:
method = getattr(principal, "has", None)
if callable(method):
return bool(method(scope))
return scopes_grant_compatible(
frozenset(getattr(principal, "scopes", ()) or ()),
scope,
)
def _require_scope(principal: object, scope: str) -> None:
if not _has_scope(principal, scope):
raise PermissionError(f"Missing scope: {scope}")
def _tenant(principal: object) -> str:
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
if not tenant_id:
raise ReportingOperationError(
"Reporting operations require a tenant-bound principal."
)
return tenant_id
def _actor(principal: object) -> str | None:
for value in (
getattr(principal, "account_id", None),
getattr(principal, "identity_id", None),
getattr(principal, "membership_id", None),
):
if str(value or "").strip():
return str(value)
return None
def _session(value: object) -> Session:
if not isinstance(value, Session):
raise TypeError("Reporting operations require a SQLAlchemy session.")
return value
def _required(value: object, label: str, maximum: int) -> str:
result = str(value or "").strip()
if not result:
raise ReportingOperationError(f"{label} is required.")
if len(result) > maximum:
raise ReportingOperationError(f"{label} is limited to {maximum} characters.")
return result
def _aware(value: datetime, label: str) -> None:
if value.tzinfo is None or value.utcoffset() is None:
raise ReportingOperationError(f"{label} must include a timezone.")
def _sha256(value: object) -> str:
return hashlib.sha256(
json.dumps(
_json_value(value),
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
).encode("utf-8")
).hexdigest()
def _json_value(value: object) -> Any:
if isinstance(value, datetime):
return value.isoformat()
if hasattr(value, "model_dump"):
return _json_value(value.model_dump(mode="json"))
if isinstance(value, Mapping):
return {str(key): _json_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_value(item) for item in value]
return value
def _datetime_text(value: datetime | None) -> str | None:
if value is None:
return None
if value.tzinfo is None:
value = value.replace(tzinfo=UTC)
return value.isoformat()
__all__ = [
"IMPORT_SCOPE",
"PUBLISH_SCOPE",
"SCHEDULE_SCOPE",
"ReportingOperationError",
"SqlReportingScheduler",
"assess_import",
"delete_saved_view",
"dispatch_due_schedules",
"export_execution",
"list_import_assessments",
"list_saved_views",
"list_schedules",
"publish_execution",
"upsert_saved_view",
"upsert_schedule",
]
@@ -0,0 +1,494 @@
from __future__ import annotations
from collections import defaultdict
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import date, datetime
from decimal import Decimal
from typing import Any
from govoplan_reporting.backend.schemas import (
FilterClause,
MeasureDefinition,
ReportQuery,
SemanticModelDefinition,
TypedExpression,
VisualizationDefinition,
)
QUERY_ENGINE_VERSION = "reporting-query-v1"
class ReportingQueryError(ValueError):
pass
@dataclass(frozen=True, slots=True)
class QueryResult:
rows: tuple[dict[str, Any], ...]
total_rows: int
schema: tuple[dict[str, Any], ...]
truncated: bool
diagnostics: tuple[dict[str, Any], ...] = ()
def execute_semantic_query(
rows: Sequence[Mapping[str, object]],
semantic_model: SemanticModelDefinition,
query: ReportQuery,
) -> QueryResult:
dimensions = {item.key: item for item in semantic_model.dimensions}
measures = {item.key: item for item in semantic_model.measures}
selected_dimensions = tuple(query.dimensions or semantic_model.default_dimensions)
selected_measures = tuple(query.measures or semantic_model.default_measures)
if query.mode == "pivot" and query.pivot is not None:
selected_dimensions = tuple(
dict.fromkeys((*query.pivot.rows, *query.pivot.columns))
)
selected_measures = tuple(query.pivot.measures or selected_measures)
_require_known(selected_dimensions, dimensions, "dimensions")
_require_known(selected_measures, measures, "measures")
_require_known(
tuple(item.dimension for item in query.filters),
dimensions,
"filter dimensions",
)
filtered = tuple(
dict(row)
for row in rows
if all(
_matches_filter(row.get(dimensions[item.dimension].field), item)
for item in query.filters
)
)
if query.mode == "detail":
result_rows = _detail_rows(filtered, selected_dimensions, dimensions)
else:
result_rows = _summary_rows(
filtered,
selected_dimensions,
selected_measures,
dimensions,
measures,
)
if query.mode == "pivot" and query.pivot is not None:
result_rows = _pivot_rows(
result_rows,
row_dimensions=tuple(query.pivot.rows),
column_dimensions=tuple(query.pivot.columns),
measures=selected_measures,
include_totals=query.pivot.include_totals,
)
sorted_rows = _sort_rows(result_rows, query)
total = len(sorted_rows)
selected = sorted_rows[query.offset : query.offset + query.limit]
return QueryResult(
rows=tuple(selected),
total_rows=total,
schema=_infer_schema(selected or sorted_rows[:1]),
truncated=query.offset + len(selected) < total,
)
class DefaultChartRenderer:
"""Build a provider-neutral chart model with a mandatory table fallback."""
def render(
self,
*,
visualization: VisualizationDefinition,
result: QueryResult,
) -> Mapping[str, object]:
category = visualization.category_dimension
measure_keys = tuple(visualization.measures)
missing: set[str] = set()
if visualization.kind not in {"table", "pivot", "metric"}:
if not category or not measure_keys:
missing = {"chart category and measure configuration"}
else:
available = {str(item.get("name")) for item in result.schema}
missing = {category, *measure_keys} - available
if missing and not visualization.tabular_fallback:
raise ReportingQueryError(
"Chart references unavailable result fields: "
+ ", ".join(sorted(missing))
)
return {
"contract_version": "1",
"kind": "table" if missing else visualization.kind,
"requested_kind": visualization.kind,
"category": category,
"series": visualization.series_dimension,
"measures": list(measure_keys),
"options": dict(visualization.options),
"fallback_reason": (
"The selected query does not expose the fields required by the "
"saved visualization. Showing its accessible table fallback."
if missing
else None
),
"rows": list(result.rows),
"schema": list(result.schema),
"tabular_fallback": (
{
"rows": list(result.rows),
"schema": list(result.schema),
}
if visualization.tabular_fallback or missing
else None
),
}
def _detail_rows(
rows: Sequence[Mapping[str, object]],
selected_dimensions: Sequence[str],
dimensions: Mapping[str, object],
) -> list[dict[str, Any]]:
if not selected_dimensions:
return [
{str(key): _json_value(value) for key, value in row.items()} for row in rows
]
return [
{
key: _json_value(row.get(getattr(dimensions[key], "field")))
for key in selected_dimensions
}
for row in rows
]
def _summary_rows(
rows: Sequence[Mapping[str, object]],
selected_dimensions: Sequence[str],
selected_measures: Sequence[str],
dimensions: Mapping[str, object],
measures: Mapping[str, MeasureDefinition],
) -> list[dict[str, Any]]:
grouped: dict[tuple[object, ...], list[Mapping[str, object]]] = defaultdict(list)
if selected_dimensions:
for row in rows:
key = tuple(
row.get(getattr(dimensions[item], "field"))
for item in selected_dimensions
)
grouped[key].append(row)
else:
grouped[()] = list(rows)
result: list[dict[str, Any]] = []
for group_key, group_rows in grouped.items():
item: dict[str, Any] = {
dimension: _json_value(value)
for dimension, value in zip(selected_dimensions, group_key, strict=True)
}
pending: list[MeasureDefinition] = []
for key in selected_measures:
measure = measures[key]
if measure.aggregation == "calculated":
pending.append(measure)
else:
item[key] = _aggregate(group_rows, measure)
for measure in pending:
item[measure.key] = _evaluate_expression(
measure.expression,
row=None,
measures=item,
)
result.append(item)
return result
def _aggregate(
rows: Sequence[Mapping[str, object]],
measure: MeasureDefinition,
) -> object:
if measure.aggregation == "count":
if measure.field is None:
return len(rows)
return sum(row.get(measure.field) is not None for row in rows)
values = [
row.get(measure.field or "")
for row in rows
if row.get(measure.field or "") is not None
]
if measure.aggregation == "count_distinct":
return len({_hashable(value) for value in values})
if not values:
return None
if measure.aggregation == "sum":
return _numeric_result(sum(_number(value) for value in values))
if measure.aggregation == "average":
return _numeric_result(sum(_number(value) for value in values) / len(values))
if measure.aggregation == "minimum":
return _json_value(min(values))
if measure.aggregation == "maximum":
return _json_value(max(values))
raise ReportingQueryError(
f"Unsupported measure aggregation: {measure.aggregation!r}."
)
def _evaluate_expression(
expression: TypedExpression | None,
*,
row: Mapping[str, object] | None,
measures: Mapping[str, object],
) -> object:
if expression is None:
return None
if expression.op == "literal":
return expression.value
if expression.op == "field":
return row.get(expression.ref or "") if row is not None else None
if expression.op == "measure":
return measures.get(expression.ref or "")
values = [
_evaluate_expression(item, row=row, measures=measures)
for item in expression.args
]
if expression.op == "add":
return _numeric_result(sum(_number(item) for item in values))
if expression.op == "subtract":
_arity(values, 2, expression.op)
return _numeric_result(_number(values[0]) - _number(values[1]))
if expression.op == "multiply":
total = Decimal(1)
for value in values:
total *= _number(value)
return _numeric_result(total)
if expression.op == "divide":
_arity(values, 2, expression.op)
divisor = _number(values[1])
return None if divisor == 0 else _numeric_result(_number(values[0]) / divisor)
if expression.op == "coalesce":
return next((item for item in values if item is not None), None)
if expression.op == "case":
if len(values) < 3:
raise ReportingQueryError(
"Case expressions require condition, value, and default."
)
pairs = values[:-1]
for index in range(0, len(pairs) - 1, 2):
if bool(pairs[index]):
return pairs[index + 1]
return values[-1]
if expression.op in {"eq", "ne", "gt", "gte", "lt", "lte"}:
_arity(values, 2, expression.op)
return _compare(values[0], values[1], expression.op)
if expression.op == "and":
return all(bool(item) for item in values)
if expression.op == "or":
return any(bool(item) for item in values)
if expression.op == "not":
_arity(values, 1, expression.op)
return not bool(values[0])
raise ReportingQueryError(f"Unsupported expression operator: {expression.op!r}.")
def _pivot_rows(
rows: Sequence[Mapping[str, object]],
*,
row_dimensions: Sequence[str],
column_dimensions: Sequence[str],
measures: Sequence[str],
include_totals: bool,
) -> list[dict[str, Any]]:
grouped: dict[tuple[object, ...], dict[str, Any]] = {}
totals: dict[tuple[object, ...], dict[str, Decimal]] = defaultdict(
lambda: defaultdict(Decimal)
)
for source in rows:
row_key = tuple(source.get(item) for item in row_dimensions)
target = grouped.setdefault(
row_key,
{
item: _json_value(value)
for item, value in zip(row_dimensions, row_key, strict=True)
},
)
column_key = (
" / ".join(
str(source.get(item) if source.get(item) is not None else "(blank)")
for item in column_dimensions
)
or "value"
)
for measure in measures:
field_name = f"{column_key}.{measure}"
value = source.get(measure)
target[field_name] = _json_value(value)
if include_totals and value is not None:
try:
totals[row_key][measure] += _number(value)
except ReportingQueryError:
pass
if include_totals:
for row_key, target in grouped.items():
for measure in measures:
if measure in totals[row_key]:
target[f"total.{measure}"] = _numeric_result(
totals[row_key][measure]
)
return list(grouped.values())
def _matches_filter(value: object, clause: FilterClause) -> bool:
expected = clause.value
if clause.operator == "is_null":
return value is None
if clause.operator == "not_null":
return value is not None
if clause.operator == "eq":
return value == expected
if clause.operator == "ne":
return value != expected
if clause.operator in {"in", "not_in"}:
if not isinstance(expected, (list, tuple, set, frozenset)):
raise ReportingQueryError("Set filters require a list value.")
result = value in expected
return result if clause.operator == "in" else not result
if clause.operator == "contains":
return str(expected or "").casefold() in str(value or "").casefold()
if clause.operator == "starts_with":
return str(value or "").casefold().startswith(str(expected or "").casefold())
if clause.operator == "between":
if not isinstance(expected, (list, tuple)) or len(expected) != 2:
raise ReportingQueryError("Between filters require two values.")
return value is not None and expected[0] <= value <= expected[1]
if clause.operator in {"gt", "gte", "lt", "lte"}:
if value is None or expected is None:
return False
return bool(_compare(value, expected, clause.operator))
raise ReportingQueryError(f"Unsupported filter operator: {clause.operator!r}.")
def _sort_rows(
rows: Sequence[dict[str, Any]],
query: ReportQuery,
) -> list[dict[str, Any]]:
result = list(rows)
for clause in reversed(query.sort):
result.sort(
key=lambda item: _sort_key(item.get(clause.key)),
reverse=clause.direction == "desc",
)
return result
def _infer_schema(rows: Sequence[Mapping[str, object]]) -> tuple[dict[str, Any], ...]:
names = tuple(dict.fromkeys(str(key) for row in rows for key in row))
return tuple(
{
"name": name,
"type": _value_type(
next((row.get(name) for row in rows if row.get(name) is not None), None)
),
"nullable": any(row.get(name) is None for row in rows),
}
for name in names
)
def _value_type(value: object) -> str:
if value is None:
return "string"
if isinstance(value, bool):
return "boolean"
if isinstance(value, int):
return "integer"
if isinstance(value, (float, Decimal)):
return "number"
if isinstance(value, datetime):
return "datetime"
if isinstance(value, date):
return "date"
if isinstance(value, (dict, list, tuple)):
return "json"
return "string"
def _require_known(
keys: Sequence[str],
available: Mapping[str, object],
label: str,
) -> None:
unknown = set(keys) - set(available)
if unknown:
raise ReportingQueryError(
f"Report query references unknown {label}: " + ", ".join(sorted(unknown))
)
def _compare(left: object, right: object, operator: str) -> bool:
if operator == "eq":
return left == right
if operator == "ne":
return left != right
try:
if operator == "gt":
return left > right # type: ignore[operator]
if operator == "gte":
return left >= right # type: ignore[operator]
if operator == "lt":
return left < right # type: ignore[operator]
if operator == "lte":
return left <= right # type: ignore[operator]
except TypeError as exc:
raise ReportingQueryError("Report comparison values are incompatible.") from exc
raise ReportingQueryError(f"Unsupported comparison operator: {operator!r}.")
def _arity(values: Sequence[object], count: int, operator: str) -> None:
if len(values) != count:
raise ReportingQueryError(
f"Expression {operator} requires exactly {count} arguments."
)
def _number(value: object) -> Decimal:
if value is None or isinstance(value, bool):
raise ReportingQueryError("A numeric report expression received a non-number.")
try:
return Decimal(str(value))
except Exception as exc:
raise ReportingQueryError("A numeric report expression is invalid.") from exc
def _numeric_result(value: Decimal) -> int | float:
if value == value.to_integral_value():
return int(value)
return float(value)
def _sort_key(value: object) -> tuple[bool, str, str]:
return value is None, type(value).__name__, str(value).casefold()
def _hashable(value: object) -> object:
if isinstance(value, Mapping):
return tuple(sorted((str(key), _hashable(item)) for key, item in value.items()))
if isinstance(value, (list, tuple)):
return tuple(_hashable(item) for item in value)
return value
def _json_value(value: object) -> Any:
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, Decimal):
return _numeric_result(value)
if isinstance(value, Mapping):
return {str(key): _json_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_value(item) for item in value]
return value
__all__ = [
"QUERY_ENGINE_VERSION",
"DefaultChartRenderer",
"QueryResult",
"ReportingQueryError",
"execute_semantic_query",
]
@@ -0,0 +1,81 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from govoplan_reporting.backend.definitions import (
create_definition,
get_definition,
list_definitions,
update_definition,
)
class SqlReportingRegistry:
def get_definition(
self,
session: object,
principal: object,
*,
definition_kind: str,
definition_id: str,
revision: int | None = None,
):
return get_definition(
session, # type: ignore[arg-type]
principal,
definition_kind=definition_kind,
definition_id=definition_id,
revision=revision,
)
def list_definitions(
self,
session: object,
principal: object,
*,
definition_kinds: Sequence[str] | None = None,
limit: int = 100,
):
records, _total = list_definitions(
session, # type: ignore[arg-type]
principal,
definition_kinds=definition_kinds,
limit=limit,
)
return records
def create_definition(
self,
session: object,
principal: object,
**values,
):
return create_definition(session, principal, **values) # type: ignore[arg-type]
def update_definition(
self,
session: object,
principal: object,
*,
definition_kind: str,
definition_id: str,
expected_revision: int,
recorded_at,
change_reason: str,
idempotency_key: str,
changes: Mapping[str, object],
):
return update_definition(
session, # type: ignore[arg-type]
principal,
definition_kind=definition_kind,
definition_id=definition_id,
expected_revision=expected_revision,
recorded_at=recorded_at,
change_reason=change_reason,
idempotency_key=idempotency_key,
changes=changes,
)
__all__ = ["SqlReportingRegistry"]
+526
View File
@@ -0,0 +1,526 @@
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
from govoplan_core.core.dataflows import dataflow_dataset_output
from govoplan_core.db.session import get_session
from govoplan_reporting.backend.definitions import (
READ_SCOPE,
WRITE_SCOPE,
ReportingDefinitionError,
create_definition,
definition_history,
get_definition,
list_definitions,
update_definition,
)
from govoplan_reporting.backend.execution import (
QUALITY_SCOPE,
RUN_SCOPE,
ReportingExecutionError,
ReportingExecutionFailure,
execute_report,
get_execution,
list_executions,
run_quality_plan,
)
from govoplan_reporting.backend.operations import (
IMPORT_SCOPE,
PUBLISH_SCOPE,
SCHEDULE_SCOPE,
ReportingOperationError,
assess_import,
delete_saved_view,
dispatch_due_schedules,
export_execution,
list_import_assessments,
list_saved_views,
list_schedules,
publish_execution,
upsert_saved_view,
upsert_schedule,
)
from govoplan_reporting.backend.query_engine import ReportingQueryError
from govoplan_reporting.backend.schemas import (
DefinitionUpdateRequest,
DefinitionWriteRequest,
ImportAssessmentRequest,
PublicationRequest,
QualityRunRequest,
ReportExecutionRequest,
SavedViewWriteRequest,
ScheduleWriteRequest,
)
def create_router(registry: object | None) -> APIRouter:
router = APIRouter(prefix="/reporting", tags=["reporting"])
@router.get("/definitions")
def api_list_definitions(
definition_kind: list[str] | None = Query(default=None),
definition_status: list[str] | None = Query(default=None, alias="status"),
query: str = "",
offset: int = Query(default=0, ge=0),
limit: int = Query(default=100, ge=1, le=200),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, READ_SCOPE)
try:
records, total = list_definitions(
session,
principal,
definition_kinds=definition_kind,
statuses=definition_status,
query=query,
offset=offset,
limit=limit,
)
except ReportingDefinitionError as exc:
raise _error(exc) from exc
return {
"definitions": [item.to_dict() for item in records],
"total": total,
"offset": offset,
"limit": limit,
}
@router.post(
"/definitions",
status_code=status.HTTP_201_CREATED,
)
def api_create_definition(
payload: DefinitionWriteRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, Any]:
_require(principal, WRITE_SCOPE)
try:
record = create_definition(
session,
principal,
**payload.model_dump(),
)
session.commit()
except (ReportingDefinitionError, ValueError) as exc:
session.rollback()
raise _error(exc) from exc
return record.to_dict()
@router.get("/definitions/{definition_kind}/{definition_id}")
def api_get_definition(
definition_kind: str,
definition_id: str,
revision: int | None = Query(default=None, ge=1),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, Any]:
_require(principal, READ_SCOPE)
try:
record = get_definition(
session,
principal,
definition_kind=definition_kind,
definition_id=definition_id,
revision=revision,
)
except ReportingDefinitionError as exc:
raise _error(exc) from exc
if record is None:
raise HTTPException(
status_code=404, detail="Reporting definition not found"
)
return record.to_dict()
@router.patch("/definitions/{definition_kind}/{definition_id}")
def api_update_definition(
definition_kind: str,
definition_id: str,
payload: DefinitionUpdateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, Any]:
_require(principal, WRITE_SCOPE)
changes = payload.model_dump(
exclude={
"expected_revision",
"recorded_at",
"change_reason",
"idempotency_key",
},
exclude_unset=True,
)
try:
record = update_definition(
session,
principal,
definition_kind=definition_kind,
definition_id=definition_id,
expected_revision=payload.expected_revision,
recorded_at=payload.recorded_at,
change_reason=payload.change_reason,
idempotency_key=payload.idempotency_key,
changes=changes,
)
session.commit()
except (
ReportingDefinitionError,
PermissionError,
LookupError,
ValueError,
) as exc:
session.rollback()
raise _error(exc) from exc
return record.to_dict()
@router.get("/definitions/{definition_kind}/{definition_id}/history")
def api_definition_history(
definition_kind: str,
definition_id: str,
limit: int = Query(default=100, ge=1, le=200),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, READ_SCOPE)
records = definition_history(
session,
principal,
definition_kind=definition_kind,
definition_id=definition_id,
limit=limit,
)
if not records:
raise HTTPException(
status_code=404, detail="Reporting definition not found"
)
return {"revisions": [item.to_dict() for item in records]}
@router.get("/sources/dataflow")
def api_dataflow_sources(
query: str = "",
limit: int = Query(default=100, ge=1, le=200),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, WRITE_SCOPE)
provider = dataflow_dataset_output(registry)
if provider is None:
return {
"available": False,
"reason": "The Dataflow dataset provider is not enabled.",
"sources": [],
}
sources = provider.list_outputs(
session,
principal,
query=query,
limit=limit,
)
return {
"available": True,
"reason": None,
"sources": [
{
"pipeline_ref": item.pipeline_ref,
"name": item.name,
"description": item.description,
"revision": item.revision,
"definition_hash": item.definition_hash,
"status": item.status,
"updated_at": item.updated_at.isoformat()
if item.updated_at
else None,
"parameters": dict(item.parameters),
"provenance": dict(item.provenance),
}
for item in sources
],
}
@router.post("/reports/{report_id}/executions")
def api_execute_report(
report_id: str,
payload: ReportExecutionRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, RUN_SCOPE)
try:
result = execute_report(
session,
principal,
registry=registry,
report_id=report_id,
report_revision=payload.report_revision,
parameters=payload.parameters,
query=payload.query,
idempotency_key=payload.idempotency_key,
)
session.commit()
except ReportingExecutionFailure as exc:
session.commit()
raise HTTPException(
status_code=422,
detail={"message": str(exc), "execution_id": exc.execution_id},
) from exc
except (
ReportingExecutionError,
ReportingQueryError,
PermissionError,
LookupError,
ValueError,
) as exc:
session.rollback()
raise _error(exc) from exc
return result
@router.get("/reports/{report_id}/executions")
def api_list_executions(
report_id: str,
limit: int = Query(default=100, ge=1, le=200),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, RUN_SCOPE)
return {
"executions": list(
list_executions(session, principal, report_id=report_id, limit=limit)
)
}
@router.get("/executions/{execution_id}")
def api_get_execution(
execution_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, RUN_SCOPE)
result = get_execution(session, principal, execution_id=execution_id)
if result is None:
raise HTTPException(status_code=404, detail="Reporting execution not found")
return result
@router.get("/executions/{execution_id}/export")
def api_export_execution(
execution_id: str,
format: str = Query(default="csv", pattern="^(csv|json)$"),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> Response:
_require(principal, RUN_SCOPE)
try:
content, media_type, filename = export_execution(
session,
principal,
execution_id=execution_id,
format=format,
)
except (ReportingOperationError, LookupError) as exc:
raise _error(exc) from exc
return Response(
content=content,
media_type=media_type,
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@router.post("/executions/{execution_id}/publications")
def api_publish_execution(
execution_id: str,
payload: PublicationRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, PUBLISH_SCOPE)
try:
result = publish_execution(
session,
principal,
registry=registry,
execution_id=execution_id,
**payload.model_dump(),
)
session.commit()
except (ReportingOperationError, PermissionError, LookupError) as exc:
session.commit()
raise _error(exc) from exc
return result
@router.get("/reports/{report_id}/saved-views")
def api_list_saved_views(
report_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, READ_SCOPE)
return {
"views": list(list_saved_views(session, principal, report_id=report_id))
}
@router.put("/reports/{report_id}/saved-views/{view_id}")
def api_upsert_saved_view(
report_id: str,
view_id: str,
payload: SavedViewWriteRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, READ_SCOPE)
if view_id != payload.view_id:
raise HTTPException(status_code=400, detail="Saved-view identifiers differ")
try:
result = upsert_saved_view(
session,
principal,
report_id=report_id,
**payload.model_dump(),
)
session.commit()
except (ReportingOperationError, PermissionError, LookupError) as exc:
session.rollback()
raise _error(exc) from exc
return result
@router.delete("/saved-views/{view_id}", status_code=204)
def api_delete_saved_view(
view_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> Response:
_require(principal, READ_SCOPE)
try:
deleted = delete_saved_view(session, principal, view_id=view_id)
session.commit()
except PermissionError as exc:
session.rollback()
raise _error(exc) from exc
if not deleted:
raise HTTPException(status_code=404, detail="Saved view not found")
return Response(status_code=204)
@router.get("/schedules")
def api_list_schedules(
report_id: str | None = None,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, SCHEDULE_SCOPE)
return {
"schedules": list(list_schedules(session, principal, report_id=report_id))
}
@router.put("/schedules/{schedule_id}")
def api_upsert_schedule(
schedule_id: str,
payload: ScheduleWriteRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, SCHEDULE_SCOPE)
if schedule_id != payload.schedule_id:
raise HTTPException(status_code=400, detail="Schedule identifiers differ")
try:
result = upsert_schedule(session, principal, **payload.model_dump())
session.commit()
except (ReportingOperationError, PermissionError, LookupError) as exc:
session.rollback()
raise _error(exc) from exc
return result
@router.post("/schedules/dispatch")
def api_dispatch_schedules(
limit: int = Query(default=20, ge=1, le=100),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, SCHEDULE_SCOPE)
result = dispatch_due_schedules(
session,
principal,
registry=registry,
now=None,
limit=limit,
)
session.commit()
return result
@router.post("/quality-plans/{quality_plan_id}/runs")
def api_run_quality_plan(
quality_plan_id: str,
payload: QualityRunRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, QUALITY_SCOPE)
try:
result = run_quality_plan(
session,
principal,
registry=registry,
quality_plan_id=quality_plan_id,
quality_plan_revision=payload.quality_plan_revision,
parameters=payload.parameters,
)
session.commit()
except (ReportingExecutionError, PermissionError, LookupError) as exc:
session.rollback()
raise _error(exc) from exc
return result
@router.post("/imports/assessments", status_code=201)
def api_assess_import(
payload: ImportAssessmentRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, IMPORT_SCOPE)
try:
result = assess_import(session, principal, **payload.model_dump())
session.commit()
except (ReportingOperationError, PermissionError) as exc:
session.rollback()
raise _error(exc) from exc
return result
@router.get("/imports/assessments")
def api_list_import_assessments(
limit: int = Query(default=100, ge=1, le=200),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, IMPORT_SCOPE)
return {
"assessments": list(
list_import_assessments(session, principal, limit=limit)
)
}
return router
def _require(principal: ApiPrincipal, scope: str) -> None:
if not has_scope(principal, scope):
raise HTTPException(status_code=403, detail=f"Missing scope: {scope}")
def _error(exc: Exception) -> HTTPException:
message = str(exc)
lowered = message.casefold()
if isinstance(exc, LookupError):
code = 404
elif isinstance(exc, PermissionError):
code = 403
elif any(word in lowered for word in ("conflict", "already", "stale")):
code = 409
elif "unavailable" in lowered or "not enabled" in lowered:
code = 503
else:
code = 400
return HTTPException(status_code=code, detail=message)
__all__ = ["create_router"]
+503
View File
@@ -0,0 +1,503 @@
from __future__ import annotations
from datetime import datetime
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
DefinitionKind = Literal[
"dataset",
"semantic_model",
"report",
"quality_plan",
]
FieldType = Literal[
"string",
"integer",
"number",
"boolean",
"date",
"datetime",
"json",
]
class ReportingReference(BaseModel):
model_config = ConfigDict(extra="forbid")
owner_module: str = Field(min_length=1, max_length=100)
resource_type: str = Field(min_length=1, max_length=100)
resource_id: str = Field(min_length=1, max_length=255)
revision: str | None = Field(default=None, max_length=255)
relationship: str = Field(default="related", min_length=1, max_length=80)
label: str | None = Field(default=None, max_length=500)
class DatasetField(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=255)
type: FieldType = "string"
label: str | None = Field(default=None, max_length=500)
nullable: bool = True
description: str | None = Field(default=None, max_length=4_000)
classification: Literal[
"public",
"internal",
"confidential",
"restricted",
] = "internal"
class FreshnessPolicy(BaseModel):
model_config = ConfigDict(extra="forbid")
max_age_seconds: int | None = Field(default=None, ge=1, le=31_536_000)
stale_action: Literal["allow", "warn", "block"] = "warn"
require_source_fingerprints: bool = True
class DatasetDefinition(BaseModel):
model_config = ConfigDict(extra="forbid", populate_by_name=True)
source_kind: Literal["dataflow", "read_model", "static"]
source_ref: str = Field(min_length=1, max_length=500)
source_revision: int | None = Field(default=None, ge=1)
definition_hash: str | None = Field(default=None, min_length=1, max_length=128)
source_parameters: dict[str, Any] = Field(default_factory=dict)
static_rows: list[dict[str, Any]] = Field(default_factory=list, max_length=2_000)
expected_source_fingerprints: list[dict[str, Any]] = Field(
default_factory=list,
max_length=100,
)
fields: list[DatasetField] = Field(
default_factory=list,
max_length=500,
validation_alias="schema",
serialization_alias="schema",
)
freshness: FreshnessPolicy = Field(default_factory=FreshnessPolicy)
row_policy_ref: str | None = Field(default=None, max_length=500)
purpose: str = Field(min_length=1, max_length=4_000)
privacy: dict[str, Any] = Field(default_factory=dict)
retention: dict[str, Any] = Field(default_factory=dict)
policy_provenance: dict[str, Any] = Field(default_factory=dict)
institutional_references: list[ReportingReference] = Field(
default_factory=list,
max_length=200,
)
@model_validator(mode="after")
def validate_source_pin(self) -> "DatasetDefinition":
if self.source_kind == "dataflow" and self.source_revision is None:
raise ValueError("Dataflow datasets require a pinned source revision.")
if self.source_kind == "static" and not self.static_rows:
raise ValueError("Static analytical datasets require static_rows.")
names = [item.name for item in self.fields]
if len(names) != len(set(names)):
raise ValueError("Dataset schema field names must be unique.")
return self
class DimensionDefinition(BaseModel):
model_config = ConfigDict(extra="forbid")
key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$")
field: str = Field(min_length=1, max_length=255)
label: str = Field(min_length=1, max_length=500)
type: FieldType = "string"
format: str | None = Field(default=None, max_length=255)
default_sort: Literal["asc", "desc"] | None = None
class HierarchyLevel(BaseModel):
model_config = ConfigDict(extra="forbid")
dimension: str = Field(min_length=1, max_length=120)
label: str | None = Field(default=None, max_length=500)
class HierarchyDefinition(BaseModel):
model_config = ConfigDict(extra="forbid")
key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$")
label: str = Field(min_length=1, max_length=500)
levels: list[HierarchyLevel] = Field(min_length=1, max_length=20)
class TypedExpression(BaseModel):
model_config = ConfigDict(extra="forbid")
op: Literal[
"literal",
"field",
"measure",
"add",
"subtract",
"multiply",
"divide",
"coalesce",
"case",
"eq",
"ne",
"gt",
"gte",
"lt",
"lte",
"and",
"or",
"not",
]
value: Any = None
ref: str | None = Field(default=None, max_length=255)
args: list["TypedExpression"] = Field(default_factory=list, max_length=50)
result_type: FieldType | None = None
@model_validator(mode="after")
def validate_shape(self) -> "TypedExpression":
if self.op in {"field", "measure"} and not self.ref:
raise ValueError(f"Expression {self.op} requires ref.")
if self.op == "literal" and self.args:
raise ValueError("Literal expressions cannot have arguments.")
if self.op not in {"literal", "field", "measure"} and not self.args:
raise ValueError(f"Expression {self.op} requires arguments.")
return self
class MeasureDefinition(BaseModel):
model_config = ConfigDict(extra="forbid")
key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$")
label: str = Field(min_length=1, max_length=500)
aggregation: Literal[
"sum",
"count",
"count_distinct",
"average",
"minimum",
"maximum",
"calculated",
]
field: str | None = Field(default=None, max_length=255)
expression: TypedExpression | None = None
format: str | None = Field(default=None, max_length=255)
@model_validator(mode="after")
def validate_measure(self) -> "MeasureDefinition":
if self.aggregation == "calculated" and self.expression is None:
raise ValueError("Calculated measures require an expression.")
if self.aggregation not in {"count", "calculated"} and not self.field:
raise ValueError(f"{self.aggregation} measures require a field.")
return self
class SemanticModelDefinition(BaseModel):
model_config = ConfigDict(extra="forbid")
dataset_id: str = Field(min_length=1, max_length=255)
dataset_revision: int = Field(ge=1)
dimensions: list[DimensionDefinition] = Field(default_factory=list, max_length=300)
hierarchies: list[HierarchyDefinition] = Field(default_factory=list, max_length=100)
measures: list[MeasureDefinition] = Field(default_factory=list, max_length=300)
default_dimensions: list[str] = Field(default_factory=list, max_length=50)
default_measures: list[str] = Field(default_factory=list, max_length=50)
metadata: dict[str, Any] = Field(default_factory=dict)
@model_validator(mode="after")
def validate_semantics(self) -> "SemanticModelDefinition":
dimensions = {item.key for item in self.dimensions}
measures = {item.key for item in self.measures}
if len(dimensions) != len(self.dimensions):
raise ValueError("Semantic dimensions require unique keys.")
if len(measures) != len(self.measures):
raise ValueError("Semantic measures require unique keys.")
unknown_dimensions = set(self.default_dimensions) - dimensions
unknown_measures = set(self.default_measures) - measures
hierarchy_dimensions = {
level.dimension
for hierarchy in self.hierarchies
for level in hierarchy.levels
}
unknown_dimensions |= hierarchy_dimensions - dimensions
if unknown_dimensions or unknown_measures:
raise ValueError(
"Semantic model references unknown dimensions or measures: "
+ ", ".join(sorted(unknown_dimensions | unknown_measures))
)
return self
class ParameterDefinition(BaseModel):
model_config = ConfigDict(extra="forbid")
key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$")
label: str = Field(min_length=1, max_length=500)
type: FieldType = "string"
required: bool = False
default: Any = None
allowed_values: list[Any] = Field(default_factory=list, max_length=500)
class FilterClause(BaseModel):
model_config = ConfigDict(extra="forbid")
dimension: str = Field(min_length=1, max_length=120)
operator: Literal[
"eq",
"ne",
"in",
"not_in",
"contains",
"starts_with",
"gt",
"gte",
"lt",
"lte",
"between",
"is_null",
"not_null",
] = "eq"
value: Any = None
class SortClause(BaseModel):
model_config = ConfigDict(extra="forbid")
key: str = Field(min_length=1, max_length=120)
direction: Literal["asc", "desc"] = "asc"
class PivotDefinition(BaseModel):
model_config = ConfigDict(extra="forbid")
rows: list[str] = Field(default_factory=list, max_length=10)
columns: list[str] = Field(default_factory=list, max_length=5)
measures: list[str] = Field(default_factory=list, max_length=20)
include_totals: bool = True
class ReportQuery(BaseModel):
model_config = ConfigDict(extra="forbid")
mode: Literal["summary", "detail", "pivot"] = "summary"
dimensions: list[str] = Field(default_factory=list, max_length=50)
measures: list[str] = Field(default_factory=list, max_length=50)
filters: list[FilterClause] = Field(default_factory=list, max_length=100)
sort: list[SortClause] = Field(default_factory=list, max_length=20)
pivot: PivotDefinition | None = None
offset: int = Field(default=0, ge=0)
limit: int = Field(default=200, ge=1, le=2_000)
@model_validator(mode="after")
def validate_pivot(self) -> "ReportQuery":
if self.mode == "pivot" and self.pivot is None:
raise ValueError("Pivot mode requires a pivot definition.")
return self
class VisualizationDefinition(BaseModel):
model_config = ConfigDict(extra="forbid")
kind: Literal[
"table",
"pivot",
"bar",
"line",
"area",
"column",
"pie",
"metric",
] = "table"
category_dimension: str | None = Field(default=None, max_length=120)
series_dimension: str | None = Field(default=None, max_length=120)
measures: list[str] = Field(default_factory=list, max_length=20)
options: dict[str, Any] = Field(default_factory=dict)
tabular_fallback: bool = True
class ReportDefinition(BaseModel):
model_config = ConfigDict(extra="forbid")
semantic_model_id: str = Field(min_length=1, max_length=255)
semantic_model_revision: int = Field(ge=1)
parameters: list[ParameterDefinition] = Field(default_factory=list, max_length=100)
default_query: ReportQuery = Field(default_factory=ReportQuery)
visualization: VisualizationDefinition = Field(
default_factory=VisualizationDefinition
)
layout: dict[str, Any] = Field(default_factory=dict)
access_policy: dict[str, Any] = Field(default_factory=dict)
publication_defaults: dict[str, Any] = Field(default_factory=dict)
institutional_references: list[ReportingReference] = Field(
default_factory=list,
max_length=200,
)
class QualityAssertion(BaseModel):
model_config = ConfigDict(extra="forbid")
key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$")
kind: Literal[
"not_null",
"unique",
"range",
"accepted_values",
"row_count",
"comparison",
]
field: str | None = Field(default=None, max_length=255)
severity: Literal["info", "warning", "error", "blocker"] = "error"
config: dict[str, Any] = Field(default_factory=dict)
class QualityPlanDefinition(BaseModel):
model_config = ConfigDict(extra="forbid")
dataset_id: str = Field(min_length=1, max_length=255)
dataset_revision: int = Field(ge=1)
assertions: list[QualityAssertion] = Field(min_length=1, max_length=200)
block_report_execution: bool = True
DEFINITION_PAYLOAD_TYPES = {
"dataset": DatasetDefinition,
"semantic_model": SemanticModelDefinition,
"report": ReportDefinition,
"quality_plan": QualityPlanDefinition,
}
def validate_definition_payload(
kind: str,
payload: dict[str, Any],
) -> dict[str, Any]:
model = DEFINITION_PAYLOAD_TYPES.get(kind)
if model is None:
raise ValueError(f"Unsupported Reporting definition kind: {kind!r}.")
return model.model_validate(payload).model_dump(mode="json", by_alias=True)
class DefinitionWriteRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
definition_kind: DefinitionKind
definition_id: str = Field(min_length=1, max_length=255)
definition_key: str = Field(
min_length=1,
max_length=120,
pattern=r"^[a-z0-9._-]+$",
)
name: str = Field(min_length=1, max_length=500)
description: str | None = Field(default=None, max_length=100_000)
status: Literal["draft", "active", "retired"] = "draft"
visibility: Literal["tenant", "restricted"] = "tenant"
recorded_at: datetime
change_reason: str = Field(min_length=1, max_length=1_000)
payload: dict[str, Any]
idempotency_key: str = Field(min_length=1, max_length=255)
class DefinitionUpdateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
expected_revision: int = Field(ge=1)
recorded_at: datetime
change_reason: str = Field(min_length=1, max_length=1_000)
idempotency_key: str = Field(min_length=1, max_length=255)
name: str | None = Field(default=None, min_length=1, max_length=500)
description: str | None = Field(default=None, max_length=100_000)
status: Literal["draft", "active", "retired"] | None = None
visibility: Literal["tenant", "restricted"] | None = None
payload: dict[str, Any] | None = None
class ReportExecutionRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
report_revision: int | None = Field(default=None, ge=1)
parameters: dict[str, Any] = Field(default_factory=dict)
query: ReportQuery | None = None
idempotency_key: str = Field(min_length=1, max_length=255)
class SavedViewWriteRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
view_id: str = Field(min_length=1, max_length=36)
report_revision: int = Field(ge=1)
name: str = Field(min_length=1, max_length=500)
state: dict[str, Any]
shared: bool = False
access: dict[str, Any] = Field(default_factory=dict)
expected_revision: int | None = Field(default=None, ge=1)
class ScheduleWriteRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
schedule_id: str = Field(min_length=1, max_length=36)
report_id: str = Field(min_length=1, max_length=255)
report_revision: int = Field(ge=1)
name: str = Field(min_length=1, max_length=500)
trigger_kind: Literal["scheduled", "interval"]
trigger_config: dict[str, Any]
parameters: dict[str, Any] = Field(default_factory=dict)
query: ReportQuery = Field(default_factory=ReportQuery)
publication_target: dict[str, Any] = Field(default_factory=dict)
enabled: bool = True
next_run_at: datetime | None = None
expected_revision: int | None = Field(default=None, ge=1)
class PublicationRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
target_capability: str = Field(min_length=1, max_length=255)
target_ref: str | None = Field(default=None, max_length=1_000)
format: Literal["json", "csv", "xlsx", "html", "pdf"] = "csv"
idempotency_key: str = Field(min_length=1, max_length=255)
options: dict[str, Any] = Field(default_factory=dict)
class QualityRunRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
quality_plan_revision: int | None = Field(default=None, ge=1)
parameters: dict[str, Any] = Field(default_factory=dict)
class ImportAssessmentRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
source_system: str = Field(min_length=1, max_length=255)
source_id: str = Field(min_length=1, max_length=500)
metadata: dict[str, Any]
accepted_approximations: list[str] = Field(default_factory=list, max_length=500)
TypedExpression.model_rebuild()
__all__ = [
"DatasetDefinition",
"DefinitionUpdateRequest",
"DefinitionWriteRequest",
"DimensionDefinition",
"FilterClause",
"ImportAssessmentRequest",
"MeasureDefinition",
"PublicationRequest",
"QualityPlanDefinition",
"QualityRunRequest",
"ReportDefinition",
"ReportExecutionRequest",
"ReportQuery",
"SavedViewWriteRequest",
"ScheduleWriteRequest",
"SemanticModelDefinition",
"TypedExpression",
"VisualizationDefinition",
"validate_definition_payload",
]
@@ -0,0 +1,202 @@
from __future__ import annotations
from collections import defaultdict
from collections.abc import Mapping, Sequence
from urllib.parse import quote
from sqlalchemy import func
from sqlalchemy.orm import Session
from govoplan_core.core.modules import ModuleContext
from govoplan_core.core.search import (
SearchAuthorizationRequest,
SearchBackfillPage,
SearchBackfillRequest,
SearchDocument,
SearchResourceType,
)
from govoplan_reporting.backend.db.models import (
ReportingDefinitionGrant,
ReportingDefinitionIdentity,
ReportingDefinitionRevision,
)
from govoplan_reporting.backend.definitions import can_read_definition
PROVIDER_ID = "reporting.reports"
RESOURCE_TYPE = "report"
class ReportingSearchSource:
def resource_types(self) -> Sequence[SearchResourceType]:
return (
SearchResourceType(
provider_id=PROVIDER_ID,
module_id="reporting",
resource_type=RESOURCE_TYPE,
label="Reports",
requires_authorization_recheck=True,
),
)
def backfill(
self,
session: object,
*,
request: SearchBackfillRequest,
) -> SearchBackfillPage:
if request.provider_id != PROVIDER_ID or request.resource_type != RESOURCE_TYPE:
raise ValueError("Unsupported Reporting search source.")
db = _session(session)
query = db.query(ReportingDefinitionRevision).filter(
ReportingDefinitionRevision.tenant_id == request.tenant_id,
ReportingDefinitionRevision.definition_kind == "report",
ReportingDefinitionRevision.status != "retired",
ReportingDefinitionRevision.superseded_at.is_(None),
)
if request.cursor:
query = query.filter(ReportingDefinitionRevision.id > request.cursor)
rows = (
query.order_by(ReportingDefinitionRevision.id.asc())
.limit(request.limit + 1)
.all()
)
has_more = len(rows) > request.limit
selected = rows[: request.limit]
tokens = _acl_tokens(db, selected)
high_watermark = (
db.query(func.max(ReportingDefinitionRevision.updated_at))
.filter(
ReportingDefinitionRevision.tenant_id == request.tenant_id,
ReportingDefinitionRevision.definition_kind == "report",
ReportingDefinitionRevision.superseded_at.is_(None),
)
.scalar()
)
return SearchBackfillPage(
documents=tuple(
_document(row, tokens[row.definition_id]) for row in selected
),
next_cursor=selected[-1].id if has_more and selected else None,
complete=not has_more,
high_watermark=(
high_watermark.isoformat() if high_watermark is not None else None
),
)
def authorize(
self,
session: object,
principal: object,
*,
requests: Sequence[SearchAuthorizationRequest],
) -> Mapping[str, bool]:
db = _session(session)
tenant_id = str(getattr(principal, "tenant_id", "") or "")
decisions = {request.reference.key: False for request in requests}
for request in requests:
reference = request.reference
if (
reference.tenant_id != tenant_id
or reference.module_id != "reporting"
or reference.resource_type != RESOURCE_TYPE
):
continue
decisions[reference.key] = can_read_definition(
db,
principal,
definition_kind="report",
definition_id=reference.resource_id,
)
return decisions
def create_reporting_search_source(
context: ModuleContext,
) -> ReportingSearchSource:
del context
return ReportingSearchSource()
def _document(
row: ReportingDefinitionRevision,
tokens: tuple[str, ...],
) -> SearchDocument:
restricted_tokens = tuple(
dict.fromkeys((*tokens, "scope:reporting:definition:admin"))
)
return SearchDocument(
tenant_id=row.tenant_id,
module_id="reporting",
provider_id=PROVIDER_ID,
resource_type=RESOURCE_TYPE,
resource_id=row.definition_id,
title=row.name,
url=f"/reporting?reportId={quote(row.definition_id, safe='')}",
summary=row.description,
body=row.description,
keywords=(row.definition_key, row.status),
visibility=row.visibility,
acl_tokens=restricted_tokens if row.visibility == "restricted" else (),
source_revision=str(row.revision),
source_updated_at=row.updated_at or row.recorded_at,
metadata={
"definition_key": row.definition_key,
"status": row.status,
"content_hash": row.content_hash,
},
requires_authorization_recheck=True,
)
def _acl_tokens(
session: Session,
rows: Sequence[ReportingDefinitionRevision],
) -> Mapping[str, tuple[str, ...]]:
result: dict[str, list[str]] = defaultdict(list)
if not rows:
return result
identity_ids = {row.identity_id for row in rows}
for identity in (
session.query(ReportingDefinitionIdentity)
.filter(ReportingDefinitionIdentity.id.in_(identity_ids))
.all()
):
if identity.created_by:
result[identity.definition_id].append(f"account:{identity.created_by}")
definition_ids = {row.definition_id for row in rows}
grants = (
session.query(ReportingDefinitionGrant)
.filter(
ReportingDefinitionGrant.tenant_id == rows[0].tenant_id,
ReportingDefinitionGrant.definition_kind == "report",
ReportingDefinitionGrant.definition_id.in_(definition_ids),
ReportingDefinitionGrant.active.is_(True),
)
.all()
)
for grant in grants:
prefix = (
"function"
if grant.subject_kind == "function_assignment"
else grant.subject_kind
)
result[grant.definition_id].append(f"{prefix}:{grant.subject_id}")
return {
definition_id: tuple(dict.fromkeys(values))
for definition_id, values in result.items()
}
def _session(value: object) -> Session:
if not isinstance(value, Session):
raise TypeError("Reporting search requires a SQLAlchemy session.")
return value
__all__ = [
"PROVIDER_ID",
"RESOURCE_TYPE",
"ReportingSearchSource",
"create_reporting_search_source",
]
+1
View File
@@ -0,0 +1 @@