feat: implement governed reporting vertical
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Reporting backend package."""
|
||||
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from govoplan_core.auth import has_scope
|
||||
from govoplan_core.core.modules import AccessDecision
|
||||
|
||||
|
||||
class ReportingScopeAclProvider:
|
||||
def __init__(self, resource_type: str) -> None:
|
||||
self.resource_type = resource_type
|
||||
|
||||
def can_read(self, principal: object, resource_id: str) -> bool:
|
||||
del resource_id
|
||||
return has_scope(principal, "reporting:definition:read")
|
||||
|
||||
def can_write(self, principal: object, resource_id: str) -> bool:
|
||||
del resource_id
|
||||
return has_scope(principal, "reporting:definition:write")
|
||||
|
||||
def explain(self, principal: object, resource_id: str) -> AccessDecision:
|
||||
del resource_id
|
||||
allowed = self.can_read(principal, "")
|
||||
return AccessDecision(
|
||||
allowed=allowed,
|
||||
reason=None if allowed else "Missing scope: reporting:definition:read",
|
||||
requirements=("reporting:definition:read",),
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["ReportingScopeAclProvider"]
|
||||
@@ -0,0 +1,135 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
CAPABILITY_REPORTING_REGISTRY = "reporting.registry"
|
||||
CAPABILITY_REPORTING_RUNNER = "reporting.runner"
|
||||
CAPABILITY_REPORTING_SCHEDULER = "reporting.scheduler"
|
||||
CAPABILITY_REPORTING_CHART_RENDERER = "reporting.chart_renderer"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReportingDatasetReadRequest:
|
||||
source_ref: str
|
||||
source_revision: int | None
|
||||
parameters: Mapping[str, object] = field(default_factory=dict)
|
||||
row_limit: int = 2_000
|
||||
expected_definition_hash: str | None = None
|
||||
expected_source_fingerprints: tuple[Mapping[str, object], ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReportingDatasetReadResult:
|
||||
rows: tuple[Mapping[str, object], ...]
|
||||
total_rows: int
|
||||
truncated: bool
|
||||
output_hash: str
|
||||
executor_version: str
|
||||
definition_hash: str | None = None
|
||||
source_fingerprints: tuple[Mapping[str, object], ...] = ()
|
||||
diagnostics: tuple[Mapping[str, object], ...] = ()
|
||||
generated_at: datetime | None = None
|
||||
provenance: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ReportingReadModelProvider(Protocol):
|
||||
def read_dataset(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
request: ReportingDatasetReadRequest,
|
||||
) -> ReportingDatasetReadResult: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReportingRowPolicyRequest:
|
||||
dataset_id: str
|
||||
dataset_revision: int
|
||||
policy_ref: str
|
||||
rows: tuple[Mapping[str, object], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReportingRowPolicyResult:
|
||||
rows: tuple[Mapping[str, object], ...]
|
||||
decision_ref: str
|
||||
provenance: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ReportingRowPolicyProvider(Protocol):
|
||||
def authorize_rows(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
request: ReportingRowPolicyRequest,
|
||||
) -> ReportingRowPolicyResult: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReportingPublicationPayload:
|
||||
publication_id: str
|
||||
execution_id: str
|
||||
tenant_id: str
|
||||
report_id: str
|
||||
report_revision: int
|
||||
format: str
|
||||
target_ref: str | None
|
||||
rows: tuple[Mapping[str, object], ...]
|
||||
schema: tuple[Mapping[str, object], ...]
|
||||
output_hash: str
|
||||
options: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ReportingPublicationTarget(Protocol):
|
||||
def publish_report(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
payload: ReportingPublicationPayload,
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ReportingChartRenderer(Protocol):
|
||||
def render(
|
||||
self, *, visualization: object, result: object
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
|
||||
def capability(registry: object | None, name: str) -> object | None:
|
||||
if (
|
||||
registry is None
|
||||
or not hasattr(registry, "has_capability")
|
||||
or not hasattr(registry, "capability")
|
||||
or not registry.has_capability(name)
|
||||
):
|
||||
return None
|
||||
return registry.capability(name)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CAPABILITY_REPORTING_CHART_RENDERER",
|
||||
"CAPABILITY_REPORTING_REGISTRY",
|
||||
"CAPABILITY_REPORTING_RUNNER",
|
||||
"CAPABILITY_REPORTING_SCHEDULER",
|
||||
"ReportingChartRenderer",
|
||||
"ReportingDatasetReadRequest",
|
||||
"ReportingDatasetReadResult",
|
||||
"ReportingPublicationPayload",
|
||||
"ReportingPublicationTarget",
|
||||
"ReportingReadModelProvider",
|
||||
"ReportingRowPolicyProvider",
|
||||
"ReportingRowPolicyRequest",
|
||||
"ReportingRowPolicyResult",
|
||||
"capability",
|
||||
]
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Reporting database models."""
|
||||
|
||||
from govoplan_reporting.backend.db.models import (
|
||||
ReportingDefinitionGrant,
|
||||
ReportingDefinitionIdentity,
|
||||
ReportingDefinitionRevision,
|
||||
ReportingExecution,
|
||||
ReportingImportAssessment,
|
||||
ReportingPublication,
|
||||
ReportingQualityResult,
|
||||
ReportingSavedView,
|
||||
ReportingSchedule,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ReportingDefinitionGrant",
|
||||
"ReportingDefinitionIdentity",
|
||||
"ReportingDefinitionRevision",
|
||||
"ReportingExecution",
|
||||
"ReportingImportAssessment",
|
||||
"ReportingPublication",
|
||||
"ReportingQualityResult",
|
||||
"ReportingSavedView",
|
||||
"ReportingSchedule",
|
||||
]
|
||||
@@ -0,0 +1,441 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from govoplan_core.db.base import Base, TimestampMixin
|
||||
|
||||
|
||||
def new_uuid() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
class ReportingDefinitionIdentity(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_definition_identities"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
name="uq_reporting_definition_identity",
|
||||
),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_key",
|
||||
name="uq_reporting_definition_key",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_definition_catalog",
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_key",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
definition_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||
definition_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
definition_key: Mapped[str] = mapped_column(String(120), nullable=False, index=True)
|
||||
created_by: Mapped[str | None] = mapped_column(
|
||||
String(255), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
class ReportingDefinitionRevision(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_definition_revisions"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
"revision",
|
||||
name="uq_reporting_definition_revision",
|
||||
),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"idempotency_key",
|
||||
name="uq_reporting_definition_idempotency",
|
||||
),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"event_id",
|
||||
name="uq_reporting_definition_event",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_definition_current",
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
"superseded_at",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_definition_parent",
|
||||
"tenant_id",
|
||||
"parent_kind",
|
||||
"parent_id",
|
||||
"parent_revision",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_definition_list",
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"status",
|
||||
"recorded_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
identity_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("reporting_definition_identities.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
definition_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||
definition_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
definition_key: Mapped[str] = mapped_column(String(120), nullable=False, index=True)
|
||||
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
previous_revision_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("reporting_definition_revisions.id", ondelete="RESTRICT"),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
parent_kind: Mapped[str | None] = mapped_column(
|
||||
String(40), nullable=True, index=True
|
||||
)
|
||||
parent_id: Mapped[str | None] = mapped_column(
|
||||
String(255), nullable=True, index=True
|
||||
)
|
||||
parent_revision: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
name: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||
visibility: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
change_reason: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
request_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
event_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
recorded_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True
|
||||
)
|
||||
superseded_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
changed_by: Mapped[str | None] = mapped_column(
|
||||
String(255), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
class ReportingExecution(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_executions"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("tenant_id", "execution_id", name="uq_reporting_execution"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"idempotency_key",
|
||||
name="uq_reporting_execution_idempotency",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_execution_history",
|
||||
"tenant_id",
|
||||
"report_id",
|
||||
"started_at",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_execution_status",
|
||||
"tenant_id",
|
||||
"status",
|
||||
"started_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
report_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
report_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
semantic_model_id: Mapped[str] = mapped_column(
|
||||
String(255), nullable=False, index=True
|
||||
)
|
||||
semantic_model_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
dataset_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
dataset_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
request_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
parameters: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
query: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||
source_fingerprints: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
definition_hashes: Mapped[dict[str, str]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
output_hash: Mapped[str | None] = mapped_column(
|
||||
String(64), nullable=True, index=True
|
||||
)
|
||||
executor_version: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
result_schema: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
result_rows: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
total_rows: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
truncated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
||||
diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
provenance: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||
|
||||
|
||||
class ReportingDefinitionGrant(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_definition_grants"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
"subject_kind",
|
||||
"subject_id",
|
||||
name="uq_reporting_definition_grant",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_definition_grant_subject",
|
||||
"tenant_id",
|
||||
"subject_kind",
|
||||
"subject_id",
|
||||
"active",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_definition_grant_object",
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
"active",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
definition_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||
definition_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
subject_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||
subject_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
permissions: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
|
||||
active: Mapped[bool] = mapped_column(
|
||||
Boolean, default=True, nullable=False, index=True
|
||||
)
|
||||
source_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
|
||||
|
||||
class ReportingSavedView(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_saved_views"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("tenant_id", "view_id", name="uq_reporting_saved_view"),
|
||||
Index(
|
||||
"ix_reporting_saved_view_catalog",
|
||||
"tenant_id",
|
||||
"report_id",
|
||||
"owner_id",
|
||||
"shared",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
view_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
report_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
report_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
owner_kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||
owner_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
name: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||
state: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||
shared: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, nullable=False, index=True
|
||||
)
|
||||
access: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||
|
||||
|
||||
class ReportingSchedule(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_schedules"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("tenant_id", "schedule_id", name="uq_reporting_schedule"),
|
||||
Index(
|
||||
"ix_reporting_schedule_due",
|
||||
"enabled",
|
||||
"next_run_at",
|
||||
"tenant_id",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
schedule_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
report_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
report_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||
trigger_kind: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
trigger_config: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
parameters: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
query: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||
publication_target: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, default=True, nullable=False, index=True
|
||||
)
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
last_execution_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
|
||||
|
||||
class ReportingPublication(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_publications"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id", "publication_id", name="uq_reporting_publication"
|
||||
),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"idempotency_key",
|
||||
name="uq_reporting_publication_idempotency",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_publication_history",
|
||||
"tenant_id",
|
||||
"execution_id",
|
||||
"created_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
publication_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
target_capability: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
target_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||
format: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
|
||||
|
||||
class ReportingQualityResult(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_quality_results"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("tenant_id", "result_id", name="uq_reporting_quality_result"),
|
||||
Index(
|
||||
"ix_reporting_quality_history",
|
||||
"tenant_id",
|
||||
"quality_plan_id",
|
||||
"evaluated_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
result_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
quality_plan_id: Mapped[str] = mapped_column(
|
||||
String(255), nullable=False, index=True
|
||||
)
|
||||
quality_plan_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
dataset_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
dataset_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||
output_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
assertions: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
source_fingerprints: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
evaluated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True
|
||||
)
|
||||
actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
|
||||
|
||||
class ReportingImportAssessment(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_import_assessments"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id", "assessment_id", name="uq_reporting_import_assessment"
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_import_history",
|
||||
"tenant_id",
|
||||
"source_system",
|
||||
"created_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
assessment_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
source_system: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
source_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
mapping_report: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||
accepted_approximations: Mapped[list[str]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
assessed_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ReportingDefinitionGrant",
|
||||
"ReportingDefinitionIdentity",
|
||||
"ReportingDefinitionRevision",
|
||||
"ReportingExecution",
|
||||
"ReportingImportAssessment",
|
||||
"ReportingPublication",
|
||||
"ReportingQualityResult",
|
||||
"ReportingSavedView",
|
||||
"ReportingSchedule",
|
||||
]
|
||||
@@ -0,0 +1,955 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import and_, exists, func, or_
|
||||
from sqlalchemy.orm import Query, Session
|
||||
|
||||
from govoplan_core.core.events import (
|
||||
EventActorRef,
|
||||
EventObjectRef,
|
||||
EventTenantRef,
|
||||
PlatformEvent,
|
||||
emit_platform_event,
|
||||
)
|
||||
from govoplan_core.security.module_permissions import scopes_grant_compatible
|
||||
from govoplan_reporting.backend.db.models import (
|
||||
ReportingDefinitionGrant,
|
||||
ReportingDefinitionIdentity,
|
||||
ReportingDefinitionRevision,
|
||||
)
|
||||
from govoplan_reporting.backend.domain import (
|
||||
ReportingDefinitionRecord,
|
||||
definition_from_row,
|
||||
)
|
||||
from govoplan_reporting.backend.schemas import validate_definition_payload
|
||||
|
||||
|
||||
READ_SCOPE = "reporting:definition:read"
|
||||
WRITE_SCOPE = "reporting:definition:write"
|
||||
ADMIN_SCOPE = "reporting:definition:admin"
|
||||
|
||||
DEFINITION_KINDS = frozenset({"dataset", "semantic_model", "report", "quality_plan"})
|
||||
STATUS_TRANSITIONS = {
|
||||
"draft": frozenset({"active", "retired"}),
|
||||
"active": frozenset({"draft", "retired"}),
|
||||
"retired": frozenset({"draft"}),
|
||||
}
|
||||
SUBJECT_KINDS = frozenset(
|
||||
{
|
||||
"account",
|
||||
"identity",
|
||||
"group",
|
||||
"role",
|
||||
"function",
|
||||
"function_assignment",
|
||||
"organization_unit",
|
||||
"service_account",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ReportingDefinitionError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def create_definition(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
definition_key: str,
|
||||
name: str,
|
||||
description: str | None,
|
||||
status: str,
|
||||
visibility: str,
|
||||
recorded_at: datetime,
|
||||
change_reason: str,
|
||||
payload: Mapping[str, Any],
|
||||
idempotency_key: str,
|
||||
) -> ReportingDefinitionRecord:
|
||||
_require_scope(principal, WRITE_SCOPE)
|
||||
tenant_id = _principal_tenant(principal)
|
||||
kind = _definition_kind(definition_kind)
|
||||
clean_id = _required(definition_id, "Reporting definition identifier", 255)
|
||||
clean_key = _key(definition_key, "Reporting definition key")
|
||||
clean_name = _required(name, "Reporting definition name", 500)
|
||||
clean_description = _optional(description, "Reporting description", 100_000)
|
||||
clean_status = _status(status)
|
||||
clean_visibility = _visibility(visibility)
|
||||
clean_reason = _required(change_reason, "Reporting change reason", 1_000)
|
||||
_aware(recorded_at, "Reporting recorded_at")
|
||||
validated_payload = validate_definition_payload(kind, dict(payload))
|
||||
parent_kind, parent_id, parent_revision = _parent_reference(
|
||||
kind,
|
||||
validated_payload,
|
||||
)
|
||||
_validate_parent(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
child_status=clean_status,
|
||||
parent_kind=parent_kind,
|
||||
parent_id=parent_id,
|
||||
parent_revision=parent_revision,
|
||||
)
|
||||
request = {
|
||||
"definition_kind": kind,
|
||||
"definition_id": clean_id,
|
||||
"definition_key": clean_key,
|
||||
"name": clean_name,
|
||||
"description": clean_description,
|
||||
"status": clean_status,
|
||||
"visibility": clean_visibility,
|
||||
"recorded_at": recorded_at,
|
||||
"change_reason": clean_reason,
|
||||
"payload": validated_payload,
|
||||
}
|
||||
request_sha256 = _sha256(request)
|
||||
replay = _replay(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
idempotency_key=idempotency_key,
|
||||
request_sha256=request_sha256,
|
||||
)
|
||||
if replay is not None:
|
||||
return replay
|
||||
if _identity(session, tenant_id, kind, clean_id) is not None:
|
||||
raise ReportingDefinitionError("Reporting definition already exists.")
|
||||
duplicate = (
|
||||
session.query(ReportingDefinitionIdentity.id)
|
||||
.filter(
|
||||
ReportingDefinitionIdentity.tenant_id == tenant_id,
|
||||
ReportingDefinitionIdentity.definition_kind == kind,
|
||||
ReportingDefinitionIdentity.definition_key == clean_key,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if duplicate is not None:
|
||||
raise ReportingDefinitionError("Reporting definition key already exists.")
|
||||
identity = ReportingDefinitionIdentity(
|
||||
tenant_id=tenant_id,
|
||||
definition_kind=kind,
|
||||
definition_id=clean_id,
|
||||
definition_key=clean_key,
|
||||
created_by=_actor(principal),
|
||||
)
|
||||
session.add(identity)
|
||||
session.flush()
|
||||
return _write_revision(
|
||||
session,
|
||||
principal,
|
||||
identity=identity,
|
||||
current=None,
|
||||
revision=1,
|
||||
name=clean_name,
|
||||
description=clean_description,
|
||||
status=clean_status,
|
||||
visibility=clean_visibility,
|
||||
recorded_at=recorded_at,
|
||||
change_reason=clean_reason,
|
||||
payload=validated_payload,
|
||||
parent_kind=parent_kind,
|
||||
parent_id=parent_id,
|
||||
parent_revision=parent_revision,
|
||||
idempotency_key=idempotency_key,
|
||||
request_sha256=request_sha256,
|
||||
operation="created",
|
||||
)
|
||||
|
||||
|
||||
def update_definition(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
expected_revision: int,
|
||||
recorded_at: datetime,
|
||||
change_reason: str,
|
||||
idempotency_key: str,
|
||||
changes: Mapping[str, object],
|
||||
) -> ReportingDefinitionRecord:
|
||||
tenant_id = _principal_tenant(principal)
|
||||
kind = _definition_kind(definition_kind)
|
||||
current_row = _current_row(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
definition_kind=kind,
|
||||
definition_id=definition_id,
|
||||
lock=True,
|
||||
)
|
||||
if current_row is None:
|
||||
raise LookupError("Reporting definition not found.")
|
||||
if not can_write_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind=kind,
|
||||
definition_id=definition_id,
|
||||
):
|
||||
raise PermissionError("Reporting definition write access is denied.")
|
||||
current = definition_from_row(current_row)
|
||||
unknown = set(changes) - {
|
||||
"name",
|
||||
"description",
|
||||
"status",
|
||||
"visibility",
|
||||
"payload",
|
||||
}
|
||||
if unknown:
|
||||
raise ReportingDefinitionError(
|
||||
"Unsupported Reporting definition fields: " + ", ".join(sorted(unknown))
|
||||
)
|
||||
_aware(recorded_at, "Reporting recorded_at")
|
||||
clean_reason = _required(change_reason, "Reporting change reason", 1_000)
|
||||
request_sha256 = _sha256(
|
||||
{
|
||||
"definition_kind": kind,
|
||||
"definition_id": definition_id,
|
||||
"expected_revision": expected_revision,
|
||||
"recorded_at": recorded_at,
|
||||
"change_reason": clean_reason,
|
||||
"changes": changes,
|
||||
}
|
||||
)
|
||||
replay = _replay(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
idempotency_key=idempotency_key,
|
||||
request_sha256=request_sha256,
|
||||
)
|
||||
if replay is not None:
|
||||
return replay
|
||||
if current.revision != expected_revision:
|
||||
raise ReportingDefinitionError(
|
||||
"Reporting definition revision conflict: the expected revision is stale."
|
||||
)
|
||||
next_status = _status(str(changes.get("status", current.status)))
|
||||
if (
|
||||
next_status != current.status
|
||||
and next_status not in STATUS_TRANSITIONS[current.status]
|
||||
):
|
||||
raise ReportingDefinitionError(
|
||||
f"Cannot move Reporting definition from {current.status!r} to {next_status!r}."
|
||||
)
|
||||
next_payload = validate_definition_payload(
|
||||
kind,
|
||||
dict(changes.get("payload", current.payload)),
|
||||
)
|
||||
parent_kind, parent_id, parent_revision = _parent_reference(kind, next_payload)
|
||||
_validate_parent(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
child_status=next_status,
|
||||
parent_kind=parent_kind,
|
||||
parent_id=parent_id,
|
||||
parent_revision=parent_revision,
|
||||
)
|
||||
identity = _identity(session, tenant_id, kind, definition_id)
|
||||
if identity is None:
|
||||
raise ReportingDefinitionError("Reporting definition identity is missing.")
|
||||
return _write_revision(
|
||||
session,
|
||||
principal,
|
||||
identity=identity,
|
||||
current=current_row,
|
||||
revision=current.revision + 1,
|
||||
name=_required(
|
||||
changes.get("name", current.name), "Reporting definition name", 500
|
||||
),
|
||||
description=_optional(
|
||||
changes.get("description", current.description),
|
||||
"Reporting description",
|
||||
100_000,
|
||||
),
|
||||
status=next_status,
|
||||
visibility=_visibility(str(changes.get("visibility", current.visibility))),
|
||||
recorded_at=recorded_at,
|
||||
change_reason=clean_reason,
|
||||
payload=next_payload,
|
||||
parent_kind=parent_kind,
|
||||
parent_id=parent_id,
|
||||
parent_revision=parent_revision,
|
||||
idempotency_key=idempotency_key,
|
||||
request_sha256=request_sha256,
|
||||
operation="updated" if next_status == current.status else "state_changed",
|
||||
)
|
||||
|
||||
|
||||
def get_definition(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
revision: int | None = None,
|
||||
) -> ReportingDefinitionRecord | None:
|
||||
tenant_id = _principal_tenant(principal)
|
||||
kind = _definition_kind(definition_kind)
|
||||
if not can_read_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind=kind,
|
||||
definition_id=definition_id,
|
||||
):
|
||||
return None
|
||||
query = session.query(ReportingDefinitionRevision).filter(
|
||||
ReportingDefinitionRevision.tenant_id == tenant_id,
|
||||
ReportingDefinitionRevision.definition_kind == kind,
|
||||
ReportingDefinitionRevision.definition_id == definition_id,
|
||||
)
|
||||
if revision is None:
|
||||
query = query.filter(ReportingDefinitionRevision.superseded_at.is_(None))
|
||||
else:
|
||||
query = query.filter(ReportingDefinitionRevision.revision == revision)
|
||||
row = query.order_by(ReportingDefinitionRevision.revision.desc()).first()
|
||||
return definition_from_row(row) if row is not None else None
|
||||
|
||||
|
||||
def list_definitions(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kinds: Sequence[str] | None = None,
|
||||
statuses: Sequence[str] | None = None,
|
||||
query: str = "",
|
||||
offset: int = 0,
|
||||
limit: int = 100,
|
||||
) -> tuple[tuple[ReportingDefinitionRecord, ...], int]:
|
||||
_require_scope(principal, READ_SCOPE)
|
||||
if offset < 0 or not 1 <= limit <= 200:
|
||||
raise ReportingDefinitionError(
|
||||
"Reporting list offset must be non-negative and limit between 1 and 200."
|
||||
)
|
||||
kinds = tuple(dict.fromkeys(definition_kinds or DEFINITION_KINDS))
|
||||
if any(item not in DEFINITION_KINDS for item in kinds):
|
||||
raise ReportingDefinitionError("Unsupported Reporting definition kind filter.")
|
||||
statement = session.query(ReportingDefinitionRevision).filter(
|
||||
ReportingDefinitionRevision.tenant_id == _principal_tenant(principal),
|
||||
ReportingDefinitionRevision.superseded_at.is_(None),
|
||||
ReportingDefinitionRevision.definition_kind.in_(kinds),
|
||||
)
|
||||
statement = _filter_accessible(statement, principal)
|
||||
if statuses:
|
||||
statement = statement.filter(
|
||||
ReportingDefinitionRevision.status.in_(tuple(dict.fromkeys(statuses)))
|
||||
)
|
||||
clean_query = query.strip().casefold()
|
||||
if clean_query:
|
||||
pattern = f"%{clean_query}%"
|
||||
statement = statement.filter(
|
||||
or_(
|
||||
func.lower(ReportingDefinitionRevision.name).like(pattern),
|
||||
func.lower(ReportingDefinitionRevision.definition_key).like(pattern),
|
||||
func.lower(ReportingDefinitionRevision.description).like(pattern),
|
||||
)
|
||||
)
|
||||
total = int(statement.with_entities(func.count()).scalar() or 0)
|
||||
rows = (
|
||||
statement.order_by(
|
||||
ReportingDefinitionRevision.definition_kind.asc(),
|
||||
ReportingDefinitionRevision.name.asc(),
|
||||
)
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return tuple(definition_from_row(row) for row in rows), total
|
||||
|
||||
|
||||
def definition_history(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
limit: int = 100,
|
||||
) -> tuple[ReportingDefinitionRecord, ...]:
|
||||
if not can_read_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind=definition_kind,
|
||||
definition_id=definition_id,
|
||||
):
|
||||
return ()
|
||||
rows = (
|
||||
session.query(ReportingDefinitionRevision)
|
||||
.filter(
|
||||
ReportingDefinitionRevision.tenant_id == _principal_tenant(principal),
|
||||
ReportingDefinitionRevision.definition_kind == definition_kind,
|
||||
ReportingDefinitionRevision.definition_id == definition_id,
|
||||
)
|
||||
.order_by(ReportingDefinitionRevision.revision.desc())
|
||||
.limit(max(1, min(limit, 200)))
|
||||
.all()
|
||||
)
|
||||
return tuple(definition_from_row(row) for row in rows)
|
||||
|
||||
|
||||
def can_read_definition(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
) -> bool:
|
||||
if not _has_scope(principal, READ_SCOPE):
|
||||
return False
|
||||
return _can_access(
|
||||
session,
|
||||
principal,
|
||||
definition_kind=definition_kind,
|
||||
definition_id=definition_id,
|
||||
permission="read",
|
||||
)
|
||||
|
||||
|
||||
def can_write_definition(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
) -> bool:
|
||||
if not _has_scope(principal, WRITE_SCOPE):
|
||||
return False
|
||||
return _can_access(
|
||||
session,
|
||||
principal,
|
||||
definition_kind=definition_kind,
|
||||
definition_id=definition_id,
|
||||
permission="write",
|
||||
)
|
||||
|
||||
|
||||
def _write_revision(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
identity: ReportingDefinitionIdentity,
|
||||
current: ReportingDefinitionRevision | None,
|
||||
revision: int,
|
||||
name: str,
|
||||
description: str | None,
|
||||
status: str,
|
||||
visibility: str,
|
||||
recorded_at: datetime,
|
||||
change_reason: str,
|
||||
payload: dict[str, Any],
|
||||
parent_kind: str | None,
|
||||
parent_id: str | None,
|
||||
parent_revision: int | None,
|
||||
idempotency_key: str,
|
||||
request_sha256: str,
|
||||
operation: str,
|
||||
) -> ReportingDefinitionRecord:
|
||||
if current is not None:
|
||||
current.superseded_at = recorded_at
|
||||
content_hash = _sha256(
|
||||
{
|
||||
"name": name,
|
||||
"description": description,
|
||||
"status": status,
|
||||
"visibility": visibility,
|
||||
"parent_kind": parent_kind,
|
||||
"parent_id": parent_id,
|
||||
"parent_revision": parent_revision,
|
||||
"payload": payload,
|
||||
}
|
||||
)
|
||||
event_id = str(uuid.uuid4())
|
||||
row = ReportingDefinitionRevision(
|
||||
tenant_id=identity.tenant_id,
|
||||
identity_id=identity.id,
|
||||
definition_kind=identity.definition_kind,
|
||||
definition_id=identity.definition_id,
|
||||
definition_key=identity.definition_key,
|
||||
revision=revision,
|
||||
previous_revision_id=current.id if current else None,
|
||||
parent_kind=parent_kind,
|
||||
parent_id=parent_id,
|
||||
parent_revision=parent_revision,
|
||||
name=name,
|
||||
description=description,
|
||||
status=status,
|
||||
visibility=visibility,
|
||||
content_hash=content_hash,
|
||||
change_reason=change_reason,
|
||||
idempotency_key=_required(idempotency_key, "Reporting idempotency key", 255),
|
||||
request_sha256=request_sha256,
|
||||
event_id=event_id,
|
||||
recorded_at=recorded_at,
|
||||
payload=payload,
|
||||
changed_by=_actor(principal),
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
_sync_grants(session, row)
|
||||
event_type = f"reporting.{identity.definition_kind}.{operation}"
|
||||
emit_platform_event(
|
||||
session,
|
||||
PlatformEvent(
|
||||
event_id=event_id,
|
||||
type=event_type,
|
||||
module_id="reporting",
|
||||
payload={
|
||||
"definition_kind": identity.definition_kind,
|
||||
"definition_key": identity.definition_key,
|
||||
"revision": revision,
|
||||
"status": status,
|
||||
"content_hash": content_hash,
|
||||
"change_reason": change_reason,
|
||||
},
|
||||
occurred_at=recorded_at,
|
||||
actor=EventActorRef(type="account", id=_actor(principal)),
|
||||
tenant=EventTenantRef(id=identity.tenant_id),
|
||||
resource=EventObjectRef(
|
||||
type=f"reporting_{identity.definition_kind}",
|
||||
id=identity.definition_id,
|
||||
label=name,
|
||||
),
|
||||
classification="restricted" if visibility == "restricted" else "internal",
|
||||
),
|
||||
)
|
||||
return definition_from_row(row)
|
||||
|
||||
|
||||
def _sync_grants(session: Session, row: ReportingDefinitionRevision) -> None:
|
||||
access_policy = row.payload.get("access_policy")
|
||||
subjects = (
|
||||
access_policy.get("subjects", []) if isinstance(access_policy, Mapping) else []
|
||||
)
|
||||
desired: dict[tuple[str, str], list[str]] = {}
|
||||
if not isinstance(subjects, list):
|
||||
raise ReportingDefinitionError("Report access_policy subjects must be a list.")
|
||||
for subject in subjects:
|
||||
if not isinstance(subject, Mapping):
|
||||
raise ReportingDefinitionError("Report access subjects must be objects.")
|
||||
kind = str(subject.get("kind") or "")
|
||||
subject_id = str(subject.get("id") or "").strip()
|
||||
if kind not in SUBJECT_KINDS or not subject_id:
|
||||
raise ReportingDefinitionError("Report access subject is invalid.")
|
||||
permissions = tuple(
|
||||
dict.fromkeys(str(item) for item in subject.get("permissions", ["read"]))
|
||||
)
|
||||
if not permissions or set(permissions) - {"read", "write", "publish", "admin"}:
|
||||
raise ReportingDefinitionError("Report access permissions are invalid.")
|
||||
desired[(kind, subject_id)] = list(permissions)
|
||||
rows = (
|
||||
session.query(ReportingDefinitionGrant)
|
||||
.filter(
|
||||
ReportingDefinitionGrant.tenant_id == row.tenant_id,
|
||||
ReportingDefinitionGrant.definition_kind == row.definition_kind,
|
||||
ReportingDefinitionGrant.definition_id == row.definition_id,
|
||||
)
|
||||
.all()
|
||||
)
|
||||
existing = {(item.subject_kind, item.subject_id): item for item in rows}
|
||||
for key, item in existing.items():
|
||||
if key not in desired:
|
||||
item.active = False
|
||||
item.source_revision = row.revision
|
||||
for key, permissions in desired.items():
|
||||
item = existing.get(key)
|
||||
if item is None:
|
||||
session.add(
|
||||
ReportingDefinitionGrant(
|
||||
tenant_id=row.tenant_id,
|
||||
definition_kind=row.definition_kind,
|
||||
definition_id=row.definition_id,
|
||||
subject_kind=key[0],
|
||||
subject_id=key[1],
|
||||
permissions=permissions,
|
||||
active=True,
|
||||
source_revision=row.revision,
|
||||
)
|
||||
)
|
||||
else:
|
||||
item.permissions = permissions
|
||||
item.active = True
|
||||
item.source_revision = row.revision
|
||||
session.flush()
|
||||
|
||||
|
||||
def _validate_parent(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
child_status: str,
|
||||
parent_kind: str | None,
|
||||
parent_id: str | None,
|
||||
parent_revision: int | None,
|
||||
) -> None:
|
||||
if parent_kind is None:
|
||||
return
|
||||
row = (
|
||||
session.query(ReportingDefinitionRevision)
|
||||
.filter(
|
||||
ReportingDefinitionRevision.tenant_id == tenant_id,
|
||||
ReportingDefinitionRevision.definition_kind == parent_kind,
|
||||
ReportingDefinitionRevision.definition_id == parent_id,
|
||||
ReportingDefinitionRevision.revision == parent_revision,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if row is None:
|
||||
raise ReportingDefinitionError(
|
||||
f"Reporting definition references a missing {parent_kind} revision."
|
||||
)
|
||||
if child_status == "active" and row.status != "active":
|
||||
raise ReportingDefinitionError(
|
||||
f"An active Reporting definition requires an active {parent_kind} revision."
|
||||
)
|
||||
|
||||
|
||||
def _parent_reference(
|
||||
definition_kind: str,
|
||||
payload: Mapping[str, object],
|
||||
) -> tuple[str | None, str | None, int | None]:
|
||||
if definition_kind == "semantic_model":
|
||||
return "dataset", str(payload["dataset_id"]), int(payload["dataset_revision"])
|
||||
if definition_kind == "report":
|
||||
return (
|
||||
"semantic_model",
|
||||
str(payload["semantic_model_id"]),
|
||||
int(payload["semantic_model_revision"]),
|
||||
)
|
||||
if definition_kind == "quality_plan":
|
||||
return "dataset", str(payload["dataset_id"]), int(payload["dataset_revision"])
|
||||
return None, None, None
|
||||
|
||||
|
||||
def _replay(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
idempotency_key: str,
|
||||
request_sha256: str,
|
||||
) -> ReportingDefinitionRecord | None:
|
||||
key = _required(idempotency_key, "Reporting idempotency key", 255)
|
||||
row = (
|
||||
session.query(ReportingDefinitionRevision)
|
||||
.filter(
|
||||
ReportingDefinitionRevision.tenant_id == tenant_id,
|
||||
ReportingDefinitionRevision.idempotency_key == key,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
if row.request_sha256 != request_sha256:
|
||||
raise ReportingDefinitionError(
|
||||
"Reporting idempotency conflict: the key belongs to another request."
|
||||
)
|
||||
return definition_from_row(row)
|
||||
|
||||
|
||||
def _can_access(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
permission: str,
|
||||
) -> bool:
|
||||
tenant_id = _principal_tenant(principal)
|
||||
row = _current_row(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
definition_kind=definition_kind,
|
||||
definition_id=definition_id,
|
||||
lock=False,
|
||||
)
|
||||
if row is None:
|
||||
return False
|
||||
if _has_scope(principal, ADMIN_SCOPE):
|
||||
return True
|
||||
identity = _identity(session, tenant_id, definition_kind, definition_id)
|
||||
actor_ids = _actor_ids(principal)
|
||||
if identity is not None and identity.created_by in actor_ids:
|
||||
return True
|
||||
if permission == "read" and row.visibility == "tenant":
|
||||
return True
|
||||
subjects = _principal_subjects(principal)
|
||||
if not subjects:
|
||||
return False
|
||||
clauses = [
|
||||
and_(
|
||||
ReportingDefinitionGrant.subject_kind == kind,
|
||||
ReportingDefinitionGrant.subject_id == subject_id,
|
||||
)
|
||||
for kind, subject_id in subjects
|
||||
]
|
||||
grants = (
|
||||
session.query(ReportingDefinitionGrant)
|
||||
.filter(
|
||||
ReportingDefinitionGrant.tenant_id == tenant_id,
|
||||
ReportingDefinitionGrant.definition_kind == definition_kind,
|
||||
ReportingDefinitionGrant.definition_id == definition_id,
|
||||
ReportingDefinitionGrant.active.is_(True),
|
||||
or_(*clauses),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
return any(
|
||||
permission == "read"
|
||||
or permission in set(item.permissions or ())
|
||||
or "admin" in set(item.permissions or ())
|
||||
for item in grants
|
||||
)
|
||||
|
||||
|
||||
def _require_scope(principal: object, scope: str) -> None:
|
||||
if not _has_scope(principal, scope):
|
||||
raise PermissionError(f"Missing scope: {scope}")
|
||||
|
||||
|
||||
def _filter_accessible(query: Query, principal: object) -> Query:
|
||||
if _has_scope(principal, ADMIN_SCOPE):
|
||||
return query
|
||||
conditions = [ReportingDefinitionRevision.visibility == "tenant"]
|
||||
actor_ids = _actor_ids(principal)
|
||||
if actor_ids:
|
||||
conditions.append(
|
||||
exists()
|
||||
.where(
|
||||
ReportingDefinitionIdentity.id
|
||||
== ReportingDefinitionRevision.identity_id
|
||||
)
|
||||
.where(ReportingDefinitionIdentity.created_by.in_(actor_ids))
|
||||
)
|
||||
subjects = _principal_subjects(principal)
|
||||
if subjects:
|
||||
conditions.append(
|
||||
exists()
|
||||
.where(
|
||||
ReportingDefinitionGrant.tenant_id
|
||||
== ReportingDefinitionRevision.tenant_id
|
||||
)
|
||||
.where(
|
||||
ReportingDefinitionGrant.definition_kind
|
||||
== ReportingDefinitionRevision.definition_kind
|
||||
)
|
||||
.where(
|
||||
ReportingDefinitionGrant.definition_id
|
||||
== ReportingDefinitionRevision.definition_id
|
||||
)
|
||||
.where(ReportingDefinitionGrant.active.is_(True))
|
||||
.where(
|
||||
or_(
|
||||
*(
|
||||
and_(
|
||||
ReportingDefinitionGrant.subject_kind == kind,
|
||||
ReportingDefinitionGrant.subject_id == subject_id,
|
||||
)
|
||||
for kind, subject_id in subjects
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
return query.filter(or_(*conditions))
|
||||
|
||||
|
||||
def _current_row(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
lock: bool,
|
||||
) -> ReportingDefinitionRevision | None:
|
||||
query = session.query(ReportingDefinitionRevision).filter(
|
||||
ReportingDefinitionRevision.tenant_id == tenant_id,
|
||||
ReportingDefinitionRevision.definition_kind == definition_kind,
|
||||
ReportingDefinitionRevision.definition_id == definition_id,
|
||||
ReportingDefinitionRevision.superseded_at.is_(None),
|
||||
)
|
||||
if lock:
|
||||
query = query.with_for_update()
|
||||
return query.one_or_none()
|
||||
|
||||
|
||||
def _identity(
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
) -> ReportingDefinitionIdentity | None:
|
||||
return (
|
||||
session.query(ReportingDefinitionIdentity)
|
||||
.filter(
|
||||
ReportingDefinitionIdentity.tenant_id == tenant_id,
|
||||
ReportingDefinitionIdentity.definition_kind == definition_kind,
|
||||
ReportingDefinitionIdentity.definition_id == definition_id,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
|
||||
|
||||
def _principal_subjects(principal: object) -> tuple[tuple[str, str], ...]:
|
||||
values: list[tuple[str, str]] = []
|
||||
for kind, attribute in (
|
||||
("account", "account_id"),
|
||||
("identity", "identity_id"),
|
||||
("function_assignment", "acting_assignment_id"),
|
||||
("service_account", "service_account_id"),
|
||||
):
|
||||
value = getattr(principal, attribute, None)
|
||||
if str(value or "").strip():
|
||||
values.append((kind, str(value)))
|
||||
for kind, attribute in (
|
||||
("group", "group_ids"),
|
||||
("role", "role_ids"),
|
||||
("function_assignment", "function_assignment_ids"),
|
||||
):
|
||||
values.extend(
|
||||
(kind, str(value))
|
||||
for value in getattr(principal, attribute, ()) or ()
|
||||
if str(value or "").strip()
|
||||
)
|
||||
return tuple(dict.fromkeys(values))
|
||||
|
||||
|
||||
def _actor(principal: object) -> str | None:
|
||||
for value in (
|
||||
getattr(principal, "account_id", None),
|
||||
getattr(principal, "identity_id", None),
|
||||
getattr(principal, "membership_id", None),
|
||||
):
|
||||
if str(value or "").strip():
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def _actor_ids(principal: object) -> tuple[str, ...]:
|
||||
user = getattr(principal, "user", None)
|
||||
return tuple(
|
||||
dict.fromkeys(
|
||||
str(value)
|
||||
for value in (
|
||||
getattr(principal, "account_id", None),
|
||||
getattr(principal, "identity_id", None),
|
||||
getattr(principal, "membership_id", None),
|
||||
getattr(user, "id", None),
|
||||
)
|
||||
if str(value or "").strip()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _principal_tenant(principal: object) -> str:
|
||||
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
|
||||
if not tenant_id:
|
||||
raise ReportingDefinitionError(
|
||||
"Reporting operations require a tenant-bound principal."
|
||||
)
|
||||
return tenant_id
|
||||
|
||||
|
||||
def _has_scope(principal: object, scope: str) -> bool:
|
||||
method = getattr(principal, "has", None)
|
||||
if callable(method):
|
||||
return bool(method(scope))
|
||||
return scopes_grant_compatible(
|
||||
frozenset(getattr(principal, "scopes", ()) or ()),
|
||||
scope,
|
||||
)
|
||||
|
||||
|
||||
def _definition_kind(value: str) -> str:
|
||||
if value not in DEFINITION_KINDS:
|
||||
raise ReportingDefinitionError(
|
||||
f"Unsupported Reporting definition kind: {value!r}."
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def _status(value: str) -> str:
|
||||
if value not in STATUS_TRANSITIONS:
|
||||
raise ReportingDefinitionError(f"Unsupported Reporting status: {value!r}.")
|
||||
return value
|
||||
|
||||
|
||||
def _visibility(value: str) -> str:
|
||||
if value not in {"tenant", "restricted"}:
|
||||
raise ReportingDefinitionError(f"Unsupported Reporting visibility: {value!r}.")
|
||||
return value
|
||||
|
||||
|
||||
def _key(value: object, label: str) -> str:
|
||||
result = _required(value, label, 120).casefold()
|
||||
if any(
|
||||
character not in "abcdefghijklmnopqrstuvwxyz0123456789._-"
|
||||
for character in result
|
||||
):
|
||||
raise ReportingDefinitionError(
|
||||
f"{label} may contain only letters, digits, dot, underscore, and hyphen."
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _required(value: object, label: str, maximum: int) -> str:
|
||||
result = str(value or "").strip()
|
||||
if not result:
|
||||
raise ReportingDefinitionError(f"{label} is required.")
|
||||
if len(result) > maximum:
|
||||
raise ReportingDefinitionError(f"{label} is limited to {maximum} characters.")
|
||||
return result
|
||||
|
||||
|
||||
def _optional(value: object, label: str, maximum: int) -> str | None:
|
||||
result = str(value or "").strip()
|
||||
if not result:
|
||||
return None
|
||||
if len(result) > maximum:
|
||||
raise ReportingDefinitionError(f"{label} is limited to {maximum} characters.")
|
||||
return result
|
||||
|
||||
|
||||
def _aware(value: datetime, label: str) -> None:
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ReportingDefinitionError(f"{label} must include a timezone.")
|
||||
|
||||
|
||||
def _sha256(value: object) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
_json_value(value),
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _json_value(value: object) -> Any:
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
if hasattr(value, "model_dump"):
|
||||
return _json_value(value.model_dump(mode="json"))
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _json_value(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_json_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ADMIN_SCOPE",
|
||||
"DEFINITION_KINDS",
|
||||
"READ_SCOPE",
|
||||
"ReportingDefinitionError",
|
||||
"WRITE_SCOPE",
|
||||
"can_read_definition",
|
||||
"can_write_definition",
|
||||
"create_definition",
|
||||
"definition_history",
|
||||
"get_definition",
|
||||
"list_definitions",
|
||||
"update_definition",
|
||||
]
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from govoplan_reporting.backend.db.models import ReportingDefinitionRevision
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReportingDefinitionRecord:
|
||||
tenant_id: str
|
||||
definition_kind: str
|
||||
definition_id: str
|
||||
definition_key: str
|
||||
revision: int
|
||||
name: str
|
||||
description: str | None
|
||||
status: str
|
||||
visibility: str
|
||||
content_hash: str
|
||||
recorded_at: datetime
|
||||
change_reason: str
|
||||
payload: dict[str, Any]
|
||||
parent_kind: str | None = None
|
||||
parent_id: str | None = None
|
||||
parent_revision: int | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"tenant_id": self.tenant_id,
|
||||
"definition_kind": self.definition_kind,
|
||||
"definition_id": self.definition_id,
|
||||
"definition_key": self.definition_key,
|
||||
"revision": self.revision,
|
||||
"name": self.name,
|
||||
"description": self.description,
|
||||
"status": self.status,
|
||||
"visibility": self.visibility,
|
||||
"content_hash": self.content_hash,
|
||||
"parent_kind": self.parent_kind,
|
||||
"parent_id": self.parent_id,
|
||||
"parent_revision": self.parent_revision,
|
||||
"recorded_at": _datetime_text(self.recorded_at),
|
||||
"change_reason": self.change_reason,
|
||||
"payload": dict(self.payload),
|
||||
}
|
||||
|
||||
|
||||
def definition_from_row(
|
||||
row: ReportingDefinitionRevision,
|
||||
) -> ReportingDefinitionRecord:
|
||||
return ReportingDefinitionRecord(
|
||||
tenant_id=row.tenant_id,
|
||||
definition_kind=row.definition_kind,
|
||||
definition_id=row.definition_id,
|
||||
definition_key=row.definition_key,
|
||||
revision=row.revision,
|
||||
name=row.name,
|
||||
description=row.description,
|
||||
status=row.status,
|
||||
visibility=row.visibility,
|
||||
content_hash=row.content_hash,
|
||||
parent_kind=row.parent_kind,
|
||||
parent_id=row.parent_id,
|
||||
parent_revision=row.parent_revision,
|
||||
recorded_at=row.recorded_at,
|
||||
change_reason=row.change_reason,
|
||||
payload=dict(row.payload or {}),
|
||||
)
|
||||
|
||||
|
||||
def _datetime_text(value: datetime) -> str:
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=UTC)
|
||||
return value.isoformat()
|
||||
|
||||
|
||||
__all__ = ["ReportingDefinitionRecord", "definition_from_row"]
|
||||
@@ -0,0 +1,976 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import UTC, date, datetime
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.dataflows import (
|
||||
DataflowDatasetRequest,
|
||||
dataflow_dataset_output,
|
||||
)
|
||||
from govoplan_core.core.events import (
|
||||
EventActorRef,
|
||||
EventObjectRef,
|
||||
EventTenantRef,
|
||||
PlatformEvent,
|
||||
emit_platform_event,
|
||||
)
|
||||
from govoplan_core.security.module_permissions import scopes_grant_compatible
|
||||
from govoplan_core.security.time import utc_now
|
||||
from govoplan_reporting.backend.contracts import (
|
||||
CAPABILITY_REPORTING_CHART_RENDERER,
|
||||
ReportingChartRenderer,
|
||||
ReportingDatasetReadRequest,
|
||||
ReportingDatasetReadResult,
|
||||
ReportingReadModelProvider,
|
||||
ReportingRowPolicyProvider,
|
||||
ReportingRowPolicyRequest,
|
||||
capability,
|
||||
)
|
||||
from govoplan_reporting.backend.db.models import (
|
||||
ReportingExecution,
|
||||
ReportingQualityResult,
|
||||
)
|
||||
from govoplan_reporting.backend.definitions import get_definition, list_definitions
|
||||
from govoplan_reporting.backend.query_engine import (
|
||||
QUERY_ENGINE_VERSION,
|
||||
DefaultChartRenderer,
|
||||
QueryResult,
|
||||
execute_semantic_query,
|
||||
)
|
||||
from govoplan_reporting.backend.schemas import (
|
||||
DatasetDefinition,
|
||||
QualityPlanDefinition,
|
||||
ReportDefinition,
|
||||
ReportQuery,
|
||||
SemanticModelDefinition,
|
||||
)
|
||||
|
||||
|
||||
RUN_SCOPE = "reporting:report:run"
|
||||
QUALITY_SCOPE = "reporting:quality:run"
|
||||
|
||||
|
||||
class ReportingExecutionError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ReportingExecutionFailure(ReportingExecutionError):
|
||||
def __init__(self, message: str, execution_id: str) -> None:
|
||||
super().__init__(message)
|
||||
self.execution_id = execution_id
|
||||
|
||||
|
||||
class SqlReportingRunner:
|
||||
def __init__(self, registry: object | None) -> None:
|
||||
self.registry = registry
|
||||
|
||||
def execute(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
report_id: str,
|
||||
report_revision: int | None = None,
|
||||
parameters: Mapping[str, object] | None = None,
|
||||
query: ReportQuery | None = None,
|
||||
idempotency_key: str,
|
||||
) -> Mapping[str, object]:
|
||||
return execute_report(
|
||||
_session(session),
|
||||
principal,
|
||||
registry=self.registry,
|
||||
report_id=report_id,
|
||||
report_revision=report_revision,
|
||||
parameters=parameters or {},
|
||||
query=query,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
|
||||
def get_execution(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
execution_id: str,
|
||||
) -> Mapping[str, object] | None:
|
||||
return get_execution(_session(session), principal, execution_id=execution_id)
|
||||
|
||||
|
||||
def execute_report(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
report_id: str,
|
||||
report_revision: int | None,
|
||||
parameters: Mapping[str, object],
|
||||
query: ReportQuery | None,
|
||||
idempotency_key: str,
|
||||
) -> dict[str, object]:
|
||||
_require_scope(principal, RUN_SCOPE)
|
||||
report_record = get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="report",
|
||||
definition_id=report_id,
|
||||
revision=report_revision,
|
||||
)
|
||||
if report_record is None:
|
||||
raise LookupError("Reporting report definition not found.")
|
||||
if report_record.status != "active":
|
||||
raise ReportingExecutionError("Only active report definitions can run.")
|
||||
report = ReportDefinition.model_validate(report_record.payload)
|
||||
semantic_record = get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="semantic_model",
|
||||
definition_id=report.semantic_model_id,
|
||||
revision=report.semantic_model_revision,
|
||||
)
|
||||
if semantic_record is None or semantic_record.status != "active":
|
||||
raise ReportingExecutionError(
|
||||
"The report's pinned semantic model is unavailable or inactive."
|
||||
)
|
||||
semantic = SemanticModelDefinition.model_validate(semantic_record.payload)
|
||||
dataset_record = get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="dataset",
|
||||
definition_id=semantic.dataset_id,
|
||||
revision=semantic.dataset_revision,
|
||||
)
|
||||
if dataset_record is None or dataset_record.status != "active":
|
||||
raise ReportingExecutionError(
|
||||
"The report's pinned analytical dataset is unavailable or inactive."
|
||||
)
|
||||
dataset = DatasetDefinition.model_validate(dataset_record.payload)
|
||||
bound_parameters = _bind_parameters(report, parameters)
|
||||
effective_query = query or report.default_query
|
||||
clean_idempotency_key = _required(
|
||||
idempotency_key,
|
||||
"Reporting execution idempotency key",
|
||||
255,
|
||||
)
|
||||
request_sha256 = _sha256(
|
||||
{
|
||||
"report_id": report_id,
|
||||
"report_revision": report_record.revision,
|
||||
"semantic_model_id": semantic_record.definition_id,
|
||||
"semantic_model_revision": semantic_record.revision,
|
||||
"dataset_id": dataset_record.definition_id,
|
||||
"dataset_revision": dataset_record.revision,
|
||||
"parameters": bound_parameters,
|
||||
"query": effective_query.model_dump(mode="json"),
|
||||
}
|
||||
)
|
||||
replay = _execution_replay(
|
||||
session,
|
||||
tenant_id=_tenant(principal),
|
||||
idempotency_key=clean_idempotency_key,
|
||||
request_sha256=request_sha256,
|
||||
)
|
||||
if replay is not None:
|
||||
return _execution_payload(replay, report=report, registry=registry)
|
||||
started_at = utc_now()
|
||||
execution = ReportingExecution(
|
||||
tenant_id=_tenant(principal),
|
||||
execution_id=str(uuid.uuid4()),
|
||||
report_id=report_record.definition_id,
|
||||
report_revision=report_record.revision,
|
||||
semantic_model_id=semantic_record.definition_id,
|
||||
semantic_model_revision=semantic_record.revision,
|
||||
dataset_id=dataset_record.definition_id,
|
||||
dataset_revision=dataset_record.revision,
|
||||
status="running",
|
||||
idempotency_key=clean_idempotency_key,
|
||||
request_sha256=request_sha256,
|
||||
parameters=_json_value(bound_parameters),
|
||||
query=effective_query.model_dump(mode="json"),
|
||||
definition_hashes={
|
||||
"report": report_record.content_hash,
|
||||
"semantic_model": semantic_record.content_hash,
|
||||
"dataset": dataset_record.content_hash,
|
||||
},
|
||||
started_at=started_at,
|
||||
actor_id=_actor(principal),
|
||||
)
|
||||
session.add(execution)
|
||||
session.flush()
|
||||
try:
|
||||
source = _read_dataset(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
dataset=dataset,
|
||||
parameters=bound_parameters,
|
||||
)
|
||||
diagnostics = list(source.diagnostics)
|
||||
diagnostics.extend(_freshness_diagnostics(dataset, source, now=started_at))
|
||||
normalized_rows = tuple(_json_value(dict(row)) for row in source.rows)
|
||||
diagnostics.extend(_validate_schema(dataset, normalized_rows))
|
||||
authorized_rows, policy_provenance = _apply_row_policy(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
dataset_id=dataset_record.definition_id,
|
||||
dataset_revision=dataset_record.revision,
|
||||
dataset=dataset,
|
||||
rows=normalized_rows,
|
||||
)
|
||||
quality_results = _evaluate_blocking_quality_plans(
|
||||
session,
|
||||
principal,
|
||||
dataset_id=dataset_record.definition_id,
|
||||
dataset_revision=dataset_record.revision,
|
||||
rows=authorized_rows,
|
||||
output_hash=source.output_hash,
|
||||
source_fingerprints=source.source_fingerprints,
|
||||
)
|
||||
result = execute_semantic_query(authorized_rows, semantic, effective_query)
|
||||
output_hash = _sha256(
|
||||
{
|
||||
"rows": result.rows,
|
||||
"schema": result.schema,
|
||||
"query": effective_query.model_dump(mode="json"),
|
||||
"source_output_hash": source.output_hash,
|
||||
}
|
||||
)
|
||||
execution.status = "succeeded"
|
||||
execution.source_fingerprints = _json_value(source.source_fingerprints)
|
||||
execution.output_hash = output_hash
|
||||
execution.executor_version = f"{QUERY_ENGINE_VERSION}+{source.executor_version}"
|
||||
execution.result_schema = list(result.schema)
|
||||
execution.result_rows = list(result.rows)
|
||||
execution.total_rows = result.total_rows
|
||||
execution.truncated = result.truncated or source.truncated
|
||||
execution.diagnostics = _json_value(diagnostics)
|
||||
execution.provenance = {
|
||||
"source": _json_value(source.provenance),
|
||||
"source_output_hash": source.output_hash,
|
||||
"row_policy": _json_value(policy_provenance),
|
||||
"quality_result_ids": [item.result_id for item in quality_results],
|
||||
"authorized_source_rows": len(authorized_rows),
|
||||
"report_content_hash": report_record.content_hash,
|
||||
"semantic_model_content_hash": semantic_record.content_hash,
|
||||
"dataset_content_hash": dataset_record.content_hash,
|
||||
}
|
||||
execution.finished_at = utc_now()
|
||||
session.flush()
|
||||
_emit_execution_event(session, execution, report_record.name)
|
||||
return _execution_payload(execution, report=report, registry=registry)
|
||||
except Exception as exc:
|
||||
execution.status = "failed"
|
||||
execution.finished_at = utc_now()
|
||||
execution.diagnostics = [
|
||||
{
|
||||
"severity": "error",
|
||||
"code": "report_execution_failed",
|
||||
"message": str(exc),
|
||||
}
|
||||
]
|
||||
session.flush()
|
||||
_emit_execution_event(session, execution, report_record.name)
|
||||
raise ReportingExecutionFailure(str(exc), execution.execution_id) from exc
|
||||
|
||||
|
||||
def get_execution(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
execution_id: str,
|
||||
) -> dict[str, object] | None:
|
||||
row = (
|
||||
session.query(ReportingExecution)
|
||||
.filter(
|
||||
ReportingExecution.tenant_id == _tenant(principal),
|
||||
ReportingExecution.execution_id == execution_id,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
report_record = get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="report",
|
||||
definition_id=row.report_id,
|
||||
revision=row.report_revision,
|
||||
)
|
||||
if report_record is None:
|
||||
return None
|
||||
return _execution_payload(
|
||||
row,
|
||||
report=ReportDefinition.model_validate(report_record.payload),
|
||||
registry=None,
|
||||
)
|
||||
|
||||
|
||||
def list_executions(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
report_id: str,
|
||||
limit: int = 100,
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
if (
|
||||
get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="report",
|
||||
definition_id=report_id,
|
||||
)
|
||||
is None
|
||||
):
|
||||
return ()
|
||||
rows = (
|
||||
session.query(ReportingExecution)
|
||||
.filter(
|
||||
ReportingExecution.tenant_id == _tenant(principal),
|
||||
ReportingExecution.report_id == report_id,
|
||||
)
|
||||
.order_by(ReportingExecution.started_at.desc())
|
||||
.limit(max(1, min(limit, 200)))
|
||||
.all()
|
||||
)
|
||||
return tuple(_execution_payload(row, report=None, registry=None) for row in rows)
|
||||
|
||||
|
||||
def _read_dataset(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
dataset: DatasetDefinition,
|
||||
parameters: Mapping[str, object],
|
||||
) -> ReportingDatasetReadResult:
|
||||
source_parameters = {**dataset.source_parameters, **parameters}
|
||||
if dataset.source_kind == "static":
|
||||
rows = tuple(dict(item) for item in dataset.static_rows)
|
||||
return ReportingDatasetReadResult(
|
||||
rows=rows,
|
||||
total_rows=len(rows),
|
||||
truncated=False,
|
||||
output_hash=_sha256(rows),
|
||||
executor_version="reporting-static-v1",
|
||||
definition_hash=dataset.definition_hash,
|
||||
generated_at=utc_now(),
|
||||
provenance={"source_kind": "static", "source_ref": dataset.source_ref},
|
||||
)
|
||||
if dataset.source_kind == "dataflow":
|
||||
provider = dataflow_dataset_output(registry)
|
||||
if provider is None:
|
||||
raise ReportingExecutionError(
|
||||
"The Dataflow dataset provider is not enabled."
|
||||
)
|
||||
result = provider.read_output(
|
||||
session,
|
||||
principal,
|
||||
request=DataflowDatasetRequest(
|
||||
pipeline_ref=dataset.source_ref,
|
||||
revision=dataset.source_revision or 0,
|
||||
parameters=source_parameters,
|
||||
row_limit=2_000,
|
||||
expected_definition_hash=dataset.definition_hash,
|
||||
expected_source_fingerprints=tuple(
|
||||
dataset.expected_source_fingerprints
|
||||
),
|
||||
),
|
||||
)
|
||||
return ReportingDatasetReadResult(
|
||||
rows=result.rows,
|
||||
total_rows=result.total_rows,
|
||||
truncated=result.truncated,
|
||||
output_hash=result.output_hash,
|
||||
executor_version=result.executor_version,
|
||||
definition_hash=result.definition_hash,
|
||||
source_fingerprints=result.source_fingerprints,
|
||||
diagnostics=result.diagnostics,
|
||||
generated_at=result.generated_at,
|
||||
provenance=result.provenance,
|
||||
)
|
||||
provider = capability(registry, dataset.source_ref)
|
||||
if not isinstance(provider, ReportingReadModelProvider):
|
||||
raise ReportingExecutionError(
|
||||
f"Reporting read-model provider {dataset.source_ref!r} is unavailable."
|
||||
)
|
||||
return provider.read_dataset(
|
||||
session,
|
||||
principal,
|
||||
request=ReportingDatasetReadRequest(
|
||||
source_ref=dataset.source_ref,
|
||||
source_revision=dataset.source_revision,
|
||||
parameters=source_parameters,
|
||||
expected_definition_hash=dataset.definition_hash,
|
||||
expected_source_fingerprints=tuple(dataset.expected_source_fingerprints),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _apply_row_policy(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
dataset_id: str,
|
||||
dataset_revision: int,
|
||||
dataset: DatasetDefinition,
|
||||
rows: tuple[Mapping[str, object], ...],
|
||||
) -> tuple[tuple[Mapping[str, object], ...], Mapping[str, object]]:
|
||||
if dataset.row_policy_ref is None:
|
||||
return rows, {"mode": "dataset_access_only"}
|
||||
provider = capability(registry, dataset.row_policy_ref)
|
||||
if not isinstance(provider, ReportingRowPolicyProvider):
|
||||
raise ReportingExecutionError(
|
||||
f"Required row-policy provider {dataset.row_policy_ref!r} is unavailable."
|
||||
)
|
||||
result = provider.authorize_rows(
|
||||
session,
|
||||
principal,
|
||||
request=ReportingRowPolicyRequest(
|
||||
dataset_id=dataset_id,
|
||||
dataset_revision=dataset_revision,
|
||||
policy_ref=dataset.row_policy_ref,
|
||||
rows=rows,
|
||||
),
|
||||
)
|
||||
if len(result.rows) > len(rows):
|
||||
raise ReportingExecutionError(
|
||||
"A row-policy provider returned more rows than it received."
|
||||
)
|
||||
return result.rows, {
|
||||
"mode": "provider",
|
||||
"provider": dataset.row_policy_ref,
|
||||
"decision_ref": result.decision_ref,
|
||||
**dict(result.provenance),
|
||||
}
|
||||
|
||||
|
||||
def _freshness_diagnostics(
|
||||
dataset: DatasetDefinition,
|
||||
source: ReportingDatasetReadResult,
|
||||
*,
|
||||
now: datetime,
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
policy = dataset.freshness
|
||||
if (
|
||||
policy.require_source_fingerprints
|
||||
and not source.source_fingerprints
|
||||
and dataset.source_kind != "static"
|
||||
):
|
||||
raise ReportingExecutionError(
|
||||
"The analytical dataset requires source fingerprints, but the provider returned none."
|
||||
)
|
||||
if policy.max_age_seconds is None:
|
||||
return ()
|
||||
generated_at = source.generated_at
|
||||
if generated_at is None:
|
||||
message = "The dataset provider did not report a generation timestamp."
|
||||
if policy.stale_action == "block":
|
||||
raise ReportingExecutionError(message)
|
||||
return (
|
||||
{"severity": "warning", "code": "freshness_unknown", "message": message},
|
||||
)
|
||||
if generated_at.tzinfo is None:
|
||||
generated_at = generated_at.replace(tzinfo=UTC)
|
||||
age_seconds = max(0, int((now - generated_at).total_seconds()))
|
||||
if age_seconds <= policy.max_age_seconds:
|
||||
return ()
|
||||
message = (
|
||||
f"Dataset age {age_seconds}s exceeds the configured "
|
||||
f"{policy.max_age_seconds}s freshness limit."
|
||||
)
|
||||
if policy.stale_action == "block":
|
||||
raise ReportingExecutionError(message)
|
||||
if policy.stale_action == "warn":
|
||||
return ({"severity": "warning", "code": "dataset_stale", "message": message},)
|
||||
return ()
|
||||
|
||||
|
||||
def _validate_schema(
|
||||
dataset: DatasetDefinition,
|
||||
rows: Sequence[Mapping[str, object]],
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
if not dataset.fields:
|
||||
return (
|
||||
{
|
||||
"severity": "info",
|
||||
"code": "schema_inferred",
|
||||
"message": "Dataset schema is inferred because no explicit fields are pinned.",
|
||||
},
|
||||
)
|
||||
errors: list[str] = []
|
||||
for field in dataset.fields:
|
||||
for index, row in enumerate(rows):
|
||||
value = row.get(field.name)
|
||||
if value is None:
|
||||
if not field.nullable:
|
||||
errors.append(f"row {index + 1}: {field.name} is null")
|
||||
continue
|
||||
if not _matches_type(value, field.type):
|
||||
errors.append(
|
||||
f"row {index + 1}: {field.name} does not match {field.type}"
|
||||
)
|
||||
if len(errors) >= 20:
|
||||
break
|
||||
if len(errors) >= 20:
|
||||
break
|
||||
if errors:
|
||||
raise ReportingExecutionError(
|
||||
"Analytical dataset schema validation failed: " + "; ".join(errors)
|
||||
)
|
||||
return ()
|
||||
|
||||
|
||||
def _evaluate_blocking_quality_plans(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
dataset_id: str,
|
||||
dataset_revision: int,
|
||||
rows: tuple[Mapping[str, object], ...],
|
||||
output_hash: str,
|
||||
source_fingerprints: Sequence[Mapping[str, object]],
|
||||
) -> tuple[ReportingQualityResult, ...]:
|
||||
plans, _total = list_definitions(
|
||||
session,
|
||||
principal,
|
||||
definition_kinds=("quality_plan",),
|
||||
statuses=("active",),
|
||||
limit=200,
|
||||
)
|
||||
relevant = tuple(
|
||||
plan
|
||||
for plan in plans
|
||||
if plan.parent_id == dataset_id and plan.parent_revision == dataset_revision
|
||||
)
|
||||
results: list[ReportingQualityResult] = []
|
||||
for plan_record in relevant:
|
||||
plan = QualityPlanDefinition.model_validate(plan_record.payload)
|
||||
result = _record_quality_result(
|
||||
session,
|
||||
principal,
|
||||
plan_id=plan_record.definition_id,
|
||||
plan_revision=plan_record.revision,
|
||||
dataset_id=dataset_id,
|
||||
dataset_revision=dataset_revision,
|
||||
plan=plan,
|
||||
rows=rows,
|
||||
output_hash=output_hash,
|
||||
source_fingerprints=source_fingerprints,
|
||||
)
|
||||
results.append(result)
|
||||
if plan.block_report_execution and result.status == "failed":
|
||||
raise ReportingExecutionError(
|
||||
f"Quality plan {plan_record.name!r} blocked report execution."
|
||||
)
|
||||
return tuple(results)
|
||||
|
||||
|
||||
def run_quality_plan(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
quality_plan_id: str,
|
||||
quality_plan_revision: int | None,
|
||||
parameters: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
_require_scope(principal, QUALITY_SCOPE)
|
||||
record = get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="quality_plan",
|
||||
definition_id=quality_plan_id,
|
||||
revision=quality_plan_revision,
|
||||
)
|
||||
if record is None or record.status != "active":
|
||||
raise LookupError("Active Reporting quality plan not found.")
|
||||
plan = QualityPlanDefinition.model_validate(record.payload)
|
||||
dataset_record = get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="dataset",
|
||||
definition_id=plan.dataset_id,
|
||||
revision=plan.dataset_revision,
|
||||
)
|
||||
if dataset_record is None or dataset_record.status != "active":
|
||||
raise ReportingExecutionError("Quality plan dataset is unavailable.")
|
||||
dataset = DatasetDefinition.model_validate(dataset_record.payload)
|
||||
source = _read_dataset(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
dataset=dataset,
|
||||
parameters=parameters,
|
||||
)
|
||||
rows = tuple(_json_value(dict(item)) for item in source.rows)
|
||||
result = _record_quality_result(
|
||||
session,
|
||||
principal,
|
||||
plan_id=record.definition_id,
|
||||
plan_revision=record.revision,
|
||||
dataset_id=dataset_record.definition_id,
|
||||
dataset_revision=dataset_record.revision,
|
||||
plan=plan,
|
||||
rows=rows,
|
||||
output_hash=source.output_hash,
|
||||
source_fingerprints=source.source_fingerprints,
|
||||
)
|
||||
return _quality_payload(result)
|
||||
|
||||
|
||||
def _record_quality_result(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
plan_id: str,
|
||||
plan_revision: int,
|
||||
dataset_id: str,
|
||||
dataset_revision: int,
|
||||
plan: QualityPlanDefinition,
|
||||
rows: Sequence[Mapping[str, object]],
|
||||
output_hash: str,
|
||||
source_fingerprints: Sequence[Mapping[str, object]],
|
||||
) -> ReportingQualityResult:
|
||||
assertions = [_evaluate_assertion(item, rows) for item in plan.assertions]
|
||||
failed = any(
|
||||
not item["passed"] and item["severity"] in {"error", "blocker"}
|
||||
for item in assertions
|
||||
)
|
||||
warning = any(not item["passed"] for item in assertions)
|
||||
row = ReportingQualityResult(
|
||||
tenant_id=_tenant(principal),
|
||||
result_id=str(uuid.uuid4()),
|
||||
quality_plan_id=plan_id,
|
||||
quality_plan_revision=plan_revision,
|
||||
dataset_id=dataset_id,
|
||||
dataset_revision=dataset_revision,
|
||||
status="failed" if failed else "warning" if warning else "passed",
|
||||
output_hash=output_hash,
|
||||
assertions=assertions,
|
||||
source_fingerprints=_json_value(source_fingerprints),
|
||||
evaluated_at=utc_now(),
|
||||
actor_id=_actor(principal),
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return row
|
||||
|
||||
|
||||
def _evaluate_assertion(
|
||||
assertion, rows: Sequence[Mapping[str, object]]
|
||||
) -> dict[str, object]:
|
||||
field = assertion.field
|
||||
config = assertion.config
|
||||
failures = 0
|
||||
if assertion.kind == "not_null":
|
||||
failures = sum(item.get(field or "") is None for item in rows)
|
||||
elif assertion.kind == "unique":
|
||||
values = [item.get(field or "") for item in rows]
|
||||
failures = len(values) - len({_stable_value(item) for item in values})
|
||||
elif assertion.kind == "range":
|
||||
minimum = config.get("minimum")
|
||||
maximum = config.get("maximum")
|
||||
failures = sum(
|
||||
value is not None
|
||||
and (
|
||||
(minimum is not None and value < minimum)
|
||||
or (maximum is not None and value > maximum)
|
||||
)
|
||||
for value in (item.get(field or "") for item in rows)
|
||||
)
|
||||
elif assertion.kind == "accepted_values":
|
||||
accepted = {_stable_value(item) for item in config.get("values", [])}
|
||||
failures = sum(
|
||||
_stable_value(item.get(field or "")) not in accepted for item in rows
|
||||
)
|
||||
elif assertion.kind == "row_count":
|
||||
minimum = int(config.get("minimum", 0))
|
||||
maximum = int(config.get("maximum", 2_000_000_000))
|
||||
failures = 0 if minimum <= len(rows) <= maximum else 1
|
||||
elif assertion.kind == "comparison":
|
||||
other_field = str(config.get("other_field") or "")
|
||||
operator = str(config.get("operator") or "eq")
|
||||
if not field or not other_field or operator not in {"eq", "ne"}:
|
||||
raise ReportingExecutionError("Comparison quality assertion is invalid.")
|
||||
failures = sum(
|
||||
(item.get(field) == item.get(other_field)) != (operator == "eq")
|
||||
for item in rows
|
||||
)
|
||||
else:
|
||||
raise ReportingExecutionError(
|
||||
f"Unsupported quality assertion: {assertion.kind!r}."
|
||||
)
|
||||
return {
|
||||
"key": assertion.key,
|
||||
"kind": assertion.kind,
|
||||
"severity": assertion.severity,
|
||||
"passed": failures == 0,
|
||||
"failure_count": failures,
|
||||
"row_count": len(rows),
|
||||
}
|
||||
|
||||
|
||||
def _execution_payload(
|
||||
row: ReportingExecution,
|
||||
*,
|
||||
report: ReportDefinition | None,
|
||||
registry: object | None,
|
||||
) -> dict[str, object]:
|
||||
payload: dict[str, object] = {
|
||||
"execution_id": row.execution_id,
|
||||
"report_id": row.report_id,
|
||||
"report_revision": row.report_revision,
|
||||
"semantic_model_id": row.semantic_model_id,
|
||||
"semantic_model_revision": row.semantic_model_revision,
|
||||
"dataset_id": row.dataset_id,
|
||||
"dataset_revision": row.dataset_revision,
|
||||
"status": row.status,
|
||||
"parameters": dict(row.parameters or {}),
|
||||
"query": dict(row.query or {}),
|
||||
"definition_hashes": dict(row.definition_hashes or {}),
|
||||
"source_fingerprints": list(row.source_fingerprints or []),
|
||||
"output_hash": row.output_hash,
|
||||
"executor_version": row.executor_version,
|
||||
"schema": list(row.result_schema or []),
|
||||
"rows": list(row.result_rows or []),
|
||||
"total_rows": row.total_rows,
|
||||
"truncated": row.truncated,
|
||||
"diagnostics": list(row.diagnostics or []),
|
||||
"provenance": dict(row.provenance or {}),
|
||||
"started_at": _datetime_text(row.started_at),
|
||||
"finished_at": _datetime_text(row.finished_at),
|
||||
"actor_id": row.actor_id,
|
||||
}
|
||||
if row.status == "succeeded" and report is not None:
|
||||
result = QueryResult(
|
||||
rows=tuple(dict(item) for item in row.result_rows or []),
|
||||
total_rows=row.total_rows,
|
||||
schema=tuple(dict(item) for item in row.result_schema or []),
|
||||
truncated=row.truncated,
|
||||
)
|
||||
renderer = capability(registry, CAPABILITY_REPORTING_CHART_RENDERER)
|
||||
if not isinstance(renderer, ReportingChartRenderer):
|
||||
renderer = DefaultChartRenderer()
|
||||
payload["visualization"] = dict(
|
||||
renderer.render(visualization=report.visualization, result=result)
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _quality_payload(row: ReportingQualityResult) -> dict[str, object]:
|
||||
return {
|
||||
"result_id": row.result_id,
|
||||
"quality_plan_id": row.quality_plan_id,
|
||||
"quality_plan_revision": row.quality_plan_revision,
|
||||
"dataset_id": row.dataset_id,
|
||||
"dataset_revision": row.dataset_revision,
|
||||
"status": row.status,
|
||||
"output_hash": row.output_hash,
|
||||
"assertions": list(row.assertions or []),
|
||||
"source_fingerprints": list(row.source_fingerprints or []),
|
||||
"evaluated_at": _datetime_text(row.evaluated_at),
|
||||
}
|
||||
|
||||
|
||||
def _execution_replay(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
idempotency_key: str,
|
||||
request_sha256: str,
|
||||
) -> ReportingExecution | None:
|
||||
row = (
|
||||
session.query(ReportingExecution)
|
||||
.filter(
|
||||
ReportingExecution.tenant_id == tenant_id,
|
||||
ReportingExecution.idempotency_key == idempotency_key,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if row is not None and row.request_sha256 != request_sha256:
|
||||
raise ReportingExecutionError(
|
||||
"Reporting execution idempotency conflict: the key belongs to another request."
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
def _bind_parameters(
|
||||
report: ReportDefinition,
|
||||
supplied: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
definitions = {item.key: item for item in report.parameters}
|
||||
unknown = set(supplied) - set(definitions)
|
||||
if unknown:
|
||||
raise ReportingExecutionError(
|
||||
"Unknown report parameters: " + ", ".join(sorted(unknown))
|
||||
)
|
||||
result: dict[str, object] = {}
|
||||
for key, definition in definitions.items():
|
||||
value = supplied.get(key, definition.default)
|
||||
if value is None and definition.required:
|
||||
raise ReportingExecutionError(f"Report parameter {key!r} is required.")
|
||||
if definition.allowed_values and value not in definition.allowed_values:
|
||||
raise ReportingExecutionError(
|
||||
f"Report parameter {key!r} is outside its allowed values."
|
||||
)
|
||||
if value is not None:
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _emit_execution_event(
|
||||
session: Session,
|
||||
execution: ReportingExecution,
|
||||
report_name: str,
|
||||
) -> None:
|
||||
emit_platform_event(
|
||||
session,
|
||||
PlatformEvent(
|
||||
type=f"reporting.execution.{execution.status}",
|
||||
module_id="reporting",
|
||||
payload={
|
||||
"execution_id": execution.execution_id,
|
||||
"report_id": execution.report_id,
|
||||
"report_revision": execution.report_revision,
|
||||
"output_hash": execution.output_hash,
|
||||
"total_rows": execution.total_rows,
|
||||
"truncated": execution.truncated,
|
||||
},
|
||||
occurred_at=execution.finished_at or execution.started_at,
|
||||
actor=EventActorRef(type="account", id=execution.actor_id),
|
||||
tenant=EventTenantRef(id=execution.tenant_id),
|
||||
resource=EventObjectRef(
|
||||
type="report_execution",
|
||||
id=execution.execution_id,
|
||||
label=report_name,
|
||||
),
|
||||
classification="internal",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _matches_type(value: object, field_type: str) -> bool:
|
||||
if field_type == "string":
|
||||
return isinstance(value, str)
|
||||
if field_type == "integer":
|
||||
return isinstance(value, int) and not isinstance(value, bool)
|
||||
if field_type == "number":
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
||||
if field_type == "boolean":
|
||||
return isinstance(value, bool)
|
||||
if field_type in {"date", "datetime"}:
|
||||
if isinstance(value, (date, datetime)):
|
||||
return True
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
return False
|
||||
if field_type == "json":
|
||||
return isinstance(value, (dict, list, tuple))
|
||||
return False
|
||||
|
||||
|
||||
def _stable_value(value: object) -> str:
|
||||
return json.dumps(_json_value(value), sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _required(value: object, label: str, maximum: int) -> str:
|
||||
result = str(value or "").strip()
|
||||
if not result:
|
||||
raise ReportingExecutionError(f"{label} is required.")
|
||||
if len(result) > maximum:
|
||||
raise ReportingExecutionError(f"{label} is limited to {maximum} characters.")
|
||||
return result
|
||||
|
||||
|
||||
def _require_scope(principal: object, scope: str) -> None:
|
||||
method = getattr(principal, "has", None)
|
||||
allowed = (
|
||||
bool(method(scope))
|
||||
if callable(method)
|
||||
else scopes_grant_compatible(
|
||||
frozenset(getattr(principal, "scopes", ()) or ()), scope
|
||||
)
|
||||
)
|
||||
if not allowed:
|
||||
raise PermissionError(f"Missing scope: {scope}")
|
||||
|
||||
|
||||
def _tenant(principal: object) -> str:
|
||||
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
|
||||
if not tenant_id:
|
||||
raise ReportingExecutionError(
|
||||
"Reporting execution requires a tenant-bound principal."
|
||||
)
|
||||
return tenant_id
|
||||
|
||||
|
||||
def _actor(principal: object) -> str | None:
|
||||
for value in (
|
||||
getattr(principal, "account_id", None),
|
||||
getattr(principal, "identity_id", None),
|
||||
getattr(principal, "membership_id", None),
|
||||
):
|
||||
if str(value or "").strip():
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def _session(value: object) -> Session:
|
||||
if not isinstance(value, Session):
|
||||
raise TypeError("Reporting execution requires a SQLAlchemy session.")
|
||||
return value
|
||||
|
||||
|
||||
def _sha256(value: object) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
_json_value(value),
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _json_value(value: object) -> Any:
|
||||
if isinstance(value, (date, datetime)):
|
||||
return value.isoformat()
|
||||
if hasattr(value, "model_dump"):
|
||||
return _json_value(value.model_dump(mode="json"))
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _json_value(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_json_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _datetime_text(value: datetime | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=UTC)
|
||||
return value.isoformat()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QUALITY_SCOPE",
|
||||
"RUN_SCOPE",
|
||||
"ReportingExecutionError",
|
||||
"ReportingExecutionFailure",
|
||||
"SqlReportingRunner",
|
||||
"execute_report",
|
||||
"get_execution",
|
||||
"list_executions",
|
||||
"run_quality_plan",
|
||||
]
|
||||
@@ -0,0 +1,478 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from govoplan_core.core.access import (
|
||||
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
||||
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
||||
)
|
||||
from govoplan_core.core.dataflows import CAPABILITY_DATAFLOW_DATASET_OUTPUT
|
||||
from govoplan_core.core.module_guards import (
|
||||
drop_table_retirement_provider,
|
||||
persistent_table_uninstall_guard,
|
||||
)
|
||||
from govoplan_core.core.modules import (
|
||||
CapabilityDocumentation,
|
||||
DocumentationLink,
|
||||
DocumentationTopic,
|
||||
FrontendModule,
|
||||
FrontendRoute,
|
||||
MigrationSpec,
|
||||
ModuleContext,
|
||||
ModuleInterfaceProvider,
|
||||
ModuleInterfaceRequirement,
|
||||
ModuleManifest,
|
||||
NavItem,
|
||||
PermissionDefinition,
|
||||
RoleTemplate,
|
||||
)
|
||||
from govoplan_core.core.provider_governance import (
|
||||
ModuleArchitectureDeclaration,
|
||||
ModuleArchitectureDocumentation,
|
||||
ModuleMaturityEvidence,
|
||||
)
|
||||
from govoplan_core.core.search import SearchSourceProviderRegistration
|
||||
from govoplan_core.core.views import ViewSurface
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_reporting.backend.acl import ReportingScopeAclProvider
|
||||
from govoplan_reporting.backend.contracts import (
|
||||
CAPABILITY_REPORTING_CHART_RENDERER,
|
||||
CAPABILITY_REPORTING_REGISTRY,
|
||||
CAPABILITY_REPORTING_RUNNER,
|
||||
CAPABILITY_REPORTING_SCHEDULER,
|
||||
)
|
||||
from govoplan_reporting.backend.db import models as reporting_models
|
||||
from govoplan_reporting.backend.definitions import (
|
||||
ADMIN_SCOPE,
|
||||
READ_SCOPE,
|
||||
WRITE_SCOPE,
|
||||
)
|
||||
from govoplan_reporting.backend.execution import (
|
||||
QUALITY_SCOPE,
|
||||
RUN_SCOPE,
|
||||
SqlReportingRunner,
|
||||
)
|
||||
from govoplan_reporting.backend.operations import (
|
||||
IMPORT_SCOPE,
|
||||
PUBLISH_SCOPE,
|
||||
SCHEDULE_SCOPE,
|
||||
SqlReportingScheduler,
|
||||
)
|
||||
from govoplan_reporting.backend.query_engine import DefaultChartRenderer
|
||||
from govoplan_reporting.backend.registry import SqlReportingRegistry
|
||||
from govoplan_reporting.backend.search_source import create_reporting_search_source
|
||||
|
||||
|
||||
MODULE_ID = "reporting"
|
||||
MODULE_NAME = "Reporting"
|
||||
MODULE_VERSION = "0.1.14"
|
||||
|
||||
|
||||
def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
|
||||
module_id, resource, action = scope.split(":", 2)
|
||||
return PermissionDefinition(
|
||||
scope=scope,
|
||||
label=label,
|
||||
description=description,
|
||||
category=MODULE_NAME,
|
||||
level="tenant",
|
||||
module_id=module_id,
|
||||
resource=resource,
|
||||
action=action,
|
||||
)
|
||||
|
||||
|
||||
PERMISSIONS = (
|
||||
_permission(
|
||||
READ_SCOPE,
|
||||
"View reporting definitions",
|
||||
"Read accessible datasets, semantic models, reports, and quality plans.",
|
||||
),
|
||||
_permission(
|
||||
WRITE_SCOPE,
|
||||
"Manage reporting definitions",
|
||||
"Create immutable revisions of Reporting definitions.",
|
||||
),
|
||||
_permission(
|
||||
ADMIN_SCOPE,
|
||||
"Administer reporting",
|
||||
"Manage restricted definitions and Reporting governance.",
|
||||
),
|
||||
_permission(
|
||||
RUN_SCOPE,
|
||||
"Run reports",
|
||||
"Execute accessible report revisions and export their authorized result.",
|
||||
),
|
||||
_permission(
|
||||
PUBLISH_SCOPE,
|
||||
"Publish reports",
|
||||
"Send successful report results to configured publication providers.",
|
||||
),
|
||||
_permission(
|
||||
SCHEDULE_SCOPE,
|
||||
"Schedule reports",
|
||||
"Create schedules and dispatch due report runs.",
|
||||
),
|
||||
_permission(
|
||||
QUALITY_SCOPE,
|
||||
"Run report quality plans",
|
||||
"Evaluate dataset quality plans and inspect evidence.",
|
||||
),
|
||||
_permission(
|
||||
IMPORT_SCOPE,
|
||||
"Assess report imports",
|
||||
"Assess external BI metadata and accept bounded approximations.",
|
||||
),
|
||||
)
|
||||
|
||||
ROLE_TEMPLATES = (
|
||||
RoleTemplate(
|
||||
slug="reporting_analyst",
|
||||
name="Reporting analyst",
|
||||
description="Define semantic reports, run them, and save analytical views.",
|
||||
permissions=(READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, QUALITY_SCOPE),
|
||||
),
|
||||
RoleTemplate(
|
||||
slug="reporting_publisher",
|
||||
name="Reporting publisher",
|
||||
description="Run, schedule, export, and publish accessible reports.",
|
||||
permissions=(READ_SCOPE, RUN_SCOPE, PUBLISH_SCOPE, SCHEDULE_SCOPE),
|
||||
),
|
||||
RoleTemplate(
|
||||
slug="reporting_administrator",
|
||||
name="Reporting administrator",
|
||||
description="Administer definitions, imports, quality, schedules, and publications.",
|
||||
permissions=tuple(item.scope for item in PERMISSIONS),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _router(context: ModuleContext):
|
||||
from govoplan_reporting.backend.router import create_router
|
||||
|
||||
return create_router(context.registry)
|
||||
|
||||
|
||||
def _registry(context: ModuleContext) -> SqlReportingRegistry:
|
||||
del context
|
||||
return SqlReportingRegistry()
|
||||
|
||||
|
||||
def _runner(context: ModuleContext) -> SqlReportingRunner:
|
||||
return SqlReportingRunner(context.registry)
|
||||
|
||||
|
||||
def _scheduler(context: ModuleContext) -> SqlReportingScheduler:
|
||||
return SqlReportingScheduler(context.registry)
|
||||
|
||||
|
||||
def _chart_renderer(context: ModuleContext) -> DefaultChartRenderer:
|
||||
del context
|
||||
return DefaultChartRenderer()
|
||||
|
||||
|
||||
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
|
||||
definitions = (
|
||||
session.query(reporting_models.ReportingDefinitionRevision)
|
||||
.filter(
|
||||
reporting_models.ReportingDefinitionRevision.tenant_id == tenant_id,
|
||||
reporting_models.ReportingDefinitionRevision.superseded_at.is_(None),
|
||||
)
|
||||
.count()
|
||||
)
|
||||
reports = (
|
||||
session.query(reporting_models.ReportingDefinitionRevision)
|
||||
.filter(
|
||||
reporting_models.ReportingDefinitionRevision.tenant_id == tenant_id,
|
||||
reporting_models.ReportingDefinitionRevision.definition_kind == "report",
|
||||
reporting_models.ReportingDefinitionRevision.superseded_at.is_(None),
|
||||
)
|
||||
.count()
|
||||
)
|
||||
executions = (
|
||||
session.query(reporting_models.ReportingExecution)
|
||||
.filter(reporting_models.ReportingExecution.tenant_id == tenant_id)
|
||||
.count()
|
||||
)
|
||||
schedules = (
|
||||
session.query(reporting_models.ReportingSchedule)
|
||||
.filter(
|
||||
reporting_models.ReportingSchedule.tenant_id == tenant_id,
|
||||
reporting_models.ReportingSchedule.enabled.is_(True),
|
||||
)
|
||||
.count()
|
||||
)
|
||||
return {
|
||||
"reporting_definitions": definitions,
|
||||
"reports": reports,
|
||||
"report_executions": executions,
|
||||
"active_report_schedules": schedules,
|
||||
}
|
||||
|
||||
|
||||
manifest = ModuleManifest(
|
||||
id=MODULE_ID,
|
||||
name=MODULE_NAME,
|
||||
version=MODULE_VERSION,
|
||||
dependencies=("access",),
|
||||
optional_dependencies=(
|
||||
"dataflow",
|
||||
"datasources",
|
||||
"connectors",
|
||||
"dashboard",
|
||||
"files",
|
||||
"mail",
|
||||
"templates",
|
||||
"workflow_engine",
|
||||
"policy",
|
||||
"search",
|
||||
"notifications",
|
||||
),
|
||||
required_capabilities=(
|
||||
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
||||
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
||||
),
|
||||
optional_capabilities=(CAPABILITY_DATAFLOW_DATASET_OUTPUT,),
|
||||
permissions=PERMISSIONS,
|
||||
role_templates=ROLE_TEMPLATES,
|
||||
route_factory=_router,
|
||||
nav_items=(
|
||||
NavItem(
|
||||
path="/reporting",
|
||||
label="Reporting",
|
||||
icon="clipboard-pen-line",
|
||||
required_any=(READ_SCOPE,),
|
||||
order=74,
|
||||
),
|
||||
),
|
||||
frontend=FrontendModule(
|
||||
module_id=MODULE_ID,
|
||||
package_name="@govoplan/reporting-webui",
|
||||
routes=(
|
||||
FrontendRoute(
|
||||
path="/reporting",
|
||||
component="ReportingPage",
|
||||
required_any=(READ_SCOPE,),
|
||||
order=74,
|
||||
),
|
||||
),
|
||||
nav_items=(
|
||||
NavItem(
|
||||
path="/reporting",
|
||||
label="Reporting",
|
||||
icon="clipboard-pen-line",
|
||||
required_any=(READ_SCOPE,),
|
||||
order=74,
|
||||
),
|
||||
),
|
||||
view_surfaces=(
|
||||
ViewSurface(
|
||||
id="reporting.navigation",
|
||||
module_id=MODULE_ID,
|
||||
kind="navigation",
|
||||
label="Reporting navigation",
|
||||
order=10,
|
||||
),
|
||||
ViewSurface(
|
||||
id="reporting.workspace",
|
||||
module_id=MODULE_ID,
|
||||
kind="route",
|
||||
label="Reporting workspace",
|
||||
order=20,
|
||||
),
|
||||
ViewSurface(
|
||||
id="reporting.parameters",
|
||||
module_id=MODULE_ID,
|
||||
kind="section",
|
||||
label="Report parameters and filters",
|
||||
parent_id="reporting.workspace",
|
||||
order=30,
|
||||
),
|
||||
ViewSurface(
|
||||
id="reporting.results",
|
||||
module_id=MODULE_ID,
|
||||
kind="section",
|
||||
label="Authorized report results",
|
||||
parent_id="reporting.workspace",
|
||||
order=40,
|
||||
),
|
||||
),
|
||||
),
|
||||
provides_interfaces=(
|
||||
ModuleInterfaceProvider(name="reporting.registry", version="0.1.0"),
|
||||
ModuleInterfaceProvider(name="reporting.runner", version="0.1.0"),
|
||||
ModuleInterfaceProvider(name="reporting.scheduler", version="0.1.0"),
|
||||
ModuleInterfaceProvider(name="reporting.chart_renderer", version="0.1.0"),
|
||||
),
|
||||
requires_interfaces=(
|
||||
ModuleInterfaceRequirement(
|
||||
name="dataflow.dataset_output",
|
||||
version_min="0.1.0",
|
||||
version_max_exclusive="0.2.0",
|
||||
optional=True,
|
||||
),
|
||||
),
|
||||
capability_factories={
|
||||
CAPABILITY_REPORTING_REGISTRY: _registry,
|
||||
CAPABILITY_REPORTING_RUNNER: _runner,
|
||||
CAPABILITY_REPORTING_SCHEDULER: _scheduler,
|
||||
CAPABILITY_REPORTING_CHART_RENDERER: _chart_renderer,
|
||||
},
|
||||
capability_documentation={
|
||||
CAPABILITY_REPORTING_REGISTRY: CapabilityDocumentation(
|
||||
label="Reporting definition registry",
|
||||
summary="Stores versioned datasets, semantic models, reports, and quality plans.",
|
||||
contract_version="0.1.0",
|
||||
),
|
||||
CAPABILITY_REPORTING_RUNNER: CapabilityDocumentation(
|
||||
label="Governed report runner",
|
||||
summary="Executes a pinned report graph over an authorized provider-owned dataset.",
|
||||
contract_version="0.1.0",
|
||||
),
|
||||
CAPABILITY_REPORTING_SCHEDULER: CapabilityDocumentation(
|
||||
label="Report schedule dispatcher",
|
||||
summary="Claims due report schedules and records run/publication evidence.",
|
||||
contract_version="0.1.0",
|
||||
),
|
||||
CAPABILITY_REPORTING_CHART_RENDERER: CapabilityDocumentation(
|
||||
label="Report chart renderer",
|
||||
summary="Builds provider-neutral chart models with an accessible tabular fallback.",
|
||||
contract_version="0.1.0",
|
||||
),
|
||||
},
|
||||
search_sources=(
|
||||
SearchSourceProviderRegistration(
|
||||
id="reporting.reports",
|
||||
factory=create_reporting_search_source,
|
||||
),
|
||||
),
|
||||
migration_spec=MigrationSpec(
|
||||
module_id=MODULE_ID,
|
||||
metadata=Base.metadata,
|
||||
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
||||
retirement_supported=True,
|
||||
retirement_provider=drop_table_retirement_provider(
|
||||
reporting_models.ReportingImportAssessment,
|
||||
reporting_models.ReportingQualityResult,
|
||||
reporting_models.ReportingPublication,
|
||||
reporting_models.ReportingSchedule,
|
||||
reporting_models.ReportingSavedView,
|
||||
reporting_models.ReportingDefinitionGrant,
|
||||
reporting_models.ReportingExecution,
|
||||
reporting_models.ReportingDefinitionRevision,
|
||||
reporting_models.ReportingDefinitionIdentity,
|
||||
label="Reporting",
|
||||
),
|
||||
retirement_notes=(
|
||||
"Destructive retirement requires a database snapshot and removes "
|
||||
"Reporting definitions, results, quality evidence, schedules, and publications."
|
||||
),
|
||||
),
|
||||
uninstall_guard_providers=(
|
||||
persistent_table_uninstall_guard(
|
||||
reporting_models.ReportingDefinitionIdentity,
|
||||
reporting_models.ReportingDefinitionRevision,
|
||||
reporting_models.ReportingDefinitionGrant,
|
||||
reporting_models.ReportingExecution,
|
||||
reporting_models.ReportingSavedView,
|
||||
reporting_models.ReportingSchedule,
|
||||
reporting_models.ReportingPublication,
|
||||
reporting_models.ReportingQualityResult,
|
||||
reporting_models.ReportingImportAssessment,
|
||||
label="Reporting",
|
||||
),
|
||||
),
|
||||
resource_acl_providers=(
|
||||
ReportingScopeAclProvider("analytical_dataset"),
|
||||
ReportingScopeAclProvider("semantic_model"),
|
||||
ReportingScopeAclProvider("report"),
|
||||
ReportingScopeAclProvider("report_execution"),
|
||||
),
|
||||
tenant_summary_providers=(_tenant_summary,),
|
||||
documentation=(
|
||||
DocumentationTopic(
|
||||
id="reporting.governed-bi",
|
||||
title="Governed reporting and semantic BI",
|
||||
summary="Build reproducible reports over provider-owned datasets without bypassing module or row-level access.",
|
||||
body=(
|
||||
"Reporting pins dataset, semantic-model, and report revisions. Runs retain "
|
||||
"definition hashes, source fingerprints, policy provenance, quality evidence, "
|
||||
"authorized result rows, diagnostics, and output hashes. Safe dimensions, "
|
||||
"aggregations, typed expressions, filters, pivots, saved views, chart models, "
|
||||
"schedules, exports, and publication providers replace unchecked SQL in the "
|
||||
"presentation layer. Dataflow and module read models remain source owners."
|
||||
),
|
||||
layer="available",
|
||||
documentation_types=("admin", "user"),
|
||||
audience=("user", "operator", "module_admin", "product_owner"),
|
||||
links=(
|
||||
DocumentationLink(
|
||||
label="Reporting module boundary",
|
||||
href="govoplan-reporting/docs/REPORTING_BOUNDARY.md",
|
||||
kind="repository",
|
||||
),
|
||||
DocumentationLink(
|
||||
label="SuperX capability assessment",
|
||||
href="govoplan-reporting/docs/SUPERX_CAPABILITY_ASSESSMENT.md",
|
||||
kind="repository",
|
||||
),
|
||||
DocumentationLink(
|
||||
label="Reporting user guide",
|
||||
href="govoplan-reporting/docs/USER_GUIDE.md",
|
||||
kind="repository",
|
||||
),
|
||||
DocumentationLink(
|
||||
label="Reporting administration guide",
|
||||
href="govoplan-reporting/docs/ADMIN_GUIDE.md",
|
||||
kind="repository",
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
architecture=ModuleArchitectureDeclaration(
|
||||
layer="data_reporting_integration",
|
||||
kind="domain",
|
||||
maturity="vertical_slice",
|
||||
evidence=(
|
||||
ModuleMaturityEvidence(
|
||||
kind="test",
|
||||
reference="tests/test_reporting_service.py",
|
||||
summary="Proves revision pinning, safe semantic execution, quality gates, access, replay, exports, and import blocking.",
|
||||
),
|
||||
ModuleMaturityEvidence(
|
||||
kind="documentation",
|
||||
reference="docs/REPORTING_BOUNDARY.md",
|
||||
summary="Defines governed analytical source, semantic, execution, and publication ownership.",
|
||||
),
|
||||
),
|
||||
known_limits=(
|
||||
"Dataflow is the first live dataset adapter; additional module read models use the provider-neutral contract.",
|
||||
"Direct browser export supports CSV and JSON; XLSX, PDF, Files, Mail, and DMS delivery require an optional publication provider.",
|
||||
"Import assessment produces blocking diagnostics but does not execute source SQL or automatically activate generated definitions.",
|
||||
"The initial chart provider emits a renderer-neutral model and accessible table; richer visual renderers remain replaceable adapters.",
|
||||
),
|
||||
owned_concepts=(
|
||||
"analytical dataset binding",
|
||||
"semantic dimension hierarchy and measure",
|
||||
"report definition and saved view",
|
||||
"report execution and publication evidence",
|
||||
"report quality plan and import assessment",
|
||||
),
|
||||
non_owned_concepts=(
|
||||
"raw datasource ingestion",
|
||||
"data transformation pipeline",
|
||||
"source module authorization",
|
||||
"template document rendering",
|
||||
"file or DMS storage",
|
||||
),
|
||||
documentation=ModuleArchitectureDocumentation(
|
||||
operations=("docs/OPERATIONS.md",),
|
||||
recovery=("docs/OPERATIONS.md",),
|
||||
security=("docs/OPERATIONS.md",),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_manifest() -> ModuleManifest:
|
||||
return manifest
|
||||
@@ -0,0 +1 @@
|
||||
"""Reporting Alembic revisions."""
|
||||
@@ -0,0 +1 @@
|
||||
"""Reporting migration versions."""
|
||||
+477
@@ -0,0 +1,477 @@
|
||||
"""v0.1.14 governed Reporting baseline.
|
||||
|
||||
Revision ID: e5b2c9d4f7a1
|
||||
Revises: None
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "e5b2c9d4f7a1"
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = "4f2a9c8e7b6d"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"reporting_definition_identities",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("definition_kind", sa.String(length=40), nullable=False),
|
||||
sa.Column("definition_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("definition_key", sa.String(length=120), nullable=False),
|
||||
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_definition_identities")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
name="uq_reporting_definition_identity",
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_key",
|
||||
name="uq_reporting_definition_key",
|
||||
),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_definition_identities",
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
"definition_key",
|
||||
"created_by",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_definition_catalog",
|
||||
"reporting_definition_identities",
|
||||
["tenant_id", "definition_kind", "definition_key"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"reporting_definition_revisions",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("identity_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("definition_kind", sa.String(length=40), nullable=False),
|
||||
sa.Column("definition_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("definition_key", sa.String(length=120), nullable=False),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("previous_revision_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("parent_kind", sa.String(length=40), nullable=True),
|
||||
sa.Column("parent_id", sa.String(length=255), nullable=True),
|
||||
sa.Column("parent_revision", sa.Integer(), nullable=True),
|
||||
sa.Column("name", sa.String(length=500), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
sa.Column("status", sa.String(length=30), nullable=False),
|
||||
sa.Column("visibility", sa.String(length=30), nullable=False),
|
||||
sa.Column("content_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("change_reason", sa.String(length=1000), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("request_sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("event_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("recorded_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("superseded_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("payload", sa.JSON(), nullable=False),
|
||||
sa.Column("changed_by", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["identity_id"],
|
||||
["reporting_definition_identities.id"],
|
||||
name=op.f(
|
||||
"fk_reporting_definition_revisions_identity_id_reporting_definition_identities"
|
||||
),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["previous_revision_id"],
|
||||
["reporting_definition_revisions.id"],
|
||||
name=op.f(
|
||||
"fk_reporting_definition_revisions_previous_revision_id_reporting_definition_revisions"
|
||||
),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_definition_revisions")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
"revision",
|
||||
name="uq_reporting_definition_revision",
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"idempotency_key",
|
||||
name="uq_reporting_definition_idempotency",
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"event_id",
|
||||
name="uq_reporting_definition_event",
|
||||
),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_definition_revisions",
|
||||
"tenant_id",
|
||||
"identity_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
"definition_key",
|
||||
"previous_revision_id",
|
||||
"parent_kind",
|
||||
"parent_id",
|
||||
"status",
|
||||
"visibility",
|
||||
"content_hash",
|
||||
"event_id",
|
||||
"recorded_at",
|
||||
"superseded_at",
|
||||
"changed_by",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_definition_current",
|
||||
"reporting_definition_revisions",
|
||||
["tenant_id", "definition_kind", "definition_id", "superseded_at"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_definition_parent",
|
||||
"reporting_definition_revisions",
|
||||
["tenant_id", "parent_kind", "parent_id", "parent_revision"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_definition_list",
|
||||
"reporting_definition_revisions",
|
||||
["tenant_id", "definition_kind", "status", "recorded_at"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"reporting_definition_grants",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("definition_kind", sa.String(length=40), nullable=False),
|
||||
sa.Column("definition_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("subject_kind", sa.String(length=40), nullable=False),
|
||||
sa.Column("subject_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("permissions", sa.JSON(), nullable=False),
|
||||
sa.Column("active", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_definition_grants")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
"subject_kind",
|
||||
"subject_id",
|
||||
name="uq_reporting_definition_grant",
|
||||
),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_definition_grants",
|
||||
"tenant_id",
|
||||
"definition_kind",
|
||||
"definition_id",
|
||||
"subject_kind",
|
||||
"subject_id",
|
||||
"active",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_definition_grant_subject",
|
||||
"reporting_definition_grants",
|
||||
["tenant_id", "subject_kind", "subject_id", "active"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_definition_grant_object",
|
||||
"reporting_definition_grants",
|
||||
["tenant_id", "definition_kind", "definition_id", "active"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"reporting_executions",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("execution_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("report_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("report_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("semantic_model_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("semantic_model_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("dataset_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("dataset_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("status", sa.String(length=30), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("request_sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("parameters", sa.JSON(), nullable=False),
|
||||
sa.Column("query", sa.JSON(), nullable=False),
|
||||
sa.Column("source_fingerprints", sa.JSON(), nullable=False),
|
||||
sa.Column("definition_hashes", sa.JSON(), nullable=False),
|
||||
sa.Column("output_hash", sa.String(length=64), nullable=True),
|
||||
sa.Column("executor_version", sa.String(length=255), nullable=True),
|
||||
sa.Column("result_schema", sa.JSON(), nullable=False),
|
||||
sa.Column("result_rows", sa.JSON(), nullable=False),
|
||||
sa.Column("total_rows", sa.Integer(), nullable=False),
|
||||
sa.Column("truncated", sa.Boolean(), nullable=False),
|
||||
sa.Column("diagnostics", sa.JSON(), nullable=False),
|
||||
sa.Column("provenance", sa.JSON(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("actor_id", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_executions")),
|
||||
sa.UniqueConstraint("tenant_id", "execution_id", name="uq_reporting_execution"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"idempotency_key",
|
||||
name="uq_reporting_execution_idempotency",
|
||||
),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_executions",
|
||||
"tenant_id",
|
||||
"execution_id",
|
||||
"report_id",
|
||||
"semantic_model_id",
|
||||
"dataset_id",
|
||||
"status",
|
||||
"output_hash",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
"actor_id",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_execution_history",
|
||||
"reporting_executions",
|
||||
["tenant_id", "report_id", "started_at"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_execution_status",
|
||||
"reporting_executions",
|
||||
["tenant_id", "status", "started_at"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"reporting_saved_views",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("view_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("report_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("report_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("owner_kind", sa.String(length=40), nullable=False),
|
||||
sa.Column("owner_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("name", sa.String(length=500), nullable=False),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("state", sa.JSON(), nullable=False),
|
||||
sa.Column("shared", sa.Boolean(), nullable=False),
|
||||
sa.Column("access", sa.JSON(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_saved_views")),
|
||||
sa.UniqueConstraint("tenant_id", "view_id", name="uq_reporting_saved_view"),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_saved_views",
|
||||
"tenant_id",
|
||||
"view_id",
|
||||
"report_id",
|
||||
"owner_kind",
|
||||
"owner_id",
|
||||
"shared",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_saved_view_catalog",
|
||||
"reporting_saved_views",
|
||||
["tenant_id", "report_id", "owner_id", "shared"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"reporting_schedules",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("schedule_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("report_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("report_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("name", sa.String(length=500), nullable=False),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("trigger_kind", sa.String(length=30), nullable=False),
|
||||
sa.Column("trigger_config", sa.JSON(), nullable=False),
|
||||
sa.Column("parameters", sa.JSON(), nullable=False),
|
||||
sa.Column("query", sa.JSON(), nullable=False),
|
||||
sa.Column("publication_target", sa.JSON(), nullable=False),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False),
|
||||
sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_run_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_execution_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_schedules")),
|
||||
sa.UniqueConstraint("tenant_id", "schedule_id", name="uq_reporting_schedule"),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_schedules",
|
||||
"tenant_id",
|
||||
"schedule_id",
|
||||
"report_id",
|
||||
"enabled",
|
||||
"next_run_at",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_schedule_due",
|
||||
"reporting_schedules",
|
||||
["enabled", "next_run_at", "tenant_id"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"reporting_publications",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("publication_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("execution_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("target_capability", sa.String(length=255), nullable=False),
|
||||
sa.Column("target_ref", sa.String(length=1000), nullable=True),
|
||||
sa.Column("format", sa.String(length=30), nullable=False),
|
||||
sa.Column("status", sa.String(length=30), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_publications")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id", "publication_id", name="uq_reporting_publication"
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"idempotency_key",
|
||||
name="uq_reporting_publication_idempotency",
|
||||
),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_publications",
|
||||
"tenant_id",
|
||||
"publication_id",
|
||||
"execution_id",
|
||||
"status",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_publication_history",
|
||||
"reporting_publications",
|
||||
["tenant_id", "execution_id", "created_at"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"reporting_quality_results",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("result_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("quality_plan_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("quality_plan_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("dataset_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("dataset_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("status", sa.String(length=30), nullable=False),
|
||||
sa.Column("output_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("assertions", sa.JSON(), nullable=False),
|
||||
sa.Column("source_fingerprints", sa.JSON(), nullable=False),
|
||||
sa.Column("evaluated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("actor_id", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_quality_results")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id", "result_id", name="uq_reporting_quality_result"
|
||||
),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_quality_results",
|
||||
"tenant_id",
|
||||
"result_id",
|
||||
"quality_plan_id",
|
||||
"dataset_id",
|
||||
"status",
|
||||
"output_hash",
|
||||
"evaluated_at",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_quality_history",
|
||||
"reporting_quality_results",
|
||||
["tenant_id", "quality_plan_id", "evaluated_at"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"reporting_import_assessments",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("assessment_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("source_system", sa.String(length=255), nullable=False),
|
||||
sa.Column("source_id", sa.String(length=500), nullable=False),
|
||||
sa.Column("source_fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("mapping_report", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(length=30), nullable=False),
|
||||
sa.Column("accepted_approximations", sa.JSON(), nullable=False),
|
||||
sa.Column("assessed_by", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_import_assessments")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"assessment_id",
|
||||
name="uq_reporting_import_assessment",
|
||||
),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_import_assessments",
|
||||
"tenant_id",
|
||||
"assessment_id",
|
||||
"source_system",
|
||||
"status",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_import_history",
|
||||
"reporting_import_assessments",
|
||||
["tenant_id", "source_system", "created_at"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("reporting_import_assessments")
|
||||
op.drop_table("reporting_quality_results")
|
||||
op.drop_table("reporting_publications")
|
||||
op.drop_table("reporting_schedules")
|
||||
op.drop_table("reporting_saved_views")
|
||||
op.drop_table("reporting_definition_grants")
|
||||
op.drop_table("reporting_executions")
|
||||
op.drop_table("reporting_definition_revisions")
|
||||
op.drop_table("reporting_definition_identities")
|
||||
|
||||
|
||||
def _indexes(table: str, *columns: str) -> None:
|
||||
for column in columns:
|
||||
op.create_index(
|
||||
op.f(f"ix_{table}_{column}"),
|
||||
table,
|
||||
[column],
|
||||
unique=False,
|
||||
)
|
||||
@@ -0,0 +1,858 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
import csv
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from io import StringIO
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.events import (
|
||||
EventActorRef,
|
||||
EventObjectRef,
|
||||
EventTenantRef,
|
||||
PlatformEvent,
|
||||
emit_platform_event,
|
||||
)
|
||||
from govoplan_core.security.module_permissions import scopes_grant_compatible
|
||||
from govoplan_core.security.time import utc_now
|
||||
from govoplan_reporting.backend.contracts import (
|
||||
ReportingPublicationPayload,
|
||||
ReportingPublicationTarget,
|
||||
capability,
|
||||
)
|
||||
from govoplan_reporting.backend.db.models import (
|
||||
ReportingImportAssessment,
|
||||
ReportingPublication,
|
||||
ReportingSavedView,
|
||||
ReportingSchedule,
|
||||
)
|
||||
from govoplan_reporting.backend.definitions import ADMIN_SCOPE, get_definition
|
||||
from govoplan_reporting.backend.execution import (
|
||||
ReportingExecutionFailure,
|
||||
execute_report,
|
||||
get_execution,
|
||||
)
|
||||
from govoplan_reporting.backend.schemas import ReportQuery
|
||||
|
||||
|
||||
PUBLISH_SCOPE = "reporting:report:publish"
|
||||
SCHEDULE_SCOPE = "reporting:schedule:write"
|
||||
IMPORT_SCOPE = "reporting:import:assess"
|
||||
|
||||
|
||||
class ReportingOperationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class SqlReportingScheduler:
|
||||
def __init__(self, registry: object | None) -> None:
|
||||
self.registry = registry
|
||||
|
||||
def dispatch_due(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
limit: int = 20,
|
||||
) -> Mapping[str, object]:
|
||||
return dispatch_due_schedules(
|
||||
_session(session),
|
||||
principal,
|
||||
registry=self.registry,
|
||||
now=now,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
def upsert_saved_view(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
view_id: str,
|
||||
report_id: str,
|
||||
report_revision: int,
|
||||
name: str,
|
||||
state: Mapping[str, object],
|
||||
shared: bool,
|
||||
access: Mapping[str, object],
|
||||
expected_revision: int | None,
|
||||
) -> dict[str, object]:
|
||||
report = get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="report",
|
||||
definition_id=report_id,
|
||||
revision=report_revision,
|
||||
)
|
||||
if report is None:
|
||||
raise LookupError("Reporting report definition not found.")
|
||||
_validate_saved_view_state(state)
|
||||
owner_id = _actor(principal)
|
||||
if owner_id is None:
|
||||
raise ReportingOperationError("Saved views require an account owner.")
|
||||
row = (
|
||||
session.query(ReportingSavedView)
|
||||
.filter(
|
||||
ReportingSavedView.tenant_id == _tenant(principal),
|
||||
ReportingSavedView.view_id == view_id,
|
||||
)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
if row is None:
|
||||
if expected_revision is not None:
|
||||
raise ReportingOperationError(
|
||||
"Saved-view revision conflict: no view exists."
|
||||
)
|
||||
row = ReportingSavedView(
|
||||
tenant_id=_tenant(principal),
|
||||
view_id=_required(view_id, "Saved-view identifier", 36),
|
||||
report_id=report_id,
|
||||
report_revision=report_revision,
|
||||
owner_kind="account",
|
||||
owner_id=owner_id,
|
||||
name=_required(name, "Saved-view name", 500),
|
||||
revision=1,
|
||||
state=_json_value(state),
|
||||
shared=shared,
|
||||
access=_json_value(access),
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
if row.owner_id != owner_id and not _has_scope(principal, ADMIN_SCOPE):
|
||||
raise PermissionError("Only the saved-view owner or an admin may edit it.")
|
||||
if expected_revision != row.revision:
|
||||
raise ReportingOperationError(
|
||||
"Saved-view revision conflict: the expected revision is stale."
|
||||
)
|
||||
row.report_id = report_id
|
||||
row.report_revision = report_revision
|
||||
row.name = _required(name, "Saved-view name", 500)
|
||||
row.state = _json_value(state)
|
||||
row.shared = shared
|
||||
row.access = _json_value(access)
|
||||
row.revision += 1
|
||||
session.flush()
|
||||
return _saved_view_payload(row)
|
||||
|
||||
|
||||
def list_saved_views(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
report_id: str,
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
if (
|
||||
get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="report",
|
||||
definition_id=report_id,
|
||||
)
|
||||
is None
|
||||
):
|
||||
return ()
|
||||
owner_id = _actor(principal)
|
||||
rows = (
|
||||
session.query(ReportingSavedView)
|
||||
.filter(
|
||||
ReportingSavedView.tenant_id == _tenant(principal),
|
||||
ReportingSavedView.report_id == report_id,
|
||||
or_(
|
||||
ReportingSavedView.shared.is_(True),
|
||||
ReportingSavedView.owner_id == owner_id,
|
||||
),
|
||||
)
|
||||
.order_by(ReportingSavedView.name.asc())
|
||||
.all()
|
||||
)
|
||||
return tuple(_saved_view_payload(row) for row in rows)
|
||||
|
||||
|
||||
def delete_saved_view(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
view_id: str,
|
||||
) -> bool:
|
||||
row = (
|
||||
session.query(ReportingSavedView)
|
||||
.filter(
|
||||
ReportingSavedView.tenant_id == _tenant(principal),
|
||||
ReportingSavedView.view_id == view_id,
|
||||
)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
if row is None:
|
||||
return False
|
||||
if row.owner_id != _actor(principal) and not _has_scope(principal, ADMIN_SCOPE):
|
||||
raise PermissionError("Only the saved-view owner or an admin may delete it.")
|
||||
session.delete(row)
|
||||
session.flush()
|
||||
return True
|
||||
|
||||
|
||||
def upsert_schedule(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
schedule_id: str,
|
||||
report_id: str,
|
||||
report_revision: int,
|
||||
name: str,
|
||||
trigger_kind: str,
|
||||
trigger_config: Mapping[str, object],
|
||||
parameters: Mapping[str, object],
|
||||
query: ReportQuery,
|
||||
publication_target: Mapping[str, object],
|
||||
enabled: bool,
|
||||
next_run_at: datetime | None,
|
||||
expected_revision: int | None,
|
||||
) -> dict[str, object]:
|
||||
_require_scope(principal, SCHEDULE_SCOPE)
|
||||
report = get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="report",
|
||||
definition_id=report_id,
|
||||
revision=report_revision,
|
||||
)
|
||||
if report is None or report.status != "active":
|
||||
raise ReportingOperationError("Schedules require an active report revision.")
|
||||
normalized_next = _validate_trigger(
|
||||
trigger_kind,
|
||||
trigger_config,
|
||||
next_run_at=next_run_at,
|
||||
)
|
||||
row = (
|
||||
session.query(ReportingSchedule)
|
||||
.filter(
|
||||
ReportingSchedule.tenant_id == _tenant(principal),
|
||||
ReportingSchedule.schedule_id == schedule_id,
|
||||
)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
if row is None:
|
||||
if expected_revision is not None:
|
||||
raise ReportingOperationError(
|
||||
"Reporting schedule revision conflict: no schedule exists."
|
||||
)
|
||||
row = ReportingSchedule(
|
||||
tenant_id=_tenant(principal),
|
||||
schedule_id=_required(schedule_id, "Reporting schedule identifier", 36),
|
||||
report_id=report_id,
|
||||
report_revision=report_revision,
|
||||
name=_required(name, "Reporting schedule name", 500),
|
||||
revision=1,
|
||||
trigger_kind=trigger_kind,
|
||||
trigger_config=_json_value(trigger_config),
|
||||
parameters=_json_value(parameters),
|
||||
query=query.model_dump(mode="json"),
|
||||
publication_target=_json_value(publication_target),
|
||||
enabled=enabled,
|
||||
next_run_at=normalized_next,
|
||||
created_by=_actor(principal),
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
if expected_revision != row.revision:
|
||||
raise ReportingOperationError(
|
||||
"Reporting schedule revision conflict: the expected revision is stale."
|
||||
)
|
||||
row.report_id = report_id
|
||||
row.report_revision = report_revision
|
||||
row.name = _required(name, "Reporting schedule name", 500)
|
||||
row.trigger_kind = trigger_kind
|
||||
row.trigger_config = _json_value(trigger_config)
|
||||
row.parameters = _json_value(parameters)
|
||||
row.query = query.model_dump(mode="json")
|
||||
row.publication_target = _json_value(publication_target)
|
||||
row.enabled = enabled
|
||||
row.next_run_at = normalized_next
|
||||
row.revision += 1
|
||||
session.flush()
|
||||
return _schedule_payload(row)
|
||||
|
||||
|
||||
def list_schedules(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
report_id: str | None = None,
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
statement = session.query(ReportingSchedule).filter(
|
||||
ReportingSchedule.tenant_id == _tenant(principal)
|
||||
)
|
||||
if report_id:
|
||||
if (
|
||||
get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind="report",
|
||||
definition_id=report_id,
|
||||
)
|
||||
is None
|
||||
):
|
||||
return ()
|
||||
statement = statement.filter(ReportingSchedule.report_id == report_id)
|
||||
return tuple(
|
||||
_schedule_payload(row)
|
||||
for row in statement.order_by(ReportingSchedule.name.asc()).all()
|
||||
)
|
||||
|
||||
|
||||
def dispatch_due_schedules(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
now: datetime | None,
|
||||
limit: int,
|
||||
) -> dict[str, object]:
|
||||
_require_scope(principal, SCHEDULE_SCOPE)
|
||||
current = now or utc_now()
|
||||
_aware(current, "Reporting scheduler time")
|
||||
rows = (
|
||||
session.query(ReportingSchedule)
|
||||
.filter(
|
||||
ReportingSchedule.enabled.is_(True),
|
||||
ReportingSchedule.next_run_at.is_not(None),
|
||||
ReportingSchedule.next_run_at <= current,
|
||||
)
|
||||
.order_by(ReportingSchedule.next_run_at.asc())
|
||||
.limit(max(1, min(limit, 100)))
|
||||
.with_for_update(skip_locked=True)
|
||||
.all()
|
||||
)
|
||||
succeeded = 0
|
||||
failed = 0
|
||||
execution_ids: list[str] = []
|
||||
for row in rows:
|
||||
scheduled_for = row.next_run_at or current
|
||||
try:
|
||||
execution = execute_report(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
report_id=row.report_id,
|
||||
report_revision=row.report_revision,
|
||||
parameters=dict(row.parameters or {}),
|
||||
query=ReportQuery.model_validate(row.query or {}),
|
||||
idempotency_key=f"schedule:{row.schedule_id}:{scheduled_for.isoformat()}",
|
||||
)
|
||||
execution_id = str(execution["execution_id"])
|
||||
execution_ids.append(execution_id)
|
||||
target = dict(row.publication_target or {})
|
||||
if target:
|
||||
publish_execution(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
execution_id=execution_id,
|
||||
target_capability=str(target.get("target_capability") or ""),
|
||||
target_ref=(
|
||||
str(target["target_ref"])
|
||||
if target.get("target_ref") is not None
|
||||
else None
|
||||
),
|
||||
format=str(target.get("format") or "csv"),
|
||||
idempotency_key=f"schedule-publication:{row.schedule_id}:{scheduled_for.isoformat()}",
|
||||
options=dict(target.get("options") or {}),
|
||||
)
|
||||
succeeded += 1
|
||||
row.last_execution_id = execution_id
|
||||
except (ReportingExecutionFailure, ReportingOperationError, LookupError):
|
||||
failed += 1
|
||||
row.last_run_at = current
|
||||
_advance_schedule(row, scheduled_for)
|
||||
session.flush()
|
||||
return {
|
||||
"claimed": len(rows),
|
||||
"succeeded": succeeded,
|
||||
"failed": failed,
|
||||
"execution_ids": execution_ids,
|
||||
}
|
||||
|
||||
|
||||
def publish_execution(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
execution_id: str,
|
||||
target_capability: str,
|
||||
target_ref: str | None,
|
||||
format: str,
|
||||
idempotency_key: str,
|
||||
options: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
_require_scope(principal, PUBLISH_SCOPE)
|
||||
execution_payload = get_execution(session, principal, execution_id=execution_id)
|
||||
if execution_payload is None:
|
||||
raise LookupError("Reporting execution not found.")
|
||||
if execution_payload["status"] != "succeeded":
|
||||
raise ReportingOperationError("Only successful report executions can publish.")
|
||||
clean_capability = _required(
|
||||
target_capability,
|
||||
"Reporting publication target capability",
|
||||
255,
|
||||
)
|
||||
clean_format = str(format).casefold()
|
||||
if clean_format not in {"json", "csv", "xlsx", "html", "pdf"}:
|
||||
raise ReportingOperationError("Unsupported Reporting publication format.")
|
||||
clean_key = _required(idempotency_key, "Reporting publication idempotency key", 255)
|
||||
existing = (
|
||||
session.query(ReportingPublication)
|
||||
.filter(
|
||||
ReportingPublication.tenant_id == _tenant(principal),
|
||||
ReportingPublication.idempotency_key == clean_key,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if existing is not None:
|
||||
if (
|
||||
existing.execution_id != execution_id
|
||||
or existing.target_capability != clean_capability
|
||||
or existing.target_ref != target_ref
|
||||
or existing.format != clean_format
|
||||
):
|
||||
raise ReportingOperationError("Reporting publication idempotency conflict.")
|
||||
return _publication_payload(existing)
|
||||
provider = capability(registry, clean_capability)
|
||||
if not isinstance(provider, ReportingPublicationTarget):
|
||||
raise ReportingOperationError(
|
||||
f"Reporting publication provider {clean_capability!r} is unavailable."
|
||||
)
|
||||
publication = ReportingPublication(
|
||||
tenant_id=_tenant(principal),
|
||||
publication_id=str(uuid.uuid4()),
|
||||
execution_id=execution_id,
|
||||
target_capability=clean_capability,
|
||||
target_ref=target_ref,
|
||||
format=clean_format,
|
||||
status="running",
|
||||
idempotency_key=clean_key,
|
||||
)
|
||||
session.add(publication)
|
||||
session.flush()
|
||||
try:
|
||||
evidence = provider.publish_report(
|
||||
session,
|
||||
principal,
|
||||
payload=ReportingPublicationPayload(
|
||||
publication_id=publication.publication_id,
|
||||
execution_id=execution_id,
|
||||
tenant_id=_tenant(principal),
|
||||
report_id=str(execution_payload["report_id"]),
|
||||
report_revision=int(execution_payload["report_revision"]),
|
||||
format=clean_format,
|
||||
target_ref=target_ref,
|
||||
rows=tuple(execution_payload["rows"]), # type: ignore[arg-type]
|
||||
schema=tuple(execution_payload["schema"]), # type: ignore[arg-type]
|
||||
output_hash=str(execution_payload["output_hash"]),
|
||||
options=dict(options),
|
||||
),
|
||||
)
|
||||
publication.status = "succeeded"
|
||||
publication.evidence = _json_value(evidence)
|
||||
publication.completed_at = utc_now()
|
||||
except Exception as exc:
|
||||
publication.status = "failed"
|
||||
publication.error = str(exc)
|
||||
publication.completed_at = utc_now()
|
||||
session.flush()
|
||||
raise ReportingOperationError(str(exc)) from exc
|
||||
session.flush()
|
||||
emit_platform_event(
|
||||
session,
|
||||
PlatformEvent(
|
||||
type=f"reporting.publication.{publication.status}",
|
||||
module_id="reporting",
|
||||
payload={
|
||||
"publication_id": publication.publication_id,
|
||||
"execution_id": publication.execution_id,
|
||||
"target_capability": publication.target_capability,
|
||||
"target_ref": publication.target_ref,
|
||||
"format": publication.format,
|
||||
"evidence": dict(publication.evidence or {}),
|
||||
},
|
||||
actor=EventActorRef(type="account", id=_actor(principal)),
|
||||
tenant=EventTenantRef(id=publication.tenant_id),
|
||||
resource=EventObjectRef(
|
||||
type="report_publication",
|
||||
id=publication.publication_id,
|
||||
),
|
||||
classification="internal",
|
||||
),
|
||||
)
|
||||
return _publication_payload(publication)
|
||||
|
||||
|
||||
def export_execution(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
execution_id: str,
|
||||
format: str,
|
||||
) -> tuple[bytes, str, str]:
|
||||
payload = get_execution(session, principal, execution_id=execution_id)
|
||||
if payload is None:
|
||||
raise LookupError("Reporting execution not found.")
|
||||
if payload["status"] != "succeeded":
|
||||
raise ReportingOperationError("Only successful executions can be exported.")
|
||||
rows = tuple(payload["rows"]) # type: ignore[arg-type]
|
||||
if format == "json":
|
||||
content = json.dumps(
|
||||
{"schema": payload["schema"], "rows": rows},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return content, "application/json", f"report-{execution_id}.json"
|
||||
if format != "csv":
|
||||
raise ReportingOperationError("Direct export supports CSV or JSON.")
|
||||
fields = tuple(
|
||||
dict.fromkeys(
|
||||
str(key) for row in rows if isinstance(row, Mapping) for key in row
|
||||
)
|
||||
)
|
||||
stream = StringIO(newline="")
|
||||
writer = csv.DictWriter(stream, fieldnames=fields, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
if isinstance(row, Mapping):
|
||||
writer.writerow({key: _safe_csv_cell(row.get(key)) for key in fields})
|
||||
return (
|
||||
stream.getvalue().encode("utf-8-sig"),
|
||||
"text/csv; charset=utf-8",
|
||||
f"report-{execution_id}.csv",
|
||||
)
|
||||
|
||||
|
||||
def assess_import(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
source_system: str,
|
||||
source_id: str,
|
||||
metadata: Mapping[str, object],
|
||||
accepted_approximations: list[str],
|
||||
) -> dict[str, object]:
|
||||
_require_scope(principal, IMPORT_SCOPE)
|
||||
raw_features = metadata.get("features", [])
|
||||
if not isinstance(raw_features, list):
|
||||
raise ReportingOperationError("Import metadata features must be a list.")
|
||||
features = tuple(dict.fromkeys(str(item) for item in raw_features))
|
||||
exact_features = {
|
||||
"dataset",
|
||||
"dimension",
|
||||
"hierarchy",
|
||||
"measure",
|
||||
"parameter",
|
||||
"table",
|
||||
"pivot",
|
||||
"chart",
|
||||
"quality_assertion",
|
||||
"saved_view",
|
||||
}
|
||||
approximated_features = {
|
||||
"provider_specific_format",
|
||||
"dialect_function",
|
||||
"dashboard_layout",
|
||||
}
|
||||
unsupported_features = {
|
||||
"raw_sql",
|
||||
"stored_procedure",
|
||||
"runtime_script",
|
||||
"implicit_authorization",
|
||||
"unchecked_custom_function",
|
||||
}
|
||||
exact = sorted(set(features) & exact_features)
|
||||
approximated = sorted(set(features) & approximated_features)
|
||||
unsupported = sorted(
|
||||
(set(features) & unsupported_features)
|
||||
| (
|
||||
set(features)
|
||||
- exact_features
|
||||
- approximated_features
|
||||
- unsupported_features
|
||||
)
|
||||
)
|
||||
accepted = sorted(set(accepted_approximations) & set(approximated))
|
||||
pending = sorted(set(approximated) - set(accepted))
|
||||
status = "ready" if not unsupported and not pending else "blocked"
|
||||
mapping_report = {
|
||||
"contract_version": "1",
|
||||
"source_system": source_system,
|
||||
"source_id": source_id,
|
||||
"exact": exact,
|
||||
"approximated": approximated,
|
||||
"accepted_approximations": accepted,
|
||||
"pending_approximations": pending,
|
||||
"unsupported": unsupported,
|
||||
"manual_bindings": list(metadata.get("manual_bindings", [])),
|
||||
"provider_assumptions": list(metadata.get("provider_assumptions", [])),
|
||||
"activation_allowed": status == "ready",
|
||||
}
|
||||
row = ReportingImportAssessment(
|
||||
tenant_id=_tenant(principal),
|
||||
assessment_id=str(uuid.uuid4()),
|
||||
source_system=_required(source_system, "Import source system", 255),
|
||||
source_id=_required(source_id, "Import source identifier", 500),
|
||||
source_fingerprint=_sha256(metadata),
|
||||
mapping_report=mapping_report,
|
||||
status=status,
|
||||
accepted_approximations=accepted,
|
||||
assessed_by=_actor(principal),
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return _assessment_payload(row)
|
||||
|
||||
|
||||
def list_import_assessments(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
limit: int = 100,
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
_require_scope(principal, IMPORT_SCOPE)
|
||||
rows = (
|
||||
session.query(ReportingImportAssessment)
|
||||
.filter(ReportingImportAssessment.tenant_id == _tenant(principal))
|
||||
.order_by(ReportingImportAssessment.created_at.desc())
|
||||
.limit(max(1, min(limit, 200)))
|
||||
.all()
|
||||
)
|
||||
return tuple(_assessment_payload(row) for row in rows)
|
||||
|
||||
|
||||
def _validate_saved_view_state(state: Mapping[str, object]) -> None:
|
||||
query = state.get("query")
|
||||
if query is not None:
|
||||
if not isinstance(query, Mapping):
|
||||
raise ReportingOperationError("Saved-view query must be an object.")
|
||||
ReportQuery.model_validate(query)
|
||||
if len(state) > 100:
|
||||
raise ReportingOperationError("Saved-view state is limited to 100 entries.")
|
||||
|
||||
|
||||
def _validate_trigger(
|
||||
trigger_kind: str,
|
||||
trigger_config: Mapping[str, object],
|
||||
*,
|
||||
next_run_at: datetime | None,
|
||||
) -> datetime | None:
|
||||
if trigger_kind not in {"scheduled", "interval"}:
|
||||
raise ReportingOperationError("Unsupported Reporting schedule trigger.")
|
||||
if next_run_at is not None:
|
||||
_aware(next_run_at, "Reporting next_run_at")
|
||||
if trigger_kind == "scheduled":
|
||||
if next_run_at is None:
|
||||
raise ReportingOperationError("Scheduled reports require next_run_at.")
|
||||
return next_run_at
|
||||
seconds = int(trigger_config.get("seconds", 0))
|
||||
if not 60 <= seconds <= 31_536_000:
|
||||
raise ReportingOperationError(
|
||||
"Reporting intervals must be between 60 seconds and one year."
|
||||
)
|
||||
return next_run_at or utc_now() + timedelta(seconds=seconds)
|
||||
|
||||
|
||||
def _advance_schedule(row: ReportingSchedule, scheduled_for: datetime) -> None:
|
||||
if row.trigger_kind == "scheduled":
|
||||
row.enabled = False
|
||||
row.next_run_at = None
|
||||
return
|
||||
if scheduled_for.tzinfo is None or scheduled_for.utcoffset() is None:
|
||||
scheduled_for = scheduled_for.replace(tzinfo=UTC)
|
||||
seconds = int((row.trigger_config or {}).get("seconds", 0))
|
||||
next_run = scheduled_for + timedelta(seconds=seconds)
|
||||
now = utc_now()
|
||||
while next_run <= now:
|
||||
next_run += timedelta(seconds=seconds)
|
||||
row.next_run_at = next_run
|
||||
|
||||
|
||||
def _saved_view_payload(row: ReportingSavedView) -> dict[str, object]:
|
||||
return {
|
||||
"view_id": row.view_id,
|
||||
"report_id": row.report_id,
|
||||
"report_revision": row.report_revision,
|
||||
"owner_kind": row.owner_kind,
|
||||
"owner_id": row.owner_id,
|
||||
"name": row.name,
|
||||
"revision": row.revision,
|
||||
"state": dict(row.state or {}),
|
||||
"shared": row.shared,
|
||||
"access": dict(row.access or {}),
|
||||
"updated_at": _datetime_text(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
def _schedule_payload(row: ReportingSchedule) -> dict[str, object]:
|
||||
return {
|
||||
"schedule_id": row.schedule_id,
|
||||
"report_id": row.report_id,
|
||||
"report_revision": row.report_revision,
|
||||
"name": row.name,
|
||||
"revision": row.revision,
|
||||
"trigger_kind": row.trigger_kind,
|
||||
"trigger_config": dict(row.trigger_config or {}),
|
||||
"parameters": dict(row.parameters or {}),
|
||||
"query": dict(row.query or {}),
|
||||
"publication_target": dict(row.publication_target or {}),
|
||||
"enabled": row.enabled,
|
||||
"next_run_at": _datetime_text(row.next_run_at),
|
||||
"last_run_at": _datetime_text(row.last_run_at),
|
||||
"last_execution_id": row.last_execution_id,
|
||||
}
|
||||
|
||||
|
||||
def _publication_payload(row: ReportingPublication) -> dict[str, object]:
|
||||
return {
|
||||
"publication_id": row.publication_id,
|
||||
"execution_id": row.execution_id,
|
||||
"target_capability": row.target_capability,
|
||||
"target_ref": row.target_ref,
|
||||
"format": row.format,
|
||||
"status": row.status,
|
||||
"evidence": dict(row.evidence or {}),
|
||||
"error": row.error,
|
||||
"completed_at": _datetime_text(row.completed_at),
|
||||
}
|
||||
|
||||
|
||||
def _assessment_payload(row: ReportingImportAssessment) -> dict[str, object]:
|
||||
return {
|
||||
"assessment_id": row.assessment_id,
|
||||
"source_system": row.source_system,
|
||||
"source_id": row.source_id,
|
||||
"source_fingerprint": row.source_fingerprint,
|
||||
"mapping_report": dict(row.mapping_report or {}),
|
||||
"status": row.status,
|
||||
"accepted_approximations": list(row.accepted_approximations or []),
|
||||
"assessed_by": row.assessed_by,
|
||||
"created_at": _datetime_text(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
def _safe_csv_cell(value: object) -> object:
|
||||
if isinstance(value, (dict, list, tuple)):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
if isinstance(value, str) and value.startswith(("=", "+", "-", "@")):
|
||||
return f"'{value}"
|
||||
return value
|
||||
|
||||
|
||||
def _has_scope(principal: object, scope: str) -> bool:
|
||||
method = getattr(principal, "has", None)
|
||||
if callable(method):
|
||||
return bool(method(scope))
|
||||
return scopes_grant_compatible(
|
||||
frozenset(getattr(principal, "scopes", ()) or ()),
|
||||
scope,
|
||||
)
|
||||
|
||||
|
||||
def _require_scope(principal: object, scope: str) -> None:
|
||||
if not _has_scope(principal, scope):
|
||||
raise PermissionError(f"Missing scope: {scope}")
|
||||
|
||||
|
||||
def _tenant(principal: object) -> str:
|
||||
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
|
||||
if not tenant_id:
|
||||
raise ReportingOperationError(
|
||||
"Reporting operations require a tenant-bound principal."
|
||||
)
|
||||
return tenant_id
|
||||
|
||||
|
||||
def _actor(principal: object) -> str | None:
|
||||
for value in (
|
||||
getattr(principal, "account_id", None),
|
||||
getattr(principal, "identity_id", None),
|
||||
getattr(principal, "membership_id", None),
|
||||
):
|
||||
if str(value or "").strip():
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def _session(value: object) -> Session:
|
||||
if not isinstance(value, Session):
|
||||
raise TypeError("Reporting operations require a SQLAlchemy session.")
|
||||
return value
|
||||
|
||||
|
||||
def _required(value: object, label: str, maximum: int) -> str:
|
||||
result = str(value or "").strip()
|
||||
if not result:
|
||||
raise ReportingOperationError(f"{label} is required.")
|
||||
if len(result) > maximum:
|
||||
raise ReportingOperationError(f"{label} is limited to {maximum} characters.")
|
||||
return result
|
||||
|
||||
|
||||
def _aware(value: datetime, label: str) -> None:
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ReportingOperationError(f"{label} must include a timezone.")
|
||||
|
||||
|
||||
def _sha256(value: object) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
_json_value(value),
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _json_value(value: object) -> Any:
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
if hasattr(value, "model_dump"):
|
||||
return _json_value(value.model_dump(mode="json"))
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _json_value(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_json_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _datetime_text(value: datetime | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=UTC)
|
||||
return value.isoformat()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"IMPORT_SCOPE",
|
||||
"PUBLISH_SCOPE",
|
||||
"SCHEDULE_SCOPE",
|
||||
"ReportingOperationError",
|
||||
"SqlReportingScheduler",
|
||||
"assess_import",
|
||||
"delete_saved_view",
|
||||
"dispatch_due_schedules",
|
||||
"export_execution",
|
||||
"list_import_assessments",
|
||||
"list_saved_views",
|
||||
"list_schedules",
|
||||
"publish_execution",
|
||||
"upsert_saved_view",
|
||||
"upsert_schedule",
|
||||
]
|
||||
@@ -0,0 +1,494 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from govoplan_reporting.backend.schemas import (
|
||||
FilterClause,
|
||||
MeasureDefinition,
|
||||
ReportQuery,
|
||||
SemanticModelDefinition,
|
||||
TypedExpression,
|
||||
VisualizationDefinition,
|
||||
)
|
||||
|
||||
|
||||
QUERY_ENGINE_VERSION = "reporting-query-v1"
|
||||
|
||||
|
||||
class ReportingQueryError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class QueryResult:
|
||||
rows: tuple[dict[str, Any], ...]
|
||||
total_rows: int
|
||||
schema: tuple[dict[str, Any], ...]
|
||||
truncated: bool
|
||||
diagnostics: tuple[dict[str, Any], ...] = ()
|
||||
|
||||
|
||||
def execute_semantic_query(
|
||||
rows: Sequence[Mapping[str, object]],
|
||||
semantic_model: SemanticModelDefinition,
|
||||
query: ReportQuery,
|
||||
) -> QueryResult:
|
||||
dimensions = {item.key: item for item in semantic_model.dimensions}
|
||||
measures = {item.key: item for item in semantic_model.measures}
|
||||
selected_dimensions = tuple(query.dimensions or semantic_model.default_dimensions)
|
||||
selected_measures = tuple(query.measures or semantic_model.default_measures)
|
||||
if query.mode == "pivot" and query.pivot is not None:
|
||||
selected_dimensions = tuple(
|
||||
dict.fromkeys((*query.pivot.rows, *query.pivot.columns))
|
||||
)
|
||||
selected_measures = tuple(query.pivot.measures or selected_measures)
|
||||
_require_known(selected_dimensions, dimensions, "dimensions")
|
||||
_require_known(selected_measures, measures, "measures")
|
||||
_require_known(
|
||||
tuple(item.dimension for item in query.filters),
|
||||
dimensions,
|
||||
"filter dimensions",
|
||||
)
|
||||
filtered = tuple(
|
||||
dict(row)
|
||||
for row in rows
|
||||
if all(
|
||||
_matches_filter(row.get(dimensions[item.dimension].field), item)
|
||||
for item in query.filters
|
||||
)
|
||||
)
|
||||
if query.mode == "detail":
|
||||
result_rows = _detail_rows(filtered, selected_dimensions, dimensions)
|
||||
else:
|
||||
result_rows = _summary_rows(
|
||||
filtered,
|
||||
selected_dimensions,
|
||||
selected_measures,
|
||||
dimensions,
|
||||
measures,
|
||||
)
|
||||
if query.mode == "pivot" and query.pivot is not None:
|
||||
result_rows = _pivot_rows(
|
||||
result_rows,
|
||||
row_dimensions=tuple(query.pivot.rows),
|
||||
column_dimensions=tuple(query.pivot.columns),
|
||||
measures=selected_measures,
|
||||
include_totals=query.pivot.include_totals,
|
||||
)
|
||||
sorted_rows = _sort_rows(result_rows, query)
|
||||
total = len(sorted_rows)
|
||||
selected = sorted_rows[query.offset : query.offset + query.limit]
|
||||
return QueryResult(
|
||||
rows=tuple(selected),
|
||||
total_rows=total,
|
||||
schema=_infer_schema(selected or sorted_rows[:1]),
|
||||
truncated=query.offset + len(selected) < total,
|
||||
)
|
||||
|
||||
|
||||
class DefaultChartRenderer:
|
||||
"""Build a provider-neutral chart model with a mandatory table fallback."""
|
||||
|
||||
def render(
|
||||
self,
|
||||
*,
|
||||
visualization: VisualizationDefinition,
|
||||
result: QueryResult,
|
||||
) -> Mapping[str, object]:
|
||||
category = visualization.category_dimension
|
||||
measure_keys = tuple(visualization.measures)
|
||||
missing: set[str] = set()
|
||||
if visualization.kind not in {"table", "pivot", "metric"}:
|
||||
if not category or not measure_keys:
|
||||
missing = {"chart category and measure configuration"}
|
||||
else:
|
||||
available = {str(item.get("name")) for item in result.schema}
|
||||
missing = {category, *measure_keys} - available
|
||||
if missing and not visualization.tabular_fallback:
|
||||
raise ReportingQueryError(
|
||||
"Chart references unavailable result fields: "
|
||||
+ ", ".join(sorted(missing))
|
||||
)
|
||||
return {
|
||||
"contract_version": "1",
|
||||
"kind": "table" if missing else visualization.kind,
|
||||
"requested_kind": visualization.kind,
|
||||
"category": category,
|
||||
"series": visualization.series_dimension,
|
||||
"measures": list(measure_keys),
|
||||
"options": dict(visualization.options),
|
||||
"fallback_reason": (
|
||||
"The selected query does not expose the fields required by the "
|
||||
"saved visualization. Showing its accessible table fallback."
|
||||
if missing
|
||||
else None
|
||||
),
|
||||
"rows": list(result.rows),
|
||||
"schema": list(result.schema),
|
||||
"tabular_fallback": (
|
||||
{
|
||||
"rows": list(result.rows),
|
||||
"schema": list(result.schema),
|
||||
}
|
||||
if visualization.tabular_fallback or missing
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _detail_rows(
|
||||
rows: Sequence[Mapping[str, object]],
|
||||
selected_dimensions: Sequence[str],
|
||||
dimensions: Mapping[str, object],
|
||||
) -> list[dict[str, Any]]:
|
||||
if not selected_dimensions:
|
||||
return [
|
||||
{str(key): _json_value(value) for key, value in row.items()} for row in rows
|
||||
]
|
||||
return [
|
||||
{
|
||||
key: _json_value(row.get(getattr(dimensions[key], "field")))
|
||||
for key in selected_dimensions
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def _summary_rows(
|
||||
rows: Sequence[Mapping[str, object]],
|
||||
selected_dimensions: Sequence[str],
|
||||
selected_measures: Sequence[str],
|
||||
dimensions: Mapping[str, object],
|
||||
measures: Mapping[str, MeasureDefinition],
|
||||
) -> list[dict[str, Any]]:
|
||||
grouped: dict[tuple[object, ...], list[Mapping[str, object]]] = defaultdict(list)
|
||||
if selected_dimensions:
|
||||
for row in rows:
|
||||
key = tuple(
|
||||
row.get(getattr(dimensions[item], "field"))
|
||||
for item in selected_dimensions
|
||||
)
|
||||
grouped[key].append(row)
|
||||
else:
|
||||
grouped[()] = list(rows)
|
||||
result: list[dict[str, Any]] = []
|
||||
for group_key, group_rows in grouped.items():
|
||||
item: dict[str, Any] = {
|
||||
dimension: _json_value(value)
|
||||
for dimension, value in zip(selected_dimensions, group_key, strict=True)
|
||||
}
|
||||
pending: list[MeasureDefinition] = []
|
||||
for key in selected_measures:
|
||||
measure = measures[key]
|
||||
if measure.aggregation == "calculated":
|
||||
pending.append(measure)
|
||||
else:
|
||||
item[key] = _aggregate(group_rows, measure)
|
||||
for measure in pending:
|
||||
item[measure.key] = _evaluate_expression(
|
||||
measure.expression,
|
||||
row=None,
|
||||
measures=item,
|
||||
)
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def _aggregate(
|
||||
rows: Sequence[Mapping[str, object]],
|
||||
measure: MeasureDefinition,
|
||||
) -> object:
|
||||
if measure.aggregation == "count":
|
||||
if measure.field is None:
|
||||
return len(rows)
|
||||
return sum(row.get(measure.field) is not None for row in rows)
|
||||
values = [
|
||||
row.get(measure.field or "")
|
||||
for row in rows
|
||||
if row.get(measure.field or "") is not None
|
||||
]
|
||||
if measure.aggregation == "count_distinct":
|
||||
return len({_hashable(value) for value in values})
|
||||
if not values:
|
||||
return None
|
||||
if measure.aggregation == "sum":
|
||||
return _numeric_result(sum(_number(value) for value in values))
|
||||
if measure.aggregation == "average":
|
||||
return _numeric_result(sum(_number(value) for value in values) / len(values))
|
||||
if measure.aggregation == "minimum":
|
||||
return _json_value(min(values))
|
||||
if measure.aggregation == "maximum":
|
||||
return _json_value(max(values))
|
||||
raise ReportingQueryError(
|
||||
f"Unsupported measure aggregation: {measure.aggregation!r}."
|
||||
)
|
||||
|
||||
|
||||
def _evaluate_expression(
|
||||
expression: TypedExpression | None,
|
||||
*,
|
||||
row: Mapping[str, object] | None,
|
||||
measures: Mapping[str, object],
|
||||
) -> object:
|
||||
if expression is None:
|
||||
return None
|
||||
if expression.op == "literal":
|
||||
return expression.value
|
||||
if expression.op == "field":
|
||||
return row.get(expression.ref or "") if row is not None else None
|
||||
if expression.op == "measure":
|
||||
return measures.get(expression.ref or "")
|
||||
values = [
|
||||
_evaluate_expression(item, row=row, measures=measures)
|
||||
for item in expression.args
|
||||
]
|
||||
if expression.op == "add":
|
||||
return _numeric_result(sum(_number(item) for item in values))
|
||||
if expression.op == "subtract":
|
||||
_arity(values, 2, expression.op)
|
||||
return _numeric_result(_number(values[0]) - _number(values[1]))
|
||||
if expression.op == "multiply":
|
||||
total = Decimal(1)
|
||||
for value in values:
|
||||
total *= _number(value)
|
||||
return _numeric_result(total)
|
||||
if expression.op == "divide":
|
||||
_arity(values, 2, expression.op)
|
||||
divisor = _number(values[1])
|
||||
return None if divisor == 0 else _numeric_result(_number(values[0]) / divisor)
|
||||
if expression.op == "coalesce":
|
||||
return next((item for item in values if item is not None), None)
|
||||
if expression.op == "case":
|
||||
if len(values) < 3:
|
||||
raise ReportingQueryError(
|
||||
"Case expressions require condition, value, and default."
|
||||
)
|
||||
pairs = values[:-1]
|
||||
for index in range(0, len(pairs) - 1, 2):
|
||||
if bool(pairs[index]):
|
||||
return pairs[index + 1]
|
||||
return values[-1]
|
||||
if expression.op in {"eq", "ne", "gt", "gte", "lt", "lte"}:
|
||||
_arity(values, 2, expression.op)
|
||||
return _compare(values[0], values[1], expression.op)
|
||||
if expression.op == "and":
|
||||
return all(bool(item) for item in values)
|
||||
if expression.op == "or":
|
||||
return any(bool(item) for item in values)
|
||||
if expression.op == "not":
|
||||
_arity(values, 1, expression.op)
|
||||
return not bool(values[0])
|
||||
raise ReportingQueryError(f"Unsupported expression operator: {expression.op!r}.")
|
||||
|
||||
|
||||
def _pivot_rows(
|
||||
rows: Sequence[Mapping[str, object]],
|
||||
*,
|
||||
row_dimensions: Sequence[str],
|
||||
column_dimensions: Sequence[str],
|
||||
measures: Sequence[str],
|
||||
include_totals: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
grouped: dict[tuple[object, ...], dict[str, Any]] = {}
|
||||
totals: dict[tuple[object, ...], dict[str, Decimal]] = defaultdict(
|
||||
lambda: defaultdict(Decimal)
|
||||
)
|
||||
for source in rows:
|
||||
row_key = tuple(source.get(item) for item in row_dimensions)
|
||||
target = grouped.setdefault(
|
||||
row_key,
|
||||
{
|
||||
item: _json_value(value)
|
||||
for item, value in zip(row_dimensions, row_key, strict=True)
|
||||
},
|
||||
)
|
||||
column_key = (
|
||||
" / ".join(
|
||||
str(source.get(item) if source.get(item) is not None else "(blank)")
|
||||
for item in column_dimensions
|
||||
)
|
||||
or "value"
|
||||
)
|
||||
for measure in measures:
|
||||
field_name = f"{column_key}.{measure}"
|
||||
value = source.get(measure)
|
||||
target[field_name] = _json_value(value)
|
||||
if include_totals and value is not None:
|
||||
try:
|
||||
totals[row_key][measure] += _number(value)
|
||||
except ReportingQueryError:
|
||||
pass
|
||||
if include_totals:
|
||||
for row_key, target in grouped.items():
|
||||
for measure in measures:
|
||||
if measure in totals[row_key]:
|
||||
target[f"total.{measure}"] = _numeric_result(
|
||||
totals[row_key][measure]
|
||||
)
|
||||
return list(grouped.values())
|
||||
|
||||
|
||||
def _matches_filter(value: object, clause: FilterClause) -> bool:
|
||||
expected = clause.value
|
||||
if clause.operator == "is_null":
|
||||
return value is None
|
||||
if clause.operator == "not_null":
|
||||
return value is not None
|
||||
if clause.operator == "eq":
|
||||
return value == expected
|
||||
if clause.operator == "ne":
|
||||
return value != expected
|
||||
if clause.operator in {"in", "not_in"}:
|
||||
if not isinstance(expected, (list, tuple, set, frozenset)):
|
||||
raise ReportingQueryError("Set filters require a list value.")
|
||||
result = value in expected
|
||||
return result if clause.operator == "in" else not result
|
||||
if clause.operator == "contains":
|
||||
return str(expected or "").casefold() in str(value or "").casefold()
|
||||
if clause.operator == "starts_with":
|
||||
return str(value or "").casefold().startswith(str(expected or "").casefold())
|
||||
if clause.operator == "between":
|
||||
if not isinstance(expected, (list, tuple)) or len(expected) != 2:
|
||||
raise ReportingQueryError("Between filters require two values.")
|
||||
return value is not None and expected[0] <= value <= expected[1]
|
||||
if clause.operator in {"gt", "gte", "lt", "lte"}:
|
||||
if value is None or expected is None:
|
||||
return False
|
||||
return bool(_compare(value, expected, clause.operator))
|
||||
raise ReportingQueryError(f"Unsupported filter operator: {clause.operator!r}.")
|
||||
|
||||
|
||||
def _sort_rows(
|
||||
rows: Sequence[dict[str, Any]],
|
||||
query: ReportQuery,
|
||||
) -> list[dict[str, Any]]:
|
||||
result = list(rows)
|
||||
for clause in reversed(query.sort):
|
||||
result.sort(
|
||||
key=lambda item: _sort_key(item.get(clause.key)),
|
||||
reverse=clause.direction == "desc",
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _infer_schema(rows: Sequence[Mapping[str, object]]) -> tuple[dict[str, Any], ...]:
|
||||
names = tuple(dict.fromkeys(str(key) for row in rows for key in row))
|
||||
return tuple(
|
||||
{
|
||||
"name": name,
|
||||
"type": _value_type(
|
||||
next((row.get(name) for row in rows if row.get(name) is not None), None)
|
||||
),
|
||||
"nullable": any(row.get(name) is None for row in rows),
|
||||
}
|
||||
for name in names
|
||||
)
|
||||
|
||||
|
||||
def _value_type(value: object) -> str:
|
||||
if value is None:
|
||||
return "string"
|
||||
if isinstance(value, bool):
|
||||
return "boolean"
|
||||
if isinstance(value, int):
|
||||
return "integer"
|
||||
if isinstance(value, (float, Decimal)):
|
||||
return "number"
|
||||
if isinstance(value, datetime):
|
||||
return "datetime"
|
||||
if isinstance(value, date):
|
||||
return "date"
|
||||
if isinstance(value, (dict, list, tuple)):
|
||||
return "json"
|
||||
return "string"
|
||||
|
||||
|
||||
def _require_known(
|
||||
keys: Sequence[str],
|
||||
available: Mapping[str, object],
|
||||
label: str,
|
||||
) -> None:
|
||||
unknown = set(keys) - set(available)
|
||||
if unknown:
|
||||
raise ReportingQueryError(
|
||||
f"Report query references unknown {label}: " + ", ".join(sorted(unknown))
|
||||
)
|
||||
|
||||
|
||||
def _compare(left: object, right: object, operator: str) -> bool:
|
||||
if operator == "eq":
|
||||
return left == right
|
||||
if operator == "ne":
|
||||
return left != right
|
||||
try:
|
||||
if operator == "gt":
|
||||
return left > right # type: ignore[operator]
|
||||
if operator == "gte":
|
||||
return left >= right # type: ignore[operator]
|
||||
if operator == "lt":
|
||||
return left < right # type: ignore[operator]
|
||||
if operator == "lte":
|
||||
return left <= right # type: ignore[operator]
|
||||
except TypeError as exc:
|
||||
raise ReportingQueryError("Report comparison values are incompatible.") from exc
|
||||
raise ReportingQueryError(f"Unsupported comparison operator: {operator!r}.")
|
||||
|
||||
|
||||
def _arity(values: Sequence[object], count: int, operator: str) -> None:
|
||||
if len(values) != count:
|
||||
raise ReportingQueryError(
|
||||
f"Expression {operator} requires exactly {count} arguments."
|
||||
)
|
||||
|
||||
|
||||
def _number(value: object) -> Decimal:
|
||||
if value is None or isinstance(value, bool):
|
||||
raise ReportingQueryError("A numeric report expression received a non-number.")
|
||||
try:
|
||||
return Decimal(str(value))
|
||||
except Exception as exc:
|
||||
raise ReportingQueryError("A numeric report expression is invalid.") from exc
|
||||
|
||||
|
||||
def _numeric_result(value: Decimal) -> int | float:
|
||||
if value == value.to_integral_value():
|
||||
return int(value)
|
||||
return float(value)
|
||||
|
||||
|
||||
def _sort_key(value: object) -> tuple[bool, str, str]:
|
||||
return value is None, type(value).__name__, str(value).casefold()
|
||||
|
||||
|
||||
def _hashable(value: object) -> object:
|
||||
if isinstance(value, Mapping):
|
||||
return tuple(sorted((str(key), _hashable(item)) for key, item in value.items()))
|
||||
if isinstance(value, (list, tuple)):
|
||||
return tuple(_hashable(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def _json_value(value: object) -> Any:
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, Decimal):
|
||||
return _numeric_result(value)
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _json_value(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_json_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QUERY_ENGINE_VERSION",
|
||||
"DefaultChartRenderer",
|
||||
"QueryResult",
|
||||
"ReportingQueryError",
|
||||
"execute_semantic_query",
|
||||
]
|
||||
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
from govoplan_reporting.backend.definitions import (
|
||||
create_definition,
|
||||
get_definition,
|
||||
list_definitions,
|
||||
update_definition,
|
||||
)
|
||||
|
||||
|
||||
class SqlReportingRegistry:
|
||||
def get_definition(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
revision: int | None = None,
|
||||
):
|
||||
return get_definition(
|
||||
session, # type: ignore[arg-type]
|
||||
principal,
|
||||
definition_kind=definition_kind,
|
||||
definition_id=definition_id,
|
||||
revision=revision,
|
||||
)
|
||||
|
||||
def list_definitions(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kinds: Sequence[str] | None = None,
|
||||
limit: int = 100,
|
||||
):
|
||||
records, _total = list_definitions(
|
||||
session, # type: ignore[arg-type]
|
||||
principal,
|
||||
definition_kinds=definition_kinds,
|
||||
limit=limit,
|
||||
)
|
||||
return records
|
||||
|
||||
def create_definition(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
**values,
|
||||
):
|
||||
return create_definition(session, principal, **values) # type: ignore[arg-type]
|
||||
|
||||
def update_definition(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
expected_revision: int,
|
||||
recorded_at,
|
||||
change_reason: str,
|
||||
idempotency_key: str,
|
||||
changes: Mapping[str, object],
|
||||
):
|
||||
return update_definition(
|
||||
session, # type: ignore[arg-type]
|
||||
principal,
|
||||
definition_kind=definition_kind,
|
||||
definition_id=definition_id,
|
||||
expected_revision=expected_revision,
|
||||
recorded_at=recorded_at,
|
||||
change_reason=change_reason,
|
||||
idempotency_key=idempotency_key,
|
||||
changes=changes,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["SqlReportingRegistry"]
|
||||
@@ -0,0 +1,526 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
|
||||
from govoplan_core.core.dataflows import dataflow_dataset_output
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_reporting.backend.definitions import (
|
||||
READ_SCOPE,
|
||||
WRITE_SCOPE,
|
||||
ReportingDefinitionError,
|
||||
create_definition,
|
||||
definition_history,
|
||||
get_definition,
|
||||
list_definitions,
|
||||
update_definition,
|
||||
)
|
||||
from govoplan_reporting.backend.execution import (
|
||||
QUALITY_SCOPE,
|
||||
RUN_SCOPE,
|
||||
ReportingExecutionError,
|
||||
ReportingExecutionFailure,
|
||||
execute_report,
|
||||
get_execution,
|
||||
list_executions,
|
||||
run_quality_plan,
|
||||
)
|
||||
from govoplan_reporting.backend.operations import (
|
||||
IMPORT_SCOPE,
|
||||
PUBLISH_SCOPE,
|
||||
SCHEDULE_SCOPE,
|
||||
ReportingOperationError,
|
||||
assess_import,
|
||||
delete_saved_view,
|
||||
dispatch_due_schedules,
|
||||
export_execution,
|
||||
list_import_assessments,
|
||||
list_saved_views,
|
||||
list_schedules,
|
||||
publish_execution,
|
||||
upsert_saved_view,
|
||||
upsert_schedule,
|
||||
)
|
||||
from govoplan_reporting.backend.query_engine import ReportingQueryError
|
||||
from govoplan_reporting.backend.schemas import (
|
||||
DefinitionUpdateRequest,
|
||||
DefinitionWriteRequest,
|
||||
ImportAssessmentRequest,
|
||||
PublicationRequest,
|
||||
QualityRunRequest,
|
||||
ReportExecutionRequest,
|
||||
SavedViewWriteRequest,
|
||||
ScheduleWriteRequest,
|
||||
)
|
||||
|
||||
|
||||
def create_router(registry: object | None) -> APIRouter:
|
||||
router = APIRouter(prefix="/reporting", tags=["reporting"])
|
||||
|
||||
@router.get("/definitions")
|
||||
def api_list_definitions(
|
||||
definition_kind: list[str] | None = Query(default=None),
|
||||
definition_status: list[str] | None = Query(default=None, alias="status"),
|
||||
query: str = "",
|
||||
offset: int = Query(default=0, ge=0),
|
||||
limit: int = Query(default=100, ge=1, le=200),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, READ_SCOPE)
|
||||
try:
|
||||
records, total = list_definitions(
|
||||
session,
|
||||
principal,
|
||||
definition_kinds=definition_kind,
|
||||
statuses=definition_status,
|
||||
query=query,
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
)
|
||||
except ReportingDefinitionError as exc:
|
||||
raise _error(exc) from exc
|
||||
return {
|
||||
"definitions": [item.to_dict() for item in records],
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
@router.post(
|
||||
"/definitions",
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def api_create_definition(
|
||||
payload: DefinitionWriteRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, Any]:
|
||||
_require(principal, WRITE_SCOPE)
|
||||
try:
|
||||
record = create_definition(
|
||||
session,
|
||||
principal,
|
||||
**payload.model_dump(),
|
||||
)
|
||||
session.commit()
|
||||
except (ReportingDefinitionError, ValueError) as exc:
|
||||
session.rollback()
|
||||
raise _error(exc) from exc
|
||||
return record.to_dict()
|
||||
|
||||
@router.get("/definitions/{definition_kind}/{definition_id}")
|
||||
def api_get_definition(
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
revision: int | None = Query(default=None, ge=1),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, Any]:
|
||||
_require(principal, READ_SCOPE)
|
||||
try:
|
||||
record = get_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind=definition_kind,
|
||||
definition_id=definition_id,
|
||||
revision=revision,
|
||||
)
|
||||
except ReportingDefinitionError as exc:
|
||||
raise _error(exc) from exc
|
||||
if record is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail="Reporting definition not found"
|
||||
)
|
||||
return record.to_dict()
|
||||
|
||||
@router.patch("/definitions/{definition_kind}/{definition_id}")
|
||||
def api_update_definition(
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
payload: DefinitionUpdateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, Any]:
|
||||
_require(principal, WRITE_SCOPE)
|
||||
changes = payload.model_dump(
|
||||
exclude={
|
||||
"expected_revision",
|
||||
"recorded_at",
|
||||
"change_reason",
|
||||
"idempotency_key",
|
||||
},
|
||||
exclude_unset=True,
|
||||
)
|
||||
try:
|
||||
record = update_definition(
|
||||
session,
|
||||
principal,
|
||||
definition_kind=definition_kind,
|
||||
definition_id=definition_id,
|
||||
expected_revision=payload.expected_revision,
|
||||
recorded_at=payload.recorded_at,
|
||||
change_reason=payload.change_reason,
|
||||
idempotency_key=payload.idempotency_key,
|
||||
changes=changes,
|
||||
)
|
||||
session.commit()
|
||||
except (
|
||||
ReportingDefinitionError,
|
||||
PermissionError,
|
||||
LookupError,
|
||||
ValueError,
|
||||
) as exc:
|
||||
session.rollback()
|
||||
raise _error(exc) from exc
|
||||
return record.to_dict()
|
||||
|
||||
@router.get("/definitions/{definition_kind}/{definition_id}/history")
|
||||
def api_definition_history(
|
||||
definition_kind: str,
|
||||
definition_id: str,
|
||||
limit: int = Query(default=100, ge=1, le=200),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, READ_SCOPE)
|
||||
records = definition_history(
|
||||
session,
|
||||
principal,
|
||||
definition_kind=definition_kind,
|
||||
definition_id=definition_id,
|
||||
limit=limit,
|
||||
)
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
status_code=404, detail="Reporting definition not found"
|
||||
)
|
||||
return {"revisions": [item.to_dict() for item in records]}
|
||||
|
||||
@router.get("/sources/dataflow")
|
||||
def api_dataflow_sources(
|
||||
query: str = "",
|
||||
limit: int = Query(default=100, ge=1, le=200),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, WRITE_SCOPE)
|
||||
provider = dataflow_dataset_output(registry)
|
||||
if provider is None:
|
||||
return {
|
||||
"available": False,
|
||||
"reason": "The Dataflow dataset provider is not enabled.",
|
||||
"sources": [],
|
||||
}
|
||||
sources = provider.list_outputs(
|
||||
session,
|
||||
principal,
|
||||
query=query,
|
||||
limit=limit,
|
||||
)
|
||||
return {
|
||||
"available": True,
|
||||
"reason": None,
|
||||
"sources": [
|
||||
{
|
||||
"pipeline_ref": item.pipeline_ref,
|
||||
"name": item.name,
|
||||
"description": item.description,
|
||||
"revision": item.revision,
|
||||
"definition_hash": item.definition_hash,
|
||||
"status": item.status,
|
||||
"updated_at": item.updated_at.isoformat()
|
||||
if item.updated_at
|
||||
else None,
|
||||
"parameters": dict(item.parameters),
|
||||
"provenance": dict(item.provenance),
|
||||
}
|
||||
for item in sources
|
||||
],
|
||||
}
|
||||
|
||||
@router.post("/reports/{report_id}/executions")
|
||||
def api_execute_report(
|
||||
report_id: str,
|
||||
payload: ReportExecutionRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, RUN_SCOPE)
|
||||
try:
|
||||
result = execute_report(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
report_id=report_id,
|
||||
report_revision=payload.report_revision,
|
||||
parameters=payload.parameters,
|
||||
query=payload.query,
|
||||
idempotency_key=payload.idempotency_key,
|
||||
)
|
||||
session.commit()
|
||||
except ReportingExecutionFailure as exc:
|
||||
session.commit()
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail={"message": str(exc), "execution_id": exc.execution_id},
|
||||
) from exc
|
||||
except (
|
||||
ReportingExecutionError,
|
||||
ReportingQueryError,
|
||||
PermissionError,
|
||||
LookupError,
|
||||
ValueError,
|
||||
) as exc:
|
||||
session.rollback()
|
||||
raise _error(exc) from exc
|
||||
return result
|
||||
|
||||
@router.get("/reports/{report_id}/executions")
|
||||
def api_list_executions(
|
||||
report_id: str,
|
||||
limit: int = Query(default=100, ge=1, le=200),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, RUN_SCOPE)
|
||||
return {
|
||||
"executions": list(
|
||||
list_executions(session, principal, report_id=report_id, limit=limit)
|
||||
)
|
||||
}
|
||||
|
||||
@router.get("/executions/{execution_id}")
|
||||
def api_get_execution(
|
||||
execution_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, RUN_SCOPE)
|
||||
result = get_execution(session, principal, execution_id=execution_id)
|
||||
if result is None:
|
||||
raise HTTPException(status_code=404, detail="Reporting execution not found")
|
||||
return result
|
||||
|
||||
@router.get("/executions/{execution_id}/export")
|
||||
def api_export_execution(
|
||||
execution_id: str,
|
||||
format: str = Query(default="csv", pattern="^(csv|json)$"),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> Response:
|
||||
_require(principal, RUN_SCOPE)
|
||||
try:
|
||||
content, media_type, filename = export_execution(
|
||||
session,
|
||||
principal,
|
||||
execution_id=execution_id,
|
||||
format=format,
|
||||
)
|
||||
except (ReportingOperationError, LookupError) as exc:
|
||||
raise _error(exc) from exc
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=media_type,
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
@router.post("/executions/{execution_id}/publications")
|
||||
def api_publish_execution(
|
||||
execution_id: str,
|
||||
payload: PublicationRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, PUBLISH_SCOPE)
|
||||
try:
|
||||
result = publish_execution(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
execution_id=execution_id,
|
||||
**payload.model_dump(),
|
||||
)
|
||||
session.commit()
|
||||
except (ReportingOperationError, PermissionError, LookupError) as exc:
|
||||
session.commit()
|
||||
raise _error(exc) from exc
|
||||
return result
|
||||
|
||||
@router.get("/reports/{report_id}/saved-views")
|
||||
def api_list_saved_views(
|
||||
report_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, READ_SCOPE)
|
||||
return {
|
||||
"views": list(list_saved_views(session, principal, report_id=report_id))
|
||||
}
|
||||
|
||||
@router.put("/reports/{report_id}/saved-views/{view_id}")
|
||||
def api_upsert_saved_view(
|
||||
report_id: str,
|
||||
view_id: str,
|
||||
payload: SavedViewWriteRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, READ_SCOPE)
|
||||
if view_id != payload.view_id:
|
||||
raise HTTPException(status_code=400, detail="Saved-view identifiers differ")
|
||||
try:
|
||||
result = upsert_saved_view(
|
||||
session,
|
||||
principal,
|
||||
report_id=report_id,
|
||||
**payload.model_dump(),
|
||||
)
|
||||
session.commit()
|
||||
except (ReportingOperationError, PermissionError, LookupError) as exc:
|
||||
session.rollback()
|
||||
raise _error(exc) from exc
|
||||
return result
|
||||
|
||||
@router.delete("/saved-views/{view_id}", status_code=204)
|
||||
def api_delete_saved_view(
|
||||
view_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> Response:
|
||||
_require(principal, READ_SCOPE)
|
||||
try:
|
||||
deleted = delete_saved_view(session, principal, view_id=view_id)
|
||||
session.commit()
|
||||
except PermissionError as exc:
|
||||
session.rollback()
|
||||
raise _error(exc) from exc
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail="Saved view not found")
|
||||
return Response(status_code=204)
|
||||
|
||||
@router.get("/schedules")
|
||||
def api_list_schedules(
|
||||
report_id: str | None = None,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, SCHEDULE_SCOPE)
|
||||
return {
|
||||
"schedules": list(list_schedules(session, principal, report_id=report_id))
|
||||
}
|
||||
|
||||
@router.put("/schedules/{schedule_id}")
|
||||
def api_upsert_schedule(
|
||||
schedule_id: str,
|
||||
payload: ScheduleWriteRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, SCHEDULE_SCOPE)
|
||||
if schedule_id != payload.schedule_id:
|
||||
raise HTTPException(status_code=400, detail="Schedule identifiers differ")
|
||||
try:
|
||||
result = upsert_schedule(session, principal, **payload.model_dump())
|
||||
session.commit()
|
||||
except (ReportingOperationError, PermissionError, LookupError) as exc:
|
||||
session.rollback()
|
||||
raise _error(exc) from exc
|
||||
return result
|
||||
|
||||
@router.post("/schedules/dispatch")
|
||||
def api_dispatch_schedules(
|
||||
limit: int = Query(default=20, ge=1, le=100),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, SCHEDULE_SCOPE)
|
||||
result = dispatch_due_schedules(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
now=None,
|
||||
limit=limit,
|
||||
)
|
||||
session.commit()
|
||||
return result
|
||||
|
||||
@router.post("/quality-plans/{quality_plan_id}/runs")
|
||||
def api_run_quality_plan(
|
||||
quality_plan_id: str,
|
||||
payload: QualityRunRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, QUALITY_SCOPE)
|
||||
try:
|
||||
result = run_quality_plan(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
quality_plan_id=quality_plan_id,
|
||||
quality_plan_revision=payload.quality_plan_revision,
|
||||
parameters=payload.parameters,
|
||||
)
|
||||
session.commit()
|
||||
except (ReportingExecutionError, PermissionError, LookupError) as exc:
|
||||
session.rollback()
|
||||
raise _error(exc) from exc
|
||||
return result
|
||||
|
||||
@router.post("/imports/assessments", status_code=201)
|
||||
def api_assess_import(
|
||||
payload: ImportAssessmentRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, IMPORT_SCOPE)
|
||||
try:
|
||||
result = assess_import(session, principal, **payload.model_dump())
|
||||
session.commit()
|
||||
except (ReportingOperationError, PermissionError) as exc:
|
||||
session.rollback()
|
||||
raise _error(exc) from exc
|
||||
return result
|
||||
|
||||
@router.get("/imports/assessments")
|
||||
def api_list_import_assessments(
|
||||
limit: int = Query(default=100, ge=1, le=200),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, IMPORT_SCOPE)
|
||||
return {
|
||||
"assessments": list(
|
||||
list_import_assessments(session, principal, limit=limit)
|
||||
)
|
||||
}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def _require(principal: ApiPrincipal, scope: str) -> None:
|
||||
if not has_scope(principal, scope):
|
||||
raise HTTPException(status_code=403, detail=f"Missing scope: {scope}")
|
||||
|
||||
|
||||
def _error(exc: Exception) -> HTTPException:
|
||||
message = str(exc)
|
||||
lowered = message.casefold()
|
||||
if isinstance(exc, LookupError):
|
||||
code = 404
|
||||
elif isinstance(exc, PermissionError):
|
||||
code = 403
|
||||
elif any(word in lowered for word in ("conflict", "already", "stale")):
|
||||
code = 409
|
||||
elif "unavailable" in lowered or "not enabled" in lowered:
|
||||
code = 503
|
||||
else:
|
||||
code = 400
|
||||
return HTTPException(status_code=code, detail=message)
|
||||
|
||||
|
||||
__all__ = ["create_router"]
|
||||
@@ -0,0 +1,503 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
DefinitionKind = Literal[
|
||||
"dataset",
|
||||
"semantic_model",
|
||||
"report",
|
||||
"quality_plan",
|
||||
]
|
||||
FieldType = Literal[
|
||||
"string",
|
||||
"integer",
|
||||
"number",
|
||||
"boolean",
|
||||
"date",
|
||||
"datetime",
|
||||
"json",
|
||||
]
|
||||
|
||||
|
||||
class ReportingReference(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
owner_module: str = Field(min_length=1, max_length=100)
|
||||
resource_type: str = Field(min_length=1, max_length=100)
|
||||
resource_id: str = Field(min_length=1, max_length=255)
|
||||
revision: str | None = Field(default=None, max_length=255)
|
||||
relationship: str = Field(default="related", min_length=1, max_length=80)
|
||||
label: str | None = Field(default=None, max_length=500)
|
||||
|
||||
|
||||
class DatasetField(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str = Field(min_length=1, max_length=255)
|
||||
type: FieldType = "string"
|
||||
label: str | None = Field(default=None, max_length=500)
|
||||
nullable: bool = True
|
||||
description: str | None = Field(default=None, max_length=4_000)
|
||||
classification: Literal[
|
||||
"public",
|
||||
"internal",
|
||||
"confidential",
|
||||
"restricted",
|
||||
] = "internal"
|
||||
|
||||
|
||||
class FreshnessPolicy(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
max_age_seconds: int | None = Field(default=None, ge=1, le=31_536_000)
|
||||
stale_action: Literal["allow", "warn", "block"] = "warn"
|
||||
require_source_fingerprints: bool = True
|
||||
|
||||
|
||||
class DatasetDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", populate_by_name=True)
|
||||
|
||||
source_kind: Literal["dataflow", "read_model", "static"]
|
||||
source_ref: str = Field(min_length=1, max_length=500)
|
||||
source_revision: int | None = Field(default=None, ge=1)
|
||||
definition_hash: str | None = Field(default=None, min_length=1, max_length=128)
|
||||
source_parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
static_rows: list[dict[str, Any]] = Field(default_factory=list, max_length=2_000)
|
||||
expected_source_fingerprints: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
max_length=100,
|
||||
)
|
||||
fields: list[DatasetField] = Field(
|
||||
default_factory=list,
|
||||
max_length=500,
|
||||
validation_alias="schema",
|
||||
serialization_alias="schema",
|
||||
)
|
||||
freshness: FreshnessPolicy = Field(default_factory=FreshnessPolicy)
|
||||
row_policy_ref: str | None = Field(default=None, max_length=500)
|
||||
purpose: str = Field(min_length=1, max_length=4_000)
|
||||
privacy: dict[str, Any] = Field(default_factory=dict)
|
||||
retention: dict[str, Any] = Field(default_factory=dict)
|
||||
policy_provenance: dict[str, Any] = Field(default_factory=dict)
|
||||
institutional_references: list[ReportingReference] = Field(
|
||||
default_factory=list,
|
||||
max_length=200,
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_source_pin(self) -> "DatasetDefinition":
|
||||
if self.source_kind == "dataflow" and self.source_revision is None:
|
||||
raise ValueError("Dataflow datasets require a pinned source revision.")
|
||||
if self.source_kind == "static" and not self.static_rows:
|
||||
raise ValueError("Static analytical datasets require static_rows.")
|
||||
names = [item.name for item in self.fields]
|
||||
if len(names) != len(set(names)):
|
||||
raise ValueError("Dataset schema field names must be unique.")
|
||||
return self
|
||||
|
||||
|
||||
class DimensionDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$")
|
||||
field: str = Field(min_length=1, max_length=255)
|
||||
label: str = Field(min_length=1, max_length=500)
|
||||
type: FieldType = "string"
|
||||
format: str | None = Field(default=None, max_length=255)
|
||||
default_sort: Literal["asc", "desc"] | None = None
|
||||
|
||||
|
||||
class HierarchyLevel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
dimension: str = Field(min_length=1, max_length=120)
|
||||
label: str | None = Field(default=None, max_length=500)
|
||||
|
||||
|
||||
class HierarchyDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$")
|
||||
label: str = Field(min_length=1, max_length=500)
|
||||
levels: list[HierarchyLevel] = Field(min_length=1, max_length=20)
|
||||
|
||||
|
||||
class TypedExpression(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
op: Literal[
|
||||
"literal",
|
||||
"field",
|
||||
"measure",
|
||||
"add",
|
||||
"subtract",
|
||||
"multiply",
|
||||
"divide",
|
||||
"coalesce",
|
||||
"case",
|
||||
"eq",
|
||||
"ne",
|
||||
"gt",
|
||||
"gte",
|
||||
"lt",
|
||||
"lte",
|
||||
"and",
|
||||
"or",
|
||||
"not",
|
||||
]
|
||||
value: Any = None
|
||||
ref: str | None = Field(default=None, max_length=255)
|
||||
args: list["TypedExpression"] = Field(default_factory=list, max_length=50)
|
||||
result_type: FieldType | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_shape(self) -> "TypedExpression":
|
||||
if self.op in {"field", "measure"} and not self.ref:
|
||||
raise ValueError(f"Expression {self.op} requires ref.")
|
||||
if self.op == "literal" and self.args:
|
||||
raise ValueError("Literal expressions cannot have arguments.")
|
||||
if self.op not in {"literal", "field", "measure"} and not self.args:
|
||||
raise ValueError(f"Expression {self.op} requires arguments.")
|
||||
return self
|
||||
|
||||
|
||||
class MeasureDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$")
|
||||
label: str = Field(min_length=1, max_length=500)
|
||||
aggregation: Literal[
|
||||
"sum",
|
||||
"count",
|
||||
"count_distinct",
|
||||
"average",
|
||||
"minimum",
|
||||
"maximum",
|
||||
"calculated",
|
||||
]
|
||||
field: str | None = Field(default=None, max_length=255)
|
||||
expression: TypedExpression | None = None
|
||||
format: str | None = Field(default=None, max_length=255)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_measure(self) -> "MeasureDefinition":
|
||||
if self.aggregation == "calculated" and self.expression is None:
|
||||
raise ValueError("Calculated measures require an expression.")
|
||||
if self.aggregation not in {"count", "calculated"} and not self.field:
|
||||
raise ValueError(f"{self.aggregation} measures require a field.")
|
||||
return self
|
||||
|
||||
|
||||
class SemanticModelDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
dataset_id: str = Field(min_length=1, max_length=255)
|
||||
dataset_revision: int = Field(ge=1)
|
||||
dimensions: list[DimensionDefinition] = Field(default_factory=list, max_length=300)
|
||||
hierarchies: list[HierarchyDefinition] = Field(default_factory=list, max_length=100)
|
||||
measures: list[MeasureDefinition] = Field(default_factory=list, max_length=300)
|
||||
default_dimensions: list[str] = Field(default_factory=list, max_length=50)
|
||||
default_measures: list[str] = Field(default_factory=list, max_length=50)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_semantics(self) -> "SemanticModelDefinition":
|
||||
dimensions = {item.key for item in self.dimensions}
|
||||
measures = {item.key for item in self.measures}
|
||||
if len(dimensions) != len(self.dimensions):
|
||||
raise ValueError("Semantic dimensions require unique keys.")
|
||||
if len(measures) != len(self.measures):
|
||||
raise ValueError("Semantic measures require unique keys.")
|
||||
unknown_dimensions = set(self.default_dimensions) - dimensions
|
||||
unknown_measures = set(self.default_measures) - measures
|
||||
hierarchy_dimensions = {
|
||||
level.dimension
|
||||
for hierarchy in self.hierarchies
|
||||
for level in hierarchy.levels
|
||||
}
|
||||
unknown_dimensions |= hierarchy_dimensions - dimensions
|
||||
if unknown_dimensions or unknown_measures:
|
||||
raise ValueError(
|
||||
"Semantic model references unknown dimensions or measures: "
|
||||
+ ", ".join(sorted(unknown_dimensions | unknown_measures))
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class ParameterDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$")
|
||||
label: str = Field(min_length=1, max_length=500)
|
||||
type: FieldType = "string"
|
||||
required: bool = False
|
||||
default: Any = None
|
||||
allowed_values: list[Any] = Field(default_factory=list, max_length=500)
|
||||
|
||||
|
||||
class FilterClause(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
dimension: str = Field(min_length=1, max_length=120)
|
||||
operator: Literal[
|
||||
"eq",
|
||||
"ne",
|
||||
"in",
|
||||
"not_in",
|
||||
"contains",
|
||||
"starts_with",
|
||||
"gt",
|
||||
"gte",
|
||||
"lt",
|
||||
"lte",
|
||||
"between",
|
||||
"is_null",
|
||||
"not_null",
|
||||
] = "eq"
|
||||
value: Any = None
|
||||
|
||||
|
||||
class SortClause(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
key: str = Field(min_length=1, max_length=120)
|
||||
direction: Literal["asc", "desc"] = "asc"
|
||||
|
||||
|
||||
class PivotDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
rows: list[str] = Field(default_factory=list, max_length=10)
|
||||
columns: list[str] = Field(default_factory=list, max_length=5)
|
||||
measures: list[str] = Field(default_factory=list, max_length=20)
|
||||
include_totals: bool = True
|
||||
|
||||
|
||||
class ReportQuery(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
mode: Literal["summary", "detail", "pivot"] = "summary"
|
||||
dimensions: list[str] = Field(default_factory=list, max_length=50)
|
||||
measures: list[str] = Field(default_factory=list, max_length=50)
|
||||
filters: list[FilterClause] = Field(default_factory=list, max_length=100)
|
||||
sort: list[SortClause] = Field(default_factory=list, max_length=20)
|
||||
pivot: PivotDefinition | None = None
|
||||
offset: int = Field(default=0, ge=0)
|
||||
limit: int = Field(default=200, ge=1, le=2_000)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_pivot(self) -> "ReportQuery":
|
||||
if self.mode == "pivot" and self.pivot is None:
|
||||
raise ValueError("Pivot mode requires a pivot definition.")
|
||||
return self
|
||||
|
||||
|
||||
class VisualizationDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
kind: Literal[
|
||||
"table",
|
||||
"pivot",
|
||||
"bar",
|
||||
"line",
|
||||
"area",
|
||||
"column",
|
||||
"pie",
|
||||
"metric",
|
||||
] = "table"
|
||||
category_dimension: str | None = Field(default=None, max_length=120)
|
||||
series_dimension: str | None = Field(default=None, max_length=120)
|
||||
measures: list[str] = Field(default_factory=list, max_length=20)
|
||||
options: dict[str, Any] = Field(default_factory=dict)
|
||||
tabular_fallback: bool = True
|
||||
|
||||
|
||||
class ReportDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
semantic_model_id: str = Field(min_length=1, max_length=255)
|
||||
semantic_model_revision: int = Field(ge=1)
|
||||
parameters: list[ParameterDefinition] = Field(default_factory=list, max_length=100)
|
||||
default_query: ReportQuery = Field(default_factory=ReportQuery)
|
||||
visualization: VisualizationDefinition = Field(
|
||||
default_factory=VisualizationDefinition
|
||||
)
|
||||
layout: dict[str, Any] = Field(default_factory=dict)
|
||||
access_policy: dict[str, Any] = Field(default_factory=dict)
|
||||
publication_defaults: dict[str, Any] = Field(default_factory=dict)
|
||||
institutional_references: list[ReportingReference] = Field(
|
||||
default_factory=list,
|
||||
max_length=200,
|
||||
)
|
||||
|
||||
|
||||
class QualityAssertion(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9._-]+$")
|
||||
kind: Literal[
|
||||
"not_null",
|
||||
"unique",
|
||||
"range",
|
||||
"accepted_values",
|
||||
"row_count",
|
||||
"comparison",
|
||||
]
|
||||
field: str | None = Field(default=None, max_length=255)
|
||||
severity: Literal["info", "warning", "error", "blocker"] = "error"
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class QualityPlanDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
dataset_id: str = Field(min_length=1, max_length=255)
|
||||
dataset_revision: int = Field(ge=1)
|
||||
assertions: list[QualityAssertion] = Field(min_length=1, max_length=200)
|
||||
block_report_execution: bool = True
|
||||
|
||||
|
||||
DEFINITION_PAYLOAD_TYPES = {
|
||||
"dataset": DatasetDefinition,
|
||||
"semantic_model": SemanticModelDefinition,
|
||||
"report": ReportDefinition,
|
||||
"quality_plan": QualityPlanDefinition,
|
||||
}
|
||||
|
||||
|
||||
def validate_definition_payload(
|
||||
kind: str,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
model = DEFINITION_PAYLOAD_TYPES.get(kind)
|
||||
if model is None:
|
||||
raise ValueError(f"Unsupported Reporting definition kind: {kind!r}.")
|
||||
return model.model_validate(payload).model_dump(mode="json", by_alias=True)
|
||||
|
||||
|
||||
class DefinitionWriteRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
definition_kind: DefinitionKind
|
||||
definition_id: str = Field(min_length=1, max_length=255)
|
||||
definition_key: str = Field(
|
||||
min_length=1,
|
||||
max_length=120,
|
||||
pattern=r"^[a-z0-9._-]+$",
|
||||
)
|
||||
name: str = Field(min_length=1, max_length=500)
|
||||
description: str | None = Field(default=None, max_length=100_000)
|
||||
status: Literal["draft", "active", "retired"] = "draft"
|
||||
visibility: Literal["tenant", "restricted"] = "tenant"
|
||||
recorded_at: datetime
|
||||
change_reason: str = Field(min_length=1, max_length=1_000)
|
||||
payload: dict[str, Any]
|
||||
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class DefinitionUpdateRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
expected_revision: int = Field(ge=1)
|
||||
recorded_at: datetime
|
||||
change_reason: str = Field(min_length=1, max_length=1_000)
|
||||
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||
name: str | None = Field(default=None, min_length=1, max_length=500)
|
||||
description: str | None = Field(default=None, max_length=100_000)
|
||||
status: Literal["draft", "active", "retired"] | None = None
|
||||
visibility: Literal["tenant", "restricted"] | None = None
|
||||
payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class ReportExecutionRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
report_revision: int | None = Field(default=None, ge=1)
|
||||
parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
query: ReportQuery | None = None
|
||||
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class SavedViewWriteRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
view_id: str = Field(min_length=1, max_length=36)
|
||||
report_revision: int = Field(ge=1)
|
||||
name: str = Field(min_length=1, max_length=500)
|
||||
state: dict[str, Any]
|
||||
shared: bool = False
|
||||
access: dict[str, Any] = Field(default_factory=dict)
|
||||
expected_revision: int | None = Field(default=None, ge=1)
|
||||
|
||||
|
||||
class ScheduleWriteRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
schedule_id: str = Field(min_length=1, max_length=36)
|
||||
report_id: str = Field(min_length=1, max_length=255)
|
||||
report_revision: int = Field(ge=1)
|
||||
name: str = Field(min_length=1, max_length=500)
|
||||
trigger_kind: Literal["scheduled", "interval"]
|
||||
trigger_config: dict[str, Any]
|
||||
parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
query: ReportQuery = Field(default_factory=ReportQuery)
|
||||
publication_target: dict[str, Any] = Field(default_factory=dict)
|
||||
enabled: bool = True
|
||||
next_run_at: datetime | None = None
|
||||
expected_revision: int | None = Field(default=None, ge=1)
|
||||
|
||||
|
||||
class PublicationRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
target_capability: str = Field(min_length=1, max_length=255)
|
||||
target_ref: str | None = Field(default=None, max_length=1_000)
|
||||
format: Literal["json", "csv", "xlsx", "html", "pdf"] = "csv"
|
||||
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||
options: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class QualityRunRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
quality_plan_revision: int | None = Field(default=None, ge=1)
|
||||
parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ImportAssessmentRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
source_system: str = Field(min_length=1, max_length=255)
|
||||
source_id: str = Field(min_length=1, max_length=500)
|
||||
metadata: dict[str, Any]
|
||||
accepted_approximations: list[str] = Field(default_factory=list, max_length=500)
|
||||
|
||||
|
||||
TypedExpression.model_rebuild()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DatasetDefinition",
|
||||
"DefinitionUpdateRequest",
|
||||
"DefinitionWriteRequest",
|
||||
"DimensionDefinition",
|
||||
"FilterClause",
|
||||
"ImportAssessmentRequest",
|
||||
"MeasureDefinition",
|
||||
"PublicationRequest",
|
||||
"QualityPlanDefinition",
|
||||
"QualityRunRequest",
|
||||
"ReportDefinition",
|
||||
"ReportExecutionRequest",
|
||||
"ReportQuery",
|
||||
"SavedViewWriteRequest",
|
||||
"ScheduleWriteRequest",
|
||||
"SemanticModelDefinition",
|
||||
"TypedExpression",
|
||||
"VisualizationDefinition",
|
||||
"validate_definition_payload",
|
||||
]
|
||||
@@ -0,0 +1,202 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from urllib.parse import quote
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.modules import ModuleContext
|
||||
from govoplan_core.core.search import (
|
||||
SearchAuthorizationRequest,
|
||||
SearchBackfillPage,
|
||||
SearchBackfillRequest,
|
||||
SearchDocument,
|
||||
SearchResourceType,
|
||||
)
|
||||
from govoplan_reporting.backend.db.models import (
|
||||
ReportingDefinitionGrant,
|
||||
ReportingDefinitionIdentity,
|
||||
ReportingDefinitionRevision,
|
||||
)
|
||||
from govoplan_reporting.backend.definitions import can_read_definition
|
||||
|
||||
|
||||
PROVIDER_ID = "reporting.reports"
|
||||
RESOURCE_TYPE = "report"
|
||||
|
||||
|
||||
class ReportingSearchSource:
|
||||
def resource_types(self) -> Sequence[SearchResourceType]:
|
||||
return (
|
||||
SearchResourceType(
|
||||
provider_id=PROVIDER_ID,
|
||||
module_id="reporting",
|
||||
resource_type=RESOURCE_TYPE,
|
||||
label="Reports",
|
||||
requires_authorization_recheck=True,
|
||||
),
|
||||
)
|
||||
|
||||
def backfill(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
request: SearchBackfillRequest,
|
||||
) -> SearchBackfillPage:
|
||||
if request.provider_id != PROVIDER_ID or request.resource_type != RESOURCE_TYPE:
|
||||
raise ValueError("Unsupported Reporting search source.")
|
||||
db = _session(session)
|
||||
query = db.query(ReportingDefinitionRevision).filter(
|
||||
ReportingDefinitionRevision.tenant_id == request.tenant_id,
|
||||
ReportingDefinitionRevision.definition_kind == "report",
|
||||
ReportingDefinitionRevision.status != "retired",
|
||||
ReportingDefinitionRevision.superseded_at.is_(None),
|
||||
)
|
||||
if request.cursor:
|
||||
query = query.filter(ReportingDefinitionRevision.id > request.cursor)
|
||||
rows = (
|
||||
query.order_by(ReportingDefinitionRevision.id.asc())
|
||||
.limit(request.limit + 1)
|
||||
.all()
|
||||
)
|
||||
has_more = len(rows) > request.limit
|
||||
selected = rows[: request.limit]
|
||||
tokens = _acl_tokens(db, selected)
|
||||
high_watermark = (
|
||||
db.query(func.max(ReportingDefinitionRevision.updated_at))
|
||||
.filter(
|
||||
ReportingDefinitionRevision.tenant_id == request.tenant_id,
|
||||
ReportingDefinitionRevision.definition_kind == "report",
|
||||
ReportingDefinitionRevision.superseded_at.is_(None),
|
||||
)
|
||||
.scalar()
|
||||
)
|
||||
return SearchBackfillPage(
|
||||
documents=tuple(
|
||||
_document(row, tokens[row.definition_id]) for row in selected
|
||||
),
|
||||
next_cursor=selected[-1].id if has_more and selected else None,
|
||||
complete=not has_more,
|
||||
high_watermark=(
|
||||
high_watermark.isoformat() if high_watermark is not None else None
|
||||
),
|
||||
)
|
||||
|
||||
def authorize(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
requests: Sequence[SearchAuthorizationRequest],
|
||||
) -> Mapping[str, bool]:
|
||||
db = _session(session)
|
||||
tenant_id = str(getattr(principal, "tenant_id", "") or "")
|
||||
decisions = {request.reference.key: False for request in requests}
|
||||
for request in requests:
|
||||
reference = request.reference
|
||||
if (
|
||||
reference.tenant_id != tenant_id
|
||||
or reference.module_id != "reporting"
|
||||
or reference.resource_type != RESOURCE_TYPE
|
||||
):
|
||||
continue
|
||||
decisions[reference.key] = can_read_definition(
|
||||
db,
|
||||
principal,
|
||||
definition_kind="report",
|
||||
definition_id=reference.resource_id,
|
||||
)
|
||||
return decisions
|
||||
|
||||
|
||||
def create_reporting_search_source(
|
||||
context: ModuleContext,
|
||||
) -> ReportingSearchSource:
|
||||
del context
|
||||
return ReportingSearchSource()
|
||||
|
||||
|
||||
def _document(
|
||||
row: ReportingDefinitionRevision,
|
||||
tokens: tuple[str, ...],
|
||||
) -> SearchDocument:
|
||||
restricted_tokens = tuple(
|
||||
dict.fromkeys((*tokens, "scope:reporting:definition:admin"))
|
||||
)
|
||||
return SearchDocument(
|
||||
tenant_id=row.tenant_id,
|
||||
module_id="reporting",
|
||||
provider_id=PROVIDER_ID,
|
||||
resource_type=RESOURCE_TYPE,
|
||||
resource_id=row.definition_id,
|
||||
title=row.name,
|
||||
url=f"/reporting?reportId={quote(row.definition_id, safe='')}",
|
||||
summary=row.description,
|
||||
body=row.description,
|
||||
keywords=(row.definition_key, row.status),
|
||||
visibility=row.visibility,
|
||||
acl_tokens=restricted_tokens if row.visibility == "restricted" else (),
|
||||
source_revision=str(row.revision),
|
||||
source_updated_at=row.updated_at or row.recorded_at,
|
||||
metadata={
|
||||
"definition_key": row.definition_key,
|
||||
"status": row.status,
|
||||
"content_hash": row.content_hash,
|
||||
},
|
||||
requires_authorization_recheck=True,
|
||||
)
|
||||
|
||||
|
||||
def _acl_tokens(
|
||||
session: Session,
|
||||
rows: Sequence[ReportingDefinitionRevision],
|
||||
) -> Mapping[str, tuple[str, ...]]:
|
||||
result: dict[str, list[str]] = defaultdict(list)
|
||||
if not rows:
|
||||
return result
|
||||
identity_ids = {row.identity_id for row in rows}
|
||||
for identity in (
|
||||
session.query(ReportingDefinitionIdentity)
|
||||
.filter(ReportingDefinitionIdentity.id.in_(identity_ids))
|
||||
.all()
|
||||
):
|
||||
if identity.created_by:
|
||||
result[identity.definition_id].append(f"account:{identity.created_by}")
|
||||
definition_ids = {row.definition_id for row in rows}
|
||||
grants = (
|
||||
session.query(ReportingDefinitionGrant)
|
||||
.filter(
|
||||
ReportingDefinitionGrant.tenant_id == rows[0].tenant_id,
|
||||
ReportingDefinitionGrant.definition_kind == "report",
|
||||
ReportingDefinitionGrant.definition_id.in_(definition_ids),
|
||||
ReportingDefinitionGrant.active.is_(True),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for grant in grants:
|
||||
prefix = (
|
||||
"function"
|
||||
if grant.subject_kind == "function_assignment"
|
||||
else grant.subject_kind
|
||||
)
|
||||
result[grant.definition_id].append(f"{prefix}:{grant.subject_id}")
|
||||
return {
|
||||
definition_id: tuple(dict.fromkeys(values))
|
||||
for definition_id, values in result.items()
|
||||
}
|
||||
|
||||
|
||||
def _session(value: object) -> Session:
|
||||
if not isinstance(value, Session):
|
||||
raise TypeError("Reporting search requires a SQLAlchemy session.")
|
||||
return value
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PROVIDER_ID",
|
||||
"RESOURCE_TYPE",
|
||||
"ReportingSearchSource",
|
||||
"create_reporting_search_source",
|
||||
]
|
||||
Reference in New Issue
Block a user