GovOPlaN/govoplan-reporting
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Complete governed reporting execution and publication

Browse Source
This commit is contained in:
zemion
2026-08-04 02:23:24 +02:00
parent a81151391b
commit 8337aec19a
29 changed files with 3611 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_reporting/backend/contracts.py
+4
View File
@@ -10,6 +10,8 @@ CAPABILITY_REPORTING_REGISTRY = "reporting.registry"
CAPABILITY_REPORTING_RUNNER = "reporting.runner"
CAPABILITY_REPORTING_SCHEDULER = "reporting.scheduler"
CAPABILITY_REPORTING_CHART_RENDERER = "reporting.chart_renderer"
CAPABILITY_REPORTING_PUBLICATION_FILES = "reporting.publication.files"
CAPABILITY_REPORTING_PUBLICATION_MAIL = "reporting.publication.mail"
@dataclass(frozen=True, slots=True)
@@ -119,6 +121,8 @@ def capability(registry: object | None, name: str) -> object | None:
__all__ = [
"CAPABILITY_REPORTING_CHART_RENDERER",
"CAPABILITY_REPORTING_PUBLICATION_FILES",
"CAPABILITY_REPORTING_PUBLICATION_MAIL",
"CAPABILITY_REPORTING_REGISTRY",
"CAPABILITY_REPORTING_RUNNER",
"CAPABILITY_REPORTING_SCHEDULER",
src/govoplan_reporting/backend/db/__init__.py
+2
View File
@@ -4,6 +4,7 @@ from govoplan_reporting.backend.db.models import (
ReportingDefinitionGrant,
ReportingDefinitionIdentity,
ReportingDefinitionRevision,
ReportingDrillContext,
ReportingExecution,
ReportingImportAssessment,
ReportingPublication,
@@ -16,6 +17,7 @@ __all__ = [
"ReportingDefinitionGrant",
"ReportingDefinitionIdentity",
"ReportingDefinitionRevision",
"ReportingDrillContext",
"ReportingExecution",
"ReportingImportAssessment",
"ReportingPublication",
src/govoplan_reporting/backend/db/models.py
+45
View File
@@ -484,6 +484,50 @@ class ReportingPublication(Base, TimestampMixin):
)
class ReportingDrillContext(Base, TimestampMixin):
__tablename__ = "reporting_drill_contexts"
__table_args__ = (
UniqueConstraint(
"tenant_id", "drill_context_id", name="uq_reporting_drill_context"
),
Index(
"ix_reporting_drill_context_expiry",
"tenant_id",
"expires_at",
),
Index(
"ix_reporting_drill_context_execution",
"tenant_id",
"execution_id",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
drill_context_id: Mapped[str] = mapped_column(
String(36), nullable=False, index=True
)
execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
token_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
context_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
actor_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
dimension_path: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
source_fingerprints: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
policy_provenance: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
expires_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True
)
last_accessed_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
class ReportingQualityResult(Base, TimestampMixin):
__tablename__ = "reporting_quality_results"
__table_args__ = (
@@ -551,6 +595,7 @@ __all__ = [
"ReportingDefinitionGrant",
"ReportingDefinitionIdentity",
"ReportingDefinitionRevision",
"ReportingDrillContext",
"ReportingExecution",
"ReportingImportAssessment",
"ReportingPublication",
src/govoplan_reporting/backend/definitions.py
+75 -17
View File
@@ -27,6 +27,11 @@ from govoplan_reporting.backend.domain import (
ReportingDefinitionRecord,
definition_from_row,
)
from govoplan_reporting.backend.governance import (
apply_parent_governance,
normalize_definition_governance,
scope_visible,
)
from govoplan_reporting.backend.schemas import validate_definition_payload
@@ -86,17 +91,29 @@ def create_definition(
clean_reason = _required(change_reason, "Reporting change reason", 1_000)
_aware(recorded_at, "Reporting recorded_at")
validated_payload = validate_definition_payload(kind, dict(payload))
validated_payload = validate_definition_payload(
kind,
normalize_definition_governance(
validated_payload,
principal,
administrative=_has_scope(principal, ADMIN_SCOPE),
),
)
parent_kind, parent_id, parent_revision = _parent_reference(
kind,
validated_payload,
)
_validate_parent(
session,
tenant_id=tenant_id,
child_status=clean_status,
parent_kind=parent_kind,
parent_id=parent_id,
parent_revision=parent_revision,
validated_payload = validate_definition_payload(
kind,
_validate_parent(
session,
tenant_id=tenant_id,
child_status=clean_status,
parent_kind=parent_kind,
parent_id=parent_id,
parent_revision=parent_revision,
child_payload=validated_payload,
),
)
request = {
"definition_kind": kind,
@@ -241,14 +258,26 @@ def update_definition(
kind,
dict(changes.get("payload", current.payload)),
)
next_payload = validate_definition_payload(
kind,
normalize_definition_governance(
next_payload,
principal,
administrative=_has_scope(principal, ADMIN_SCOPE),
),
)
parent_kind, parent_id, parent_revision = _parent_reference(kind, next_payload)
_validate_parent(
session,
tenant_id=tenant_id,
child_status=next_status,
parent_kind=parent_kind,
parent_id=parent_id,
parent_revision=parent_revision,
next_payload = validate_definition_payload(
kind,
_validate_parent(
session,
tenant_id=tenant_id,
child_status=next_status,
parent_kind=parent_kind,
parent_id=parent_id,
parent_revision=parent_revision,
child_payload=next_payload,
),
)
identity = _identity(session, tenant_id, kind, definition_id)
if identity is None:
@@ -584,9 +613,10 @@ def _validate_parent(
parent_kind: str | None,
parent_id: str | None,
parent_revision: int | None,
) -> None:
child_payload: Mapping[str, object],
) -> dict[str, object]:
if parent_kind is None:
return
return dict(child_payload)
row = (
session.query(ReportingDefinitionRevision)
.filter(
@@ -605,6 +635,7 @@ def _validate_parent(
raise ReportingDefinitionError(
f"An active Reporting definition requires an active {parent_kind} revision."
)
return apply_parent_governance(child_payload, row.payload)
def _parent_reference(
@@ -667,6 +698,8 @@ def _can_access(
)
if row is None:
return False
if not scope_visible(row.payload, principal):
return False
if _has_scope(principal, ADMIN_SCOPE):
return True
identity = _identity(session, tenant_id, definition_kind, definition_id)
@@ -712,6 +745,24 @@ def _require_scope(principal: object, scope: str) -> None:
def _filter_accessible(query: Query, principal: object) -> Query:
if _has_scope(principal, ADMIN_SCOPE):
return query
governance = ReportingDefinitionRevision.payload["governance"]
scope_type = governance["scope_type"].as_string()
scope_id = governance["scope_id"].as_string()
inherited = governance["inherit_to_lower_scopes"].as_boolean()
scope_conditions = [
scope_type.is_(None),
and_(
scope_type == "tenant",
or_(scope_id.is_(None), scope_id == _principal_tenant(principal)),
),
and_(scope_type == "system", inherited.is_(True)),
]
group_ids = tuple(_string_subject_ids(principal, "group_ids"))
if group_ids:
scope_conditions.append(and_(scope_type == "group", scope_id.in_(group_ids)))
user_ids = _actor_ids(principal)
if user_ids:
scope_conditions.append(and_(scope_type == "user", scope_id.in_(user_ids)))
conditions = [ReportingDefinitionRevision.visibility == "tenant"]
actor_ids = _actor_ids(principal)
if actor_ids:
@@ -752,7 +803,14 @@ def _filter_accessible(query: Query, principal: object) -> Query:
)
)
)
return query.filter(or_(*conditions))
return query.filter(and_(or_(*scope_conditions), or_(*conditions)))
def _string_subject_ids(principal: object, attribute: str) -> tuple[str, ...]:
raw = getattr(principal, attribute, ()) or ()
if isinstance(raw, (str, bytes)):
return (str(raw),) if raw else ()
return tuple(dict.fromkeys(str(value) for value in raw if str(value or "").strip()))
def _current_row(
src/govoplan_reporting/backend/drilldown.py
+388
View File
@@ -0,0 +1,388 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from datetime import UTC, datetime, timedelta
import hashlib
import hmac
import json
import secrets
from typing import Any
import uuid
from sqlalchemy.orm import Session
from govoplan_core.security.time import utc_now
from govoplan_reporting.backend.db.models import (
ReportingDrillContext,
ReportingExecution,
)
from govoplan_reporting.backend.definitions import get_definition
from govoplan_reporting.backend.execution import (
ReportingExecutionError,
_apply_row_policy,
_read_dataset,
_validate_schema,
get_execution,
)
from govoplan_reporting.backend.postgres_planner import execute_postgres_query
from govoplan_reporting.backend.query_engine import execute_semantic_query
from govoplan_reporting.backend.schemas import (
DatasetDefinition,
FilterClause,
ReportDefinition,
ReportQuery,
SemanticModelDefinition,
)
DRILL_CONTEXT_TTL = timedelta(minutes=20)
class ReportingDrillError(ValueError):
pass
def create_drill_context(
session: Session,
principal: object,
*,
registry: object | None,
execution_id: str,
aggregate_row: Mapping[str, object],
limit: int,
) -> dict[str, object]:
execution_payload = get_execution(
session,
principal,
execution_id=execution_id,
registry=registry,
)
if execution_payload is None or execution_payload.get("status") != "succeeded":
raise LookupError("Successful Reporting execution not found.")
row = _execution(session, _tenant(principal), execution_id)
normalized_aggregate = _json_value(dict(aggregate_row))
if normalized_aggregate not in [
_json_value(dict(item)) for item in row.result_rows or []
]:
raise ReportingDrillError(
"The selected aggregate row does not belong to this execution."
)
semantic_record = get_definition(
session,
principal,
definition_kind="semantic_model",
definition_id=row.semantic_model_id,
revision=row.semantic_model_revision,
)
if semantic_record is None:
raise PermissionError("The report semantic model is no longer accessible.")
semantic = SemanticModelDefinition.model_validate(semantic_record.payload)
query = ReportQuery.model_validate(row.query or {})
dimension_keys = _drill_dimensions(query, semantic)
dimension_map = {item.key: item for item in semantic.dimensions}
path = [
{
"dimension": key,
"label": dimension_map[key].label,
"value": normalized_aggregate.get(key),
}
for key in dimension_keys
if key in normalized_aggregate
]
if not path:
raise ReportingDrillError(
"This aggregate has no dimension path to drill through."
)
bounded_limit = max(1, min(int(limit), 500))
actor_id = _actor(principal)
if not actor_id:
raise ReportingDrillError("Drill-through requires an accountable actor.")
drill_context_id = str(uuid.uuid4())
secret = secrets.token_urlsafe(32)
token = f"{drill_context_id}.{secret}"
context = {
"execution_id": execution_id,
"output_hash": row.output_hash,
"actor_id": actor_id,
"dimension_path": path,
"source_fingerprints": list(row.source_fingerprints or []),
"limit": bounded_limit,
}
item = ReportingDrillContext(
tenant_id=row.tenant_id,
drill_context_id=drill_context_id,
execution_id=execution_id,
token_sha256=_sha256(token),
context_sha256=_sha256(context),
actor_id=actor_id,
dimension_path=path,
source_fingerprints=list(row.source_fingerprints or []),
policy_provenance=dict(
execution_payload.get("delivery_authorization") or {}
),
expires_at=utc_now() + DRILL_CONTEXT_TTL,
)
item.policy_provenance["limit"] = bounded_limit
session.add(item)
session.flush()
return {
"token": token,
"drill_context_id": drill_context_id,
"execution_id": execution_id,
"dimension_path": path,
"expires_at": _datetime_text(item.expires_at),
}
def resolve_drill_context(
session: Session,
principal: object,
*,
registry: object | None,
token: str,
) -> dict[str, object]:
context_id, separator, _secret = token.partition(".")
if not separator or not context_id:
raise ReportingDrillError("The drill-through context token is invalid.")
item = (
session.query(ReportingDrillContext)
.filter(
ReportingDrillContext.tenant_id == _tenant(principal),
ReportingDrillContext.drill_context_id == context_id,
)
.one_or_none()
)
if item is None or not hmac.compare_digest(item.token_sha256, _sha256(token)):
raise LookupError("Reporting drill-through context not found.")
if item.actor_id != _actor(principal):
raise PermissionError(
"This drill-through context belongs to another account."
)
if _aware(item.expires_at) <= utc_now():
raise ReportingDrillError("The drill-through context has expired.")
row = _execution(session, item.tenant_id, item.execution_id)
expected_context = {
"execution_id": row.execution_id,
"output_hash": row.output_hash,
"actor_id": item.actor_id,
"dimension_path": list(item.dimension_path or []),
"source_fingerprints": list(item.source_fingerprints or []),
"limit": int((item.policy_provenance or {}).get("limit", 200)),
}
if not hmac.compare_digest(item.context_sha256, _sha256(expected_context)):
raise ReportingDrillError(
"The persisted drill-through context failed its integrity check."
)
execution_payload = get_execution(
session,
principal,
execution_id=row.execution_id,
registry=registry,
)
if execution_payload is None:
raise LookupError("Reporting execution not found.")
report_record = get_definition(
session,
principal,
definition_kind="report",
definition_id=row.report_id,
revision=row.report_revision,
)
semantic_record = get_definition(
session,
principal,
definition_kind="semantic_model",
definition_id=row.semantic_model_id,
revision=row.semantic_model_revision,
)
dataset_record = get_definition(
session,
principal,
definition_kind="dataset",
definition_id=row.dataset_id,
revision=row.dataset_revision,
)
if report_record is None or semantic_record is None or dataset_record is None:
raise PermissionError(
"The report source graph is no longer accessible for drill-through."
)
report = ReportDefinition.model_validate(report_record.payload)
semantic = SemanticModelDefinition.model_validate(semantic_record.payload)
dataset = DatasetDefinition.model_validate(dataset_record.payload)
source = _read_dataset(
session,
principal,
registry=registry,
dataset=dataset,
parameters=dict(row.parameters or {}),
)
if not _fingerprints_equal(
item.source_fingerprints or [], source.source_fingerprints
):
raise ReportingExecutionError(
"The source fingerprint changed after the aggregate execution; run the report again before drilling through."
)
normalized_rows = tuple(_json_value(dict(source_row)) for source_row in source.rows)
_validate_schema(dataset, normalized_rows)
authorized_rows, row_policy = _apply_row_policy(
session,
principal,
registry=registry,
dataset_id=dataset_record.definition_id,
dataset_revision=dataset_record.revision,
dataset=dataset,
rows=normalized_rows,
)
original = ReportQuery.model_validate(row.query or {})
hidden_dimensions = _strings(report.access_policy.get("hidden_dimensions"))
visible_dimensions = [
dimension.key
for dimension in semantic.dimensions
if dimension.key not in hidden_dimensions
]
filters = list(original.filters)
filters.extend(
FilterClause(
dimension=str(path_item["dimension"]),
operator="eq",
value=path_item.get("value"),
)
for path_item in item.dimension_path or []
)
detail_query = ReportQuery(
mode="detail",
dimensions=visible_dimensions,
filters=filters,
limit=int((item.policy_provenance or {}).get("limit", 200)),
)
result = execute_postgres_query(
session,
rows=authorized_rows,
dataset=dataset,
semantic_model=semantic,
query=detail_query,
) or execute_semantic_query(authorized_rows, semantic, detail_query)
item.last_accessed_at = utc_now()
item.policy_provenance = {
**dict(item.policy_provenance or {}),
"resolved_row_policy": dict(row_policy),
"delivery_authorization": dict(
execution_payload.get("delivery_authorization") or {}
),
}
session.flush()
return {
"drill_context_id": item.drill_context_id,
"execution_id": item.execution_id,
"dimension_path": list(item.dimension_path or []),
"rows": list(result.rows),
"schema": list(result.schema),
"total_rows": result.total_rows,
"truncated": result.truncated or source.truncated,
"source_fingerprints": list(source.source_fingerprints),
"policy_provenance": dict(item.policy_provenance or {}),
"expires_at": _datetime_text(item.expires_at),
}
def _drill_dimensions(
query: ReportQuery,
semantic: SemanticModelDefinition,
) -> tuple[str, ...]:
if query.mode == "pivot" and query.pivot is not None:
return tuple(dict.fromkeys((*query.pivot.rows, *query.pivot.columns)))
return tuple(query.dimensions or semantic.default_dimensions)
def _execution(
session: Session,
tenant_id: str,
execution_id: str,
) -> ReportingExecution:
row = (
session.query(ReportingExecution)
.filter(
ReportingExecution.tenant_id == tenant_id,
ReportingExecution.execution_id == execution_id,
)
.one_or_none()
)
if row is None:
raise LookupError("Reporting execution not found.")
return row
def _fingerprints_equal(
expected: Sequence[Mapping[str, object]],
actual: Sequence[Mapping[str, object]],
) -> bool:
normalize = lambda values: sorted( # noqa: E731 - compact canonicalizer
json.dumps(
_json_value(dict(item)),
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
)
for item in values
)
return normalize(expected) == normalize(actual)
def _tenant(principal: object) -> str:
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
if not tenant_id:
raise ReportingDrillError("Drill-through requires a tenant-bound principal.")
return tenant_id
def _actor(principal: object) -> str | None:
for value in (
getattr(principal, "account_id", None),
getattr(principal, "identity_id", None),
getattr(principal, "membership_id", None),
):
if str(value or "").strip():
return str(value)
return None
def _strings(value: object) -> set[str]:
if not isinstance(value, (list, tuple, set, frozenset)):
return set()
return {str(item) for item in value if str(item).strip()}
def _sha256(value: object) -> str:
payload = value if isinstance(value, str) else json.dumps(
_json_value(value),
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
)
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def _json_value(value: object) -> Any:
if isinstance(value, datetime):
return _aware(value).isoformat()
if isinstance(value, Mapping):
return {str(key): _json_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_value(item) for item in value]
return value
def _aware(value: datetime) -> datetime:
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
def _datetime_text(value: datetime) -> str:
return _aware(value).isoformat()
__all__ = [
"DRILL_CONTEXT_TTL",
"ReportingDrillError",
"create_drill_context",
"resolve_drill_context",
]
src/govoplan_reporting/backend/execution.py
+278 -17
View File
@@ -37,6 +37,12 @@ from govoplan_reporting.backend.db.models import (
ReportingQualityResult,
)
from govoplan_reporting.backend.definitions import get_definition, list_definitions
from govoplan_reporting.backend.domain import ReportingDefinitionRecord
from govoplan_reporting.backend.governance import require_definition_action
from govoplan_reporting.backend.postgres_planner import (
POSTGRES_PLANNER_VERSION,
execute_postgres_query,
)
from govoplan_reporting.backend.query_engine import (
QUERY_ENGINE_VERSION,
DefaultChartRenderer,
@@ -99,7 +105,12 @@ class SqlReportingRunner:
*,
execution_id: str,
) -> Mapping[str, object] | None:
return get_execution(_session(session), principal, execution_id=execution_id)
return get_execution(
_session(session),
principal,
execution_id=execution_id,
registry=self.registry,
)
def execute_report(
@@ -126,6 +137,13 @@ def execute_report(
if report_record.status != "active":
raise ReportingExecutionError("Only active report definitions can run.")
report = ReportDefinition.model_validate(report_record.payload)
report_decision = require_definition_action(
session,
principal,
registry=registry,
record=report_record,
action="run",
)
semantic_record = get_definition(
session,
principal,
@@ -138,6 +156,13 @@ def execute_report(
"The report's pinned semantic model is unavailable or inactive."
)
semantic = SemanticModelDefinition.model_validate(semantic_record.payload)
semantic_decision = require_definition_action(
session,
principal,
registry=registry,
record=semantic_record,
action="view",
)
dataset_record = get_definition(
session,
principal,
@@ -150,8 +175,15 @@ def execute_report(
"The report's pinned analytical dataset is unavailable or inactive."
)
dataset = DatasetDefinition.model_validate(dataset_record.payload)
dataset_decision = require_definition_action(
session,
principal,
registry=registry,
record=dataset_record,
action="view",
)
bound_parameters = _bind_parameters(report, parameters)
effective_query = query or report.default_query
effective_query = _enforce_query_access(report, query or report.default_query)
clean_idempotency_key = _required(
idempotency_key,
"Reporting execution idempotency key",
@@ -176,7 +208,19 @@ def execute_report(
request_sha256=request_sha256,
)
if replay is not None:
return _execution_payload(replay, report=report, registry=registry)
delivery = _authorize_execution_delivery(
session,
principal,
registry=registry,
row=replay,
report_record=report_record,
)
return _execution_payload(
replay,
report=report,
registry=registry,
delivery_authorization=delivery,
)
started_at = utc_now()
execution = ReportingExecution(
tenant_id=_tenant(principal),
@@ -232,7 +276,14 @@ def execute_report(
output_hash=source.output_hash,
source_fingerprints=source.source_fingerprints,
)
result = execute_semantic_query(authorized_rows, semantic, effective_query)
result = execute_postgres_query(
session,
rows=authorized_rows,
dataset=dataset,
semantic_model=semantic,
query=effective_query,
) or execute_semantic_query(authorized_rows, semantic, effective_query)
diagnostics.extend(result.diagnostics)
output_hash = _sha256(
{
"rows": result.rows,
@@ -244,7 +295,12 @@ def execute_report(
execution.status = "succeeded"
execution.source_fingerprints = _json_value(source.source_fingerprints)
execution.output_hash = output_hash
execution.executor_version = f"{QUERY_ENGINE_VERSION}+{source.executor_version}"
planner_version = (
POSTGRES_PLANNER_VERSION
if any(item.get("code") == "postgresql_semantic_plan" for item in result.diagnostics)
else QUERY_ENGINE_VERSION
)
execution.executor_version = f"{planner_version}+{source.executor_version}"
execution.result_schema = list(result.schema)
execution.result_rows = list(result.rows)
execution.total_rows = result.total_rows
@@ -259,11 +315,35 @@ def execute_report(
"report_content_hash": report_record.content_hash,
"semantic_model_content_hash": semantic_record.content_hash,
"dataset_content_hash": dataset_record.content_hash,
"definition_governance": {
"report": report_decision.to_dict(),
"semantic_model": semantic_decision.to_dict(),
"dataset": dataset_decision.to_dict(),
},
"access_explanation": _access_explanation(
report,
effective_query,
source_rows=len(normalized_rows),
authorized_rows=len(authorized_rows),
row_policy=policy_provenance,
),
}
execution.finished_at = utc_now()
session.flush()
_emit_execution_event(session, execution, report_record.name)
return _execution_payload(execution, report=report, registry=registry)
delivery = _authorize_execution_delivery(
session,
principal,
registry=registry,
row=execution,
report_record=report_record,
)
return _execution_payload(
execution,
report=report,
registry=registry,
delivery_authorization=delivery,
)
except Exception as exc:
execution.status = "failed"
execution.finished_at = utc_now()
@@ -284,6 +364,7 @@ def get_execution(
principal: object,
*,
execution_id: str,
registry: object | None = None,
) -> dict[str, object] | None:
row = (
session.query(ReportingExecution)
@@ -304,10 +385,18 @@ def get_execution(
)
if report_record is None:
return None
delivery = _authorize_execution_delivery(
session,
principal,
registry=registry,
row=row,
report_record=report_record,
)
return _execution_payload(
row,
report=ReportDefinition.model_validate(report_record.payload),
registry=None,
registry=registry,
delivery_authorization=delivery,
)
@@ -317,16 +406,15 @@ def list_executions(
*,
report_id: str,
limit: int = 100,
registry: object | None = None,
) -> tuple[dict[str, object], ...]:
if (
get_definition(
session,
principal,
definition_kind="report",
definition_id=report_id,
)
is None
):
current_report = get_definition(
session,
principal,
definition_kind="report",
definition_id=report_id,
)
if current_report is None:
return ()
rows = (
session.query(ReportingExecution)
@@ -338,7 +426,46 @@ def list_executions(
.limit(max(1, min(limit, 200)))
.all()
)
return tuple(_execution_payload(row, report=None, registry=None) for row in rows)
authorization_cache: dict[tuple[int, int, int], dict[str, object]] = {}
payloads: list[dict[str, object]] = []
for row in rows:
key = (
row.report_revision,
row.semantic_model_revision,
row.dataset_revision,
)
report_record = (
current_report
if current_report.revision == row.report_revision
else get_definition(
session,
principal,
definition_kind="report",
definition_id=row.report_id,
revision=row.report_revision,
)
)
if report_record is None:
continue
delivery = authorization_cache.get(key)
if delivery is None:
delivery = _authorize_execution_delivery(
session,
principal,
registry=registry,
row=row,
report_record=report_record,
)
authorization_cache[key] = delivery
payloads.append(
_execution_payload(
row,
report=ReportDefinition.model_validate(report_record.payload),
registry=registry,
delivery_authorization=delivery,
)
)
return tuple(payloads)
def _read_dataset(
@@ -717,11 +844,144 @@ def _evaluate_assertion(
}
def _enforce_query_access(
report: ReportDefinition,
query: ReportQuery,
) -> ReportQuery:
policy = report.access_policy
hidden_dimensions = _policy_strings(policy, "hidden_dimensions")
hidden_measures = _policy_strings(policy, "hidden_measures")
requested_dimensions = set(query.dimensions)
requested_dimensions.update(item.dimension for item in query.filters)
if query.pivot is not None:
requested_dimensions.update(query.pivot.rows)
requested_dimensions.update(query.pivot.columns)
requested_measures = set(query.measures)
if query.pivot is not None:
requested_measures.update(query.pivot.measures)
blocked = (requested_dimensions & hidden_dimensions) | (
requested_measures & hidden_measures
)
if blocked:
raise PermissionError(
"Policy hides requested Reporting fields: "
+ ", ".join(sorted(blocked))
)
if "run" in _policy_strings(policy, "disabled_actions"):
raise PermissionError(_policy_reason(policy, "run"))
return query
def _access_explanation(
report: ReportDefinition,
query: ReportQuery,
*,
source_rows: int,
authorized_rows: int,
row_policy: Mapping[str, object],
) -> dict[str, object]:
policy = report.access_policy
hidden_dimensions = sorted(_policy_strings(policy, "hidden_dimensions"))
hidden_measures = sorted(_policy_strings(policy, "hidden_measures"))
disabled_actions = sorted(_policy_strings(policy, "disabled_actions"))
reasons = policy.get("reasons")
return {
"hidden_dimensions": hidden_dimensions,
"hidden_measures": hidden_measures,
"hidden_rows": max(0, source_rows - authorized_rows),
"disabled_actions": disabled_actions,
"reasons": dict(reasons) if isinstance(reasons, Mapping) else {},
"row_policy": dict(row_policy),
"effective_query": query.model_dump(mode="json"),
}
def _authorize_execution_delivery(
session: Session,
principal: object,
*,
registry: object | None,
row: ReportingExecution,
report_record: ReportingDefinitionRecord,
) -> dict[str, object]:
report_decision = require_definition_action(
session,
principal,
registry=registry,
record=report_record,
action="view",
)
semantic_record = get_definition(
session,
principal,
definition_kind="semantic_model",
definition_id=row.semantic_model_id,
revision=row.semantic_model_revision,
)
dataset_record = get_definition(
session,
principal,
definition_kind="dataset",
definition_id=row.dataset_id,
revision=row.dataset_revision,
)
if semantic_record is None or dataset_record is None:
raise PermissionError(
"The source definitions for this report result are no longer accessible."
)
semantic_decision = require_definition_action(
session,
principal,
registry=registry,
record=semantic_record,
action="view",
)
dataset_decision = require_definition_action(
session,
principal,
registry=registry,
record=dataset_record,
action="view",
)
dataset = DatasetDefinition.model_validate(dataset_record.payload)
_empty, row_policy = _apply_row_policy(
session,
principal,
registry=registry,
dataset_id=dataset_record.definition_id,
dataset_revision=dataset_record.revision,
dataset=dataset,
rows=(),
)
return {
"checked": True,
"report": report_decision.to_dict(),
"semantic_model": semantic_decision.to_dict(),
"dataset": dataset_decision.to_dict(),
"row_policy": dict(row_policy),
}
def _policy_strings(policy: Mapping[str, object], key: str) -> set[str]:
raw = policy.get(key, ())
if not isinstance(raw, (list, tuple, set, frozenset)):
return set()
return {str(item) for item in raw if str(item).strip()}
def _policy_reason(policy: Mapping[str, object], action: str) -> str:
reasons = policy.get("reasons")
if isinstance(reasons, Mapping) and str(reasons.get(action) or "").strip():
return str(reasons[action])
return f"Policy disables the Reporting {action} action."
def _execution_payload(
row: ReportingExecution,
*,
report: ReportDefinition | None,
registry: object | None,
delivery_authorization: Mapping[str, object] | None = None,
) -> dict[str, object]:
payload: dict[str, object] = {
"execution_id": row.execution_id,
@@ -747,6 +1007,7 @@ def _execution_payload(
"started_at": _datetime_text(row.started_at),
"finished_at": _datetime_text(row.finished_at),
"actor_id": row.actor_id,
"delivery_authorization": dict(delivery_authorization or {}),
}
if row.status == "succeeded" and report is not None:
result = QueryResult(
src/govoplan_reporting/backend/governance.py
+399
View File
@@ -0,0 +1,399 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import Literal, cast
from govoplan_core.core.access import PrincipalRef
from govoplan_core.core.policy import (
DefinitionGovernanceAction,
DefinitionGovernanceRequest,
DefinitionScopeRef,
PolicyDecision,
PolicySourceStep,
definition_governance_policy,
)
from govoplan_reporting.backend.domain import ReportingDefinitionRecord
from govoplan_reporting.backend.schemas import DefinitionGovernance
_LIMITS = (
"inherit_to_lower_scopes",
"allow_run",
"allow_reuse",
"allow_automation",
)
_SCOPE_RANK = {"system": 0, "tenant": 1, "group": 2, "user": 3}
class ReportingGovernanceError(ValueError):
pass
def normalize_definition_governance(
payload: Mapping[str, object],
principal: object,
*,
administrative: bool,
) -> dict[str, object]:
result = dict(payload)
raw = result.get("governance")
governance = DefinitionGovernance.model_validate(
raw if isinstance(raw, Mapping) else {}
)
scope_type = governance.scope_type
scope_id = str(governance.scope_id or "").strip() or None
tenant_id = _tenant(principal)
if scope_type == "system":
if not _has_scope(principal, "system:governance:write"):
raise PermissionError(
"System Reporting definitions require system governance permission."
)
elif scope_type == "tenant":
if scope_id not in {None, tenant_id}:
raise PermissionError(
"Reporting definitions can only target the active tenant."
)
scope_id = tenant_id
elif scope_type == "group":
if scope_id not in _string_set(getattr(principal, "group_ids", ())):
if not administrative:
raise PermissionError(
"Group Reporting definitions require membership in that group."
)
elif scope_type == "user":
own_ids = {
str(getattr(principal, "account_id", "") or ""),
str(getattr(principal, "membership_id", "") or ""),
}
if scope_id not in own_ids and not administrative:
raise PermissionError(
"User Reporting definitions can only target the current account."
)
if scope_id == str(getattr(principal, "membership_id", "") or ""):
scope_id = str(getattr(principal, "account_id", "") or "")
effective = _effective_limits(governance)
result["governance"] = governance.model_copy(
update={
"scope_id": scope_id,
"inherit_to_lower_scopes": effective["inherit_to_lower_scopes"],
"allow_run": effective["allow_run"],
"allow_reuse": effective["allow_reuse"],
"allow_automation": effective["allow_automation"],
"source_effective_limits": dict(effective),
}
).model_dump(mode="json")
return result
def validate_parent_governance(
child_payload: Mapping[str, object],
parent_payload: Mapping[str, object],
) -> None:
child = _governance(child_payload)
parent = _governance(parent_payload)
child_scope = _scope(child)
parent_scope = _scope(parent)
if _SCOPE_RANK[child_scope.scope_type] < _SCOPE_RANK[parent_scope.scope_type]:
raise ReportingGovernanceError(
"A Reporting definition cannot broaden the scope of its parent."
)
if child_scope != parent_scope and not parent.inherit_to_lower_scopes:
raise ReportingGovernanceError(
"The parent Reporting definition is not inherited by lower scopes."
)
parent_limits = _effective_limits(parent)
child_limits = _effective_limits(child)
broadened = [key for key in _LIMITS if child_limits[key] and not parent_limits[key]]
if broadened:
raise ReportingGovernanceError(
"A child Reporting definition cannot broaden inherited limits: "
+ ", ".join(sorted(broadened))
)
def apply_parent_governance(
child_payload: Mapping[str, object],
parent_payload: Mapping[str, object],
) -> dict[str, object]:
"""Persist the effective parent restriction and its immediate provenance."""
validate_parent_governance(child_payload, parent_payload)
child = _governance(child_payload)
parent = _governance(parent_payload)
parent_limits = _effective_limits(parent)
effective = {
key: bool(getattr(child, key)) and parent_limits[key] for key in _LIMITS
}
parent_scope = {
"scope_type": parent.scope_type,
"scope_id": parent.scope_id,
}
if parent.source_scope:
parent_scope["inherited_from"] = dict(parent.source_scope)
result = dict(child_payload)
result["governance"] = child.model_copy(
update={
"inherit_to_lower_scopes": effective["inherit_to_lower_scopes"],
"allow_run": effective["allow_run"],
"allow_reuse": effective["allow_reuse"],
"allow_automation": effective["allow_automation"],
"source_scope": parent_scope,
"source_effective_limits": effective,
"derivation_provenance": {
**dict(child.derivation_provenance),
"parent_scope": parent_scope,
"restriction_mode": "intersection",
},
}
).model_dump(mode="json")
return result
def definition_decision(
session: object,
principal: object,
*,
registry: object | None,
record: ReportingDefinitionRecord,
action: DefinitionGovernanceAction,
) -> PolicyDecision:
governance = _governance(record.payload)
source = _scope(governance)
target = _target_scope(source, principal)
request = DefinitionGovernanceRequest(
module_id="reporting",
definition_ref=f"{record.definition_kind}:{record.definition_id}:{record.revision}",
tenant_id=_tenant(principal),
definition_scope=source,
target_scope=target,
definition_kind=cast(Literal["flow", "template"], "flow"),
action=action,
actor=_principal_ref(principal),
status=record.status,
inherit_to_lower_scopes=governance.inherit_to_lower_scopes,
allow_run=governance.allow_run,
allow_reuse=governance.allow_reuse,
allow_automation=governance.allow_automation,
context={
"ancestor_limits": dict(governance.source_effective_limits),
"ancestor_source": dict(governance.source_scope or {}),
"reporting_definition_kind": record.definition_kind,
},
)
provider = definition_governance_policy(registry)
if provider is not None:
return provider.resolve_definition_action(session, request=request)
return _fallback_decision(request)
def require_definition_action(
session: object,
principal: object,
*,
registry: object | None,
record: ReportingDefinitionRecord,
action: DefinitionGovernanceAction,
) -> PolicyDecision:
decision = definition_decision(
session,
principal,
registry=registry,
record=record,
action=action,
)
if not decision.allowed:
raise PermissionError(
decision.reason or f"Reporting definition action is denied: {action}."
)
return decision
def governance_payload(payload: Mapping[str, object]) -> dict[str, object]:
governance = _governance(payload)
return {
**governance.model_dump(mode="json"),
"effective_limits": _effective_limits(governance),
}
def scope_visible(payload: Mapping[str, object], principal: object) -> bool:
governance = _governance(payload)
scope = _scope(governance)
if scope.scope_type == "system":
return governance.inherit_to_lower_scopes or _has_scope(
principal, "reporting:definition:admin"
)
if scope.scope_type == "tenant":
return scope.scope_id in {None, _tenant(principal)}
if scope.scope_type == "group":
return scope.scope_id in _string_set(getattr(principal, "group_ids", ()))
return scope.scope_id in {
str(getattr(principal, "account_id", "") or ""),
str(getattr(principal, "membership_id", "") or ""),
}
def _fallback_decision(request: DefinitionGovernanceRequest) -> PolicyDecision:
source = request.definition_scope
target = request.target_scope
same_scope = source == target
inherited = (
_SCOPE_RANK[target.scope_type] >= _SCOPE_RANK[source.scope_type]
and request.inherit_to_lower_scopes
)
visible = same_scope or inherited
if request.action == "view":
allowed = visible
elif request.action == "edit":
allowed = same_scope
elif request.action == "run":
allowed = visible and request.status == "active" and request.allow_run
elif request.action == "reuse":
allowed = visible and request.allow_reuse
elif request.action == "automate":
allowed = visible and request.allow_automation
else:
allowed = visible and request.allow_reuse
reason = (
None
if allowed
else (
"The Reporting definition's scope or inherited limits do not allow this action."
)
)
return PolicyDecision(
allowed=allowed,
reason=reason,
source_path=(
PolicySourceStep(
scope_type=source.scope_type,
scope_id=source.scope_id,
label="Reporting definition governance",
applied_fields=_LIMITS,
policy={
"inherit_to_lower_scopes": request.inherit_to_lower_scopes,
"allow_run": request.allow_run,
"allow_reuse": request.allow_reuse,
"allow_automation": request.allow_automation,
},
),
),
requirements=() if allowed else (f"reporting.definition.{request.action}",),
details={
"provider": "reporting.conservative_fallback",
"definition_scope": source.path,
"target_scope": target.path,
"action": request.action,
},
)
def _governance(payload: Mapping[str, object]) -> DefinitionGovernance:
raw = payload.get("governance")
return DefinitionGovernance.model_validate(raw if isinstance(raw, Mapping) else {})
def _scope(governance: DefinitionGovernance) -> DefinitionScopeRef:
return DefinitionScopeRef(
scope_type=governance.scope_type,
scope_id=governance.scope_id,
)
def _target_scope(source: DefinitionScopeRef, principal: object) -> DefinitionScopeRef:
if source.scope_type == "group" and source.scope_id in _string_set(
getattr(principal, "group_ids", ())
):
return source
own_ids = {
str(getattr(principal, "account_id", "") or ""),
str(getattr(principal, "membership_id", "") or ""),
}
if source.scope_type == "user" and source.scope_id in own_ids:
return source
return DefinitionScopeRef("tenant", _tenant(principal))
def _effective_limits(governance: DefinitionGovernance) -> dict[str, bool]:
source = governance.source_effective_limits
return {
key: bool(getattr(governance, key)) and source.get(key, True) is True
for key in _LIMITS
}
def _principal_ref(principal: object) -> PrincipalRef:
converter = getattr(principal, "to_platform_principal", None)
if callable(converter):
converted = converter()
if isinstance(converted, PrincipalRef):
return converted
return PrincipalRef(
account_id=str(getattr(principal, "account_id", "") or "system"),
membership_id=_optional(getattr(principal, "membership_id", None)),
tenant_id=_tenant(principal),
identity_id=_optional(getattr(principal, "identity_id", None)),
scopes=frozenset(_string_set(getattr(principal, "scopes", ()))),
group_ids=frozenset(_string_set(getattr(principal, "group_ids", ()))),
role_ids=frozenset(_string_set(getattr(principal, "role_ids", ()))),
function_assignment_ids=frozenset(
_string_set(getattr(principal, "function_assignment_ids", ()))
),
service_account_id=_optional(getattr(principal, "service_account_id", None)),
acting_assignment_id=_optional(
getattr(principal, "acting_assignment_id", None)
),
)
def _has_scope(principal: object, scope: str) -> bool:
method = getattr(principal, "has", None)
if callable(method):
return bool(method(scope))
return scope in _string_set(getattr(principal, "scopes", ()))
def _tenant(principal: object) -> str:
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
if not tenant_id:
raise ReportingGovernanceError(
"Reporting governance requires a tenant-bound principal."
)
return tenant_id
def _string_set(value: object) -> set[str]:
if isinstance(value, (str, bytes)):
return {str(value)} if value else set()
try:
return {str(item) for item in value or () if str(item).strip()} # type: ignore[union-attr]
except TypeError:
return set()
def _optional(value: object) -> str | None:
clean = str(value or "").strip()
return clean or None
__all__ = [
"ReportingGovernanceError",
"apply_parent_governance",
"definition_decision",
"governance_payload",
"normalize_definition_governance",
"require_definition_action",
"scope_visible",
"validate_parent_governance",
]
__all__ = [
"ReportingGovernanceError",
"definition_decision",
"governance_payload",
"normalize_definition_governance",
"require_definition_action",
"scope_visible",
"validate_parent_governance",
]
src/govoplan_reporting/backend/manifest.py
+62 -3
View File
@@ -7,6 +7,8 @@ from govoplan_core.core.access import (
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
)
from govoplan_core.core.dataflows import CAPABILITY_DATAFLOW_DATASET_OUTPUT
from govoplan_core.core.files import CAPABILITY_FILES_ARTIFACT_STORE
from govoplan_core.core.mail import CAPABILITY_MAIL_NOTIFICATION_DELIVERY
from govoplan_core.core.module_guards import (
drop_table_retirement_provider,
persistent_table_uninstall_guard,
@@ -41,6 +43,8 @@ from govoplan_core.db.base import Base
from govoplan_reporting.backend.acl import ReportingScopeAclProvider
from govoplan_reporting.backend.contracts import (
CAPABILITY_REPORTING_CHART_RENDERER,
CAPABILITY_REPORTING_PUBLICATION_FILES,
CAPABILITY_REPORTING_PUBLICATION_MAIL,
CAPABILITY_REPORTING_REGISTRY,
CAPABILITY_REPORTING_RUNNER,
CAPABILITY_REPORTING_SCHEDULER,
@@ -63,6 +67,10 @@ from govoplan_reporting.backend.operations import (
SqlReportingScheduler,
)
from govoplan_reporting.backend.query_engine import DefaultChartRenderer
from govoplan_reporting.backend.publication_targets import (
FilesReportingPublicationTarget,
MailReportingPublicationTarget,
)
from govoplan_reporting.backend.registry import SqlReportingRegistry
from govoplan_reporting.backend.search_source import create_reporting_search_source
@@ -175,6 +183,14 @@ def _chart_renderer(context: ModuleContext) -> DefaultChartRenderer:
return DefaultChartRenderer()
def _files_publication(context: ModuleContext) -> FilesReportingPublicationTarget:
return FilesReportingPublicationTarget(context.registry)
def _mail_publication(context: ModuleContext) -> MailReportingPublicationTarget:
return MailReportingPublicationTarget(context.registry)
def _retention(context: ModuleContext):
del context
from govoplan_reporting.backend.retention import ReportingRetentionService
@@ -246,6 +262,8 @@ manifest = ModuleManifest(
optional_capabilities=(
CAPABILITY_DATAFLOW_DATASET_OUTPUT,
CAPABILITY_POLICY_REPORTING_GOVERNANCE,
CAPABILITY_FILES_ARTIFACT_STORE,
CAPABILITY_MAIL_NOTIFICATION_DELIVERY,
),
permissions=PERMISSIONS,
role_templates=ROLE_TEMPLATES,
@@ -306,6 +324,13 @@ manifest = ModuleManifest(
parent_id="reporting.workspace",
order=40,
),
ViewSurface(
id="reporting.widget.reports",
module_id=MODULE_ID,
kind="section",
label="Reports dashboard widget",
order=75,
),
),
),
provides_interfaces=(
@@ -313,6 +338,12 @@ manifest = ModuleManifest(
ModuleInterfaceProvider(name="reporting.runner", version="0.1.0"),
ModuleInterfaceProvider(name="reporting.scheduler", version="0.1.0"),
ModuleInterfaceProvider(name="reporting.chart_renderer", version="0.1.0"),
ModuleInterfaceProvider(
name=CAPABILITY_REPORTING_PUBLICATION_FILES, version="1.0.0"
),
ModuleInterfaceProvider(
name=CAPABILITY_REPORTING_PUBLICATION_MAIL, version="1.0.0"
),
ModuleInterfaceProvider(name=CAPABILITY_REPORTING_RETENTION, version="1.0.0"),
),
requires_interfaces=(
@@ -328,12 +359,26 @@ manifest = ModuleManifest(
version_max_exclusive="2.0.0",
optional=True,
),
ModuleInterfaceRequirement(
name=CAPABILITY_FILES_ARTIFACT_STORE,
version_min="0.1.14",
version_max_exclusive="0.2.0",
optional=True,
),
ModuleInterfaceRequirement(
name="mail.notification_delivery",
version_min="0.1.0",
version_max_exclusive="2.0.0",
optional=True,
),
),
capability_factories={
CAPABILITY_REPORTING_REGISTRY: _registry,
CAPABILITY_REPORTING_RUNNER: _runner,
CAPABILITY_REPORTING_SCHEDULER: _scheduler,
CAPABILITY_REPORTING_CHART_RENDERER: _chart_renderer,
CAPABILITY_REPORTING_PUBLICATION_FILES: _files_publication,
CAPABILITY_REPORTING_PUBLICATION_MAIL: _mail_publication,
CAPABILITY_REPORTING_RETENTION: _retention,
},
capability_documentation={
@@ -357,6 +402,16 @@ manifest = ModuleManifest(
summary="Builds provider-neutral chart models with an accessible tabular fallback.",
contract_version="0.1.0",
),
CAPABILITY_REPORTING_PUBLICATION_FILES: CapabilityDocumentation(
label="Files report publication",
summary="Stores an immutable authorized report output through Files managed artifact storage.",
contract_version="1.0.0",
),
CAPABILITY_REPORTING_PUBLICATION_MAIL: CapabilityDocumentation(
label="Mail report publication",
summary="Submits an idempotent report notice through Mail's durable delivery outbox.",
contract_version="1.0.0",
),
CAPABILITY_REPORTING_RETENTION: CapabilityDocumentation(
label="Reporting result retention",
summary="Minimizes expired provider-report detail while retaining audit hashes and provenance.",
@@ -384,6 +439,7 @@ manifest = ModuleManifest(
reporting_models.ReportingSavedView,
reporting_models.ReportingDefinitionGrant,
reporting_models.ReportingExecution,
reporting_models.ReportingDrillContext,
reporting_models.ReportingProviderExport,
reporting_models.ReportingProviderExecution,
reporting_models.ReportingDefinitionRevision,
@@ -401,6 +457,7 @@ manifest = ModuleManifest(
reporting_models.ReportingDefinitionRevision,
reporting_models.ReportingDefinitionGrant,
reporting_models.ReportingExecution,
reporting_models.ReportingDrillContext,
reporting_models.ReportingProviderExecution,
reporting_models.ReportingProviderExport,
reporting_models.ReportingSavedView,
@@ -429,7 +486,9 @@ manifest = ModuleManifest(
"authorized result rows, diagnostics, and output hashes. Safe dimensions, "
"aggregations, typed expressions, filters, pivots, saved views, chart models, "
"schedules, exports, and publication providers replace unchecked SQL in the "
"presentation layer. Dataflow and module read models remain source owners."
"presentation layer. PostgreSQL executes bounded semantic plans when available. "
"Signed drill contexts reauthorize contributor rows, and Files/Mail publication "
"adapters retain idempotent evidence. Dataflow and module read models remain source owners."
),
layer="available",
documentation_types=("admin", "user"),
@@ -481,9 +540,9 @@ manifest = ModuleManifest(
),
known_limits=(
"Dataflow is the first live dataset adapter; additional module read models use the provider-neutral contract.",
"Direct browser export supports CSV and JSON; XLSX, PDF, Files, Mail, and DMS delivery require an optional publication provider.",
"Direct browser export supports CSV and JSON. Files supports CSV, JSON, and HTML publication; Mail submits a bounded report notice. XLSX/PDF require a renderer provider.",
"Import assessment produces blocking diagnostics but does not execute source SQL or automatically activate generated definitions.",
"The initial chart provider emits a renderer-neutral model and accessible table; richer visual renderers remain replaceable adapters.",
"The built-in chart catalogue covers bounded bar, column, line, area, pie, donut, and metric views; specialized visual renderers remain replaceable adapters.",
),
owned_concepts=(
"analytical dataset binding",
src/govoplan_reporting/backend/migrations/versions/c8d5e2f6a9b3_reporting_drill_contexts.py
+71
View File
@@ -0,0 +1,71 @@
"""Add authorization-bound Reporting drill contexts.
Revision ID: c8d5e2f6a9b3
Revises: b7c4e1a9d2f6
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "c8d5e2f6a9b3"
down_revision = "b7c4e1a9d2f6"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"reporting_drill_contexts",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("drill_context_id", sa.String(length=36), nullable=False),
sa.Column("execution_id", sa.String(length=36), nullable=False),
sa.Column("token_sha256", sa.String(length=64), nullable=False),
sa.Column("context_sha256", sa.String(length=64), nullable=False),
sa.Column("actor_id", sa.String(length=255), nullable=False),
sa.Column("dimension_path", sa.JSON(), nullable=False),
sa.Column("source_fingerprints", sa.JSON(), nullable=False),
sa.Column("policy_provenance", sa.JSON(), nullable=False),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("last_accessed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_drill_contexts")),
sa.UniqueConstraint(
"tenant_id",
"drill_context_id",
name="uq_reporting_drill_context",
),
)
for column in (
"tenant_id",
"drill_context_id",
"execution_id",
"actor_id",
"expires_at",
):
op.create_index(
op.f(f"ix_reporting_drill_contexts_{column}"),
"reporting_drill_contexts",
[column],
unique=False,
)
op.create_index(
"ix_reporting_drill_context_expiry",
"reporting_drill_contexts",
["tenant_id", "expires_at"],
unique=False,
)
op.create_index(
"ix_reporting_drill_context_execution",
"reporting_drill_contexts",
["tenant_id", "execution_id"],
unique=False,
)
def downgrade() -> None:
op.drop_table("reporting_drill_contexts")
src/govoplan_reporting/backend/operations.py
+48 -2
View File
@@ -396,7 +396,12 @@ def publish_execution(
options: Mapping[str, object],
) -> dict[str, object]:
_require_scope(principal, PUBLISH_SCOPE)
execution_payload = get_execution(session, principal, execution_id=execution_id)
execution_payload = get_execution(
session,
principal,
execution_id=execution_id,
registry=registry,
)
if execution_payload is None:
raise LookupError("Reporting execution not found.")
if execution_payload["status"] != "succeeded":
@@ -497,14 +502,54 @@ def publish_execution(
return _publication_payload(publication)
def list_publications(
session: Session,
principal: object,
*,
execution_id: str | None = None,
limit: int = 100,
registry: object | None = None,
) -> tuple[dict[str, object], ...]:
_require_scope(principal, PUBLISH_SCOPE)
statement = session.query(ReportingPublication).filter(
ReportingPublication.tenant_id == _tenant(principal)
)
if execution_id:
if (
get_execution(
session,
principal,
execution_id=execution_id,
registry=registry,
)
is None
):
return ()
statement = statement.filter(
ReportingPublication.execution_id == execution_id
)
rows = (
statement.order_by(ReportingPublication.created_at.desc())
.limit(max(1, min(limit, 200)))
.all()
)
return tuple(_publication_payload(row) for row in rows)
def export_execution(
session: Session,
principal: object,
*,
execution_id: str,
format: str,
registry: object | None = None,
) -> tuple[bytes, str, str]:
payload = get_execution(session, principal, execution_id=execution_id)
payload = get_execution(
session,
principal,
execution_id=execution_id,
registry=registry,
)
if payload is None:
raise LookupError("Reporting execution not found.")
if payload["status"] != "succeeded":
@@ -850,6 +895,7 @@ __all__ = [
"dispatch_due_schedules",
"export_execution",
"list_import_assessments",
"list_publications",
"list_saved_views",
"list_schedules",
"publish_execution",
src/govoplan_reporting/backend/postgres_planner.py
+509
View File
@@ -0,0 +1,509 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from datetime import date, datetime
from decimal import Decimal
import json
import re
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
from govoplan_reporting.backend.query_engine import (
QueryResult,
ReportingQueryError,
infer_query_schema,
)
from govoplan_reporting.backend.schemas import (
DatasetDefinition,
DimensionDefinition,
FilterClause,
MeasureDefinition,
ReportQuery,
SemanticModelDefinition,
TypedExpression,
)
POSTGRES_PLANNER_VERSION = "reporting-postgresql-v1"
_IDENTIFIER = re.compile(r"^[a-z0-9._-]{1,120}$")
class PostgresPlanningError(ReportingQueryError):
pass
def execute_postgres_query(
session: Session,
*,
rows: Sequence[Mapping[str, object]],
dataset: DatasetDefinition,
semantic_model: SemanticModelDefinition,
query: ReportQuery,
) -> QueryResult | None:
"""Execute a bounded semantic plan in PostgreSQL, or return None for fallback."""
if session.bind is None or session.bind.dialect.name != "postgresql":
return None
if query.mode == "pivot":
return None
plan = compile_postgres_query(dataset, semantic_model, query)
parameters = {
**plan.parameters,
"rows_json": json.dumps(
[_json_value(dict(item)) for item in rows],
ensure_ascii=True,
separators=(",", ":"),
sort_keys=True,
),
"result_limit": query.limit,
"result_offset": query.offset,
}
result = session.execute(text(plan.sql), parameters).mappings().all()
total_rows = int(result[0]["__reporting_total"]) if result else 0
output = tuple(
{
str(key): _json_value(value)
for key, value in item.items()
if key != "__reporting_total"
}
for item in result
)
return QueryResult(
rows=output,
total_rows=total_rows,
schema=infer_query_schema(output),
truncated=query.offset + len(output) < total_rows,
diagnostics=(
{
"severity": "info",
"code": "postgresql_semantic_plan",
"message": "Filters, grouping, measures, ordering, and bounds were executed by the PostgreSQL Reporting planner.",
"planner_version": POSTGRES_PLANNER_VERSION,
},
),
)
class CompiledPostgresPlan:
__slots__ = ("sql", "parameters")
def __init__(self, sql: str, parameters: Mapping[str, object]) -> None:
self.sql = sql
self.parameters = dict(parameters)
def compile_postgres_query(
dataset: DatasetDefinition,
semantic_model: SemanticModelDefinition,
query: ReportQuery,
) -> CompiledPostgresPlan:
dimensions = {item.key: item for item in semantic_model.dimensions}
measures = {item.key: item for item in semantic_model.measures}
selected_dimensions = tuple(query.dimensions or semantic_model.default_dimensions)
selected_measures = tuple(query.measures or semantic_model.default_measures)
_known(selected_dimensions, dimensions, "dimensions")
_known(selected_measures, measures, "measures")
_known(
tuple(item.dimension for item in query.filters), dimensions, "filter dimensions"
)
selected_keys = set(selected_dimensions)
if query.mode != "detail":
selected_keys.update(selected_measures)
_known(
tuple(item.key for item in query.sort),
{key: True for key in selected_keys},
"sort fields",
)
parameters: dict[str, object] = {}
source = (
"WITH source AS ("
"SELECT value AS source_row "
"FROM jsonb_array_elements(CAST(:rows_json AS jsonb)) AS source_items(value)"
")"
)
where = _filter_sql(query.filters, dimensions, parameters)
if query.mode == "detail":
fields = selected_dimensions
if not fields:
if not dataset.fields:
raise PostgresPlanningError(
"PostgreSQL detail planning requires selected dimensions or a pinned dataset schema."
)
field_types = {item.name: item.type for item in dataset.fields}
projections = [
f"{_source_value(item.name, item.type, parameters, f'detail_{index}')} AS {_quote(item.name)}"
for index, item in enumerate(dataset.fields)
]
selected_keys = set(field_types)
else:
projections = [
f"{_dimension_value(dimensions[key], parameters, f'detail_{index}')} AS {_quote(key)}"
for index, key in enumerate(fields)
]
body = "SELECT " + ", ".join(projections) + " FROM source" + where
else:
dimension_projections = [
(
key,
_dimension_value(dimensions[key], parameters, f"dimension_{index}"),
)
for index, key in enumerate(selected_dimensions)
]
selected_base_keys = [
key
for key in selected_measures
if measures[key].aggregation != "calculated"
]
calculated = [
measures[key]
for key in selected_measures
if measures[key].aggregation == "calculated"
]
dependency_keys = list(
dict.fromkeys(
dependency
for item in calculated
for dependency in _calculated_dependencies(
item.expression, measures, stack=(item.key,)
)
)
)
base_measure_keys = list(dict.fromkeys((*selected_base_keys, *dependency_keys)))
base_measures = [measures[key] for key in base_measure_keys]
grouped_select = [
f"{expression} AS {_quote(key)}"
for key, expression in dimension_projections
] + [
f"{_aggregate_sql(item, parameters, index)} AS {_quote(item.key)}"
for index, item in enumerate(base_measures)
]
if not grouped_select:
raise PostgresPlanningError(
"Summary queries require at least one dimension or measure."
)
grouped = "SELECT " + ", ".join(grouped_select) + " FROM source" + where
if dimension_projections:
grouped += " GROUP BY " + ", ".join(
expression for _key, expression in dimension_projections
)
if calculated:
outer = [_quote(key) for key in selected_dimensions] + [
_quote(key) for key in selected_base_keys
]
outer.extend(
f"{_calculated_sql(item.expression, parameters, f'calculated_{index}', measures=measures, stack=(item.key,))} AS {_quote(item.key)}"
for index, item in enumerate(calculated)
)
body = "SELECT " + ", ".join(outer) + f" FROM ({grouped}) AS grouped"
else:
body = grouped
order = ""
if query.sort:
order = " ORDER BY " + ", ".join(
f"{_quote(item.key)} {item.direction.upper()} NULLS LAST"
for item in query.sort
)
sql = (
source
+ " SELECT planned.*, COUNT(*) OVER() AS __reporting_total FROM ("
+ body
+ ") AS planned"
+ order
+ " LIMIT :result_limit OFFSET :result_offset"
)
return CompiledPostgresPlan(sql, parameters)
def _filter_sql(
filters: Sequence[FilterClause],
dimensions: Mapping[str, DimensionDefinition],
parameters: dict[str, object],
) -> str:
clauses: list[str] = []
for index, clause in enumerate(filters):
value = _dimension_value(
dimensions[clause.dimension], parameters, f"filter_field_{index}"
)
prefix = f"filter_{index}"
if clause.operator == "is_null":
clauses.append(f"{value} IS NULL")
continue
if clause.operator == "not_null":
clauses.append(f"{value} IS NOT NULL")
continue
if clause.operator in {"in", "not_in"}:
if not isinstance(clause.value, (list, tuple)):
raise PostgresPlanningError("Set filters require a list value.")
if not clause.value or len(clause.value) > 500:
raise PostgresPlanningError(
"Set filters require between 1 and 500 values."
)
names: list[str] = []
for item_index, item in enumerate(clause.value):
name = f"{prefix}_{item_index}"
parameters[name] = item
names.append(f":{name}")
operator = "NOT IN" if clause.operator == "not_in" else "IN"
clauses.append(f"{value} {operator} ({', '.join(names)})")
continue
if clause.operator == "between":
if not isinstance(clause.value, (list, tuple)) or len(clause.value) != 2:
raise PostgresPlanningError("Between filters require two values.")
parameters[f"{prefix}_low"] = clause.value[0]
parameters[f"{prefix}_high"] = clause.value[1]
clauses.append(f"{value} BETWEEN :{prefix}_low AND :{prefix}_high")
continue
parameters[prefix] = clause.value
if clause.operator == "contains":
parameters[prefix] = f"%{_like(str(clause.value or ''))}%"
clauses.append(
f"LOWER(CAST({value} AS text)) LIKE LOWER(:{prefix}) ESCAPE '\\'"
)
elif clause.operator == "starts_with":
parameters[prefix] = f"{_like(str(clause.value or ''))}%"
clauses.append(
f"LOWER(CAST({value} AS text)) LIKE LOWER(:{prefix}) ESCAPE '\\'"
)
else:
operator = {
"eq": "=",
"ne": "<>",
"gt": ">",
"gte": ">=",
"lt": "<",
"lte": "<=",
}.get(clause.operator)
if operator is None:
raise PostgresPlanningError(
f"Unsupported PostgreSQL filter operator: {clause.operator}."
)
clauses.append(f"{value} {operator} :{prefix}")
return " WHERE " + " AND ".join(clauses) if clauses else ""
def _aggregate_sql(
measure: MeasureDefinition,
parameters: dict[str, object],
index: int,
) -> str:
if measure.aggregation == "count" and measure.field is None:
return "COUNT(*)"
field = _source_value(
measure.field or "",
"number" if measure.aggregation in {"sum", "average"} else "string",
parameters,
f"measure_{index}",
)
if measure.aggregation == "count":
return f"COUNT({field})"
if measure.aggregation == "count_distinct":
return f"COUNT(DISTINCT {field})"
function = {
"sum": "SUM",
"average": "AVG",
"minimum": "MIN",
"maximum": "MAX",
}.get(measure.aggregation)
if function is None:
raise PostgresPlanningError(
f"Unsupported PostgreSQL aggregation: {measure.aggregation}."
)
return f"{function}({field})"
def _calculated_sql(
expression: TypedExpression | None,
parameters: dict[str, object],
prefix: str,
*,
measures: Mapping[str, MeasureDefinition],
stack: tuple[str, ...],
) -> str:
if expression is None:
return "NULL"
if expression.op == "literal":
parameters[prefix] = expression.value
return f":{prefix}"
if expression.op == "measure":
reference = expression.ref or ""
target = measures.get(reference)
if target is None:
raise PostgresPlanningError(
f"Calculated measure references unknown measure: {reference}."
)
if target.aggregation != "calculated":
return _quote(reference)
if reference in stack:
raise PostgresPlanningError(
"Calculated measure dependency cycle: "
+ " -> ".join((*stack, reference))
)
return _calculated_sql(
target.expression,
parameters,
prefix + "_" + reference,
measures=measures,
stack=(*stack, reference),
)
if expression.op == "field":
raise PostgresPlanningError(
"Calculated aggregate measures may reference measures, not source fields."
)
values = [
_calculated_sql(
item,
parameters,
f"{prefix}_{index}",
measures=measures,
stack=stack,
)
for index, item in enumerate(expression.args)
]
if expression.op in {"add", "multiply", "and", "or"}:
operator = {"add": "+", "multiply": "*", "and": "AND", "or": "OR"}[
expression.op
]
return "(" + f" {operator} ".join(values) + ")"
if expression.op in {"subtract", "divide", "eq", "ne", "gt", "gte", "lt", "lte"}:
if len(values) != 2:
raise PostgresPlanningError(
f"Expression {expression.op} requires exactly two arguments."
)
operator = {
"subtract": "-",
"divide": "/",
"eq": "=",
"ne": "<>",
"gt": ">",
"gte": ">=",
"lt": "<",
"lte": "<=",
}[expression.op]
right = f"NULLIF({values[1]}, 0)" if expression.op == "divide" else values[1]
return f"({values[0]} {operator} {right})"
if expression.op == "not":
if len(values) != 1:
raise PostgresPlanningError("Expression not requires one argument.")
return f"(NOT {values[0]})"
if expression.op == "coalesce":
return "COALESCE(" + ", ".join(values) + ")"
if expression.op == "case":
if len(values) < 3 or len(values) % 2 == 0:
raise PostgresPlanningError(
"Case expressions require condition/value pairs and a default."
)
branches = " ".join(
f"WHEN {values[index]} THEN {values[index + 1]}"
for index in range(0, len(values) - 1, 2)
)
return f"(CASE {branches} ELSE {values[-1]} END)"
raise PostgresPlanningError(
f"Unsupported PostgreSQL expression operator: {expression.op}."
)
def _calculated_dependencies(
expression: TypedExpression | None,
measures: Mapping[str, MeasureDefinition],
*,
stack: tuple[str, ...],
) -> tuple[str, ...]:
if expression is None:
return ()
if expression.op == "measure":
reference = expression.ref or ""
target = measures.get(reference)
if target is None:
raise PostgresPlanningError(
f"Calculated measure references unknown measure: {reference}."
)
if target.aggregation != "calculated":
return (reference,)
if reference in stack:
raise PostgresPlanningError(
"Calculated measure dependency cycle: "
+ " -> ".join((*stack, reference))
)
return _calculated_dependencies(
target.expression,
measures,
stack=(*stack, reference),
)
dependencies: list[str] = []
for item in expression.args:
dependencies.extend(_calculated_dependencies(item, measures, stack=stack))
return tuple(dict.fromkeys(dependencies))
def _dimension_value(
dimension: DimensionDefinition,
parameters: dict[str, object],
prefix: str,
) -> str:
return _source_value(dimension.field, dimension.type, parameters, prefix)
def _source_value(
field: str,
field_type: str,
parameters: dict[str, object],
prefix: str,
) -> str:
parameters[prefix] = field
raw = f"source_row ->> :{prefix}"
if field_type == "integer":
return f"NULLIF({raw}, '')::bigint"
if field_type == "number":
return f"NULLIF({raw}, '')::numeric"
if field_type == "boolean":
return f"NULLIF({raw}, '')::boolean"
if field_type == "date":
return f"NULLIF({raw}, '')::date"
if field_type == "datetime":
return f"NULLIF({raw}, '')::timestamptz"
if field_type == "json":
return f"source_row -> :{prefix}"
return raw
def _known(keys: Sequence[str], available: Mapping[str, object], label: str) -> None:
unknown = set(keys) - set(available)
if unknown:
raise PostgresPlanningError(
f"Report query references unknown {label}: " + ", ".join(sorted(unknown))
)
def _quote(value: str) -> str:
if not _IDENTIFIER.fullmatch(value):
raise PostgresPlanningError(f"Unsafe Reporting identifier: {value!r}.")
return '"' + value.replace('"', '""') + '"'
def _like(value: str) -> str:
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
def _json_value(value: object) -> Any:
if isinstance(value, Decimal):
integral = value.to_integral_value()
return int(integral) if value == integral else float(value)
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, Mapping):
return {str(key): _json_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_value(item) for item in value]
return value
__all__ = [
"POSTGRES_PLANNER_VERSION",
"CompiledPostgresPlan",
"PostgresPlanningError",
"compile_postgres_query",
"execute_postgres_query",
]
src/govoplan_reporting/backend/publication_targets.py
+303
View File
@@ -0,0 +1,303 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
import csv
from html import escape
from io import StringIO
import json
import re
from govoplan_core.core.files import (
CAPABILITY_FILES_ARTIFACT_STORE,
ManagedArtifactStore,
ManagedArtifactWriteRequest,
)
from govoplan_core.core.mail import (
CAPABILITY_MAIL_NOTIFICATION_DELIVERY,
NotificationMailDeliveryProvider,
NotificationMailDeliveryRequest,
)
from govoplan_reporting.backend.contracts import (
CAPABILITY_REPORTING_PUBLICATION_FILES,
CAPABILITY_REPORTING_PUBLICATION_MAIL,
ReportingPublicationPayload,
capability,
)
class FilesReportingPublicationTarget:
def __init__(self, registry: object | None) -> None:
self.registry = registry
def publish_report(
self,
session: object,
principal: object,
*,
payload: ReportingPublicationPayload,
) -> Mapping[str, object]:
provider = capability(self.registry, CAPABILITY_FILES_ARTIFACT_STORE)
if not isinstance(provider, ManagedArtifactStore):
raise RuntimeError(
"Files publication requires the enabled files.artifact_store capability."
)
content, content_type, extension = _serialize(payload)
filename = _filename(payload, extension)
folder = str(payload.target_ref or "Generated/Reports").strip()
stored = provider.store_artifact(
session,
principal,
request=ManagedArtifactWriteRequest(
filename=filename,
payload=content,
content_type=content_type,
folder=folder,
description=(
f"Reporting publication for {payload.report_id} revision "
f"{payload.report_revision}."
),
idempotency_key=f"reporting:{payload.publication_id}",
metadata={
"producer_module": "reporting",
"publication_id": payload.publication_id,
"execution_id": payload.execution_id,
"report_id": payload.report_id,
"report_revision": payload.report_revision,
"output_hash": payload.output_hash,
},
),
)
return {
"provider": CAPABILITY_FILES_ARTIFACT_STORE,
"status": "stored",
"file_asset_id": stored.file_asset_id,
"file_version_id": stored.file_version_id,
"filename": stored.filename,
"display_path": stored.display_path,
"sha256": stored.sha256,
"size_bytes": stored.size_bytes,
"output_hash": payload.output_hash,
}
class MailReportingPublicationTarget:
def __init__(self, registry: object | None) -> None:
self.registry = registry
def publish_report(
self,
session: object,
principal: object,
*,
payload: ReportingPublicationPayload,
) -> Mapping[str, object]:
provider = capability(self.registry, CAPABILITY_MAIL_NOTIFICATION_DELIVERY)
if not isinstance(provider, NotificationMailDeliveryProvider):
raise RuntimeError(
"Mail publication requires the enabled mail.notificationDelivery capability."
)
recipient = str(payload.target_ref or "").strip()
if not recipient:
raise ValueError("Mail publication requires a recipient address.")
options = dict(payload.options)
profile_id = _required_option(options, "mail_profile_id", "Mail profile")
from_address = _required_option(options, "from_address", "Sender address")
subject = str(
options.get("subject")
or f"Report {payload.report_id} revision {payload.report_revision}"
).strip()
action_url = str(options.get("action_url") or "").strip() or None
preview = _text_preview(payload.rows, payload.schema)
result = provider.submit_notification_mail(
session,
NotificationMailDeliveryRequest(
tenant_id=payload.tenant_id,
notification_id=f"reporting-publication:{payload.publication_id}",
recipient=recipient,
subject=subject,
body_text=(
f"Report: {payload.report_id}\n"
f"Revision: {payload.report_revision}\n"
f"Rows: {len(payload.rows)}\n"
f"Output hash: {payload.output_hash}\n\n"
f"{preview}"
),
action_url=action_url,
mail_profile_id=profile_id,
from_address=from_address,
smtp_server_id=_optional(options.get("smtp_server_id")),
smtp_credential_id=_optional(options.get("smtp_credential_id")),
metadata={
"producer_module": "reporting",
"publication_id": payload.publication_id,
"execution_id": payload.execution_id,
"report_id": payload.report_id,
"report_revision": payload.report_revision,
"output_hash": payload.output_hash,
},
),
)
status = str(result.get("status") or "").casefold()
if status not in {"accepted", "queued", "submitted", "succeeded"}:
raise RuntimeError(
str(result.get("error") or "Mail did not accept the report publication.")
)
return {
**dict(result),
"publication_id": payload.publication_id,
"recipient": recipient,
"output_hash": payload.output_hash,
}
def publication_target_catalog(registry: object | None) -> tuple[dict[str, object], ...]:
files_available = isinstance(
capability(registry, CAPABILITY_FILES_ARTIFACT_STORE), ManagedArtifactStore
)
mail_available = isinstance(
capability(registry, CAPABILITY_MAIL_NOTIFICATION_DELIVERY),
NotificationMailDeliveryProvider,
)
return (
{
"capability": CAPABILITY_REPORTING_PUBLICATION_FILES,
"label": "Files",
"available": files_available,
"reason": None
if files_available
else "Enable Files with managed artifact storage to publish durable report files.",
"formats": ["csv", "json", "html"],
"target_label": "Folder",
"target_required": False,
"required_options": [],
},
{
"capability": CAPABILITY_REPORTING_PUBLICATION_MAIL,
"label": "Mail",
"available": mail_available,
"reason": None
if mail_available
else "Enable Mail and configure its notification-delivery capability to publish report notices.",
"formats": ["html"],
"target_label": "Recipient",
"target_required": True,
"required_options": ["mail_profile_id", "from_address"],
},
)
def _serialize(payload: ReportingPublicationPayload) -> tuple[bytes, str, str]:
if payload.format == "json":
content = json.dumps(
{
"report_id": payload.report_id,
"report_revision": payload.report_revision,
"execution_id": payload.execution_id,
"output_hash": payload.output_hash,
"schema": list(payload.schema),
"rows": list(payload.rows),
},
ensure_ascii=False,
indent=2,
default=str,
).encode("utf-8")
return content, "application/json", "json"
if payload.format == "csv":
fields = _fields(payload.rows, payload.schema)
stream = StringIO(newline="")
writer = csv.DictWriter(stream, fieldnames=fields, extrasaction="ignore")
writer.writeheader()
for row in payload.rows:
writer.writerow({key: _safe_csv(row.get(key)) for key in fields})
return (
stream.getvalue().encode("utf-8-sig"),
"text/csv; charset=utf-8",
"csv",
)
if payload.format == "html":
fields = _fields(payload.rows, payload.schema)
headers = "".join(f"<th scope=\"col\">{escape(key)}</th>" for key in fields)
body = "".join(
"<tr>"
+ "".join(
f"<td>{escape(_display(row.get(key)))}</td>" for key in fields
)
+ "</tr>"
for row in payload.rows
)
content = (
"<!doctype html><html><head><meta charset=\"utf-8\"><title>"
+ escape(payload.report_id)
+ "</title></head><body><h1>"
+ escape(payload.report_id)
+ f"</h1><p>Revision {payload.report_revision}; output {escape(payload.output_hash)}</p>"
+ f"<table><thead><tr>{headers}</tr></thead><tbody>{body}</tbody></table>"
+ "</body></html>"
)
return content.encode("utf-8"), "text/html; charset=utf-8", "html"
raise ValueError(
"This publication target supports CSV, JSON, and accessible HTML. "
"XLSX and PDF require a renderer provider."
)
def _filename(payload: ReportingPublicationPayload, extension: str) -> str:
configured = str(payload.options.get("filename") or "").strip()
stem = configured.rsplit(".", 1)[0] if configured else payload.report_id
safe = re.sub(r"[^A-Za-z0-9._-]+", "-", stem).strip(".-") or "report"
return f"{safe}-r{payload.report_revision}.{extension}"
def _fields(
rows: Sequence[Mapping[str, object]], schema: Sequence[Mapping[str, object]]
) -> list[str]:
fields = [str(item.get("name")) for item in schema if item.get("name")]
if fields:
return fields
return list(dict.fromkeys(str(key) for row in rows for key in row))
def _safe_csv(value: object) -> object:
if isinstance(value, (dict, list, tuple)):
value = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
if isinstance(value, str) and value.startswith(("=", "+", "-", "@")):
return "'" + value
return value
def _display(value: object) -> str:
if value is None:
return ""
if isinstance(value, (dict, list, tuple)):
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
return str(value)
def _text_preview(
rows: Sequence[Mapping[str, object]], schema: Sequence[Mapping[str, object]]
) -> str:
fields = _fields(rows, schema)[:8]
lines = [" | ".join(fields)]
lines.extend(" | ".join(_display(row.get(key)) for key in fields) for row in rows[:10])
if len(rows) > 10:
lines.append(f"... {len(rows) - 10} more rows")
return "\n".join(lines)
def _required_option(options: Mapping[str, object], key: str, label: str) -> str:
value = str(options.get(key) or "").strip()
if not value:
raise ValueError(f"{label} is required for Mail publication.")
return value
def _optional(value: object) -> str | None:
clean = str(value or "").strip()
return clean or None
__all__ = [
"FilesReportingPublicationTarget",
"MailReportingPublicationTarget",
"publication_target_catalog",
]
src/govoplan_reporting/backend/query_engine.py
+3 -2
View File
@@ -86,7 +86,7 @@ def execute_semantic_query(
return QueryResult(
rows=tuple(selected),
total_rows=total,
schema=_infer_schema(selected or sorted_rows[:1]),
schema=infer_query_schema(selected or sorted_rows[:1]),
truncated=query.offset + len(selected) < total,
)
@@ -376,7 +376,7 @@ def _sort_rows(
return result
def _infer_schema(rows: Sequence[Mapping[str, object]]) -> tuple[dict[str, Any], ...]:
def infer_query_schema(rows: Sequence[Mapping[str, object]]) -> tuple[dict[str, Any], ...]:
names = tuple(dict.fromkeys(str(key) for row in rows for key in row))
return tuple(
{
@@ -491,4 +491,5 @@ __all__ = [
"QueryResult",
"ReportingQueryError",
"execute_semantic_query",
"infer_query_schema",
]
src/govoplan_reporting/backend/router.py
+102 -2
View File
@@ -18,6 +18,11 @@ from govoplan_reporting.backend.definitions import (
list_definitions,
update_definition,
)
from govoplan_reporting.backend.drilldown import (
ReportingDrillError,
create_drill_context,
resolve_drill_context,
)
from govoplan_reporting.backend.execution import (
QUALITY_SCOPE,
RUN_SCOPE,
@@ -38,6 +43,7 @@ from govoplan_reporting.backend.operations import (
dispatch_due_schedules,
export_execution,
list_import_assessments,
list_publications,
list_saved_views,
list_schedules,
publish_execution,
@@ -53,10 +59,12 @@ from govoplan_reporting.backend.provider_reports import (
list_provider_reports,
provider_parameter_options,
)
from govoplan_reporting.backend.publication_targets import publication_target_catalog
from govoplan_reporting.backend.query_engine import ReportingQueryError
from govoplan_reporting.backend.schemas import (
DefinitionUpdateRequest,
DefinitionWriteRequest,
DrillContextCreateRequest,
ImportAssessmentRequest,
PublicationRequest,
ProviderReportExecutionRequest,
@@ -432,7 +440,13 @@ def create_router(registry: object | None) -> APIRouter:
_require(principal, RUN_SCOPE)
return {
"executions": list(
list_executions(session, principal, report_id=report_id, limit=limit)
list_executions(
session,
principal,
report_id=report_id,
limit=limit,
registry=registry,
)
)
}
@@ -443,11 +457,69 @@ def create_router(registry: object | None) -> APIRouter:
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, RUN_SCOPE)
result = get_execution(session, principal, execution_id=execution_id)
result = get_execution(
session,
principal,
execution_id=execution_id,
registry=registry,
)
if result is None:
raise HTTPException(status_code=404, detail="Reporting execution not found")
return result
@router.post("/executions/{execution_id}/drill-contexts", status_code=201)
def api_create_drill_context(
execution_id: str,
payload: DrillContextCreateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, RUN_SCOPE)
try:
result = create_drill_context(
session,
principal,
registry=registry,
execution_id=execution_id,
aggregate_row=payload.aggregate_row,
limit=payload.limit,
)
session.commit()
except (
ReportingDrillError,
ReportingExecutionError,
PermissionError,
LookupError,
) as exc:
session.rollback()
raise _error(exc) from exc
return result
@router.get("/drill-contexts/{token}")
def api_resolve_drill_context(
token: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, RUN_SCOPE)
try:
result = resolve_drill_context(
session,
principal,
registry=registry,
token=token,
)
session.commit()
except (
ReportingDrillError,
ReportingExecutionError,
PermissionError,
LookupError,
) as exc:
session.rollback()
raise _error(exc) from exc
return result
@router.get("/executions/{execution_id}/export")
def api_export_execution(
execution_id: str,
@@ -462,6 +534,7 @@ def create_router(registry: object | None) -> APIRouter:
principal,
execution_id=execution_id,
format=format,
registry=registry,
)
except (ReportingOperationError, LookupError) as exc:
raise _error(exc) from exc
@@ -493,6 +566,33 @@ def create_router(registry: object | None) -> APIRouter:
raise _error(exc) from exc
return result
@router.get("/publication-targets")
def api_publication_targets(
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, PUBLISH_SCOPE)
return {"targets": list(publication_target_catalog(registry))}
@router.get("/publications")
def api_list_publications(
execution_id: str | None = None,
limit: int = Query(default=100, ge=1, le=200),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> dict[str, object]:
_require(principal, PUBLISH_SCOPE)
return {
"publications": list(
list_publications(
session,
principal,
execution_id=execution_id,
limit=limit,
registry=registry,
)
)
}
@router.get("/reports/{report_id}/saved-views")
def api_list_saved_views(
report_id: str,
src/govoplan_reporting/backend/schemas.py
+51
View File
@@ -58,6 +58,43 @@ class FreshnessPolicy(BaseModel):
require_source_fingerprints: bool = True
class DefinitionGovernance(BaseModel):
"""Versioned scope and restrictive inheritance metadata for a definition."""
model_config = ConfigDict(extra="forbid")
scope_type: Literal["system", "tenant", "group", "user"] = "tenant"
scope_id: str | None = Field(default=None, max_length=255)
inherit_to_lower_scopes: bool = False
allow_run: bool = True
allow_reuse: bool = False
allow_automation: bool = False
source_scope: dict[str, Any] | None = None
source_effective_limits: dict[str, bool] = Field(default_factory=dict)
derivation_provenance: dict[str, Any] = Field(default_factory=dict)
@model_validator(mode="after")
def validate_scope(self) -> "DefinitionGovernance":
if self.scope_type == "system":
if self.scope_id:
raise ValueError("System Reporting definitions do not carry a scope ID.")
elif self.scope_type in {"group", "user"} and not str(self.scope_id or "").strip():
raise ValueError(
f"{self.scope_type.capitalize()} Reporting definitions require a scope ID."
)
unknown = set(self.source_effective_limits) - {
"inherit_to_lower_scopes",
"allow_run",
"allow_reuse",
"allow_automation",
}
if unknown:
raise ValueError(
"Unknown inherited Reporting limits: " + ", ".join(sorted(unknown))
)
return self
class DatasetDefinition(BaseModel):
model_config = ConfigDict(extra="forbid", populate_by_name=True)
@@ -87,6 +124,7 @@ class DatasetDefinition(BaseModel):
default_factory=list,
max_length=200,
)
governance: DefinitionGovernance = Field(default_factory=DefinitionGovernance)
@model_validator(mode="after")
def validate_source_pin(self) -> "DatasetDefinition":
@@ -203,6 +241,7 @@ class SemanticModelDefinition(BaseModel):
default_dimensions: list[str] = Field(default_factory=list, max_length=50)
default_measures: list[str] = Field(default_factory=list, max_length=50)
metadata: dict[str, Any] = Field(default_factory=dict)
governance: DefinitionGovernance = Field(default_factory=DefinitionGovernance)
@model_validator(mode="after")
def validate_semantics(self) -> "SemanticModelDefinition":
@@ -307,6 +346,7 @@ class VisualizationDefinition(BaseModel):
"area",
"column",
"pie",
"donut",
"metric",
] = "table"
category_dimension: str | None = Field(default=None, max_length=120)
@@ -333,6 +373,7 @@ class ReportDefinition(BaseModel):
default_factory=list,
max_length=200,
)
governance: DefinitionGovernance = Field(default_factory=DefinitionGovernance)
class QualityAssertion(BaseModel):
@@ -359,6 +400,7 @@ class QualityPlanDefinition(BaseModel):
dataset_revision: int = Field(ge=1)
assertions: list[QualityAssertion] = Field(min_length=1, max_length=200)
block_report_execution: bool = True
governance: DefinitionGovernance = Field(default_factory=DefinitionGovernance)
DEFINITION_PAYLOAD_TYPES = {
@@ -478,6 +520,13 @@ class PublicationRequest(BaseModel):
options: dict[str, Any] = Field(default_factory=dict)
class DrillContextCreateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
aggregate_row: dict[str, Any] = Field(max_length=500)
limit: int = Field(default=200, ge=1, le=500)
class QualityRunRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
@@ -499,9 +548,11 @@ TypedExpression.model_rebuild()
__all__ = [
"DatasetDefinition",
"DefinitionGovernance",
"DefinitionUpdateRequest",
"DefinitionWriteRequest",
"DimensionDefinition",
"DrillContextCreateRequest",
"FilterClause",
"ImportAssessmentRequest",
"MeasureDefinition",