feat: add governed cross-module reporting
This commit is contained in:
@@ -218,6 +218,125 @@ class ReportingExecution(Base, TimestampMixin):
|
||||
actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||
|
||||
|
||||
class ReportingProviderExecution(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_provider_executions"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"execution_id",
|
||||
name="uq_reporting_provider_execution",
|
||||
),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"idempotency_key",
|
||||
name="uq_reporting_provider_execution_idempotency",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_provider_execution_history",
|
||||
"tenant_id",
|
||||
"provider_id",
|
||||
"report_id",
|
||||
"generated_at",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_provider_execution_retention",
|
||||
"tenant_id",
|
||||
"expires_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
provider_id: Mapped[str] = mapped_column(String(120), nullable=False, index=True)
|
||||
report_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
report_revision: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
contract_version: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
request_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
purpose: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
audience_scope: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
parameters: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
result_schema: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
result_payload: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
source_revisions: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
effective_scope: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
privacy_transforms: Mapped[list[str]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
provenance: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
governance_provenance: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
retention_class: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
retention_days: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
expires_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
retention_redacted_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
output_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
generated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True
|
||||
)
|
||||
actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||
|
||||
|
||||
class ReportingProviderExport(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_provider_exports"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"export_id",
|
||||
name="uq_reporting_provider_export",
|
||||
),
|
||||
Index(
|
||||
"ix_reporting_provider_export_history",
|
||||
"tenant_id",
|
||||
"provider_execution_id",
|
||||
"exported_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
export_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
provider_execution_id: Mapped[str] = mapped_column(
|
||||
ForeignKey(
|
||||
"reporting_provider_executions.id",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
format: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
purpose: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
audience_scope: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, default=dict, nullable=False
|
||||
)
|
||||
output_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
exported_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True
|
||||
)
|
||||
actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||
|
||||
|
||||
class ReportingDefinitionGrant(Base, TimestampMixin):
|
||||
__tablename__ = "reporting_definition_grants"
|
||||
__table_args__ = (
|
||||
@@ -435,6 +554,8 @@ __all__ = [
|
||||
"ReportingExecution",
|
||||
"ReportingImportAssessment",
|
||||
"ReportingPublication",
|
||||
"ReportingProviderExecution",
|
||||
"ReportingProviderExport",
|
||||
"ReportingQualityResult",
|
||||
"ReportingSavedView",
|
||||
"ReportingSchedule",
|
||||
|
||||
@@ -31,6 +31,10 @@ from govoplan_core.core.provider_governance import (
|
||||
ModuleArchitectureDocumentation,
|
||||
ModuleMaturityEvidence,
|
||||
)
|
||||
from govoplan_core.core.reporting import (
|
||||
CAPABILITY_POLICY_REPORTING_GOVERNANCE,
|
||||
CAPABILITY_REPORTING_RETENTION,
|
||||
)
|
||||
from govoplan_core.core.search import SearchSourceProviderRegistration
|
||||
from govoplan_core.core.views import ViewSurface
|
||||
from govoplan_core.db.base import Base
|
||||
@@ -171,6 +175,13 @@ def _chart_renderer(context: ModuleContext) -> DefaultChartRenderer:
|
||||
return DefaultChartRenderer()
|
||||
|
||||
|
||||
def _retention(context: ModuleContext):
|
||||
del context
|
||||
from govoplan_reporting.backend.retention import ReportingRetentionService
|
||||
|
||||
return ReportingRetentionService()
|
||||
|
||||
|
||||
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
|
||||
definitions = (
|
||||
session.query(reporting_models.ReportingDefinitionRevision)
|
||||
@@ -232,17 +243,21 @@ manifest = ModuleManifest(
|
||||
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
||||
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
||||
),
|
||||
optional_capabilities=(CAPABILITY_DATAFLOW_DATASET_OUTPUT,),
|
||||
optional_capabilities=(
|
||||
CAPABILITY_DATAFLOW_DATASET_OUTPUT,
|
||||
CAPABILITY_POLICY_REPORTING_GOVERNANCE,
|
||||
),
|
||||
permissions=PERMISSIONS,
|
||||
role_templates=ROLE_TEMPLATES,
|
||||
route_factory=_router,
|
||||
nav_items=(
|
||||
NavItem(
|
||||
path="/reporting",
|
||||
path="/reports",
|
||||
label="Reporting",
|
||||
icon="clipboard-pen-line",
|
||||
required_any=(READ_SCOPE,),
|
||||
order=74,
|
||||
surface_id="reporting.navigation",
|
||||
),
|
||||
),
|
||||
frontend=FrontendModule(
|
||||
@@ -250,36 +265,31 @@ manifest = ModuleManifest(
|
||||
package_name="@govoplan/reporting-webui",
|
||||
routes=(
|
||||
FrontendRoute(
|
||||
path="/reporting",
|
||||
path="/reports",
|
||||
component="ReportingPage",
|
||||
required_any=(READ_SCOPE,),
|
||||
order=74,
|
||||
surface_id="reporting.workspace",
|
||||
),
|
||||
FrontendRoute(
|
||||
path="/reporting",
|
||||
component="ReportingPage",
|
||||
required_any=(READ_SCOPE,),
|
||||
order=175,
|
||||
surface_id="reporting.compatibility",
|
||||
),
|
||||
),
|
||||
nav_items=(
|
||||
NavItem(
|
||||
path="/reporting",
|
||||
path="/reports",
|
||||
label="Reporting",
|
||||
icon="clipboard-pen-line",
|
||||
required_any=(READ_SCOPE,),
|
||||
order=74,
|
||||
surface_id="reporting.navigation",
|
||||
),
|
||||
),
|
||||
view_surfaces=(
|
||||
ViewSurface(
|
||||
id="reporting.navigation",
|
||||
module_id=MODULE_ID,
|
||||
kind="navigation",
|
||||
label="Reporting navigation",
|
||||
order=10,
|
||||
),
|
||||
ViewSurface(
|
||||
id="reporting.workspace",
|
||||
module_id=MODULE_ID,
|
||||
kind="route",
|
||||
label="Reporting workspace",
|
||||
order=20,
|
||||
),
|
||||
ViewSurface(
|
||||
id="reporting.parameters",
|
||||
module_id=MODULE_ID,
|
||||
@@ -303,6 +313,7 @@ manifest = ModuleManifest(
|
||||
ModuleInterfaceProvider(name="reporting.runner", version="0.1.0"),
|
||||
ModuleInterfaceProvider(name="reporting.scheduler", version="0.1.0"),
|
||||
ModuleInterfaceProvider(name="reporting.chart_renderer", version="0.1.0"),
|
||||
ModuleInterfaceProvider(name=CAPABILITY_REPORTING_RETENTION, version="1.0.0"),
|
||||
),
|
||||
requires_interfaces=(
|
||||
ModuleInterfaceRequirement(
|
||||
@@ -311,12 +322,19 @@ manifest = ModuleManifest(
|
||||
version_max_exclusive="0.2.0",
|
||||
optional=True,
|
||||
),
|
||||
ModuleInterfaceRequirement(
|
||||
name=CAPABILITY_POLICY_REPORTING_GOVERNANCE,
|
||||
version_min="1.0.0",
|
||||
version_max_exclusive="2.0.0",
|
||||
optional=True,
|
||||
),
|
||||
),
|
||||
capability_factories={
|
||||
CAPABILITY_REPORTING_REGISTRY: _registry,
|
||||
CAPABILITY_REPORTING_RUNNER: _runner,
|
||||
CAPABILITY_REPORTING_SCHEDULER: _scheduler,
|
||||
CAPABILITY_REPORTING_CHART_RENDERER: _chart_renderer,
|
||||
CAPABILITY_REPORTING_RETENTION: _retention,
|
||||
},
|
||||
capability_documentation={
|
||||
CAPABILITY_REPORTING_REGISTRY: CapabilityDocumentation(
|
||||
@@ -339,6 +357,13 @@ manifest = ModuleManifest(
|
||||
summary="Builds provider-neutral chart models with an accessible tabular fallback.",
|
||||
contract_version="0.1.0",
|
||||
),
|
||||
CAPABILITY_REPORTING_RETENTION: CapabilityDocumentation(
|
||||
label="Reporting result retention",
|
||||
summary="Minimizes expired provider-report detail while retaining audit hashes and provenance.",
|
||||
contract_version="1.0",
|
||||
documentation_types=("admin",),
|
||||
audience=("privacy_officer", "operator", "system_admin"),
|
||||
),
|
||||
},
|
||||
search_sources=(
|
||||
SearchSourceProviderRegistration(
|
||||
@@ -359,6 +384,8 @@ manifest = ModuleManifest(
|
||||
reporting_models.ReportingSavedView,
|
||||
reporting_models.ReportingDefinitionGrant,
|
||||
reporting_models.ReportingExecution,
|
||||
reporting_models.ReportingProviderExport,
|
||||
reporting_models.ReportingProviderExecution,
|
||||
reporting_models.ReportingDefinitionRevision,
|
||||
reporting_models.ReportingDefinitionIdentity,
|
||||
label="Reporting",
|
||||
@@ -374,6 +401,8 @@ manifest = ModuleManifest(
|
||||
reporting_models.ReportingDefinitionRevision,
|
||||
reporting_models.ReportingDefinitionGrant,
|
||||
reporting_models.ReportingExecution,
|
||||
reporting_models.ReportingProviderExecution,
|
||||
reporting_models.ReportingProviderExport,
|
||||
reporting_models.ReportingSavedView,
|
||||
reporting_models.ReportingSchedule,
|
||||
reporting_models.ReportingPublication,
|
||||
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
"""Add governed cross-module report executions and export history.
|
||||
|
||||
Revision ID: b7c4e1a9d2f6
|
||||
Revises: e5b2c9d4f7a1
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "b7c4e1a9d2f6"
|
||||
down_revision = "e5b2c9d4f7a1"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"reporting_provider_executions",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("execution_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("provider_id", sa.String(length=120), nullable=False),
|
||||
sa.Column("report_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("report_revision", sa.String(length=255), nullable=False),
|
||||
sa.Column("contract_version", sa.String(length=30), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("request_sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("purpose", sa.Text(), nullable=False),
|
||||
sa.Column("audience_scope", sa.JSON(), nullable=False),
|
||||
sa.Column("parameters", sa.JSON(), nullable=False),
|
||||
sa.Column("result_schema", sa.JSON(), nullable=False),
|
||||
sa.Column("result_payload", sa.JSON(), nullable=False),
|
||||
sa.Column("source_revisions", sa.JSON(), nullable=False),
|
||||
sa.Column("effective_scope", sa.JSON(), nullable=False),
|
||||
sa.Column("privacy_transforms", sa.JSON(), nullable=False),
|
||||
sa.Column("provenance", sa.JSON(), nullable=False),
|
||||
sa.Column("governance_provenance", sa.JSON(), nullable=False),
|
||||
sa.Column("retention_class", sa.String(length=120), nullable=False),
|
||||
sa.Column("retention_days", sa.Integer(), nullable=True),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("retention_redacted_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("output_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("generated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("actor_id", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_provider_executions")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"execution_id",
|
||||
name="uq_reporting_provider_execution",
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"idempotency_key",
|
||||
name="uq_reporting_provider_execution_idempotency",
|
||||
),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_provider_executions",
|
||||
"tenant_id",
|
||||
"execution_id",
|
||||
"provider_id",
|
||||
"report_id",
|
||||
"expires_at",
|
||||
"retention_redacted_at",
|
||||
"output_hash",
|
||||
"generated_at",
|
||||
"actor_id",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_provider_execution_history",
|
||||
"reporting_provider_executions",
|
||||
["tenant_id", "provider_id", "report_id", "generated_at"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_provider_execution_retention",
|
||||
"reporting_provider_executions",
|
||||
["tenant_id", "expires_at"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"reporting_provider_exports",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("export_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("provider_execution_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("execution_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("format", sa.String(length=30), nullable=False),
|
||||
sa.Column("purpose", sa.Text(), nullable=False),
|
||||
sa.Column("audience_scope", sa.JSON(), nullable=False),
|
||||
sa.Column("output_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("exported_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("actor_id", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["provider_execution_id"],
|
||||
["reporting_provider_executions.id"],
|
||||
name=op.f(
|
||||
"fk_reporting_provider_exports_provider_execution_id_reporting_provider_executions"
|
||||
),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_reporting_provider_exports")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"export_id",
|
||||
name="uq_reporting_provider_export",
|
||||
),
|
||||
)
|
||||
_indexes(
|
||||
"reporting_provider_exports",
|
||||
"tenant_id",
|
||||
"export_id",
|
||||
"provider_execution_id",
|
||||
"execution_id",
|
||||
"output_hash",
|
||||
"exported_at",
|
||||
"actor_id",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reporting_provider_export_history",
|
||||
"reporting_provider_exports",
|
||||
["tenant_id", "provider_execution_id", "exported_at"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("reporting_provider_exports")
|
||||
op.drop_table("reporting_provider_executions")
|
||||
|
||||
|
||||
def _indexes(table: str, *columns: str) -> None:
|
||||
for column in columns:
|
||||
op.create_index(op.f(f"ix_{table}_{column}"), table, [column], unique=False)
|
||||
@@ -0,0 +1,835 @@
|
||||
"""Governed execution boundary for module-contributed reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
import csv
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from hashlib import sha256
|
||||
import io
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.audit.logging import audit_from_principal
|
||||
from govoplan_core.core.reporting import (
|
||||
ReportDescriptor,
|
||||
ReportParameterOption,
|
||||
ReportProvider,
|
||||
ReportProviderRequest,
|
||||
ReportingGovernanceDecision,
|
||||
ReportingGovernanceRequest,
|
||||
report_providers,
|
||||
reporting_governance_provider,
|
||||
)
|
||||
from govoplan_reporting.backend.db.models import (
|
||||
ReportingProviderExecution,
|
||||
ReportingProviderExport,
|
||||
)
|
||||
|
||||
|
||||
class ProviderReportError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def list_provider_reports(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
) -> dict[str, object]:
|
||||
reports: list[dict[str, object]] = []
|
||||
diagnostics: list[dict[str, str]] = []
|
||||
for provider_id, provider in report_providers(registry):
|
||||
try:
|
||||
for descriptor in provider.list_reports(session, principal):
|
||||
_validate_descriptor(provider_id, descriptor)
|
||||
decision = _governance_decision(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
descriptor=descriptor,
|
||||
action="catalogue",
|
||||
purpose=None,
|
||||
audience_scope={},
|
||||
)
|
||||
reports.append(
|
||||
{
|
||||
**descriptor.to_dict(),
|
||||
"available": decision.allowed,
|
||||
"unavailable_reason": decision.reason,
|
||||
"governance": _decision_payload(decision),
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - isolate optional providers.
|
||||
diagnostics.append(
|
||||
{
|
||||
"provider_id": provider_id,
|
||||
"code": "provider_catalogue_failed",
|
||||
"message": "The provider catalogue could not be loaded.",
|
||||
"error_type": type(exc).__name__,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"reports": sorted(
|
||||
reports,
|
||||
key=lambda item: (str(item["title"]), str(item["provider_id"])),
|
||||
),
|
||||
"diagnostics": diagnostics,
|
||||
}
|
||||
|
||||
|
||||
def provider_parameter_options(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
provider_id: str,
|
||||
report_id: str,
|
||||
parameter_key: str,
|
||||
query: str,
|
||||
limit: int,
|
||||
) -> tuple[ReportParameterOption, ...]:
|
||||
provider, descriptor = _provider_report(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
provider_id=provider_id,
|
||||
report_id=report_id,
|
||||
)
|
||||
parameter = next(
|
||||
(item for item in descriptor.parameters if item.key == parameter_key),
|
||||
None,
|
||||
)
|
||||
if parameter is None or not parameter.options_from_provider:
|
||||
raise ProviderReportError("This report parameter has no provider options")
|
||||
return provider.parameter_options(
|
||||
session,
|
||||
principal,
|
||||
report_id=report_id,
|
||||
parameter_key=parameter_key,
|
||||
query=query,
|
||||
limit=max(1, min(limit, 200)),
|
||||
)
|
||||
|
||||
|
||||
def execute_provider_report(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
provider_id: str,
|
||||
report_id: str,
|
||||
parameters: Mapping[str, object],
|
||||
purpose: str,
|
||||
audience_scope: Mapping[str, object],
|
||||
idempotency_key: str,
|
||||
) -> dict[str, object]:
|
||||
provider, descriptor = _provider_report(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
provider_id=provider_id,
|
||||
report_id=report_id,
|
||||
)
|
||||
clean_purpose = purpose.strip()
|
||||
clean_idempotency_key = idempotency_key.strip()
|
||||
clean_parameters = _validated_parameters(descriptor, parameters)
|
||||
clean_audience = _json_mapping(audience_scope, "audience scope")
|
||||
if descriptor.purpose_required and not clean_purpose:
|
||||
raise ProviderReportError("A report purpose is required")
|
||||
if descriptor.audience_scope_required and not clean_audience:
|
||||
raise ProviderReportError("An effective audience scope is required")
|
||||
request_hash = _hash(
|
||||
{
|
||||
"provider_id": provider_id,
|
||||
"report_id": report_id,
|
||||
"revision": descriptor.revision,
|
||||
"parameters": clean_parameters,
|
||||
"purpose": clean_purpose,
|
||||
"audience_scope": clean_audience,
|
||||
}
|
||||
)
|
||||
replay = (
|
||||
session.query(ReportingProviderExecution)
|
||||
.filter(
|
||||
ReportingProviderExecution.tenant_id == _tenant(principal),
|
||||
ReportingProviderExecution.idempotency_key == clean_idempotency_key,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if replay is not None:
|
||||
if replay.request_sha256 != request_hash:
|
||||
raise ProviderReportError(
|
||||
"The provider-report idempotency key belongs to another request"
|
||||
)
|
||||
if _expired(replay) or replay.retention_redacted_at is not None:
|
||||
raise ProviderReportError(
|
||||
"The replayed provider-report result has expired; use a new idempotency key"
|
||||
)
|
||||
_require_result_access(provider, session, principal, replay)
|
||||
return provider_execution_payload(replay)
|
||||
|
||||
preflight = _governance_decision(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
descriptor=descriptor,
|
||||
action="execute",
|
||||
purpose=clean_purpose,
|
||||
audience_scope=clean_audience,
|
||||
)
|
||||
_require_allowed(preflight)
|
||||
result = provider.execute_report(
|
||||
session,
|
||||
principal,
|
||||
request=ReportProviderRequest(
|
||||
report_id=report_id,
|
||||
parameters=clean_parameters,
|
||||
purpose=clean_purpose,
|
||||
audience_scope=clean_audience,
|
||||
),
|
||||
)
|
||||
_validate_result(descriptor, result, tenant_id=_tenant(principal))
|
||||
decision = _governance_decision(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
descriptor=descriptor,
|
||||
action="execute",
|
||||
purpose=clean_purpose,
|
||||
audience_scope=clean_audience,
|
||||
applied_privacy_transforms=result.applied_privacy_transforms,
|
||||
)
|
||||
_require_allowed(decision)
|
||||
missing_policy_transforms = set(decision.required_privacy_transforms) - set(
|
||||
result.applied_privacy_transforms
|
||||
)
|
||||
if missing_policy_transforms:
|
||||
raise ProviderReportError(
|
||||
"The report omitted Policy-required privacy transformations: "
|
||||
+ ", ".join(sorted(missing_policy_transforms))
|
||||
)
|
||||
generated_at = _aware(result.generated_at)
|
||||
result_payload = _json_mapping(result.payload, "provider report payload")
|
||||
expires_at = (
|
||||
generated_at + timedelta(days=decision.retention_days)
|
||||
if decision.retention_days is not None
|
||||
else None
|
||||
)
|
||||
row = ReportingProviderExecution(
|
||||
tenant_id=_tenant(principal),
|
||||
execution_id=str(uuid.uuid4()),
|
||||
provider_id=provider_id,
|
||||
report_id=report_id,
|
||||
report_revision=descriptor.revision,
|
||||
contract_version=descriptor.contract_version,
|
||||
idempotency_key=clean_idempotency_key,
|
||||
request_sha256=request_hash,
|
||||
purpose=clean_purpose,
|
||||
audience_scope=clean_audience,
|
||||
parameters=clean_parameters,
|
||||
result_schema=[item.to_dict() for item in descriptor.result_schema],
|
||||
result_payload=result_payload,
|
||||
source_revisions=[dict(item) for item in result.source_revisions],
|
||||
effective_scope=dict(result.effective_scope),
|
||||
privacy_transforms=list(result.applied_privacy_transforms),
|
||||
provenance=dict(result.provenance),
|
||||
governance_provenance=dict(decision.provenance),
|
||||
retention_class=descriptor.retention_class,
|
||||
retention_days=decision.retention_days,
|
||||
expires_at=expires_at,
|
||||
output_hash=_hash(result_payload),
|
||||
generated_at=generated_at,
|
||||
actor_id=_actor(principal),
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="reporting.provider_report.executed",
|
||||
object_type="reporting_provider_execution",
|
||||
object_id=row.execution_id,
|
||||
details={
|
||||
"provider_id": provider_id,
|
||||
"report_id": report_id,
|
||||
"report_revision": descriptor.revision,
|
||||
"purpose": clean_purpose,
|
||||
"audience_scope": clean_audience,
|
||||
"source_revision_count": len(row.source_revisions),
|
||||
"privacy_transforms": row.privacy_transforms,
|
||||
"retention_class": row.retention_class,
|
||||
"retention_days": row.retention_days,
|
||||
"output_hash": row.output_hash,
|
||||
},
|
||||
)
|
||||
return provider_execution_payload(row)
|
||||
|
||||
|
||||
def get_provider_execution(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
execution_id: str,
|
||||
) -> dict[str, object] | None:
|
||||
row = _provider_execution(session, principal, execution_id=execution_id)
|
||||
if row is None or _expired(row):
|
||||
return None
|
||||
provider, _descriptor = _provider_report(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
provider_id=row.provider_id,
|
||||
report_id=row.report_id,
|
||||
)
|
||||
_require_result_access(provider, session, principal, row)
|
||||
return provider_execution_payload(row)
|
||||
|
||||
|
||||
def export_provider_execution(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
execution_id: str,
|
||||
format: str,
|
||||
purpose: str,
|
||||
audience_scope: Mapping[str, object],
|
||||
) -> tuple[bytes, str, str]:
|
||||
row = _provider_execution(session, principal, execution_id=execution_id)
|
||||
if row is None or _expired(row):
|
||||
raise LookupError("Provider report execution not found")
|
||||
provider, descriptor = _provider_report(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
provider_id=row.provider_id,
|
||||
report_id=row.report_id,
|
||||
)
|
||||
_require_result_access(provider, session, principal, row)
|
||||
clean_format = format.strip().lower()
|
||||
if clean_format not in descriptor.export_formats:
|
||||
raise ProviderReportError("This report does not support the export format")
|
||||
clean_purpose = purpose.strip()
|
||||
clean_audience = _json_mapping(audience_scope, "audience scope")
|
||||
decision = _governance_decision(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
descriptor=descriptor,
|
||||
action="export",
|
||||
purpose=clean_purpose,
|
||||
audience_scope=clean_audience,
|
||||
export_format=clean_format,
|
||||
applied_privacy_transforms=tuple(row.privacy_transforms or ()),
|
||||
)
|
||||
_require_allowed(decision)
|
||||
if clean_format not in decision.export_formats:
|
||||
raise ProviderReportError("Policy does not allow this export format")
|
||||
content, media_type, extension = _serialize_export(
|
||||
clean_format,
|
||||
row.result_payload,
|
||||
row.result_schema,
|
||||
)
|
||||
output_hash = sha256(content).hexdigest()
|
||||
export = ReportingProviderExport(
|
||||
tenant_id=row.tenant_id,
|
||||
export_id=str(uuid.uuid4()),
|
||||
provider_execution_id=row.id,
|
||||
execution_id=row.execution_id,
|
||||
format=clean_format,
|
||||
purpose=clean_purpose,
|
||||
audience_scope=clean_audience,
|
||||
output_hash=output_hash,
|
||||
exported_at=datetime.now(UTC),
|
||||
actor_id=_actor(principal),
|
||||
)
|
||||
session.add(export)
|
||||
session.flush()
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="reporting.provider_report.exported",
|
||||
object_type="reporting_provider_execution",
|
||||
object_id=row.execution_id,
|
||||
details={
|
||||
"export_id": export.export_id,
|
||||
"format": clean_format,
|
||||
"purpose": clean_purpose,
|
||||
"audience_scope": clean_audience,
|
||||
"output_hash": output_hash,
|
||||
},
|
||||
)
|
||||
filename = f"{row.provider_id}-{row.report_id}-{row.execution_id}.{extension}"
|
||||
return content, media_type, filename
|
||||
|
||||
|
||||
def list_provider_exports(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
execution_id: str,
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
row = _provider_execution(session, principal, execution_id=execution_id)
|
||||
if row is None or _expired(row):
|
||||
return ()
|
||||
provider, _descriptor = _provider_report(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
provider_id=row.provider_id,
|
||||
report_id=row.report_id,
|
||||
)
|
||||
_require_result_access(provider, session, principal, row)
|
||||
exports = (
|
||||
session.query(ReportingProviderExport)
|
||||
.filter(
|
||||
ReportingProviderExport.tenant_id == row.tenant_id,
|
||||
ReportingProviderExport.provider_execution_id == row.id,
|
||||
)
|
||||
.order_by(ReportingProviderExport.exported_at.desc())
|
||||
.all()
|
||||
)
|
||||
return tuple(
|
||||
{
|
||||
"export_id": item.export_id,
|
||||
"execution_id": item.execution_id,
|
||||
"format": item.format,
|
||||
"purpose": item.purpose,
|
||||
"audience_scope": dict(item.audience_scope or {}),
|
||||
"output_hash": item.output_hash,
|
||||
"exported_at": item.exported_at.isoformat(),
|
||||
"actor_id": item.actor_id,
|
||||
}
|
||||
for item in exports
|
||||
)
|
||||
|
||||
|
||||
def provider_execution_payload(row: ReportingProviderExecution) -> dict[str, object]:
|
||||
return {
|
||||
"execution_id": row.execution_id,
|
||||
"provider_id": row.provider_id,
|
||||
"report_id": row.report_id,
|
||||
"report_revision": row.report_revision,
|
||||
"contract_version": row.contract_version,
|
||||
"purpose": row.purpose,
|
||||
"audience_scope": dict(row.audience_scope or {}),
|
||||
"parameters": dict(row.parameters or {}),
|
||||
"result_schema": list(row.result_schema or []),
|
||||
"result": dict(row.result_payload or {}),
|
||||
"source_revisions": list(row.source_revisions or []),
|
||||
"effective_scope": dict(row.effective_scope or {}),
|
||||
"privacy_transforms": list(row.privacy_transforms or []),
|
||||
"provenance": dict(row.provenance or {}),
|
||||
"governance_provenance": dict(row.governance_provenance or {}),
|
||||
"retention_class": row.retention_class,
|
||||
"retention_days": row.retention_days,
|
||||
"expires_at": row.expires_at.isoformat() if row.expires_at else None,
|
||||
"output_hash": row.output_hash,
|
||||
"generated_at": row.generated_at.isoformat(),
|
||||
"actor_id": row.actor_id,
|
||||
}
|
||||
|
||||
|
||||
def _provider_report(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
provider_id: str,
|
||||
report_id: str,
|
||||
) -> tuple[ReportProvider, ReportDescriptor]:
|
||||
provider = dict(report_providers(registry)).get(provider_id)
|
||||
if provider is None:
|
||||
raise LookupError("Report provider is unavailable")
|
||||
descriptor = next(
|
||||
(
|
||||
item
|
||||
for item in provider.list_reports(session, principal)
|
||||
if item.report_id == report_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if descriptor is None:
|
||||
raise LookupError("Provider report is unavailable")
|
||||
_validate_descriptor(provider_id, descriptor)
|
||||
return provider, descriptor
|
||||
|
||||
|
||||
def _validate_descriptor(provider_id: str, descriptor: ReportDescriptor) -> None:
|
||||
if descriptor.provider_id != provider_id:
|
||||
raise ProviderReportError("Report descriptor provider id differs")
|
||||
if not descriptor.report_id or not descriptor.revision:
|
||||
raise ProviderReportError("Report descriptors require stable ids and revisions")
|
||||
parameter_keys = [item.key for item in descriptor.parameters]
|
||||
field_paths = [item.path for item in descriptor.result_schema]
|
||||
transform_ids = [item.id for item in descriptor.privacy_transforms]
|
||||
if len(parameter_keys) != len(set(parameter_keys)):
|
||||
raise ProviderReportError("Report parameter keys must be unique")
|
||||
if not field_paths or len(field_paths) != len(set(field_paths)):
|
||||
raise ProviderReportError("Report result paths must be non-empty and unique")
|
||||
if len(transform_ids) != len(set(transform_ids)):
|
||||
raise ProviderReportError("Report privacy transforms must be unique")
|
||||
if any(item.sensitive for item in descriptor.result_schema) and (
|
||||
descriptor.reidentification_risk != "high"
|
||||
):
|
||||
raise ProviderReportError(
|
||||
"Sensitive result fields require high risk classification"
|
||||
)
|
||||
|
||||
|
||||
def _validate_result(
|
||||
descriptor: ReportDescriptor,
|
||||
result: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
) -> None:
|
||||
if not hasattr(result, "report_id") or result.report_id != descriptor.report_id:
|
||||
raise ProviderReportError("Provider result report id differs")
|
||||
if not result.source_revisions:
|
||||
raise ProviderReportError("Provider result has no source revision provenance")
|
||||
if str(result.effective_scope.get("tenant_id") or "") != tenant_id:
|
||||
raise ProviderReportError(
|
||||
"Provider result effective scope differs from the tenant"
|
||||
)
|
||||
required = {item.id for item in descriptor.privacy_transforms if item.required}
|
||||
missing = required - set(result.applied_privacy_transforms)
|
||||
if missing:
|
||||
raise ProviderReportError(
|
||||
"Provider result omitted required privacy transformations: "
|
||||
+ ", ".join(sorted(missing))
|
||||
)
|
||||
declared = {item.path: item for item in descriptor.result_schema}
|
||||
for field in descriptor.result_schema:
|
||||
if field.type != "suppressed_count":
|
||||
continue
|
||||
value = _path_value(result.payload, field.path)
|
||||
if not isinstance(value, Mapping) or set(value) != {"value", "suppressed"}:
|
||||
raise ProviderReportError(
|
||||
f"Provider result has an invalid suppressed count: {field.path}"
|
||||
)
|
||||
if value["value"] is not None and (
|
||||
not isinstance(value["value"], int) or isinstance(value["value"], bool)
|
||||
):
|
||||
raise ProviderReportError(
|
||||
f"Provider result has an invalid suppressed-count value: {field.path}"
|
||||
)
|
||||
if not isinstance(value["suppressed"], bool):
|
||||
raise ProviderReportError(
|
||||
f"Provider result has an invalid suppression flag: {field.path}"
|
||||
)
|
||||
for path in _leaf_paths(result.payload):
|
||||
if path in declared:
|
||||
continue
|
||||
if any(
|
||||
path.startswith(declared_path + ".")
|
||||
and field.type in {"object", "suppressed_count"}
|
||||
for declared_path, field in declared.items()
|
||||
):
|
||||
continue
|
||||
raise ProviderReportError(f"Provider result contains undeclared field: {path}")
|
||||
|
||||
|
||||
def _validated_parameters(
|
||||
descriptor: ReportDescriptor,
|
||||
parameters: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
clean = _json_mapping(parameters, "report parameters")
|
||||
declared = {item.key: item for item in descriptor.parameters}
|
||||
unknown = set(clean) - set(declared)
|
||||
if unknown:
|
||||
raise ProviderReportError(
|
||||
"Unknown report parameters: " + ", ".join(sorted(unknown))
|
||||
)
|
||||
missing = [
|
||||
item.key
|
||||
for item in descriptor.parameters
|
||||
if item.required and clean.get(item.key) in (None, "")
|
||||
]
|
||||
if missing:
|
||||
raise ProviderReportError(
|
||||
"Missing report parameters: " + ", ".join(sorted(missing))
|
||||
)
|
||||
invalid = [
|
||||
item.key
|
||||
for item in descriptor.parameters
|
||||
if item.key in clean
|
||||
and clean[item.key] is not None
|
||||
and not _valid_parameter_value(item.type, clean[item.key])
|
||||
]
|
||||
if invalid:
|
||||
raise ProviderReportError(
|
||||
"Invalid report parameter types: " + ", ".join(sorted(invalid))
|
||||
)
|
||||
return clean
|
||||
|
||||
|
||||
def _valid_parameter_value(parameter_type: str, value: object) -> bool:
|
||||
if parameter_type in {"string", "reference"}:
|
||||
return isinstance(value, str)
|
||||
if parameter_type == "boolean":
|
||||
return isinstance(value, bool)
|
||||
if parameter_type == "integer":
|
||||
return isinstance(value, int) and not isinstance(value, bool)
|
||||
if parameter_type == "number":
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
||||
if parameter_type == "date":
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
try:
|
||||
date.fromisoformat(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
if parameter_type == "datetime":
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
try:
|
||||
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _governance_decision(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
registry: object | None,
|
||||
descriptor: ReportDescriptor,
|
||||
action: str,
|
||||
purpose: str | None,
|
||||
audience_scope: Mapping[str, object],
|
||||
export_format: str | None = None,
|
||||
applied_privacy_transforms: tuple[str, ...] = (),
|
||||
) -> ReportingGovernanceDecision:
|
||||
required = tuple(item.id for item in descriptor.privacy_transforms if item.required)
|
||||
provider = reporting_governance_provider(registry)
|
||||
if provider is None:
|
||||
allowed = descriptor.reidentification_risk != "high"
|
||||
if descriptor.purpose_required and action != "catalogue" and not purpose:
|
||||
allowed = False
|
||||
if (
|
||||
descriptor.audience_scope_required
|
||||
and action != "catalogue"
|
||||
and not audience_scope
|
||||
):
|
||||
allowed = False
|
||||
if action == "export" and export_format not in descriptor.export_formats:
|
||||
allowed = False
|
||||
return ReportingGovernanceDecision(
|
||||
allowed=allowed,
|
||||
reason=None
|
||||
if allowed
|
||||
else "The built-in Reporting privacy baseline denied this action.",
|
||||
retention_days=30,
|
||||
export_formats=descriptor.export_formats,
|
||||
required_privacy_transforms=required,
|
||||
provenance={"provider": "reporting.built_in_baseline", "version": "1"},
|
||||
)
|
||||
decision = provider.decide_reporting_action(
|
||||
session,
|
||||
principal,
|
||||
request=ReportingGovernanceRequest(
|
||||
action=action, # type: ignore[arg-type]
|
||||
tenant_id=_tenant(principal),
|
||||
provider_id=descriptor.provider_id,
|
||||
report_id=descriptor.report_id,
|
||||
purpose=purpose,
|
||||
audience_scope=audience_scope,
|
||||
retention_class=descriptor.retention_class,
|
||||
export_format=export_format,
|
||||
reidentification_risk=descriptor.reidentification_risk,
|
||||
declared_privacy_transforms=required,
|
||||
applied_privacy_transforms=applied_privacy_transforms,
|
||||
),
|
||||
)
|
||||
unsupported = set(decision.required_privacy_transforms) - {
|
||||
item.id for item in descriptor.privacy_transforms
|
||||
}
|
||||
if not unsupported:
|
||||
return decision
|
||||
return ReportingGovernanceDecision(
|
||||
allowed=False,
|
||||
reason=(
|
||||
"Policy requires privacy transformations this report does not support: "
|
||||
+ ", ".join(sorted(unsupported))
|
||||
),
|
||||
retention_days=decision.retention_days,
|
||||
export_formats=decision.export_formats,
|
||||
required_privacy_transforms=decision.required_privacy_transforms,
|
||||
provenance={
|
||||
**dict(decision.provenance),
|
||||
"unsupported_required_privacy_transforms": sorted(unsupported),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _serialize_export(
|
||||
format: str,
|
||||
payload: Mapping[str, object],
|
||||
schema: list[dict[str, object]],
|
||||
) -> tuple[bytes, str, str]:
|
||||
if format == "json":
|
||||
return (
|
||||
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True).encode(
|
||||
"utf-8"
|
||||
),
|
||||
"application/json",
|
||||
"json",
|
||||
)
|
||||
if format != "csv":
|
||||
raise ProviderReportError("Unsupported provider report export format")
|
||||
output = io.StringIO(newline="")
|
||||
writer = csv.writer(output)
|
||||
writer.writerow([str(item["label"]) for item in schema])
|
||||
writer.writerow(
|
||||
[_csv_value(_path_value(payload, str(item["path"]))) for item in schema]
|
||||
)
|
||||
return output.getvalue().encode("utf-8-sig"), "text/csv; charset=utf-8", "csv"
|
||||
|
||||
|
||||
def _csv_value(value: object) -> object:
|
||||
if isinstance(value, Mapping):
|
||||
if value.get("suppressed") is True:
|
||||
return "suppressed"
|
||||
if set(value).issuperset({"value", "suppressed"}):
|
||||
value = value.get("value")
|
||||
else:
|
||||
value = json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
if isinstance(value, (list, tuple)):
|
||||
value = json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
if isinstance(value, str) and value.startswith(("=", "+", "-", "@", "\t", "\r")):
|
||||
return "'" + value
|
||||
return "" if value is None else value
|
||||
|
||||
|
||||
def _path_value(payload: Mapping[str, object], path: str) -> object:
|
||||
value: object = payload
|
||||
for part in path.split("."):
|
||||
if not isinstance(value, Mapping):
|
||||
return None
|
||||
value = value.get(part)
|
||||
return value
|
||||
|
||||
|
||||
def _leaf_paths(value: object, prefix: str = "") -> tuple[str, ...]:
|
||||
if isinstance(value, Mapping):
|
||||
rows: list[str] = []
|
||||
for key, item in value.items():
|
||||
path = f"{prefix}.{key}" if prefix else str(key)
|
||||
rows.extend(_leaf_paths(item, path))
|
||||
return tuple(rows)
|
||||
if isinstance(value, (list, tuple)):
|
||||
return (prefix,)
|
||||
return (prefix,)
|
||||
|
||||
|
||||
def _provider_execution(
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
execution_id: str,
|
||||
) -> ReportingProviderExecution | None:
|
||||
return (
|
||||
session.query(ReportingProviderExecution)
|
||||
.filter(
|
||||
ReportingProviderExecution.tenant_id == _tenant(principal),
|
||||
ReportingProviderExecution.execution_id == execution_id,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
|
||||
|
||||
def _require_result_access(
|
||||
provider: ReportProvider,
|
||||
session: Session,
|
||||
principal: object,
|
||||
row: ReportingProviderExecution,
|
||||
) -> None:
|
||||
if not provider.authorize_result(
|
||||
session,
|
||||
principal,
|
||||
report_id=row.report_id,
|
||||
source_revisions=tuple(row.source_revisions or ()),
|
||||
effective_scope=dict(row.effective_scope or {}),
|
||||
):
|
||||
raise PermissionError(
|
||||
"The source module no longer permits access to this report result"
|
||||
)
|
||||
|
||||
|
||||
def _expired(row: ReportingProviderExecution) -> bool:
|
||||
if row.expires_at is None:
|
||||
return False
|
||||
return _aware(row.expires_at) <= datetime.now(UTC)
|
||||
|
||||
|
||||
def _require_allowed(decision: ReportingGovernanceDecision) -> None:
|
||||
if not decision.allowed:
|
||||
raise PermissionError(decision.reason or "Reporting Policy denied this action")
|
||||
|
||||
|
||||
def _decision_payload(decision: ReportingGovernanceDecision) -> dict[str, object]:
|
||||
return {
|
||||
"allowed": decision.allowed,
|
||||
"reason": decision.reason,
|
||||
"retention_days": decision.retention_days,
|
||||
"export_formats": list(decision.export_formats),
|
||||
"required_privacy_transforms": list(decision.required_privacy_transforms),
|
||||
"provenance": dict(decision.provenance),
|
||||
}
|
||||
|
||||
|
||||
def _json_mapping(value: Mapping[str, object], label: str) -> dict[str, object]:
|
||||
try:
|
||||
payload = json.loads(json.dumps(value, sort_keys=True, separators=(",", ":")))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ProviderReportError(f"The {label} must contain JSON values") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise ProviderReportError(f"The {label} must be an object")
|
||||
return payload
|
||||
|
||||
|
||||
def _hash(value: object) -> str:
|
||||
return sha256(
|
||||
json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _tenant(principal: object) -> str:
|
||||
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
|
||||
if not tenant_id:
|
||||
raise PermissionError("A tenant principal is required")
|
||||
return tenant_id
|
||||
|
||||
|
||||
def _actor(principal: object) -> str | None:
|
||||
user = getattr(principal, "user", None)
|
||||
actor_id = getattr(user, "id", None)
|
||||
return str(actor_id) if actor_id else None
|
||||
|
||||
|
||||
def _aware(value: datetime) -> datetime:
|
||||
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ProviderReportError",
|
||||
"execute_provider_report",
|
||||
"export_provider_execution",
|
||||
"get_provider_execution",
|
||||
"list_provider_exports",
|
||||
"list_provider_reports",
|
||||
"provider_parameter_options",
|
||||
]
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Retention minimization for governed provider-report results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_reporting.backend.db.models import ReportingProviderExecution
|
||||
|
||||
|
||||
class ReportingRetentionService:
|
||||
def apply_retention(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
dry_run: bool,
|
||||
now: datetime,
|
||||
limit: int = 500,
|
||||
) -> dict[str, int]:
|
||||
if not isinstance(session, Session):
|
||||
raise TypeError("Reporting retention requires a SQLAlchemy Session")
|
||||
observed_at = _aware(now)
|
||||
rows = (
|
||||
session.query(ReportingProviderExecution)
|
||||
.filter(
|
||||
ReportingProviderExecution.expires_at.is_not(None),
|
||||
ReportingProviderExecution.expires_at <= observed_at,
|
||||
ReportingProviderExecution.retention_redacted_at.is_(None),
|
||||
)
|
||||
.order_by(
|
||||
ReportingProviderExecution.expires_at.asc(),
|
||||
ReportingProviderExecution.id.asc(),
|
||||
)
|
||||
.limit(max(1, min(int(limit), 5_000)))
|
||||
.all()
|
||||
)
|
||||
counts = {
|
||||
"eligible": len(rows),
|
||||
"redacted": 0,
|
||||
"remaining_in_batch": 0,
|
||||
}
|
||||
if dry_run:
|
||||
return counts
|
||||
for row in rows:
|
||||
# Keep immutable request/output hashes and provenance as audit
|
||||
# evidence while removing the retained report detail itself.
|
||||
row.result_payload = {}
|
||||
row.retention_redacted_at = observed_at
|
||||
counts["redacted"] += 1
|
||||
session.flush()
|
||||
counts["remaining_in_batch"] = (
|
||||
session.query(ReportingProviderExecution.id)
|
||||
.filter(
|
||||
ReportingProviderExecution.expires_at.is_not(None),
|
||||
ReportingProviderExecution.expires_at <= observed_at,
|
||||
ReportingProviderExecution.retention_redacted_at.is_(None),
|
||||
)
|
||||
.limit(1)
|
||||
.count()
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def _aware(value: datetime) -> datetime:
|
||||
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
__all__ = ["ReportingRetentionService"]
|
||||
@@ -44,12 +44,23 @@ from govoplan_reporting.backend.operations import (
|
||||
upsert_saved_view,
|
||||
upsert_schedule,
|
||||
)
|
||||
from govoplan_reporting.backend.provider_reports import (
|
||||
ProviderReportError,
|
||||
execute_provider_report,
|
||||
export_provider_execution,
|
||||
get_provider_execution,
|
||||
list_provider_exports,
|
||||
list_provider_reports,
|
||||
provider_parameter_options,
|
||||
)
|
||||
from govoplan_reporting.backend.query_engine import ReportingQueryError
|
||||
from govoplan_reporting.backend.schemas import (
|
||||
DefinitionUpdateRequest,
|
||||
DefinitionWriteRequest,
|
||||
ImportAssessmentRequest,
|
||||
PublicationRequest,
|
||||
ProviderReportExecutionRequest,
|
||||
ProviderReportExportRequest,
|
||||
QualityRunRequest,
|
||||
ReportExecutionRequest,
|
||||
SavedViewWriteRequest,
|
||||
@@ -60,6 +71,138 @@ from govoplan_reporting.backend.schemas import (
|
||||
def create_router(registry: object | None) -> APIRouter:
|
||||
router = APIRouter(prefix="/reporting", tags=["reporting"])
|
||||
|
||||
@router.get("/provider-reports")
|
||||
def api_list_provider_reports(
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, READ_SCOPE)
|
||||
try:
|
||||
return list_provider_reports(session, principal, registry=registry)
|
||||
except (ProviderReportError, TypeError, ValueError) as exc:
|
||||
raise _error(exc) from exc
|
||||
|
||||
@router.get(
|
||||
"/provider-reports/{provider_id}/{report_id}/parameters/{parameter_key}/options"
|
||||
)
|
||||
def api_provider_parameter_options(
|
||||
provider_id: str,
|
||||
report_id: str,
|
||||
parameter_key: str,
|
||||
query: str = "",
|
||||
limit: int = Query(default=100, ge=1, le=200),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, READ_SCOPE)
|
||||
try:
|
||||
options = provider_parameter_options(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
provider_id=provider_id,
|
||||
report_id=report_id,
|
||||
parameter_key=parameter_key,
|
||||
query=query,
|
||||
limit=limit,
|
||||
)
|
||||
except (ProviderReportError, PermissionError, LookupError) as exc:
|
||||
raise _error(exc) from exc
|
||||
return {"options": [item.to_dict() for item in options]}
|
||||
|
||||
@router.post("/provider-reports/{provider_id}/{report_id}/executions")
|
||||
def api_execute_provider_report(
|
||||
provider_id: str,
|
||||
report_id: str,
|
||||
payload: ProviderReportExecutionRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, RUN_SCOPE)
|
||||
try:
|
||||
result = execute_provider_report(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
provider_id=provider_id,
|
||||
report_id=report_id,
|
||||
**payload.model_dump(),
|
||||
)
|
||||
session.commit()
|
||||
except (ProviderReportError, PermissionError, LookupError, ValueError) as exc:
|
||||
session.rollback()
|
||||
raise _error(exc) from exc
|
||||
return result
|
||||
|
||||
@router.get("/provider-executions/{execution_id}")
|
||||
def api_get_provider_execution(
|
||||
execution_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, RUN_SCOPE)
|
||||
try:
|
||||
result = get_provider_execution(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
execution_id=execution_id,
|
||||
)
|
||||
except (ProviderReportError, PermissionError, LookupError) as exc:
|
||||
raise _error(exc) from exc
|
||||
if result is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail="Provider report execution not found"
|
||||
)
|
||||
return result
|
||||
|
||||
@router.get("/provider-executions/{execution_id}/exports")
|
||||
def api_list_provider_exports(
|
||||
execution_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> dict[str, object]:
|
||||
_require(principal, RUN_SCOPE)
|
||||
try:
|
||||
return {
|
||||
"exports": list(
|
||||
list_provider_exports(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
execution_id=execution_id,
|
||||
)
|
||||
)
|
||||
}
|
||||
except (ProviderReportError, PermissionError, LookupError) as exc:
|
||||
raise _error(exc) from exc
|
||||
|
||||
@router.post("/provider-executions/{execution_id}/exports")
|
||||
def api_export_provider_execution(
|
||||
execution_id: str,
|
||||
payload: ProviderReportExportRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> Response:
|
||||
_require(principal, RUN_SCOPE)
|
||||
try:
|
||||
content, media_type, filename = export_provider_execution(
|
||||
session,
|
||||
principal,
|
||||
registry=registry,
|
||||
execution_id=execution_id,
|
||||
**payload.model_dump(),
|
||||
)
|
||||
session.commit()
|
||||
except (ProviderReportError, PermissionError, LookupError) as exc:
|
||||
session.rollback()
|
||||
raise _error(exc) from exc
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=media_type,
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
@router.get("/definitions")
|
||||
def api_list_definitions(
|
||||
definition_kind: list[str] | None = Query(default=None),
|
||||
|
||||
@@ -422,6 +422,23 @@ class ReportExecutionRequest(BaseModel):
|
||||
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class ProviderReportExecutionRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
purpose: str = Field(min_length=1, max_length=1_000)
|
||||
audience_scope: dict[str, Any] = Field(min_length=1, max_length=50)
|
||||
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class ProviderReportExportRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
format: Literal["json", "csv"] = "json"
|
||||
purpose: str = Field(min_length=1, max_length=1_000)
|
||||
audience_scope: dict[str, Any] = Field(min_length=1, max_length=50)
|
||||
|
||||
|
||||
class SavedViewWriteRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@@ -489,6 +506,8 @@ __all__ = [
|
||||
"ImportAssessmentRequest",
|
||||
"MeasureDefinition",
|
||||
"PublicationRequest",
|
||||
"ProviderReportExecutionRequest",
|
||||
"ProviderReportExportRequest",
|
||||
"QualityPlanDefinition",
|
||||
"QualityRunRequest",
|
||||
"ReportDefinition",
|
||||
|
||||
Reference in New Issue
Block a user