"""Governed execution boundary for module-contributed reports.""" from __future__ import annotations from collections.abc import Mapping import csv from datetime import UTC, date, datetime, timedelta from hashlib import sha256 import io import json import uuid from sqlalchemy.orm import Session from govoplan_core.audit.logging import audit_from_principal from govoplan_core.core.reporting import ( ReportDescriptor, ReportParameterOption, ReportProvider, ReportProviderRequest, ReportingGovernanceDecision, ReportingGovernanceRequest, report_providers, reporting_governance_provider, ) from govoplan_reporting.backend.db.models import ( ReportingProviderExecution, ReportingProviderExport, ) class ProviderReportError(RuntimeError): pass def list_provider_reports( session: Session, principal: object, *, registry: object | None, ) -> dict[str, object]: reports: list[dict[str, object]] = [] diagnostics: list[dict[str, str]] = [] for provider_id, provider in report_providers(registry): try: for descriptor in provider.list_reports(session, principal): _validate_descriptor(provider_id, descriptor) decision = _governance_decision( session, principal, registry=registry, descriptor=descriptor, action="catalogue", purpose=None, audience_scope={}, ) reports.append( { **descriptor.to_dict(), "available": decision.allowed, "unavailable_reason": decision.reason, "governance": _decision_payload(decision), } ) except Exception as exc: # noqa: BLE001 - isolate optional providers. diagnostics.append( { "provider_id": provider_id, "code": "provider_catalogue_failed", "message": "The provider catalogue could not be loaded.", "error_type": type(exc).__name__, } ) return { "reports": sorted( reports, key=lambda item: (str(item["title"]), str(item["provider_id"])), ), "diagnostics": diagnostics, } def provider_parameter_options( session: Session, principal: object, *, registry: object | None, provider_id: str, report_id: str, parameter_key: str, query: str, limit: int, ) -> tuple[ReportParameterOption, ...]: provider, descriptor = _provider_report( session, principal, registry=registry, provider_id=provider_id, report_id=report_id, ) parameter = next( (item for item in descriptor.parameters if item.key == parameter_key), None, ) if parameter is None or not parameter.options_from_provider: raise ProviderReportError("This report parameter has no provider options") return provider.parameter_options( session, principal, report_id=report_id, parameter_key=parameter_key, query=query, limit=max(1, min(limit, 200)), ) def execute_provider_report( session: Session, principal: object, *, registry: object | None, provider_id: str, report_id: str, parameters: Mapping[str, object], purpose: str, audience_scope: Mapping[str, object], idempotency_key: str, ) -> dict[str, object]: provider, descriptor = _provider_report( session, principal, registry=registry, provider_id=provider_id, report_id=report_id, ) clean_purpose = purpose.strip() clean_idempotency_key = idempotency_key.strip() clean_parameters = _validated_parameters(descriptor, parameters) clean_audience = _json_mapping(audience_scope, "audience scope") if descriptor.purpose_required and not clean_purpose: raise ProviderReportError("A report purpose is required") if descriptor.audience_scope_required and not clean_audience: raise ProviderReportError("An effective audience scope is required") request_hash = _hash( { "provider_id": provider_id, "report_id": report_id, "revision": descriptor.revision, "parameters": clean_parameters, "purpose": clean_purpose, "audience_scope": clean_audience, } ) replay = ( session.query(ReportingProviderExecution) .filter( ReportingProviderExecution.tenant_id == _tenant(principal), ReportingProviderExecution.idempotency_key == clean_idempotency_key, ) .one_or_none() ) if replay is not None: if replay.request_sha256 != request_hash: raise ProviderReportError( "The provider-report idempotency key belongs to another request" ) if _expired(replay) or replay.retention_redacted_at is not None: raise ProviderReportError( "The replayed provider-report result has expired; use a new idempotency key" ) _require_result_access(provider, session, principal, replay) return provider_execution_payload(replay) preflight = _governance_decision( session, principal, registry=registry, descriptor=descriptor, action="execute", purpose=clean_purpose, audience_scope=clean_audience, ) _require_allowed(preflight) result = provider.execute_report( session, principal, request=ReportProviderRequest( report_id=report_id, parameters=clean_parameters, purpose=clean_purpose, audience_scope=clean_audience, ), ) _validate_result(descriptor, result, tenant_id=_tenant(principal)) decision = _governance_decision( session, principal, registry=registry, descriptor=descriptor, action="execute", purpose=clean_purpose, audience_scope=clean_audience, applied_privacy_transforms=result.applied_privacy_transforms, ) _require_allowed(decision) missing_policy_transforms = set(decision.required_privacy_transforms) - set( result.applied_privacy_transforms ) if missing_policy_transforms: raise ProviderReportError( "The report omitted Policy-required privacy transformations: " + ", ".join(sorted(missing_policy_transforms)) ) generated_at = _aware(result.generated_at) result_payload = _json_mapping(result.payload, "provider report payload") expires_at = ( generated_at + timedelta(days=decision.retention_days) if decision.retention_days is not None else None ) row = ReportingProviderExecution( tenant_id=_tenant(principal), execution_id=str(uuid.uuid4()), provider_id=provider_id, report_id=report_id, report_revision=descriptor.revision, contract_version=descriptor.contract_version, idempotency_key=clean_idempotency_key, request_sha256=request_hash, purpose=clean_purpose, audience_scope=clean_audience, parameters=clean_parameters, result_schema=[item.to_dict() for item in descriptor.result_schema], result_payload=result_payload, source_revisions=[dict(item) for item in result.source_revisions], effective_scope=dict(result.effective_scope), privacy_transforms=list(result.applied_privacy_transforms), provenance=dict(result.provenance), governance_provenance=dict(decision.provenance), retention_class=descriptor.retention_class, retention_days=decision.retention_days, expires_at=expires_at, output_hash=_hash(result_payload), generated_at=generated_at, actor_id=_actor(principal), ) session.add(row) session.flush() audit_from_principal( session, principal, action="reporting.provider_report.executed", object_type="reporting_provider_execution", object_id=row.execution_id, details={ "provider_id": provider_id, "report_id": report_id, "report_revision": descriptor.revision, "purpose": clean_purpose, "audience_scope": clean_audience, "source_revision_count": len(row.source_revisions), "privacy_transforms": row.privacy_transforms, "retention_class": row.retention_class, "retention_days": row.retention_days, "output_hash": row.output_hash, }, ) return provider_execution_payload(row) def get_provider_execution( session: Session, principal: object, *, registry: object | None, execution_id: str, ) -> dict[str, object] | None: row = _provider_execution(session, principal, execution_id=execution_id) if row is None or _expired(row): return None provider, _descriptor = _provider_report( session, principal, registry=registry, provider_id=row.provider_id, report_id=row.report_id, ) _require_result_access(provider, session, principal, row) return provider_execution_payload(row) def export_provider_execution( session: Session, principal: object, *, registry: object | None, execution_id: str, format: str, purpose: str, audience_scope: Mapping[str, object], ) -> tuple[bytes, str, str]: row = _provider_execution(session, principal, execution_id=execution_id) if row is None or _expired(row): raise LookupError("Provider report execution not found") provider, descriptor = _provider_report( session, principal, registry=registry, provider_id=row.provider_id, report_id=row.report_id, ) _require_result_access(provider, session, principal, row) clean_format = format.strip().lower() if clean_format not in descriptor.export_formats: raise ProviderReportError("This report does not support the export format") clean_purpose = purpose.strip() clean_audience = _json_mapping(audience_scope, "audience scope") decision = _governance_decision( session, principal, registry=registry, descriptor=descriptor, action="export", purpose=clean_purpose, audience_scope=clean_audience, export_format=clean_format, applied_privacy_transforms=tuple(row.privacy_transforms or ()), ) _require_allowed(decision) if clean_format not in decision.export_formats: raise ProviderReportError("Policy does not allow this export format") content, media_type, extension = _serialize_export( clean_format, row.result_payload, row.result_schema, ) output_hash = sha256(content).hexdigest() export = ReportingProviderExport( tenant_id=row.tenant_id, export_id=str(uuid.uuid4()), provider_execution_id=row.id, execution_id=row.execution_id, format=clean_format, purpose=clean_purpose, audience_scope=clean_audience, output_hash=output_hash, exported_at=datetime.now(UTC), actor_id=_actor(principal), ) session.add(export) session.flush() audit_from_principal( session, principal, action="reporting.provider_report.exported", object_type="reporting_provider_execution", object_id=row.execution_id, details={ "export_id": export.export_id, "format": clean_format, "purpose": clean_purpose, "audience_scope": clean_audience, "output_hash": output_hash, }, ) filename = f"{row.provider_id}-{row.report_id}-{row.execution_id}.{extension}" return content, media_type, filename def list_provider_exports( session: Session, principal: object, *, registry: object | None, execution_id: str, ) -> tuple[dict[str, object], ...]: row = _provider_execution(session, principal, execution_id=execution_id) if row is None or _expired(row): return () provider, _descriptor = _provider_report( session, principal, registry=registry, provider_id=row.provider_id, report_id=row.report_id, ) _require_result_access(provider, session, principal, row) exports = ( session.query(ReportingProviderExport) .filter( ReportingProviderExport.tenant_id == row.tenant_id, ReportingProviderExport.provider_execution_id == row.id, ) .order_by(ReportingProviderExport.exported_at.desc()) .all() ) return tuple( { "export_id": item.export_id, "execution_id": item.execution_id, "format": item.format, "purpose": item.purpose, "audience_scope": dict(item.audience_scope or {}), "output_hash": item.output_hash, "exported_at": item.exported_at.isoformat(), "actor_id": item.actor_id, } for item in exports ) def provider_execution_payload(row: ReportingProviderExecution) -> dict[str, object]: return { "execution_id": row.execution_id, "provider_id": row.provider_id, "report_id": row.report_id, "report_revision": row.report_revision, "contract_version": row.contract_version, "purpose": row.purpose, "audience_scope": dict(row.audience_scope or {}), "parameters": dict(row.parameters or {}), "result_schema": list(row.result_schema or []), "result": dict(row.result_payload or {}), "source_revisions": list(row.source_revisions or []), "effective_scope": dict(row.effective_scope or {}), "privacy_transforms": list(row.privacy_transforms or []), "provenance": dict(row.provenance or {}), "governance_provenance": dict(row.governance_provenance or {}), "retention_class": row.retention_class, "retention_days": row.retention_days, "expires_at": row.expires_at.isoformat() if row.expires_at else None, "output_hash": row.output_hash, "generated_at": row.generated_at.isoformat(), "actor_id": row.actor_id, } def _provider_report( session: Session, principal: object, *, registry: object | None, provider_id: str, report_id: str, ) -> tuple[ReportProvider, ReportDescriptor]: provider = dict(report_providers(registry)).get(provider_id) if provider is None: raise LookupError("Report provider is unavailable") descriptor = next( ( item for item in provider.list_reports(session, principal) if item.report_id == report_id ), None, ) if descriptor is None: raise LookupError("Provider report is unavailable") _validate_descriptor(provider_id, descriptor) return provider, descriptor def _validate_descriptor(provider_id: str, descriptor: ReportDescriptor) -> None: if descriptor.provider_id != provider_id: raise ProviderReportError("Report descriptor provider id differs") if not descriptor.report_id or not descriptor.revision: raise ProviderReportError("Report descriptors require stable ids and revisions") parameter_keys = [item.key for item in descriptor.parameters] field_paths = [item.path for item in descriptor.result_schema] transform_ids = [item.id for item in descriptor.privacy_transforms] if len(parameter_keys) != len(set(parameter_keys)): raise ProviderReportError("Report parameter keys must be unique") if not field_paths or len(field_paths) != len(set(field_paths)): raise ProviderReportError("Report result paths must be non-empty and unique") if len(transform_ids) != len(set(transform_ids)): raise ProviderReportError("Report privacy transforms must be unique") if any(item.sensitive for item in descriptor.result_schema) and ( descriptor.reidentification_risk != "high" ): raise ProviderReportError( "Sensitive result fields require high risk classification" ) def _validate_result( descriptor: ReportDescriptor, result: object, *, tenant_id: str, ) -> None: if not hasattr(result, "report_id") or result.report_id != descriptor.report_id: raise ProviderReportError("Provider result report id differs") if not result.source_revisions: raise ProviderReportError("Provider result has no source revision provenance") if str(result.effective_scope.get("tenant_id") or "") != tenant_id: raise ProviderReportError( "Provider result effective scope differs from the tenant" ) required = {item.id for item in descriptor.privacy_transforms if item.required} missing = required - set(result.applied_privacy_transforms) if missing: raise ProviderReportError( "Provider result omitted required privacy transformations: " + ", ".join(sorted(missing)) ) declared = {item.path: item for item in descriptor.result_schema} for field in descriptor.result_schema: if field.type != "suppressed_count": continue value = _path_value(result.payload, field.path) if not isinstance(value, Mapping) or set(value) != {"value", "suppressed"}: raise ProviderReportError( f"Provider result has an invalid suppressed count: {field.path}" ) if value["value"] is not None and ( not isinstance(value["value"], int) or isinstance(value["value"], bool) ): raise ProviderReportError( f"Provider result has an invalid suppressed-count value: {field.path}" ) if not isinstance(value["suppressed"], bool): raise ProviderReportError( f"Provider result has an invalid suppression flag: {field.path}" ) for path in _leaf_paths(result.payload): if path in declared: continue if any( path.startswith(declared_path + ".") and field.type in {"object", "suppressed_count"} for declared_path, field in declared.items() ): continue raise ProviderReportError(f"Provider result contains undeclared field: {path}") def _validated_parameters( descriptor: ReportDescriptor, parameters: Mapping[str, object], ) -> dict[str, object]: clean = _json_mapping(parameters, "report parameters") declared = {item.key: item for item in descriptor.parameters} unknown = set(clean) - set(declared) if unknown: raise ProviderReportError( "Unknown report parameters: " + ", ".join(sorted(unknown)) ) missing = [ item.key for item in descriptor.parameters if item.required and clean.get(item.key) in (None, "") ] if missing: raise ProviderReportError( "Missing report parameters: " + ", ".join(sorted(missing)) ) invalid = [ item.key for item in descriptor.parameters if item.key in clean and clean[item.key] is not None and not _valid_parameter_value(item.type, clean[item.key]) ] if invalid: raise ProviderReportError( "Invalid report parameter types: " + ", ".join(sorted(invalid)) ) return clean def _valid_parameter_value(parameter_type: str, value: object) -> bool: if parameter_type in {"string", "reference"}: return isinstance(value, str) if parameter_type == "boolean": return isinstance(value, bool) if parameter_type == "integer": return isinstance(value, int) and not isinstance(value, bool) if parameter_type == "number": return isinstance(value, (int, float)) and not isinstance(value, bool) if parameter_type == "date": if not isinstance(value, str): return False try: date.fromisoformat(value) except ValueError: return False return True if parameter_type == "datetime": if not isinstance(value, str): return False try: datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: return False return True return False def _governance_decision( session: Session, principal: object, *, registry: object | None, descriptor: ReportDescriptor, action: str, purpose: str | None, audience_scope: Mapping[str, object], export_format: str | None = None, applied_privacy_transforms: tuple[str, ...] = (), ) -> ReportingGovernanceDecision: required = tuple(item.id for item in descriptor.privacy_transforms if item.required) provider = reporting_governance_provider(registry) if provider is None: allowed = descriptor.reidentification_risk != "high" if descriptor.purpose_required and action != "catalogue" and not purpose: allowed = False if ( descriptor.audience_scope_required and action != "catalogue" and not audience_scope ): allowed = False if action == "export" and export_format not in descriptor.export_formats: allowed = False return ReportingGovernanceDecision( allowed=allowed, reason=None if allowed else "The built-in Reporting privacy baseline denied this action.", retention_days=30, export_formats=descriptor.export_formats, required_privacy_transforms=required, provenance={"provider": "reporting.built_in_baseline", "version": "1"}, ) decision = provider.decide_reporting_action( session, principal, request=ReportingGovernanceRequest( action=action, # type: ignore[arg-type] tenant_id=_tenant(principal), provider_id=descriptor.provider_id, report_id=descriptor.report_id, purpose=purpose, audience_scope=audience_scope, retention_class=descriptor.retention_class, export_format=export_format, reidentification_risk=descriptor.reidentification_risk, declared_privacy_transforms=required, applied_privacy_transforms=applied_privacy_transforms, ), ) unsupported = set(decision.required_privacy_transforms) - { item.id for item in descriptor.privacy_transforms } if not unsupported: return decision return ReportingGovernanceDecision( allowed=False, reason=( "Policy requires privacy transformations this report does not support: " + ", ".join(sorted(unsupported)) ), retention_days=decision.retention_days, export_formats=decision.export_formats, required_privacy_transforms=decision.required_privacy_transforms, provenance={ **dict(decision.provenance), "unsupported_required_privacy_transforms": sorted(unsupported), }, ) def _serialize_export( format: str, payload: Mapping[str, object], schema: list[dict[str, object]], ) -> tuple[bytes, str, str]: if format == "json": return ( json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True).encode( "utf-8" ), "application/json", "json", ) if format != "csv": raise ProviderReportError("Unsupported provider report export format") output = io.StringIO(newline="") writer = csv.writer(output) writer.writerow([str(item["label"]) for item in schema]) writer.writerow( [_csv_value(_path_value(payload, str(item["path"]))) for item in schema] ) return output.getvalue().encode("utf-8-sig"), "text/csv; charset=utf-8", "csv" def _csv_value(value: object) -> object: if isinstance(value, Mapping): if value.get("suppressed") is True: return "suppressed" if set(value).issuperset({"value", "suppressed"}): value = value.get("value") else: value = json.dumps(value, ensure_ascii=False, sort_keys=True) if isinstance(value, (list, tuple)): value = json.dumps(value, ensure_ascii=False, sort_keys=True) if isinstance(value, str) and value.startswith(("=", "+", "-", "@", "\t", "\r")): return "'" + value return "" if value is None else value def _path_value(payload: Mapping[str, object], path: str) -> object: value: object = payload for part in path.split("."): if not isinstance(value, Mapping): return None value = value.get(part) return value def _leaf_paths(value: object, prefix: str = "") -> tuple[str, ...]: if isinstance(value, Mapping): rows: list[str] = [] for key, item in value.items(): path = f"{prefix}.{key}" if prefix else str(key) rows.extend(_leaf_paths(item, path)) return tuple(rows) if isinstance(value, (list, tuple)): return (prefix,) return (prefix,) def _provider_execution( session: Session, principal: object, *, execution_id: str, ) -> ReportingProviderExecution | None: return ( session.query(ReportingProviderExecution) .filter( ReportingProviderExecution.tenant_id == _tenant(principal), ReportingProviderExecution.execution_id == execution_id, ) .one_or_none() ) def _require_result_access( provider: ReportProvider, session: Session, principal: object, row: ReportingProviderExecution, ) -> None: if not provider.authorize_result( session, principal, report_id=row.report_id, source_revisions=tuple(row.source_revisions or ()), effective_scope=dict(row.effective_scope or {}), ): raise PermissionError( "The source module no longer permits access to this report result" ) def _expired(row: ReportingProviderExecution) -> bool: if row.expires_at is None: return False return _aware(row.expires_at) <= datetime.now(UTC) def _require_allowed(decision: ReportingGovernanceDecision) -> None: if not decision.allowed: raise PermissionError(decision.reason or "Reporting Policy denied this action") def _decision_payload(decision: ReportingGovernanceDecision) -> dict[str, object]: return { "allowed": decision.allowed, "reason": decision.reason, "retention_days": decision.retention_days, "export_formats": list(decision.export_formats), "required_privacy_transforms": list(decision.required_privacy_transforms), "provenance": dict(decision.provenance), } def _json_mapping(value: Mapping[str, object], label: str) -> dict[str, object]: try: payload = json.loads(json.dumps(value, sort_keys=True, separators=(",", ":"))) except (TypeError, ValueError) as exc: raise ProviderReportError(f"The {label} must contain JSON values") from exc if not isinstance(payload, dict): raise ProviderReportError(f"The {label} must be an object") return payload def _hash(value: object) -> str: return sha256( json.dumps( value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ).encode("utf-8") ).hexdigest() def _tenant(principal: object) -> str: tenant_id = str(getattr(principal, "tenant_id", "") or "").strip() if not tenant_id: raise PermissionError("A tenant principal is required") return tenant_id def _actor(principal: object) -> str | None: user = getattr(principal, "user", None) actor_id = getattr(user, "id", None) return str(actor_id) if actor_id else None def _aware(value: datetime) -> datetime: return value if value.tzinfo is not None else value.replace(tzinfo=UTC) __all__ = [ "ProviderReportError", "execute_provider_report", "export_provider_execution", "get_provider_execution", "list_provider_exports", "list_provider_reports", "provider_parameter_options", ]