diff --git a/README.md b/README.md index e39542c..b2ccfb8 100644 --- a/README.md +++ b/README.md @@ -36,3 +36,11 @@ actions across system, tenant, group, and user scopes. Templates cannot run or be automated. Derived definitions retain ancestor ceilings, and every decision includes the ordered Policy source path and effective limits so a UI can explain why an action is available or blocked. + +Cross-module reports use `policy.reporting_governance`. System policy defines +the export, retention, privacy-transform, and high re-identification-risk +ceiling; tenant policy may only tighten it, and malformed explicit policy fails +closed. The shared privacy-retention run calls the optional +`reporting.retention` capability to clear expired provider-report payloads +without importing Reporting models, while Reporting keeps hashes and bounded +provenance as audit evidence. diff --git a/src/govoplan_policy/backend/manifest.py b/src/govoplan_policy/backend/manifest.py index 9bc8475..5227e66 100644 --- a/src/govoplan_policy/backend/manifest.py +++ b/src/govoplan_policy/backend/manifest.py @@ -21,6 +21,7 @@ from govoplan_core.core.policy import ( CAPABILITY_POLICY_VIEW_GOVERNANCE, ) from govoplan_core.core.modules import ( + CapabilityDocumentation, DocumentationTopic, FrontendModule, MigrationSpec, @@ -28,6 +29,7 @@ from govoplan_core.core.modules import ( ModuleInterfaceProvider, ModuleManifest, ) +from govoplan_core.core.reporting import CAPABILITY_POLICY_REPORTING_GOVERNANCE from govoplan_core.core.provider_governance import declared_module_architecture from govoplan_core.core.views import ViewSurface from govoplan_core.db.base import Base @@ -93,6 +95,15 @@ def _distribution_channel_policy(context: ModuleContext) -> object: return DistributionChannelPolicyProvider() +def _reporting_governance_policy(context: ModuleContext) -> object: + del context + from govoplan_policy.backend.reporting_governance import ( + ReportingGovernancePolicyProvider, + ) + + return ReportingGovernancePolicyProvider() + + manifest = ModuleManifest( id="policy", name="Policy", @@ -118,6 +129,10 @@ manifest = ModuleManifest( name=CAPABILITY_POLICY_DISTRIBUTION_CHANNELS, version="1.0.0", ), + ModuleInterfaceProvider( + name=CAPABILITY_POLICY_REPORTING_GOVERNANCE, + version="1.0.0", + ), ), route_factory=_route_factory, documentation=( @@ -134,10 +149,18 @@ manifest = ModuleManifest( id="policy.hierarchy-overrides-and-retention", title="Administer policy hierarchy and overrides", summary="Policy evaluates versioned system, tenant, group, and user rules for retention and module-owned governed actions.", - body="Administrators can simulate effective retention before saving a lower-level change. Explicit overrides retain source and provenance information and are evaluated through typed capabilities for definitions, Views, function assignments, distribution channels, scheduling privacy, and retention. Templates and inherited definitions keep their upstream ceilings when reused or derived.", + body="Administrators can simulate effective retention before saving a lower-level change. Explicit overrides retain source and provenance information and are evaluated through typed capabilities for definitions, Views, function assignments, distribution channels, scheduling privacy, cross-module reporting, and retention. Templates and inherited definitions keep their upstream ceilings when reused or derived.", documentation_types=("admin",), audience=("policy_admin", "tenant_admin", "system_admin"), - related_modules=("dataflow", "workflow_engine", "views", "idm", "dist_lists", "scheduling"), + related_modules=( + "dataflow", + "workflow_engine", + "views", + "idm", + "dist_lists", + "scheduling", + "reporting", + ), metadata={"kind": "reference"}, ), ), @@ -202,17 +225,36 @@ manifest = ModuleManifest( _function_assignment_governance_policy ), CAPABILITY_POLICY_DISTRIBUTION_CHANNELS: _distribution_channel_policy, + CAPABILITY_POLICY_REPORTING_GOVERNANCE: _reporting_governance_policy, CAPABILITY_POLICY_PRIVACY_RETENTION: _privacy_retention_service, CAPABILITY_POLICY_SCHEDULING_PARTICIPANT_PRIVACY: _scheduling_participant_privacy_policy, }, + capability_documentation={ + CAPABILITY_POLICY_REPORTING_GOVERNANCE: CapabilityDocumentation( + label="Reporting privacy governance", + summary=( + "Tightens report minimization, retention, export formats, and " + "re-identification-risk decisions across system and tenant scopes." + ), + contract_version="1.0", + documentation_types=("admin",), + audience=("policy_admin", "privacy_officer", "system_admin"), + ), + }, architecture=declared_module_architecture( layer="governance_accountability", kind="governance", maturity="vertical_slice", documentation_ref="docs/POLICY_DECISION_PROVENANCE.md", test_ref="tests/test_policy_hierarchy.py", - known_limits=("The current policy families do not yet form a universal expression or enforcement engine.",), - owned_concepts=("policy definition", "policy override", "policy decision provenance"), + known_limits=( + "The current policy families do not yet form a universal expression or enforcement engine.", + ), + owned_concepts=( + "policy definition", + "policy override", + "policy decision provenance", + ), non_owned_concepts=("application permission", "domain record", "audit record"), recovery_docs=("docs/POLICY_DECISION_PROVENANCE.md",), security_docs=("docs/POLICY_DECISION_PROVENANCE.md",), diff --git a/src/govoplan_policy/backend/reporting_governance.py b/src/govoplan_policy/backend/reporting_governance.py new file mode 100644 index 0000000..624de0b --- /dev/null +++ b/src/govoplan_policy/backend/reporting_governance.py @@ -0,0 +1,263 @@ +"""Hierarchical privacy and export policy for cross-module reports.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +from sqlalchemy.orm import Session + +from govoplan_core.admin.settings import get_system_settings +from govoplan_core.core.reporting import ( + ReportingGovernanceDecision, + ReportingGovernanceRequest, +) +from govoplan_core.tenancy.scope import Tenant +from govoplan_policy.backend.retention import effective_privacy_policy + + +REPORTING_POLICY_SETTINGS_KEY = "reporting_governance_policy" +_ALLOWED_KEYS = { + "allow_exports", + "allowed_export_formats", + "allow_high_reidentification_risk", + "max_retention_days", + "required_privacy_transforms", +} + + +@dataclass(frozen=True, slots=True) +class _Policy: + allow_exports: bool = True + allowed_export_formats: tuple[str, ...] = ("json", "csv") + allow_high_reidentification_risk: bool = False + max_retention_days: int | None = 30 + required_privacy_transforms: tuple[str, ...] = () + + +class ReportingGovernancePolicyProvider: + def decide_reporting_action( + self, + session: object, + principal: object, + *, + request: ReportingGovernanceRequest, + ) -> ReportingGovernanceDecision: + if not isinstance(session, Session): + raise TypeError("Reporting governance requires a SQLAlchemy Session") + try: + policy, sources = _effective_policy( + session, + tenant_id=request.tenant_id, + ) + retention = effective_privacy_policy( + session, + tenant_id=request.tenant_id, + ).stored_report_detail_retention_days + except (LookupError, TypeError, ValueError) as exc: + return ReportingGovernanceDecision( + allowed=False, + reason="Reporting governance Policy is malformed or unavailable.", + retention_days=0, + export_formats=(), + required_privacy_transforms=request.declared_privacy_transforms, + provenance={ + "provider": "policy.reporting_governance", + "version": "1", + "decision": "fail_closed", + "error_type": type(exc).__name__, + }, + ) + del principal + retention_days = _minimum_optional( + policy.max_retention_days, + retention, + ) + allowed = True + reason = None + if ( + request.reidentification_risk == "high" + and not policy.allow_high_reidentification_risk + ): + allowed = False + reason = "Policy blocks reports with high re-identification risk." + if request.action == "export": + if not policy.allow_exports: + allowed = False + reason = "Policy disables cross-module report exports." + elif request.export_format not in policy.allowed_export_formats: + allowed = False + reason = "Policy does not allow the requested report export format." + required = tuple( + sorted( + set(request.declared_privacy_transforms) + | set(policy.required_privacy_transforms) + ) + ) + return ReportingGovernanceDecision( + allowed=allowed, + reason=reason, + retention_days=retention_days, + export_formats=( + policy.allowed_export_formats if policy.allow_exports else () + ), + required_privacy_transforms=required, + provenance={ + "provider": "policy.reporting_governance", + "version": "1", + "sources": sources, + "effective": { + "allow_exports": policy.allow_exports, + "allowed_export_formats": list(policy.allowed_export_formats), + "allow_high_reidentification_risk": ( + policy.allow_high_reidentification_risk + ), + "max_retention_days": policy.max_retention_days, + "privacy_retention_days": retention, + "required_privacy_transforms": list(required), + }, + }, + ) + + +def _effective_policy( + session: Session, + *, + tenant_id: str, +) -> tuple[_Policy, list[dict[str, object]]]: + system_settings = get_system_settings(session).settings or {} + system_patch = _patch(system_settings, "system") + policy = _apply_patch(_Policy(), system_patch) + sources = [ + { + "scope_type": "system", + "scope_id": None, + "applied_fields": sorted(system_patch), + } + ] + tenant = session.get(Tenant, tenant_id) + if tenant is None: + raise LookupError("Tenant not found for Reporting Policy") + tenant_patch = _patch(tenant.settings or {}, "tenant") + policy = _tighten(policy, tenant_patch) + sources.append( + { + "scope_type": "tenant", + "scope_id": tenant_id, + "applied_fields": sorted(tenant_patch), + } + ) + return policy, sources + + +def _patch(settings: Mapping[str, object], source: str) -> dict[str, object]: + raw = settings.get(REPORTING_POLICY_SETTINGS_KEY) + if raw in (None, ""): + return {} + if not isinstance(raw, Mapping): + raise ValueError(f"{source} Reporting Policy must be an object") + unknown = set(raw) - _ALLOWED_KEYS + if unknown: + raise ValueError( + f"{source} Reporting Policy has unknown fields: " + + ", ".join(sorted(str(item) for item in unknown)) + ) + return dict(raw) + + +def _apply_patch(parent: _Policy, patch: Mapping[str, object]) -> _Policy: + return _Policy( + allow_exports=_boolean(patch, "allow_exports", parent.allow_exports), + allowed_export_formats=_formats( + patch.get("allowed_export_formats"), + default=parent.allowed_export_formats, + ), + allow_high_reidentification_risk=_boolean( + patch, + "allow_high_reidentification_risk", + parent.allow_high_reidentification_risk, + ), + max_retention_days=_days( + patch.get("max_retention_days"), + default=parent.max_retention_days, + ), + required_privacy_transforms=_transforms( + patch.get("required_privacy_transforms"), + default=parent.required_privacy_transforms, + ), + ) + + +def _tighten(parent: _Policy, patch: Mapping[str, object]) -> _Policy: + child = _apply_patch(parent, patch) + return _Policy( + allow_exports=parent.allow_exports and child.allow_exports, + allowed_export_formats=tuple( + item + for item in parent.allowed_export_formats + if item in child.allowed_export_formats + ), + allow_high_reidentification_risk=( + parent.allow_high_reidentification_risk + and child.allow_high_reidentification_risk + ), + max_retention_days=_minimum_optional( + parent.max_retention_days, + child.max_retention_days, + ), + required_privacy_transforms=tuple( + sorted( + set(parent.required_privacy_transforms) + | set(child.required_privacy_transforms) + ) + ), + ) + + +def _boolean( + value: Mapping[str, object], + key: str, + default: bool, +) -> bool: + raw = value.get(key) + if raw is None: + return default + if not isinstance(raw, bool): + raise ValueError(f"Reporting Policy {key} must be boolean") + return raw + + +def _formats(value: object, *, default: tuple[str, ...]) -> tuple[str, ...]: + if value is None: + return default + if not isinstance(value, list) or any( + item not in {"json", "csv"} for item in value + ): + raise ValueError("Reporting Policy export formats must contain json or csv") + return tuple(dict.fromkeys(str(item) for item in value)) + + +def _transforms(value: object, *, default: tuple[str, ...]) -> tuple[str, ...]: + if value is None: + return default + if not isinstance(value, list) or any( + not isinstance(item, str) or not item.strip() for item in value + ): + raise ValueError("Reporting Policy privacy transforms must be strings") + return tuple(sorted(set(item.strip() for item in value))) + + +def _days(value: object, *, default: int | None) -> int | None: + if value is None: + return default + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError("Reporting Policy retention days must be a positive integer") + return value + + +def _minimum_optional(left: int | None, right: int | None) -> int | None: + values = [item for item in (left, right) if item is not None] + return min(values) if values else None + + +__all__ = ["ReportingGovernancePolicyProvider"] diff --git a/src/govoplan_policy/backend/retention.py b/src/govoplan_policy/backend/retention.py index b1aabcd..63d7c07 100644 --- a/src/govoplan_policy/backend/retention.py +++ b/src/govoplan_policy/backend/retention.py @@ -24,6 +24,10 @@ from govoplan_core.core.access import ( AuditRetentionProvider, ) from govoplan_core.core.policy import PolicySourceStep, policy_source_step +from govoplan_core.core.reporting import ( + CAPABILITY_REPORTING_RETENTION, + ReportingRetentionProvider, +) from govoplan_core.core.runtime import get_registry from govoplan_core.admin.models import SystemSettings from govoplan_core.privacy.schemas import ( @@ -110,7 +114,11 @@ class PrivacyPolicyError(RuntimeError): def _campaign_policy_provider() -> CampaignPolicyContextProvider | None: registry = get_registry() - if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT): + if ( + registry is None + or not hasattr(registry, "has_capability") + or not registry.has_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT) + ): return None capability = registry.require_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT) if not isinstance(capability, CampaignPolicyContextProvider): @@ -120,7 +128,11 @@ def _campaign_policy_provider() -> CampaignPolicyContextProvider | None: def _campaign_retention_provider() -> CampaignRetentionProvider | None: registry = get_registry() - if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_RETENTION): + if ( + registry is None + or not hasattr(registry, "has_capability") + or not registry.has_capability(CAPABILITY_CAMPAIGNS_RETENTION) + ): return None capability = registry.require_capability(CAPABILITY_CAMPAIGNS_RETENTION) if not isinstance(capability, CampaignRetentionProvider): @@ -130,7 +142,11 @@ def _campaign_retention_provider() -> CampaignRetentionProvider | None: def _access_administration() -> AccessAdministration | None: registry = get_registry() - if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_ACCESS_ADMINISTRATION): + if ( + registry is None + or not hasattr(registry, "has_capability") + or not registry.has_capability(CAPABILITY_ACCESS_ADMINISTRATION) + ): return None capability = registry.require_capability(CAPABILITY_ACCESS_ADMINISTRATION) if not isinstance(capability, AccessAdministration): @@ -140,7 +156,11 @@ def _access_administration() -> AccessAdministration | None: def _audit_retention_provider() -> AuditRetentionProvider | None: registry = get_registry() - if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_AUDIT_RETENTION): + if ( + registry is None + or not hasattr(registry, "has_capability") + or not registry.has_capability(CAPABILITY_AUDIT_RETENTION) + ): return None capability = registry.require_capability(CAPABILITY_AUDIT_RETENTION) @@ -149,7 +169,23 @@ def _audit_retention_provider() -> AuditRetentionProvider | None: return capability -def _user_settings(session: Session, *, user_id: str, tenant_id: str) -> dict[str, Any] | None: +def _reporting_retention_provider() -> ReportingRetentionProvider | None: + registry = get_registry() + if ( + registry is None + or not hasattr(registry, "has_capability") + or not registry.has_capability(CAPABILITY_REPORTING_RETENTION) + ): + return None + capability = registry.require_capability(CAPABILITY_REPORTING_RETENTION) + if not isinstance(capability, ReportingRetentionProvider): + raise PrivacyPolicyError("Reporting retention capability is invalid") + return capability + + +def _user_settings( + session: Session, *, user_id: str, tenant_id: str +) -> dict[str, Any] | None: provider = _access_administration() if provider is None: return None @@ -157,15 +193,21 @@ def _user_settings(session: Session, *, user_id: str, tenant_id: str) -> dict[st return dict(payload) if payload is not None else None -def _set_user_settings(session: Session, *, user_id: str, tenant_id: str, settings_payload: dict[str, Any]) -> dict[str, Any] | None: +def _set_user_settings( + session: Session, *, user_id: str, tenant_id: str, settings_payload: dict[str, Any] +) -> dict[str, Any] | None: provider = _access_administration() if provider is None: return None - payload = provider.set_user_settings(session, user_id, tenant_id=tenant_id, settings=settings_payload) + payload = provider.set_user_settings( + session, user_id, tenant_id=tenant_id, settings=settings_payload + ) return dict(payload) if payload is not None else None -def _group_settings(session: Session, *, group_id: str, tenant_id: str) -> dict[str, Any] | None: +def _group_settings( + session: Session, *, group_id: str, tenant_id: str +) -> dict[str, Any] | None: provider = _access_administration() if provider is None: return None @@ -173,19 +215,27 @@ def _group_settings(session: Session, *, group_id: str, tenant_id: str) -> dict[ return dict(payload) if payload is not None else None -def _set_group_settings(session: Session, *, group_id: str, tenant_id: str, settings_payload: dict[str, Any]) -> dict[str, Any] | None: +def _set_group_settings( + session: Session, *, group_id: str, tenant_id: str, settings_payload: dict[str, Any] +) -> dict[str, Any] | None: provider = _access_administration() if provider is None: return None - payload = provider.set_group_settings(session, group_id, tenant_id=tenant_id, settings=settings_payload) + payload = provider.set_group_settings( + session, group_id, tenant_id=tenant_id, settings=settings_payload + ) return dict(payload) if payload is not None else None -def _campaign_policy_context(session: Session, *, campaign_id: str, tenant_id: str | None = None) -> CampaignPolicyContext: +def _campaign_policy_context( + session: Session, *, campaign_id: str, tenant_id: str | None = None +) -> CampaignPolicyContext: provider = _campaign_policy_provider() if provider is None: raise PrivacyPolicyError("Campaign module is not installed") - context = provider.get_campaign_policy_context(session, tenant_id=tenant_id, campaign_id=campaign_id) + context = provider.get_campaign_policy_context( + session, tenant_id=tenant_id, campaign_id=campaign_id + ) if context is None: raise PrivacyPolicyError("Campaign not found for privacy policy") return context @@ -208,17 +258,29 @@ def _privacy_policy_data(settings_payload: dict[str, Any] | None) -> dict[str, A return data if isinstance(data, dict) else {} -def _privacy_policy_patch_from_settings(settings_payload: dict[str, Any] | None) -> dict[str, Any]: +def _privacy_policy_patch_from_settings( + settings_payload: dict[str, Any] | None, +) -> dict[str, Any]: data = _privacy_policy_data(settings_payload) if not data: return {} - return PrivacyRetentionPolicyPatch.model_validate(data).model_dump(mode="json", exclude_none=True) + return PrivacyRetentionPolicyPatch.model_validate(data).model_dump( + mode="json", exclude_none=True + ) -def _merge_privacy_policy(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> PrivacyRetentionPolicy: +def _merge_privacy_policy( + parent: PrivacyRetentionPolicy, patch: dict[str, Any] +) -> PrivacyRetentionPolicy: payload = parent.model_dump(mode="json") - parent_allow = {**default_allow_lower_level_limits(), **(payload.get("allow_lower_level_limits") or {})} - if parent_allow["store_raw_campaign_json"] and patch.get("store_raw_campaign_json") is False: + parent_allow = { + **default_allow_lower_level_limits(), + **(payload.get("allow_lower_level_limits") or {}), + } + if ( + parent_allow["store_raw_campaign_json"] + and patch.get("store_raw_campaign_json") is False + ): payload["store_raw_campaign_json"] = False for key in RETENTION_DAY_KEYS: value = patch.get(key) @@ -227,7 +289,12 @@ def _merge_privacy_policy(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) current = payload.get(key) payload[key] = int(value) if current is None else min(int(current), int(value)) detail_level = patch.get("audit_detail_level") - if parent_allow["audit_detail_level"] and detail_level and AUDIT_DETAIL_LEVEL_ORDER[detail_level] > AUDIT_DETAIL_LEVEL_ORDER[payload["audit_detail_level"]]: + if ( + parent_allow["audit_detail_level"] + and detail_level + and AUDIT_DETAIL_LEVEL_ORDER[detail_level] + > AUDIT_DETAIL_LEVEL_ORDER[payload["audit_detail_level"]] + ): payload["audit_detail_level"] = detail_level patch_allow = patch.get("allow_lower_level_limits") or {} @@ -239,64 +306,94 @@ def _merge_privacy_policy(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) def _parent_allow_lower_level_limits(parent_payload: dict[str, Any]) -> dict[str, bool]: - return {**default_allow_lower_level_limits(), **(parent_payload.get("allow_lower_level_limits") or {})} + return { + **default_allow_lower_level_limits(), + **(parent_payload.get("allow_lower_level_limits") or {}), + } def _privacy_restriction_rules() -> tuple[PolicyRestrictionRule, ...]: return ( PolicyRestrictionRule( field="store_raw_campaign_json", - is_more_restrictive_or_equal=lambda parent_value, requested_value: not (requested_value is True and parent_value is False), + is_more_restrictive_or_equal=lambda parent_value, requested_value: ( + not (requested_value is True and parent_value is False) + ), less_restrictive_message="Raw campaign JSON storage cannot be re-enabled below a parent policy that disables it.", ), *( PolicyRestrictionRule( field=key, - is_more_restrictive_or_equal=lambda parent_value, requested_value: parent_value is None or int(requested_value) <= int(parent_value), + is_more_restrictive_or_equal=lambda parent_value, requested_value: ( + parent_value is None or int(requested_value) <= int(parent_value) + ), less_restrictive_message=f"{key} cannot be less restrictive than the parent retention policy.", ) for key in RETENTION_DAY_KEYS ), PolicyRestrictionRule( field="audit_detail_level", - is_more_restrictive_or_equal=lambda parent_value, requested_value: AUDIT_DETAIL_LEVEL_ORDER[str(requested_value)] >= AUDIT_DETAIL_LEVEL_ORDER[str(parent_value or "full")], + is_more_restrictive_or_equal=lambda parent_value, requested_value: ( + AUDIT_DETAIL_LEVEL_ORDER[str(requested_value)] + >= AUDIT_DETAIL_LEVEL_ORDER[str(parent_value or "full")] + ), less_restrictive_message="Audit detail level cannot be less restrictive than the parent retention policy.", ), ) -def _validate_privacy_patch_against_parent(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> None: +def _validate_privacy_patch_against_parent( + parent: PrivacyRetentionPolicy, patch: dict[str, Any] +) -> None: parent_payload = parent.model_dump(mode="json") issues = validate_hierarchical_policy_patch( parent_policy=parent_payload, patch=patch, field_keys=RETENTION_POLICY_FIELD_KEYS, - parent_allow_lower_level_limits=_parent_allow_lower_level_limits(parent_payload), + parent_allow_lower_level_limits=_parent_allow_lower_level_limits( + parent_payload + ), restriction_rules=_privacy_restriction_rules(), - locked_field_message=lambda key: f"{key} is locked by the parent retention policy.", - relock_message=lambda key: f"{key} limiting cannot be re-enabled below a parent retention policy lock.", + locked_field_message=lambda key: ( + f"{key} is locked by the parent retention policy." + ), + relock_message=lambda key: ( + f"{key} limiting cannot be re-enabled below a parent retention policy lock." + ), ) if issues: raise PrivacyPolicyError(issues[0].message) -def _set_settings_privacy_policy(settings_payload: dict[str, Any] | None, policy: dict[str, Any]) -> dict[str, Any]: +def _set_settings_privacy_policy( + settings_payload: dict[str, Any] | None, policy: dict[str, Any] +) -> dict[str, Any]: payload = dict(settings_payload or {}) payload[PRIVACY_POLICY_SETTINGS_KEY] = policy return payload def privacy_policy_from_settings(item: SystemSettings) -> PrivacyRetentionPolicy: - return PrivacyRetentionPolicy.model_validate(_privacy_policy_data(item.settings or {})) + return PrivacyRetentionPolicy.model_validate( + _privacy_policy_data(item.settings or {}) + ) def privacy_policy_from_session(session: Session) -> PrivacyRetentionPolicy: return privacy_policy_from_settings(get_system_settings(session)) -def set_privacy_policy(item: SystemSettings, policy: PrivacyRetentionPolicy | dict[str, Any]) -> PrivacyRetentionPolicy: - validated = policy if isinstance(policy, PrivacyRetentionPolicy) else PrivacyRetentionPolicy.model_validate(policy) - item.settings = _set_settings_privacy_policy(item.settings, validated.model_dump(mode="json")) +def set_privacy_policy( + item: SystemSettings, policy: PrivacyRetentionPolicy | dict[str, Any] +) -> PrivacyRetentionPolicy: + validated = ( + policy + if isinstance(policy, PrivacyRetentionPolicy) + else PrivacyRetentionPolicy.model_validate(policy) + ) + item.settings = _set_settings_privacy_policy( + item.settings, validated.model_dump(mode="json") + ) return validated @@ -319,21 +416,39 @@ def effective_privacy_policy( tenant = session.get(Tenant, tenant_id) if tenant is None: raise PrivacyPolicyError("Tenant not found for privacy policy") - policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {})) + policy = _merge_privacy_policy( + policy, _privacy_policy_patch_from_settings(tenant.settings or {}) + ) if owner_user_id: - user_settings = _user_settings(session, user_id=owner_user_id, tenant_id=tenant_id) if tenant_id else None + user_settings = ( + _user_settings(session, user_id=owner_user_id, tenant_id=tenant_id) + if tenant_id + else None + ) if user_settings is not None: - policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user_settings)) + policy = _merge_privacy_policy( + policy, _privacy_policy_patch_from_settings(user_settings) + ) if owner_group_id: - group_settings = _group_settings(session, group_id=owner_group_id, tenant_id=tenant_id) if tenant_id else None + group_settings = ( + _group_settings(session, group_id=owner_group_id, tenant_id=tenant_id) + if tenant_id + else None + ) if group_settings is not None: - policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group_settings)) + policy = _merge_privacy_policy( + policy, _privacy_policy_patch_from_settings(group_settings) + ) if campaign is not None: - policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(dict(campaign.settings or {}))) + policy = _merge_privacy_policy( + policy, _privacy_policy_patch_from_settings(dict(campaign.settings or {})) + ) return policy -def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> PrivacyRetentionPolicy: +def parent_privacy_policy( + session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None +) -> PrivacyRetentionPolicy: clean_scope = scope_type.strip().casefold() policy = privacy_policy_from_session(session) if clean_scope == "tenant": @@ -341,38 +456,65 @@ def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str, tenant = session.get(Tenant, tenant_id) if tenant is None: raise PrivacyPolicyError("Tenant not found for privacy policy") - policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {})) + policy = _merge_privacy_policy( + policy, _privacy_policy_patch_from_settings(tenant.settings or {}) + ) if clean_scope in {"user", "group"}: return policy if clean_scope != "campaign" or not scope_id: return policy - campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) + campaign = _campaign_policy_context( + session, tenant_id=tenant_id, campaign_id=scope_id + ) if campaign.owner_user_id: - user_settings = _user_settings(session, user_id=campaign.owner_user_id, tenant_id=tenant_id) + user_settings = _user_settings( + session, user_id=campaign.owner_user_id, tenant_id=tenant_id + ) if user_settings is not None: - policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user_settings)) + policy = _merge_privacy_policy( + policy, _privacy_policy_patch_from_settings(user_settings) + ) if campaign.owner_group_id: - group_settings = _group_settings(session, group_id=campaign.owner_group_id, tenant_id=tenant_id) + group_settings = _group_settings( + session, group_id=campaign.owner_group_id, tenant_id=tenant_id + ) if group_settings is not None: - policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group_settings)) + policy = _merge_privacy_policy( + policy, _privacy_policy_patch_from_settings(group_settings) + ) return policy - -def _retention_policy_source_fields(patch: dict[str, Any], *, baseline: bool = False) -> list[str]: +def _retention_policy_source_fields( + patch: dict[str, Any], *, baseline: bool = False +) -> list[str]: fields: list[str] = [] for key in RETENTION_POLICY_FIELD_KEYS: if key in patch and patch.get(key) is not None: fields.append(key) - if isinstance(patch.get("allow_lower_level_limits"), dict) and patch["allow_lower_level_limits"]: + if ( + isinstance(patch.get("allow_lower_level_limits"), dict) + and patch["allow_lower_level_limits"] + ): fields.append("allow_lower_level_limits") if baseline and not fields: fields.append("defaults") return fields -def _retention_policy_source_step(scope_type: str, label: str, scope_id: str | None, patch: dict[str, Any], *, baseline: bool = False) -> dict[str, Any]: - source_policy = PrivacyRetentionPolicy.model_validate(patch).model_dump(mode="json") if baseline else dict(patch) +def _retention_policy_source_step( + scope_type: str, + label: str, + scope_id: str | None, + patch: dict[str, Any], + *, + baseline: bool = False, +) -> dict[str, Any]: + source_policy = ( + PrivacyRetentionPolicy.model_validate(patch).model_dump(mode="json") + if baseline + else dict(patch) + ) return policy_source_step( scope_type, label, @@ -391,7 +533,15 @@ def effective_privacy_policy_sources( campaign_id: str | None = None, ) -> list[dict[str, Any]]: system_settings = get_system_settings(session) - sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)] + sources = [ + _retention_policy_source_step( + "system", + "System", + None, + _privacy_policy_patch_from_settings(system_settings.settings or {}), + baseline=True, + ) + ] campaign: CampaignPolicyContext | None = None if campaign_id: campaign = _campaign_policy_context(session, campaign_id=campaign_id) @@ -402,48 +552,124 @@ def effective_privacy_policy_sources( tenant = session.get(Tenant, tenant_id) if tenant is None: raise PrivacyPolicyError("Tenant not found for privacy policy") - sources.append(_retention_policy_source_step("tenant", "Tenant", tenant.id, _privacy_policy_patch_from_settings(tenant.settings or {}))) + sources.append( + _retention_policy_source_step( + "tenant", + "Tenant", + tenant.id, + _privacy_policy_patch_from_settings(tenant.settings or {}), + ) + ) if owner_user_id: - user_settings = _user_settings(session, user_id=owner_user_id, tenant_id=tenant_id) if tenant_id else None + user_settings = ( + _user_settings(session, user_id=owner_user_id, tenant_id=tenant_id) + if tenant_id + else None + ) if user_settings is not None: - sources.append(_retention_policy_source_step("user", "Owner user", owner_user_id, _privacy_policy_patch_from_settings(user_settings))) + sources.append( + _retention_policy_source_step( + "user", + "Owner user", + owner_user_id, + _privacy_policy_patch_from_settings(user_settings), + ) + ) if owner_group_id: - group_settings = _group_settings(session, group_id=owner_group_id, tenant_id=tenant_id) if tenant_id else None + group_settings = ( + _group_settings(session, group_id=owner_group_id, tenant_id=tenant_id) + if tenant_id + else None + ) if group_settings is not None: - sources.append(_retention_policy_source_step("group", "Owner group", owner_group_id, _privacy_policy_patch_from_settings(group_settings))) + sources.append( + _retention_policy_source_step( + "group", + "Owner group", + owner_group_id, + _privacy_policy_patch_from_settings(group_settings), + ) + ) if campaign is not None: - sources.append(_retention_policy_source_step("campaign", "Campaign", campaign.id, _privacy_policy_patch_from_settings(dict(campaign.settings or {})))) + sources.append( + _retention_policy_source_step( + "campaign", + "Campaign", + campaign.id, + _privacy_policy_patch_from_settings(dict(campaign.settings or {})), + ) + ) return sources -def parent_privacy_policy_sources(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> list[dict[str, Any]]: +def parent_privacy_policy_sources( + session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None +) -> list[dict[str, Any]]: clean_scope = scope_type.strip().casefold() if clean_scope == "system": return [] system_settings = get_system_settings(session) - sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)] + sources = [ + _retention_policy_source_step( + "system", + "System", + None, + _privacy_policy_patch_from_settings(system_settings.settings or {}), + baseline=True, + ) + ] if clean_scope == "tenant": return sources tenant = session.get(Tenant, tenant_id) if tenant is None: raise PrivacyPolicyError("Tenant not found for privacy policy") - sources.append(_retention_policy_source_step("tenant", "Tenant", tenant.id, _privacy_policy_patch_from_settings(tenant.settings or {}))) + sources.append( + _retention_policy_source_step( + "tenant", + "Tenant", + tenant.id, + _privacy_policy_patch_from_settings(tenant.settings or {}), + ) + ) if clean_scope in {"user", "group"}: return sources if clean_scope != "campaign" or not scope_id: return sources - campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) + campaign = _campaign_policy_context( + session, tenant_id=tenant_id, campaign_id=scope_id + ) if campaign.owner_user_id: - user_settings = _user_settings(session, user_id=campaign.owner_user_id, tenant_id=tenant_id) + user_settings = _user_settings( + session, user_id=campaign.owner_user_id, tenant_id=tenant_id + ) if user_settings is not None: - sources.append(_retention_policy_source_step("user", "Owner user", campaign.owner_user_id, _privacy_policy_patch_from_settings(user_settings))) + sources.append( + _retention_policy_source_step( + "user", + "Owner user", + campaign.owner_user_id, + _privacy_policy_patch_from_settings(user_settings), + ) + ) if campaign.owner_group_id: - group_settings = _group_settings(session, group_id=campaign.owner_group_id, tenant_id=tenant_id) + group_settings = _group_settings( + session, group_id=campaign.owner_group_id, tenant_id=tenant_id + ) if group_settings is not None: - sources.append(_retention_policy_source_step("group", "Owner group", campaign.owner_group_id, _privacy_policy_patch_from_settings(group_settings))) + sources.append( + _retention_policy_source_step( + "group", + "Owner group", + campaign.owner_group_id, + _privacy_policy_patch_from_settings(group_settings), + ) + ) return sources -def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> dict[str, Any]: + +def get_privacy_policy_for_scope( + session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None +) -> dict[str, Any]: clean_scope = scope_type.strip().casefold() if clean_scope == "system": return privacy_policy_from_session(session).model_dump(mode="json") @@ -453,21 +679,29 @@ def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type raise PrivacyPolicyError("Tenant privacy policy not found") return _privacy_policy_patch_from_settings(tenant.settings or {}) if not scope_id: - raise PrivacyPolicyError(f"{clean_scope.capitalize()} privacy policy requires scope_id") + raise PrivacyPolicyError( + f"{clean_scope.capitalize()} privacy policy requires scope_id" + ) if clean_scope == "user": user_settings = _user_settings(session, user_id=scope_id, tenant_id=tenant_id) if user_settings is None: raise PrivacyPolicyError("User privacy policy not found") return _privacy_policy_patch_from_settings(user_settings) if clean_scope == "group": - group_settings = _group_settings(session, group_id=scope_id, tenant_id=tenant_id) + group_settings = _group_settings( + session, group_id=scope_id, tenant_id=tenant_id + ) if group_settings is None: raise PrivacyPolicyError("Group privacy policy not found") return _privacy_policy_patch_from_settings(group_settings) if clean_scope == "campaign": - campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) + campaign = _campaign_policy_context( + session, tenant_id=tenant_id, campaign_id=scope_id + ) return _privacy_policy_patch_from_settings(dict(campaign.settings or {})) - raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign") + raise PrivacyPolicyError( + "Privacy policy scope must be system, tenant, user, group or campaign" + ) def set_privacy_policy_for_scope( @@ -498,15 +732,21 @@ def set_privacy_policy_for_scope( ) -def _set_system_privacy_policy(session: Session, policy: dict[str, Any] | None) -> dict[str, Any]: +def _set_system_privacy_policy( + session: Session, policy: dict[str, Any] | None +) -> dict[str, Any]: item = get_system_settings(session) - validated = set_privacy_policy(item, PrivacyRetentionPolicy.model_validate(policy or {})) + validated = set_privacy_policy( + item, PrivacyRetentionPolicy.model_validate(policy or {}) + ) session.add(item) return validated.model_dump(mode="json") def _privacy_policy_patch_payload(policy: dict[str, Any] | None) -> dict[str, Any]: - return PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(mode="json", exclude_none=True) + return PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump( + mode="json", exclude_none=True + ) def _validate_privacy_policy_patch_for_scope( @@ -517,12 +757,18 @@ def _validate_privacy_policy_patch_for_scope( scope_id: str | None, patch: dict[str, Any], ) -> None: - parent_scope_id = _parent_privacy_policy_scope_id(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) - parent = parent_privacy_policy(session, tenant_id=tenant_id, scope_type=scope_type, scope_id=parent_scope_id) + parent_scope_id = _parent_privacy_policy_scope_id( + tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id + ) + parent = parent_privacy_policy( + session, tenant_id=tenant_id, scope_type=scope_type, scope_id=parent_scope_id + ) _validate_privacy_patch_against_parent(parent, patch) -def _parent_privacy_policy_scope_id(*, tenant_id: str, scope_type: str, scope_id: str | None) -> str | None: +def _parent_privacy_policy_scope_id( + *, tenant_id: str, scope_type: str, scope_id: str | None +) -> str | None: if scope_id: return scope_id if scope_type == "tenant": @@ -539,21 +785,33 @@ def _set_scoped_privacy_policy( patch: dict[str, Any], ) -> dict[str, Any]: if scope_type == "tenant": - return _set_tenant_privacy_policy(session, tenant_id=tenant_id, scope_id=scope_id, patch=patch) + return _set_tenant_privacy_policy( + session, tenant_id=tenant_id, scope_id=scope_id, patch=patch + ) clean_scope_id = _required_privacy_policy_scope_id(scope_type, scope_id) if scope_type == "user": - return _set_user_privacy_policy(session, tenant_id=tenant_id, user_id=clean_scope_id, patch=patch) + return _set_user_privacy_policy( + session, tenant_id=tenant_id, user_id=clean_scope_id, patch=patch + ) if scope_type == "group": - return _set_group_privacy_policy(session, tenant_id=tenant_id, group_id=clean_scope_id, patch=patch) + return _set_group_privacy_policy( + session, tenant_id=tenant_id, group_id=clean_scope_id, patch=patch + ) if scope_type == "campaign": - return _set_campaign_privacy_policy(session, tenant_id=tenant_id, campaign_id=clean_scope_id, patch=patch) - raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign") + return _set_campaign_privacy_policy( + session, tenant_id=tenant_id, campaign_id=clean_scope_id, patch=patch + ) + raise PrivacyPolicyError( + "Privacy policy scope must be system, tenant, user, group or campaign" + ) def _required_privacy_policy_scope_id(scope_type: str, scope_id: str | None) -> str: if scope_id: return scope_id - raise PrivacyPolicyError(f"{scope_type.capitalize()} privacy policy requires scope_id") + raise PrivacyPolicyError( + f"{scope_type.capitalize()} privacy policy requires scope_id" + ) def _set_tenant_privacy_policy( @@ -571,34 +829,65 @@ def _set_tenant_privacy_policy( return patch -def _set_user_privacy_policy(session: Session, *, tenant_id: str, user_id: str, patch: dict[str, Any]) -> dict[str, Any]: +def _set_user_privacy_policy( + session: Session, *, tenant_id: str, user_id: str, patch: dict[str, Any] +) -> dict[str, Any]: current_settings = _user_settings(session, user_id=user_id, tenant_id=tenant_id) if current_settings is None: raise PrivacyPolicyError("User privacy policy not found") settings_payload = _set_settings_privacy_policy(current_settings, patch) - if _set_user_settings(session, user_id=user_id, tenant_id=tenant_id, settings_payload=settings_payload) is None: + if ( + _set_user_settings( + session, + user_id=user_id, + tenant_id=tenant_id, + settings_payload=settings_payload, + ) + is None + ): raise PrivacyPolicyError("User privacy policy not found") return patch -def _set_group_privacy_policy(session: Session, *, tenant_id: str, group_id: str, patch: dict[str, Any]) -> dict[str, Any]: +def _set_group_privacy_policy( + session: Session, *, tenant_id: str, group_id: str, patch: dict[str, Any] +) -> dict[str, Any]: current_settings = _group_settings(session, group_id=group_id, tenant_id=tenant_id) if current_settings is None: raise PrivacyPolicyError("Group privacy policy not found") settings_payload = _set_settings_privacy_policy(current_settings, patch) - if _set_group_settings(session, group_id=group_id, tenant_id=tenant_id, settings_payload=settings_payload) is None: + if ( + _set_group_settings( + session, + group_id=group_id, + tenant_id=tenant_id, + settings_payload=settings_payload, + ) + is None + ): raise PrivacyPolicyError("Group privacy policy not found") return patch -def _set_campaign_privacy_policy(session: Session, *, tenant_id: str, campaign_id: str, patch: dict[str, Any]) -> dict[str, Any]: +def _set_campaign_privacy_policy( + session: Session, *, tenant_id: str, campaign_id: str, patch: dict[str, Any] +) -> dict[str, Any]: provider = _campaign_policy_provider() if provider is None: raise PrivacyPolicyError("Campaign module is not installed") - campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=campaign_id) - settings_payload = _set_settings_privacy_policy(dict(campaign.settings or {}), patch) + campaign = _campaign_policy_context( + session, tenant_id=tenant_id, campaign_id=campaign_id + ) + settings_payload = _set_settings_privacy_policy( + dict(campaign.settings or {}), patch + ) try: - provider.set_campaign_settings(session, tenant_id=tenant_id, campaign_id=campaign_id, settings=settings_payload) + provider.set_campaign_settings( + session, + tenant_id=tenant_id, + campaign_id=campaign_id, + settings=settings_payload, + ) except ValueError as exc: raise PrivacyPolicyError("Campaign privacy policy not found") from exc return patch @@ -613,10 +902,16 @@ def simulate_privacy_policy_change( policy: dict[str, Any] | None = None, ) -> dict[str, Any]: clean_scope = scope_type.strip().casefold() - patch = PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(mode="json", exclude_none=True) - current = get_privacy_policy_for_scope(session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id) + patch = PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump( + mode="json", exclude_none=True + ) + current = get_privacy_policy_for_scope( + session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id + ) if clean_scope == "system": - parent_payload = PrivacyRetentionPolicy.model_validate(current).model_dump(mode="json") + parent_payload = PrivacyRetentionPolicy.model_validate(current).model_dump( + mode="json" + ) parent_allow = default_allow_lower_level_limits() parent_sources = effective_privacy_policy_sources(session) else: @@ -641,15 +936,25 @@ def simulate_privacy_policy_change( patch=patch, field_keys=RETENTION_POLICY_FIELD_KEYS, parent_allow_lower_level_limits=parent_allow, - restriction_rules=() if clean_scope == "system" else _privacy_restriction_rules(), - source_path=[PolicySourceStep.from_mapping(source) for source in parent_sources], - locked_field_message=lambda key: f"{key} is locked by the parent retention policy.", - relock_message=lambda key: f"{key} limiting cannot be re-enabled below a parent retention policy lock.", + restriction_rules=() + if clean_scope == "system" + else _privacy_restriction_rules(), + source_path=[ + PolicySourceStep.from_mapping(source) for source in parent_sources + ], + locked_field_message=lambda key: ( + f"{key} is locked by the parent retention policy." + ), + relock_message=lambda key: ( + f"{key} limiting cannot be re-enabled below a parent retention policy lock." + ), ) return simulation.to_dict() -def sanitize_audit_details_for_policy(session: Session, details: dict[str, Any]) -> dict[str, Any]: +def sanitize_audit_details_for_policy( + session: Session, details: dict[str, Any] +) -> dict[str, Any]: policy = privacy_policy_from_session(session) if policy.audit_detail_level == "full": return details @@ -675,11 +980,20 @@ def _redact_audit_value(value: Any) -> Any: return value -def _system_cutoffs(policy: PrivacyRetentionPolicy, *, now: datetime) -> dict[str, datetime | None]: +def _system_cutoffs( + policy: PrivacyRetentionPolicy, *, now: datetime +) -> dict[str, datetime | None]: return { - "raw_campaign_json": _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now), + "raw_campaign_json": _cutoff( + 0 + if not policy.store_raw_campaign_json + else policy.raw_campaign_json_retention_days, + now=now, + ), "generated_eml": _cutoff(policy.generated_eml_retention_days, now=now), - "stored_report_detail": _cutoff(policy.stored_report_detail_retention_days, now=now), + "stored_report_detail": _cutoff( + policy.stored_report_detail_retention_days, now=now + ), "mock_mailbox": _cutoff(policy.mock_mailbox_retention_days, now=now), "audit_detail": _cutoff(policy.audit_detail_retention_days, now=now), } @@ -692,9 +1006,24 @@ def apply_retention_policy(session: Session, *, dry_run: bool = True) -> dict[st policy_cache: dict[str, PrivacyRetentionPolicy] = {} campaign_retention = _campaign_retention_provider() campaign_counts = { - "raw_campaign_json": {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0}, - "generated_eml": {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0}, - "stored_report_detail": {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0}, + "raw_campaign_json": { + "eligible": 0, + "redacted": 0, + "skipped_not_final": 0, + "already_redacted": 0, + }, + "generated_eml": { + "eligible": 0, + "metadata_cleared": 0, + "files_deleted": 0, + "files_missing": 0, + "skipped_not_final": 0, + }, + "stored_report_detail": { + "eligible_versions": 0, + "summaries_redacted": 0, + "already_redacted": 0, + }, } if campaign_retention is not None: campaign_counts.update( @@ -704,27 +1033,54 @@ def apply_retention_policy(session: Session, *, dry_run: bool = True) -> dict[st session, dry_run=dry_run, now=now, - policy_for_campaign_id=lambda campaign_id: _campaign_policy_for_id(session, campaign_id, policy_cache), + policy_for_campaign_id=lambda campaign_id: _campaign_policy_for_id( + session, campaign_id, policy_cache + ), ).items() } ) + reporting_retention = _reporting_retention_provider() + reporting_counts = { + "eligible": 0, + "redacted": 0, + "remaining_in_batch": 0, + } + if reporting_retention is not None: + reporting_counts.update( + reporting_retention.apply_retention( + session, + dry_run=dry_run, + now=now, + ) + ) counts = { "raw_campaign_json": campaign_counts["raw_campaign_json"], "generated_eml": campaign_counts["generated_eml"], - "stored_report_detail": campaign_counts["stored_report_detail"], - "mock_mailbox": _apply_mock_mailbox_retention(cutoffs["mock_mailbox"], dry_run=dry_run), - "audit_detail": _apply_audit_detail_retention(session, dry_run=dry_run, now=now), + "stored_report_detail": { + **campaign_counts["stored_report_detail"], + "provider_reports": reporting_counts, + }, + "mock_mailbox": _apply_mock_mailbox_retention( + cutoffs["mock_mailbox"], dry_run=dry_run + ), + "audit_detail": _apply_audit_detail_retention( + session, dry_run=dry_run, now=now + ), } return { "dry_run": dry_run, "policy": policy.model_dump(mode="json"), - "cutoffs": {key: value.isoformat() if value else None for key, value in cutoffs.items()}, + "cutoffs": { + key: value.isoformat() if value else None for key, value in cutoffs.items() + }, "effective_policy_scope": "per-object", "counts": counts, } -def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy: +def _campaign_policy_for_id( + session: Session, campaign_id: str | None, cache: dict[str, PrivacyRetentionPolicy] +) -> PrivacyRetentionPolicy: if not campaign_id: return privacy_policy_from_session(session) if campaign_id not in cache: @@ -732,7 +1088,9 @@ def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: di return cache[campaign_id] -def _apply_mock_mailbox_retention(cutoff: datetime | None, *, dry_run: bool) -> dict[str, int]: +def _apply_mock_mailbox_retention( + cutoff: datetime | None, *, dry_run: bool +) -> dict[str, int]: result = {"eligible_records": 0, "json_deleted": 0, "eml_deleted": 0} if cutoff is None: return result @@ -774,32 +1132,47 @@ def _parse_datetime(value: Any) -> datetime | None: return parsed -def _privacy_policy_for_audit_item(session: Session, item: AuditRecordRef, campaign_cache: dict[str, PrivacyRetentionPolicy], tenant_cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy: +def _privacy_policy_for_audit_item( + session: Session, + item: AuditRecordRef, + campaign_cache: dict[str, PrivacyRetentionPolicy], + tenant_cache: dict[str, PrivacyRetentionPolicy], +) -> PrivacyRetentionPolicy: if item.object_type == "campaign" and item.object_id: provider = _campaign_policy_provider() if provider is not None: - context = provider.get_campaign_policy_context(session, campaign_id=str(item.object_id)) + context = provider.get_campaign_policy_context( + session, campaign_id=str(item.object_id) + ) if context is not None: return _campaign_policy_for_id(session, context.id, campaign_cache) if item.tenant_id: if item.tenant_id not in tenant_cache: - tenant_cache[item.tenant_id] = effective_privacy_policy(session, tenant_id=item.tenant_id) + tenant_cache[item.tenant_id] = effective_privacy_policy( + session, tenant_id=item.tenant_id + ) return tenant_cache[item.tenant_id] return privacy_policy_from_session(session) -def _apply_audit_detail_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]: +def _apply_audit_detail_retention( + session: Session, *, dry_run: bool, now: datetime +) -> dict[str, int]: provider = _audit_retention_provider() if provider is None: return {"eligible": 0, "redacted": 0, "already_redacted": 0} campaign_cache: dict[str, PrivacyRetentionPolicy] = {} tenant_cache: dict[str, PrivacyRetentionPolicy] = {} - return dict(provider.apply_detail_retention( - session, - dry_run=dry_run, - now=now, - policy_for_record=lambda item: _privacy_policy_for_audit_item(session, item, campaign_cache, tenant_cache), - )) + return dict( + provider.apply_detail_retention( + session, + dry_run=dry_run, + now=now, + policy_for_record=lambda item: _privacy_policy_for_audit_item( + session, item, campaign_cache, tenant_cache + ), + ) + ) class SqlPrivacyRetentionService: diff --git a/tests/test_policy_module_contract.py b/tests/test_policy_module_contract.py index 7e015df..48a3fbf 100644 --- a/tests/test_policy_module_contract.py +++ b/tests/test_policy_module_contract.py @@ -14,6 +14,7 @@ from govoplan_core.core.policy import ( from govoplan_core.core.distribution_lists import ( CAPABILITY_POLICY_DISTRIBUTION_CHANNELS, ) +from govoplan_core.core.reporting import CAPABILITY_POLICY_REPORTING_GOVERNANCE from govoplan_policy.backend.manifest import manifest @@ -50,6 +51,7 @@ class PolicyModuleContractTests(unittest.TestCase): CAPABILITY_POLICY_DISTRIBUTION_CHANNELS, CAPABILITY_POLICY_FUNCTION_ASSIGNMENT_GOVERNANCE, CAPABILITY_POLICY_PRIVACY_RETENTION, + CAPABILITY_POLICY_REPORTING_GOVERNANCE, CAPABILITY_POLICY_SCHEDULING_PARTICIPANT_PRIVACY, CAPABILITY_POLICY_VIEW_GOVERNANCE, }, diff --git a/tests/test_reporting_governance.py b/tests/test_reporting_governance.py new file mode 100644 index 0000000..fe0509e --- /dev/null +++ b/tests/test_reporting_governance.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from govoplan_core.core.reporting import ReportingGovernanceRequest +from govoplan_core.db.base import Base +from govoplan_core.tenancy.scope import Tenant, create_scope_tables +from govoplan_policy.backend.reporting_governance import ( + ReportingGovernancePolicyProvider, +) +from govoplan_policy.backend import retention as retention_module + + +def _request(action: str, *, export_format: str | None = None): + return ReportingGovernanceRequest( + action=action, # type: ignore[arg-type] + tenant_id="tenant-1", + provider_id="campaigns", + report_id="delivery-outcomes", + purpose="Operational review", + audience_scope={"scope_type": "tenant", "scope_id": "tenant-1"}, + retention_class="stored_report_detail", + export_format=export_format, + reidentification_risk="low", + declared_privacy_transforms=("small_cell_suppression",), + applied_privacy_transforms=("small_cell_suppression",), + ) + + +def test_tenant_reporting_policy_can_tighten_export_and_retention() -> None: + engine = create_engine("sqlite+pysqlite:///:memory:") + create_scope_tables(engine) + Base.metadata.create_all(engine) + with Session(engine) as session: + session.add( + Tenant( + id="tenant-1", + slug="tenant-1", + name="Tenant 1", + settings={ + "privacy_retention_policy": { + "stored_report_detail_retention_days": 10 + }, + "reporting_governance_policy": { + "allow_exports": False, + "required_privacy_transforms": ["explicit_denominator"], + }, + }, + ) + ) + session.commit() + provider = ReportingGovernancePolicyProvider() + + execute = provider.decide_reporting_action( + session, + object(), + request=_request("execute"), + ) + export = provider.decide_reporting_action( + session, + object(), + request=_request("export", export_format="json"), + ) + + assert execute.allowed is True + assert execute.retention_days == 10 + assert set(execute.required_privacy_transforms) == { + "small_cell_suppression", + "explicit_denominator", + } + assert export.allowed is False + assert export.export_formats == () + engine.dispose() + + +def test_malformed_reporting_policy_fails_closed() -> None: + engine = create_engine("sqlite+pysqlite:///:memory:") + create_scope_tables(engine) + Base.metadata.create_all(engine) + with Session(engine) as session: + session.add( + Tenant( + id="tenant-1", + slug="tenant-1", + name="Tenant 1", + settings={"reporting_governance_policy": {"allow_exports": "yes"}}, + ) + ) + session.commit() + + decision = ReportingGovernancePolicyProvider().decide_reporting_action( + session, + object(), + request=_request("execute"), + ) + + assert decision.allowed is False + assert decision.provenance["decision"] == "fail_closed" + engine.dispose() + + +def test_shared_retention_run_invokes_reporting_without_model_imports( + monkeypatch, +) -> None: + class _ReportingRetention: + def apply_retention(self, session, *, dry_run, now, limit=500): + del session, now, limit + return { + "eligible": 2, + "redacted": 0 if dry_run else 2, + "remaining_in_batch": 0, + } + + class _Registry: + def has_capability(self, name): + return name == "reporting.retention" + + def require_capability(self, name): + assert name == "reporting.retention" + return _ReportingRetention() + + monkeypatch.setattr(retention_module, "get_registry", lambda: _Registry()) + engine = create_engine("sqlite+pysqlite:///:memory:") + create_scope_tables(engine) + Base.metadata.create_all(engine) + with Session(engine) as session: + result = retention_module.apply_retention_policy(session, dry_run=False) + + assert result["counts"]["stored_report_detail"]["provider_reports"] == { + "eligible": 2, + "redacted": 2, + "remaining_in_batch": 0, + } + engine.dispose()