from __future__ import annotations import json from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, field_validator from sqlalchemy.orm import Session from govoplan_core.admin.settings import get_system_settings from govoplan_core.core.campaigns import ( CAPABILITY_CAMPAIGNS_POLICY_CONTEXT, CAPABILITY_CAMPAIGNS_RETENTION, CampaignPolicyContext, CampaignPolicyContextProvider, CampaignRetentionProvider, ) from govoplan_core.core.access import ( CAPABILITY_ACCESS_ADMINISTRATION, CAPABILITY_AUDIT_RETENTION, AccessAdministration, AuditRecordRef, AuditRetentionProvider, ) from govoplan_core.core.policy import PolicySourceStep, policy_source_step from govoplan_core.core.runtime import get_registry from govoplan_core.admin.models import SystemSettings from govoplan_core.settings import settings from govoplan_core.tenancy.scope import Tenant from govoplan_policy.backend.hierarchy import ( PolicyRestrictionRule, simulate_hierarchical_policy_change, validate_hierarchical_policy_patch, ) PRIVACY_POLICY_SETTINGS_KEY = "privacy_retention_policy" RETENTION_DAY_KEYS = ( "raw_campaign_json_retention_days", "generated_eml_retention_days", "stored_report_detail_retention_days", "mock_mailbox_retention_days", "audit_detail_retention_days", ) RETENTION_POLICY_FIELD_KEYS = ( "store_raw_campaign_json", *RETENTION_DAY_KEYS, "audit_detail_level", ) AUDIT_DETAIL_LEVEL_ORDER = {"full": 0, "redacted": 1, "minimal": 2} def default_allow_lower_level_limits() -> dict[str, bool]: return {key: True for key in RETENTION_POLICY_FIELD_KEYS} def _normalize_allow_lower_level_limits(value: Any, *, fill_defaults: bool) -> dict[str, bool] | None: if value in (None, ""): return default_allow_lower_level_limits() if fill_defaults else None if not isinstance(value, dict): raise ValueError("allow_lower_level_limits must be an object") normalized = default_allow_lower_level_limits() if fill_defaults else {} for key, allowed in value.items(): clean_key = str(key) if clean_key not in RETENTION_POLICY_FIELD_KEYS: raise ValueError(f"Unknown retention policy field: {clean_key}") normalized[clean_key] = bool(allowed) return normalized AUDIT_DETAIL_REDACTION_KEYS = { "attachments", "bcc", "body", "campaign_json", "cc", "content", "entries", "html", "imap", "message", "password", "raw_eml", "raw_json", "recipients", "secret", "smtp", "template", "text", "token", "to", } class PrivacyRetentionPolicy(BaseModel): model_config = ConfigDict(extra="forbid") store_raw_campaign_json: bool = True raw_campaign_json_retention_days: int | None = Field(default=None, ge=0) generated_eml_retention_days: int | None = Field(default=None, ge=0) stored_report_detail_retention_days: int | None = Field(default=None, ge=0) mock_mailbox_retention_days: int | None = Field(default=None, ge=0) audit_detail_retention_days: int | None = Field(default=None, ge=0) audit_detail_level: Literal["full", "redacted", "minimal"] = "full" allow_lower_level_limits: dict[str, bool] = Field(default_factory=default_allow_lower_level_limits) @field_validator( "raw_campaign_json_retention_days", "generated_eml_retention_days", "stored_report_detail_retention_days", "mock_mailbox_retention_days", "audit_detail_retention_days", mode="before", ) @classmethod def _blank_string_is_none(cls, value: Any) -> Any: if value == "": return None return value @field_validator("allow_lower_level_limits", mode="before") @classmethod def _normalize_allow_lower_level_limits(cls, value: Any) -> Any: return _normalize_allow_lower_level_limits(value, fill_defaults=True) class PrivacyRetentionPolicyPatch(BaseModel): model_config = ConfigDict(extra="forbid") store_raw_campaign_json: bool | None = None raw_campaign_json_retention_days: int | None = Field(default=None, ge=0) generated_eml_retention_days: int | None = Field(default=None, ge=0) stored_report_detail_retention_days: int | None = Field(default=None, ge=0) mock_mailbox_retention_days: int | None = Field(default=None, ge=0) audit_detail_retention_days: int | None = Field(default=None, ge=0) audit_detail_level: Literal["full", "redacted", "minimal"] | None = None allow_lower_level_limits: dict[str, bool] | None = None @field_validator(*RETENTION_DAY_KEYS, mode="before") @classmethod def _blank_string_is_none(cls, value: Any) -> Any: if value == "": return None return value @field_validator("allow_lower_level_limits", mode="before") @classmethod def _normalize_allow_lower_level_limits(cls, value: Any) -> Any: return _normalize_allow_lower_level_limits(value, fill_defaults=False) class PrivacyPolicyError(RuntimeError): pass def _campaign_policy_provider() -> CampaignPolicyContextProvider | None: registry = get_registry() if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT): return None capability = registry.require_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT) if not isinstance(capability, CampaignPolicyContextProvider): raise PrivacyPolicyError("Campaign policy context capability is invalid") return capability def _campaign_retention_provider() -> CampaignRetentionProvider | None: registry = get_registry() if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_RETENTION): return None capability = registry.require_capability(CAPABILITY_CAMPAIGNS_RETENTION) if not isinstance(capability, CampaignRetentionProvider): raise PrivacyPolicyError("Campaign retention capability is invalid") return capability def _access_administration() -> AccessAdministration | None: registry = get_registry() if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_ACCESS_ADMINISTRATION): return None capability = registry.require_capability(CAPABILITY_ACCESS_ADMINISTRATION) if not isinstance(capability, AccessAdministration): raise PrivacyPolicyError("Access administration capability is invalid") return capability def _audit_retention_provider() -> AuditRetentionProvider | None: registry = get_registry() if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_AUDIT_RETENTION): return None capability = registry.require_capability(CAPABILITY_AUDIT_RETENTION) if not isinstance(capability, AuditRetentionProvider): raise PrivacyPolicyError("Audit retention capability is invalid") return capability def _user_settings(session: Session, *, user_id: str, tenant_id: str) -> dict[str, Any] | None: provider = _access_administration() if provider is None: return None payload = provider.user_settings(session, user_id, tenant_id=tenant_id) return dict(payload) if payload is not None else None def _set_user_settings(session: Session, *, user_id: str, tenant_id: str, settings_payload: dict[str, Any]) -> dict[str, Any] | None: provider = _access_administration() if provider is None: return None payload = provider.set_user_settings(session, user_id, tenant_id=tenant_id, settings=settings_payload) return dict(payload) if payload is not None else None def _group_settings(session: Session, *, group_id: str, tenant_id: str) -> dict[str, Any] | None: provider = _access_administration() if provider is None: return None payload = provider.group_settings(session, group_id, tenant_id=tenant_id) return dict(payload) if payload is not None else None def _set_group_settings(session: Session, *, group_id: str, tenant_id: str, settings_payload: dict[str, Any]) -> dict[str, Any] | None: provider = _access_administration() if provider is None: return None payload = provider.set_group_settings(session, group_id, tenant_id=tenant_id, settings=settings_payload) return dict(payload) if payload is not None else None def _campaign_policy_context(session: Session, *, campaign_id: str, tenant_id: str | None = None) -> CampaignPolicyContext: provider = _campaign_policy_provider() if provider is None: raise PrivacyPolicyError("Campaign module is not installed") context = provider.get_campaign_policy_context(session, tenant_id=tenant_id, campaign_id=campaign_id) if context is None: raise PrivacyPolicyError("Campaign not found for privacy policy") return context def _utcnow() -> datetime: return datetime.now(timezone.utc) def _cutoff(days: int | None, *, now: datetime) -> datetime | None: if days is None: return None return now - timedelta(days=days) def _privacy_policy_data(settings_payload: dict[str, Any] | None) -> dict[str, Any]: if not isinstance(settings_payload, dict): return {} data = settings_payload.get(PRIVACY_POLICY_SETTINGS_KEY, {}) return data if isinstance(data, dict) else {} def _privacy_policy_patch_from_settings(settings_payload: dict[str, Any] | None) -> dict[str, Any]: data = _privacy_policy_data(settings_payload) if not data: return {} return PrivacyRetentionPolicyPatch.model_validate(data).model_dump(mode="json", exclude_none=True) def _merge_privacy_policy(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> PrivacyRetentionPolicy: payload = parent.model_dump(mode="json") parent_allow = {**default_allow_lower_level_limits(), **(payload.get("allow_lower_level_limits") or {})} if parent_allow["store_raw_campaign_json"] and patch.get("store_raw_campaign_json") is False: payload["store_raw_campaign_json"] = False for key in RETENTION_DAY_KEYS: value = patch.get(key) if value is None or not parent_allow[key]: continue current = payload.get(key) payload[key] = int(value) if current is None else min(int(current), int(value)) detail_level = patch.get("audit_detail_level") if parent_allow["audit_detail_level"] and detail_level and AUDIT_DETAIL_LEVEL_ORDER[detail_level] > AUDIT_DETAIL_LEVEL_ORDER[payload["audit_detail_level"]]: payload["audit_detail_level"] = detail_level patch_allow = patch.get("allow_lower_level_limits") or {} payload["allow_lower_level_limits"] = { key: parent_allow[key] and bool(patch_allow.get(key, parent_allow[key])) for key in RETENTION_POLICY_FIELD_KEYS } return PrivacyRetentionPolicy.model_validate(payload) def _parent_allow_lower_level_limits(parent_payload: dict[str, Any]) -> dict[str, bool]: return {**default_allow_lower_level_limits(), **(parent_payload.get("allow_lower_level_limits") or {})} def _privacy_restriction_rules() -> tuple[PolicyRestrictionRule, ...]: return ( PolicyRestrictionRule( field="store_raw_campaign_json", is_more_restrictive_or_equal=lambda parent_value, requested_value: not (requested_value is True and parent_value is False), less_restrictive_message="Raw campaign JSON storage cannot be re-enabled below a parent policy that disables it.", ), *( PolicyRestrictionRule( field=key, is_more_restrictive_or_equal=lambda parent_value, requested_value: parent_value is None or int(requested_value) <= int(parent_value), less_restrictive_message=f"{key} cannot be less restrictive than the parent retention policy.", ) for key in RETENTION_DAY_KEYS ), PolicyRestrictionRule( field="audit_detail_level", is_more_restrictive_or_equal=lambda parent_value, requested_value: AUDIT_DETAIL_LEVEL_ORDER[str(requested_value)] >= AUDIT_DETAIL_LEVEL_ORDER[str(parent_value or "full")], less_restrictive_message="Audit detail level cannot be less restrictive than the parent retention policy.", ), ) def _validate_privacy_patch_against_parent(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> None: parent_payload = parent.model_dump(mode="json") issues = validate_hierarchical_policy_patch( parent_policy=parent_payload, patch=patch, field_keys=RETENTION_POLICY_FIELD_KEYS, parent_allow_lower_level_limits=_parent_allow_lower_level_limits(parent_payload), restriction_rules=_privacy_restriction_rules(), locked_field_message=lambda key: f"{key} is locked by the parent retention policy.", relock_message=lambda key: f"{key} limiting cannot be re-enabled below a parent retention policy lock.", ) if issues: raise PrivacyPolicyError(issues[0].message) def _set_settings_privacy_policy(settings_payload: dict[str, Any] | None, policy: dict[str, Any]) -> dict[str, Any]: payload = dict(settings_payload or {}) payload[PRIVACY_POLICY_SETTINGS_KEY] = policy return payload def privacy_policy_from_settings(item: SystemSettings) -> PrivacyRetentionPolicy: return PrivacyRetentionPolicy.model_validate(_privacy_policy_data(item.settings or {})) def privacy_policy_from_session(session: Session) -> PrivacyRetentionPolicy: return privacy_policy_from_settings(get_system_settings(session)) def set_privacy_policy(item: SystemSettings, policy: PrivacyRetentionPolicy | dict[str, Any]) -> PrivacyRetentionPolicy: validated = policy if isinstance(policy, PrivacyRetentionPolicy) else PrivacyRetentionPolicy.model_validate(policy) item.settings = _set_settings_privacy_policy(item.settings, validated.model_dump(mode="json")) return validated def effective_privacy_policy( session: Session, *, tenant_id: str | None = None, owner_user_id: str | None = None, owner_group_id: str | None = None, campaign_id: str | None = None, ) -> PrivacyRetentionPolicy: policy = privacy_policy_from_session(session) campaign: CampaignPolicyContext | None = None if campaign_id: campaign = _campaign_policy_context(session, campaign_id=campaign_id) tenant_id = campaign.tenant_id owner_user_id = campaign.owner_user_id owner_group_id = campaign.owner_group_id if tenant_id: tenant = session.get(Tenant, tenant_id) if tenant is None: raise PrivacyPolicyError("Tenant not found for privacy policy") policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {})) if owner_user_id: user_settings = _user_settings(session, user_id=owner_user_id, tenant_id=tenant_id) if tenant_id else None if user_settings is not None: policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user_settings)) if owner_group_id: group_settings = _group_settings(session, group_id=owner_group_id, tenant_id=tenant_id) if tenant_id else None if group_settings is not None: policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group_settings)) if campaign is not None: policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(dict(campaign.settings or {}))) return policy def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> PrivacyRetentionPolicy: clean_scope = scope_type.strip().casefold() policy = privacy_policy_from_session(session) if clean_scope == "tenant": return policy tenant = session.get(Tenant, tenant_id) if tenant is None: raise PrivacyPolicyError("Tenant not found for privacy policy") policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {})) if clean_scope in {"user", "group"}: return policy if clean_scope != "campaign" or not scope_id: return policy campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) if campaign.owner_user_id: user_settings = _user_settings(session, user_id=campaign.owner_user_id, tenant_id=tenant_id) if user_settings is not None: policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user_settings)) if campaign.owner_group_id: group_settings = _group_settings(session, group_id=campaign.owner_group_id, tenant_id=tenant_id) if group_settings is not None: policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group_settings)) return policy def _retention_policy_source_fields(patch: dict[str, Any], *, baseline: bool = False) -> list[str]: fields: list[str] = [] for key in RETENTION_POLICY_FIELD_KEYS: if key in patch and patch.get(key) is not None: fields.append(key) if isinstance(patch.get("allow_lower_level_limits"), dict) and patch["allow_lower_level_limits"]: fields.append("allow_lower_level_limits") if baseline and not fields: fields.append("defaults") return fields def _retention_policy_source_step(scope_type: str, label: str, scope_id: str | None, patch: dict[str, Any], *, baseline: bool = False) -> dict[str, Any]: source_policy = PrivacyRetentionPolicy.model_validate(patch).model_dump(mode="json") if baseline else dict(patch) return policy_source_step( scope_type, label, scope_id, applied_fields=_retention_policy_source_fields(patch, baseline=baseline), policy=source_policy, ) def effective_privacy_policy_sources( session: Session, *, tenant_id: str | None = None, owner_user_id: str | None = None, owner_group_id: str | None = None, campaign_id: str | None = None, ) -> list[dict[str, Any]]: system_settings = get_system_settings(session) sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)] campaign: CampaignPolicyContext | None = None if campaign_id: campaign = _campaign_policy_context(session, campaign_id=campaign_id) tenant_id = campaign.tenant_id owner_user_id = campaign.owner_user_id owner_group_id = campaign.owner_group_id if tenant_id: tenant = session.get(Tenant, tenant_id) if tenant is None: raise PrivacyPolicyError("Tenant not found for privacy policy") sources.append(_retention_policy_source_step("tenant", "Tenant", tenant.id, _privacy_policy_patch_from_settings(tenant.settings or {}))) if owner_user_id: user_settings = _user_settings(session, user_id=owner_user_id, tenant_id=tenant_id) if tenant_id else None if user_settings is not None: sources.append(_retention_policy_source_step("user", "Owner user", owner_user_id, _privacy_policy_patch_from_settings(user_settings))) if owner_group_id: group_settings = _group_settings(session, group_id=owner_group_id, tenant_id=tenant_id) if tenant_id else None if group_settings is not None: sources.append(_retention_policy_source_step("group", "Owner group", owner_group_id, _privacy_policy_patch_from_settings(group_settings))) if campaign is not None: sources.append(_retention_policy_source_step("campaign", "Campaign", campaign.id, _privacy_policy_patch_from_settings(dict(campaign.settings or {})))) return sources def parent_privacy_policy_sources(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> list[dict[str, Any]]: clean_scope = scope_type.strip().casefold() if clean_scope == "system": return [] system_settings = get_system_settings(session) sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)] if clean_scope == "tenant": return sources tenant = session.get(Tenant, tenant_id) if tenant is None: raise PrivacyPolicyError("Tenant not found for privacy policy") sources.append(_retention_policy_source_step("tenant", "Tenant", tenant.id, _privacy_policy_patch_from_settings(tenant.settings or {}))) if clean_scope in {"user", "group"}: return sources if clean_scope != "campaign" or not scope_id: return sources campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) if campaign.owner_user_id: user_settings = _user_settings(session, user_id=campaign.owner_user_id, tenant_id=tenant_id) if user_settings is not None: sources.append(_retention_policy_source_step("user", "Owner user", campaign.owner_user_id, _privacy_policy_patch_from_settings(user_settings))) if campaign.owner_group_id: group_settings = _group_settings(session, group_id=campaign.owner_group_id, tenant_id=tenant_id) if group_settings is not None: sources.append(_retention_policy_source_step("group", "Owner group", campaign.owner_group_id, _privacy_policy_patch_from_settings(group_settings))) return sources def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> dict[str, Any]: clean_scope = scope_type.strip().casefold() if clean_scope == "system": return privacy_policy_from_session(session).model_dump(mode="json") if clean_scope == "tenant": tenant = session.get(Tenant, scope_id or tenant_id) if tenant is None or tenant.id != tenant_id: raise PrivacyPolicyError("Tenant privacy policy not found") return _privacy_policy_patch_from_settings(tenant.settings or {}) if not scope_id: raise PrivacyPolicyError(f"{clean_scope.capitalize()} privacy policy requires scope_id") if clean_scope == "user": user_settings = _user_settings(session, user_id=scope_id, tenant_id=tenant_id) if user_settings is None: raise PrivacyPolicyError("User privacy policy not found") return _privacy_policy_patch_from_settings(user_settings) if clean_scope == "group": group_settings = _group_settings(session, group_id=scope_id, tenant_id=tenant_id) if group_settings is None: raise PrivacyPolicyError("Group privacy policy not found") return _privacy_policy_patch_from_settings(group_settings) if clean_scope == "campaign": campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) return _privacy_policy_patch_from_settings(dict(campaign.settings or {})) raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign") def set_privacy_policy_for_scope( session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None, policy: dict[str, Any] | None = None, ) -> dict[str, Any]: clean_scope = scope_type.strip().casefold() if clean_scope == "system": item = get_system_settings(session) validated = set_privacy_policy(item, PrivacyRetentionPolicy.model_validate(policy or {})) session.add(item) return validated.model_dump(mode="json") patch = PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(mode="json", exclude_none=True) _validate_privacy_patch_against_parent(parent_privacy_policy(session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id or (tenant_id if clean_scope == "tenant" else None)), patch) if clean_scope == "tenant": tenant = session.get(Tenant, scope_id or tenant_id) if tenant is None or tenant.id != tenant_id: raise PrivacyPolicyError("Tenant privacy policy not found") tenant.settings = _set_settings_privacy_policy(tenant.settings, patch) session.add(tenant) return patch if not scope_id: raise PrivacyPolicyError(f"{clean_scope.capitalize()} privacy policy requires scope_id") if clean_scope == "user": current_settings = _user_settings(session, user_id=scope_id, tenant_id=tenant_id) if current_settings is None: raise PrivacyPolicyError("User privacy policy not found") if _set_user_settings(session, user_id=scope_id, tenant_id=tenant_id, settings_payload=_set_settings_privacy_policy(current_settings, patch)) is None: raise PrivacyPolicyError("User privacy policy not found") return patch if clean_scope == "group": current_settings = _group_settings(session, group_id=scope_id, tenant_id=tenant_id) if current_settings is None: raise PrivacyPolicyError("Group privacy policy not found") if _set_group_settings(session, group_id=scope_id, tenant_id=tenant_id, settings_payload=_set_settings_privacy_policy(current_settings, patch)) is None: raise PrivacyPolicyError("Group privacy policy not found") return patch if clean_scope == "campaign": provider = _campaign_policy_provider() if provider is None: raise PrivacyPolicyError("Campaign module is not installed") campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) settings_payload = _set_settings_privacy_policy(dict(campaign.settings or {}), patch) try: provider.set_campaign_settings(session, tenant_id=tenant_id, campaign_id=scope_id, settings=settings_payload) except ValueError as exc: raise PrivacyPolicyError("Campaign privacy policy not found") from exc return patch raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign") def simulate_privacy_policy_change( session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None, policy: dict[str, Any] | None = None, ) -> dict[str, Any]: clean_scope = scope_type.strip().casefold() patch = PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(mode="json", exclude_none=True) current = get_privacy_policy_for_scope(session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id) if clean_scope == "system": parent_payload = PrivacyRetentionPolicy.model_validate(current).model_dump(mode="json") parent_allow = default_allow_lower_level_limits() parent_sources = effective_privacy_policy_sources(session) else: parent = parent_privacy_policy( session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id or (tenant_id if clean_scope == "tenant" else None), ) parent_payload = parent.model_dump(mode="json") parent_allow = _parent_allow_lower_level_limits(parent_payload) parent_sources = parent_privacy_policy_sources( session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id or (tenant_id if clean_scope == "tenant" else None), ) simulation = simulate_hierarchical_policy_change( parent_policy=parent_payload, current_policy=current, patch=patch, field_keys=RETENTION_POLICY_FIELD_KEYS, parent_allow_lower_level_limits=parent_allow, restriction_rules=() if clean_scope == "system" else _privacy_restriction_rules(), source_path=[PolicySourceStep.from_mapping(source) for source in parent_sources], locked_field_message=lambda key: f"{key} is locked by the parent retention policy.", relock_message=lambda key: f"{key} limiting cannot be re-enabled below a parent retention policy lock.", ) return simulation.to_dict() def sanitize_audit_details_for_policy(session: Session, details: dict[str, Any]) -> dict[str, Any]: policy = privacy_policy_from_session(session) if policy.audit_detail_level == "full": return details if policy.audit_detail_level == "minimal": return { "_privacy": {"detail_level": "minimal"}, "keys": sorted(str(key) for key in details.keys()), } return _redact_audit_value(details) def _redact_audit_value(value: Any) -> Any: if isinstance(value, dict): redacted: dict[str, Any] = {} for key, item in value.items(): if str(key).lower() in AUDIT_DETAIL_REDACTION_KEYS: redacted[key] = "" else: redacted[key] = _redact_audit_value(item) return redacted if isinstance(value, list): return [_redact_audit_value(item) for item in value] return value def _system_cutoffs(policy: PrivacyRetentionPolicy, *, now: datetime) -> dict[str, datetime | None]: return { "raw_campaign_json": _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now), "generated_eml": _cutoff(policy.generated_eml_retention_days, now=now), "stored_report_detail": _cutoff(policy.stored_report_detail_retention_days, now=now), "mock_mailbox": _cutoff(policy.mock_mailbox_retention_days, now=now), "audit_detail": _cutoff(policy.audit_detail_retention_days, now=now), } def apply_retention_policy(session: Session, *, dry_run: bool = True) -> dict[str, Any]: now = _utcnow() policy = privacy_policy_from_session(session) cutoffs = _system_cutoffs(policy, now=now) policy_cache: dict[str, PrivacyRetentionPolicy] = {} campaign_retention = _campaign_retention_provider() campaign_counts = { "raw_campaign_json": {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0}, "generated_eml": {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0}, "stored_report_detail": {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0}, } if campaign_retention is not None: campaign_counts.update( { key: dict(value) for key, value in campaign_retention.apply_retention( session, dry_run=dry_run, now=now, policy_for_campaign_id=lambda campaign_id: _campaign_policy_for_id(session, campaign_id, policy_cache), ).items() } ) counts = { "raw_campaign_json": campaign_counts["raw_campaign_json"], "generated_eml": campaign_counts["generated_eml"], "stored_report_detail": campaign_counts["stored_report_detail"], "mock_mailbox": _apply_mock_mailbox_retention(cutoffs["mock_mailbox"], dry_run=dry_run), "audit_detail": _apply_audit_detail_retention(session, dry_run=dry_run, now=now), } return { "dry_run": dry_run, "policy": policy.model_dump(mode="json"), "cutoffs": {key: value.isoformat() if value else None for key, value in cutoffs.items()}, "effective_policy_scope": "per-object", "counts": counts, } def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy: if not campaign_id: return privacy_policy_from_session(session) if campaign_id not in cache: cache[campaign_id] = effective_privacy_policy(session, campaign_id=campaign_id) return cache[campaign_id] def _apply_mock_mailbox_retention(cutoff: datetime | None, *, dry_run: bool) -> dict[str, int]: result = {"eligible_records": 0, "json_deleted": 0, "eml_deleted": 0} if cutoff is None: return result message_dir = Path(settings.mock_mailbox_dir) / "messages" if not message_dir.exists(): return result for path in message_dir.glob("*.json"): try: record = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError: continue created_at = _parse_datetime(record.get("created_at")) if created_at and created_at >= cutoff: continue result["eligible_records"] += 1 if dry_run: continue raw_filename = record.get("raw_filename") if raw_filename: raw_path = message_dir / str(raw_filename) if raw_path.exists(): raw_path.unlink() result["eml_deleted"] += 1 if path.exists(): path.unlink() result["json_deleted"] += 1 return result def _parse_datetime(value: Any) -> datetime | None: if not value: return None try: parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) except ValueError: return None if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed def _privacy_policy_for_audit_item(session: Session, item: AuditRecordRef, campaign_cache: dict[str, PrivacyRetentionPolicy], tenant_cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy: if item.object_type == "campaign" and item.object_id: provider = _campaign_policy_provider() if provider is not None: context = provider.get_campaign_policy_context(session, campaign_id=str(item.object_id)) if context is not None: return _campaign_policy_for_id(session, context.id, campaign_cache) if item.tenant_id: if item.tenant_id not in tenant_cache: tenant_cache[item.tenant_id] = effective_privacy_policy(session, tenant_id=item.tenant_id) return tenant_cache[item.tenant_id] return privacy_policy_from_session(session) def _apply_audit_detail_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]: provider = _audit_retention_provider() if provider is None: return {"eligible": 0, "redacted": 0, "already_redacted": 0} campaign_cache: dict[str, PrivacyRetentionPolicy] = {} tenant_cache: dict[str, PrivacyRetentionPolicy] = {} return dict(provider.apply_detail_retention( session, dry_run=dry_run, now=now, policy_for_record=lambda item: _privacy_policy_for_audit_item(session, item, campaign_cache, tenant_cache), )) class SqlPrivacyRetentionService: def privacy_policy_from_settings(self, *args: Any, **kwargs: Any) -> Any: return privacy_policy_from_settings(*args, **kwargs) def privacy_policy_from_session(self, *args: Any, **kwargs: Any) -> Any: return privacy_policy_from_session(*args, **kwargs) def set_privacy_policy(self, *args: Any, **kwargs: Any) -> Any: return set_privacy_policy(*args, **kwargs) def effective_privacy_policy(self, *args: Any, **kwargs: Any) -> Any: return effective_privacy_policy(*args, **kwargs) def parent_privacy_policy(self, *args: Any, **kwargs: Any) -> Any: return parent_privacy_policy(*args, **kwargs) def effective_privacy_policy_sources(self, *args: Any, **kwargs: Any) -> Any: return effective_privacy_policy_sources(*args, **kwargs) def parent_privacy_policy_sources(self, *args: Any, **kwargs: Any) -> Any: return parent_privacy_policy_sources(*args, **kwargs) def get_privacy_policy_for_scope(self, *args: Any, **kwargs: Any) -> Any: return get_privacy_policy_for_scope(*args, **kwargs) def set_privacy_policy_for_scope(self, *args: Any, **kwargs: Any) -> Any: return set_privacy_policy_for_scope(*args, **kwargs) def simulate_privacy_policy_change(self, *args: Any, **kwargs: Any) -> Any: return simulate_privacy_policy_change(*args, **kwargs) def sanitize_audit_details_for_policy(self, *args: Any, **kwargs: Any) -> Any: return sanitize_audit_details_for_policy(*args, **kwargs) def apply_retention_policy(self, *args: Any, **kwargs: Any) -> Any: return apply_retention_policy(*args, **kwargs)