669 lines
29 KiB
Python
669 lines
29 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Literal
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_core.admin.settings import get_system_settings
|
|
from govoplan_core.core.campaigns import (
|
|
CAPABILITY_CAMPAIGNS_POLICY_CONTEXT,
|
|
CAPABILITY_CAMPAIGNS_RETENTION,
|
|
CampaignPolicyContext,
|
|
CampaignPolicyContextProvider,
|
|
CampaignRetentionProvider,
|
|
)
|
|
from govoplan_core.core.runtime import get_registry
|
|
from govoplan_access.backend.db.models import Group, User
|
|
from govoplan_audit.backend.db.models import AuditLog
|
|
from govoplan_core.admin.models import SystemSettings
|
|
from govoplan_core.settings import settings
|
|
from govoplan_tenancy.backend.db.models import Tenant
|
|
|
|
PRIVACY_POLICY_SETTINGS_KEY = "privacy_retention_policy"
|
|
RETENTION_DAY_KEYS = (
|
|
"raw_campaign_json_retention_days",
|
|
"generated_eml_retention_days",
|
|
"stored_report_detail_retention_days",
|
|
"mock_mailbox_retention_days",
|
|
"audit_detail_retention_days",
|
|
)
|
|
RETENTION_POLICY_FIELD_KEYS = (
|
|
"store_raw_campaign_json",
|
|
*RETENTION_DAY_KEYS,
|
|
"audit_detail_level",
|
|
)
|
|
AUDIT_DETAIL_LEVEL_ORDER = {"full": 0, "redacted": 1, "minimal": 2}
|
|
|
|
|
|
def default_allow_lower_level_limits() -> dict[str, bool]:
|
|
return {key: True for key in RETENTION_POLICY_FIELD_KEYS}
|
|
|
|
|
|
def _normalize_allow_lower_level_limits(value: Any, *, fill_defaults: bool) -> dict[str, bool] | None:
|
|
if value in (None, ""):
|
|
return default_allow_lower_level_limits() if fill_defaults else None
|
|
if not isinstance(value, dict):
|
|
raise ValueError("allow_lower_level_limits must be an object")
|
|
normalized = default_allow_lower_level_limits() if fill_defaults else {}
|
|
for key, allowed in value.items():
|
|
clean_key = str(key)
|
|
if clean_key not in RETENTION_POLICY_FIELD_KEYS:
|
|
raise ValueError(f"Unknown retention policy field: {clean_key}")
|
|
normalized[clean_key] = bool(allowed)
|
|
return normalized
|
|
|
|
AUDIT_DETAIL_REDACTION_KEYS = {
|
|
"attachments",
|
|
"bcc",
|
|
"body",
|
|
"campaign_json",
|
|
"cc",
|
|
"content",
|
|
"entries",
|
|
"html",
|
|
"imap",
|
|
"message",
|
|
"password",
|
|
"raw_eml",
|
|
"raw_json",
|
|
"recipients",
|
|
"secret",
|
|
"smtp",
|
|
"template",
|
|
"text",
|
|
"token",
|
|
"to",
|
|
}
|
|
|
|
|
|
class PrivacyRetentionPolicy(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
store_raw_campaign_json: bool = True
|
|
raw_campaign_json_retention_days: int | None = Field(default=None, ge=0)
|
|
generated_eml_retention_days: int | None = Field(default=None, ge=0)
|
|
stored_report_detail_retention_days: int | None = Field(default=None, ge=0)
|
|
mock_mailbox_retention_days: int | None = Field(default=None, ge=0)
|
|
audit_detail_retention_days: int | None = Field(default=None, ge=0)
|
|
audit_detail_level: Literal["full", "redacted", "minimal"] = "full"
|
|
allow_lower_level_limits: dict[str, bool] = Field(default_factory=default_allow_lower_level_limits)
|
|
|
|
@field_validator(
|
|
"raw_campaign_json_retention_days",
|
|
"generated_eml_retention_days",
|
|
"stored_report_detail_retention_days",
|
|
"mock_mailbox_retention_days",
|
|
"audit_detail_retention_days",
|
|
mode="before",
|
|
)
|
|
@classmethod
|
|
def _blank_string_is_none(cls, value: Any) -> Any:
|
|
if value == "":
|
|
return None
|
|
return value
|
|
|
|
@field_validator("allow_lower_level_limits", mode="before")
|
|
@classmethod
|
|
def _normalize_allow_lower_level_limits(cls, value: Any) -> Any:
|
|
return _normalize_allow_lower_level_limits(value, fill_defaults=True)
|
|
|
|
|
|
class PrivacyRetentionPolicyPatch(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
store_raw_campaign_json: bool | None = None
|
|
raw_campaign_json_retention_days: int | None = Field(default=None, ge=0)
|
|
generated_eml_retention_days: int | None = Field(default=None, ge=0)
|
|
stored_report_detail_retention_days: int | None = Field(default=None, ge=0)
|
|
mock_mailbox_retention_days: int | None = Field(default=None, ge=0)
|
|
audit_detail_retention_days: int | None = Field(default=None, ge=0)
|
|
audit_detail_level: Literal["full", "redacted", "minimal"] | None = None
|
|
allow_lower_level_limits: dict[str, bool] | None = None
|
|
|
|
@field_validator(*RETENTION_DAY_KEYS, mode="before")
|
|
@classmethod
|
|
def _blank_string_is_none(cls, value: Any) -> Any:
|
|
if value == "":
|
|
return None
|
|
return value
|
|
|
|
@field_validator("allow_lower_level_limits", mode="before")
|
|
@classmethod
|
|
def _normalize_allow_lower_level_limits(cls, value: Any) -> Any:
|
|
return _normalize_allow_lower_level_limits(value, fill_defaults=False)
|
|
|
|
|
|
class PrivacyPolicyError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def _campaign_policy_provider() -> CampaignPolicyContextProvider | None:
|
|
registry = get_registry()
|
|
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT):
|
|
return None
|
|
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT)
|
|
if not isinstance(capability, CampaignPolicyContextProvider):
|
|
raise PrivacyPolicyError("Campaign policy context capability is invalid")
|
|
return capability
|
|
|
|
|
|
def _campaign_retention_provider() -> CampaignRetentionProvider | None:
|
|
registry = get_registry()
|
|
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_RETENTION):
|
|
return None
|
|
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_RETENTION)
|
|
if not isinstance(capability, CampaignRetentionProvider):
|
|
raise PrivacyPolicyError("Campaign retention capability is invalid")
|
|
return capability
|
|
|
|
|
|
def _campaign_policy_context(session: Session, *, campaign_id: str, tenant_id: str | None = None) -> CampaignPolicyContext:
|
|
provider = _campaign_policy_provider()
|
|
if provider is None:
|
|
raise PrivacyPolicyError("Campaign module is not installed")
|
|
context = provider.get_campaign_policy_context(session, tenant_id=tenant_id, campaign_id=campaign_id)
|
|
if context is None:
|
|
raise PrivacyPolicyError("Campaign not found for privacy policy")
|
|
return context
|
|
|
|
|
|
def _utcnow() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _cutoff(days: int | None, *, now: datetime) -> datetime | None:
|
|
if days is None:
|
|
return None
|
|
return now - timedelta(days=days)
|
|
|
|
|
|
def _json_sha256(value: Any) -> str:
|
|
payload = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8")
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def _privacy_policy_data(settings_payload: dict[str, Any] | None) -> dict[str, Any]:
|
|
if not isinstance(settings_payload, dict):
|
|
return {}
|
|
data = settings_payload.get(PRIVACY_POLICY_SETTINGS_KEY, {})
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
|
|
def _privacy_policy_patch_from_settings(settings_payload: dict[str, Any] | None) -> dict[str, Any]:
|
|
data = _privacy_policy_data(settings_payload)
|
|
if not data:
|
|
return {}
|
|
return PrivacyRetentionPolicyPatch.model_validate(data).model_dump(mode="json", exclude_none=True)
|
|
|
|
|
|
def _merge_privacy_policy(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> PrivacyRetentionPolicy:
|
|
payload = parent.model_dump(mode="json")
|
|
parent_allow = {**default_allow_lower_level_limits(), **(payload.get("allow_lower_level_limits") or {})}
|
|
if parent_allow["store_raw_campaign_json"] and patch.get("store_raw_campaign_json") is False:
|
|
payload["store_raw_campaign_json"] = False
|
|
for key in RETENTION_DAY_KEYS:
|
|
value = patch.get(key)
|
|
if value is None or not parent_allow[key]:
|
|
continue
|
|
current = payload.get(key)
|
|
payload[key] = int(value) if current is None else min(int(current), int(value))
|
|
detail_level = patch.get("audit_detail_level")
|
|
if parent_allow["audit_detail_level"] and detail_level and AUDIT_DETAIL_LEVEL_ORDER[detail_level] > AUDIT_DETAIL_LEVEL_ORDER[payload["audit_detail_level"]]:
|
|
payload["audit_detail_level"] = detail_level
|
|
|
|
patch_allow = patch.get("allow_lower_level_limits") or {}
|
|
payload["allow_lower_level_limits"] = {
|
|
key: parent_allow[key] and bool(patch_allow.get(key, parent_allow[key]))
|
|
for key in RETENTION_POLICY_FIELD_KEYS
|
|
}
|
|
return PrivacyRetentionPolicy.model_validate(payload)
|
|
|
|
|
|
def _validate_privacy_patch_against_parent(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> None:
|
|
parent_payload = parent.model_dump(mode="json")
|
|
parent_allow = {**default_allow_lower_level_limits(), **(parent_payload.get("allow_lower_level_limits") or {})}
|
|
for key in RETENTION_POLICY_FIELD_KEYS:
|
|
if key in patch and patch.get(key) is not None and not parent_allow[key]:
|
|
raise PrivacyPolicyError(f"{key} is locked by the parent retention policy.")
|
|
patch_allow = patch.get("allow_lower_level_limits") or {}
|
|
for key, allowed in patch_allow.items():
|
|
if allowed and not parent_allow[key]:
|
|
raise PrivacyPolicyError(f"{key} limiting cannot be re-enabled below a parent retention policy lock.")
|
|
if patch.get("store_raw_campaign_json") is True and not parent.store_raw_campaign_json:
|
|
raise PrivacyPolicyError("Raw campaign JSON storage cannot be re-enabled below a parent policy that disables it.")
|
|
for key in RETENTION_DAY_KEYS:
|
|
value = patch.get(key)
|
|
parent_value = parent_payload.get(key)
|
|
if value is not None and parent_value is not None and int(value) > int(parent_value):
|
|
raise PrivacyPolicyError(f"{key} cannot be less restrictive than the parent retention policy.")
|
|
detail_level = patch.get("audit_detail_level")
|
|
if detail_level and AUDIT_DETAIL_LEVEL_ORDER[detail_level] < AUDIT_DETAIL_LEVEL_ORDER[parent.audit_detail_level]:
|
|
raise PrivacyPolicyError("Audit detail level cannot be less restrictive than the parent retention policy.")
|
|
|
|
|
|
def _set_settings_privacy_policy(settings_payload: dict[str, Any] | None, policy: dict[str, Any]) -> dict[str, Any]:
|
|
payload = dict(settings_payload or {})
|
|
payload[PRIVACY_POLICY_SETTINGS_KEY] = policy
|
|
return payload
|
|
|
|
|
|
def privacy_policy_from_settings(item: SystemSettings) -> PrivacyRetentionPolicy:
|
|
return PrivacyRetentionPolicy.model_validate(_privacy_policy_data(item.settings or {}))
|
|
|
|
|
|
def privacy_policy_from_session(session: Session) -> PrivacyRetentionPolicy:
|
|
return privacy_policy_from_settings(get_system_settings(session))
|
|
|
|
|
|
def set_privacy_policy(item: SystemSettings, policy: PrivacyRetentionPolicy | dict[str, Any]) -> PrivacyRetentionPolicy:
|
|
validated = policy if isinstance(policy, PrivacyRetentionPolicy) else PrivacyRetentionPolicy.model_validate(policy)
|
|
item.settings = _set_settings_privacy_policy(item.settings, validated.model_dump(mode="json"))
|
|
return validated
|
|
|
|
|
|
def effective_privacy_policy(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str | None = None,
|
|
owner_user_id: str | None = None,
|
|
owner_group_id: str | None = None,
|
|
campaign_id: str | None = None,
|
|
) -> PrivacyRetentionPolicy:
|
|
policy = privacy_policy_from_session(session)
|
|
campaign: CampaignPolicyContext | None = None
|
|
if campaign_id:
|
|
campaign = _campaign_policy_context(session, campaign_id=campaign_id)
|
|
tenant_id = campaign.tenant_id
|
|
owner_user_id = campaign.owner_user_id
|
|
owner_group_id = campaign.owner_group_id
|
|
if tenant_id:
|
|
tenant = session.get(Tenant, tenant_id)
|
|
if tenant is None:
|
|
raise PrivacyPolicyError("Tenant not found for privacy policy")
|
|
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {}))
|
|
if owner_user_id:
|
|
user = session.get(User, owner_user_id)
|
|
if user and (not tenant_id or user.tenant_id == tenant_id):
|
|
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user.settings or {}))
|
|
if owner_group_id:
|
|
group = session.get(Group, owner_group_id)
|
|
if group and (not tenant_id or group.tenant_id == tenant_id):
|
|
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group.settings or {}))
|
|
if campaign is not None:
|
|
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(dict(campaign.settings or {})))
|
|
return policy
|
|
|
|
|
|
def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> PrivacyRetentionPolicy:
|
|
clean_scope = scope_type.strip().casefold()
|
|
policy = privacy_policy_from_session(session)
|
|
if clean_scope == "tenant":
|
|
return policy
|
|
tenant = session.get(Tenant, tenant_id)
|
|
if tenant is None:
|
|
raise PrivacyPolicyError("Tenant not found for privacy policy")
|
|
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {}))
|
|
if clean_scope in {"user", "group"}:
|
|
return policy
|
|
if clean_scope != "campaign" or not scope_id:
|
|
return policy
|
|
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
|
|
if campaign.owner_user_id:
|
|
user = session.get(User, campaign.owner_user_id)
|
|
if user and user.tenant_id == tenant_id:
|
|
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user.settings or {}))
|
|
if campaign.owner_group_id:
|
|
group = session.get(Group, campaign.owner_group_id)
|
|
if group and group.tenant_id == tenant_id:
|
|
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group.settings or {}))
|
|
return policy
|
|
|
|
|
|
|
|
def _retention_policy_source_fields(patch: dict[str, Any], *, baseline: bool = False) -> list[str]:
|
|
fields: list[str] = []
|
|
for key in RETENTION_POLICY_FIELD_KEYS:
|
|
if key in patch and patch.get(key) is not None:
|
|
fields.append(key)
|
|
if isinstance(patch.get("allow_lower_level_limits"), dict) and patch["allow_lower_level_limits"]:
|
|
fields.append("allow_lower_level_limits")
|
|
if baseline and not fields:
|
|
fields.append("defaults")
|
|
return fields
|
|
|
|
|
|
def _retention_policy_source_step(scope_type: str, label: str, scope_id: str | None, patch: dict[str, Any], *, baseline: bool = False) -> dict[str, Any]:
|
|
source_policy = PrivacyRetentionPolicy.model_validate(patch).model_dump(mode="json") if baseline else dict(patch)
|
|
return {
|
|
"scope_type": scope_type,
|
|
"scope_id": scope_id,
|
|
"label": label,
|
|
"applied_fields": _retention_policy_source_fields(patch, baseline=baseline),
|
|
"policy": source_policy,
|
|
}
|
|
|
|
|
|
def effective_privacy_policy_sources(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str | None = None,
|
|
owner_user_id: str | None = None,
|
|
owner_group_id: str | None = None,
|
|
campaign_id: str | None = None,
|
|
) -> list[dict[str, Any]]:
|
|
system_settings = get_system_settings(session)
|
|
sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)]
|
|
campaign: CampaignPolicyContext | None = None
|
|
if campaign_id:
|
|
campaign = _campaign_policy_context(session, campaign_id=campaign_id)
|
|
tenant_id = campaign.tenant_id
|
|
owner_user_id = campaign.owner_user_id
|
|
owner_group_id = campaign.owner_group_id
|
|
if tenant_id:
|
|
tenant = session.get(Tenant, tenant_id)
|
|
if tenant is None:
|
|
raise PrivacyPolicyError("Tenant not found for privacy policy")
|
|
sources.append(_retention_policy_source_step("tenant", "Tenant", tenant.id, _privacy_policy_patch_from_settings(tenant.settings or {})))
|
|
if owner_user_id:
|
|
user = session.get(User, owner_user_id)
|
|
if user and (not tenant_id or user.tenant_id == tenant_id):
|
|
sources.append(_retention_policy_source_step("user", "Owner user", user.id, _privacy_policy_patch_from_settings(user.settings or {})))
|
|
if owner_group_id:
|
|
group = session.get(Group, owner_group_id)
|
|
if group and (not tenant_id or group.tenant_id == tenant_id):
|
|
sources.append(_retention_policy_source_step("group", "Owner group", group.id, _privacy_policy_patch_from_settings(group.settings or {})))
|
|
if campaign is not None:
|
|
sources.append(_retention_policy_source_step("campaign", "Campaign", campaign.id, _privacy_policy_patch_from_settings(dict(campaign.settings or {}))))
|
|
return sources
|
|
|
|
|
|
def parent_privacy_policy_sources(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> list[dict[str, Any]]:
|
|
clean_scope = scope_type.strip().casefold()
|
|
if clean_scope == "system":
|
|
return []
|
|
system_settings = get_system_settings(session)
|
|
sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)]
|
|
if clean_scope == "tenant":
|
|
return sources
|
|
tenant = session.get(Tenant, tenant_id)
|
|
if tenant is None:
|
|
raise PrivacyPolicyError("Tenant not found for privacy policy")
|
|
sources.append(_retention_policy_source_step("tenant", "Tenant", tenant.id, _privacy_policy_patch_from_settings(tenant.settings or {})))
|
|
if clean_scope in {"user", "group"}:
|
|
return sources
|
|
if clean_scope != "campaign" or not scope_id:
|
|
return sources
|
|
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
|
|
if campaign.owner_user_id:
|
|
user = session.get(User, campaign.owner_user_id)
|
|
if user and user.tenant_id == tenant_id:
|
|
sources.append(_retention_policy_source_step("user", "Owner user", user.id, _privacy_policy_patch_from_settings(user.settings or {})))
|
|
if campaign.owner_group_id:
|
|
group = session.get(Group, campaign.owner_group_id)
|
|
if group and group.tenant_id == tenant_id:
|
|
sources.append(_retention_policy_source_step("group", "Owner group", group.id, _privacy_policy_patch_from_settings(group.settings or {})))
|
|
return sources
|
|
|
|
def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> dict[str, Any]:
|
|
clean_scope = scope_type.strip().casefold()
|
|
if clean_scope == "system":
|
|
return privacy_policy_from_session(session).model_dump(mode="json")
|
|
if clean_scope == "tenant":
|
|
tenant = session.get(Tenant, scope_id or tenant_id)
|
|
if tenant is None or tenant.id != tenant_id:
|
|
raise PrivacyPolicyError("Tenant privacy policy not found")
|
|
return _privacy_policy_patch_from_settings(tenant.settings or {})
|
|
if not scope_id:
|
|
raise PrivacyPolicyError(f"{clean_scope.capitalize()} privacy policy requires scope_id")
|
|
if clean_scope == "user":
|
|
user = session.get(User, scope_id)
|
|
if user is None or user.tenant_id != tenant_id:
|
|
raise PrivacyPolicyError("User privacy policy not found")
|
|
return _privacy_policy_patch_from_settings(user.settings or {})
|
|
if clean_scope == "group":
|
|
group = session.get(Group, scope_id)
|
|
if group is None or group.tenant_id != tenant_id:
|
|
raise PrivacyPolicyError("Group privacy policy not found")
|
|
return _privacy_policy_patch_from_settings(group.settings or {})
|
|
if clean_scope == "campaign":
|
|
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
|
|
return _privacy_policy_patch_from_settings(dict(campaign.settings or {}))
|
|
raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign")
|
|
|
|
|
|
def set_privacy_policy_for_scope(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
scope_type: str,
|
|
scope_id: str | None = None,
|
|
policy: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
clean_scope = scope_type.strip().casefold()
|
|
if clean_scope == "system":
|
|
item = get_system_settings(session)
|
|
validated = set_privacy_policy(item, PrivacyRetentionPolicy.model_validate(policy or {}))
|
|
session.add(item)
|
|
return validated.model_dump(mode="json")
|
|
patch = PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(mode="json", exclude_none=True)
|
|
_validate_privacy_patch_against_parent(parent_privacy_policy(session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id or (tenant_id if clean_scope == "tenant" else None)), patch)
|
|
if clean_scope == "tenant":
|
|
tenant = session.get(Tenant, scope_id or tenant_id)
|
|
if tenant is None or tenant.id != tenant_id:
|
|
raise PrivacyPolicyError("Tenant privacy policy not found")
|
|
tenant.settings = _set_settings_privacy_policy(tenant.settings, patch)
|
|
session.add(tenant)
|
|
return patch
|
|
if not scope_id:
|
|
raise PrivacyPolicyError(f"{clean_scope.capitalize()} privacy policy requires scope_id")
|
|
if clean_scope == "user":
|
|
user = session.get(User, scope_id)
|
|
if user is None or user.tenant_id != tenant_id:
|
|
raise PrivacyPolicyError("User privacy policy not found")
|
|
user.settings = _set_settings_privacy_policy(user.settings, patch)
|
|
session.add(user)
|
|
return patch
|
|
if clean_scope == "group":
|
|
group = session.get(Group, scope_id)
|
|
if group is None or group.tenant_id != tenant_id:
|
|
raise PrivacyPolicyError("Group privacy policy not found")
|
|
group.settings = _set_settings_privacy_policy(group.settings, patch)
|
|
session.add(group)
|
|
return patch
|
|
if clean_scope == "campaign":
|
|
provider = _campaign_policy_provider()
|
|
if provider is None:
|
|
raise PrivacyPolicyError("Campaign module is not installed")
|
|
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
|
|
settings_payload = _set_settings_privacy_policy(dict(campaign.settings or {}), patch)
|
|
try:
|
|
provider.set_campaign_settings(session, tenant_id=tenant_id, campaign_id=scope_id, settings=settings_payload)
|
|
except ValueError as exc:
|
|
raise PrivacyPolicyError("Campaign privacy policy not found") from exc
|
|
return patch
|
|
raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign")
|
|
|
|
|
|
def sanitize_audit_details_for_policy(session: Session, details: dict[str, Any]) -> dict[str, Any]:
|
|
policy = privacy_policy_from_session(session)
|
|
if policy.audit_detail_level == "full":
|
|
return details
|
|
if policy.audit_detail_level == "minimal":
|
|
return {
|
|
"_privacy": {"detail_level": "minimal"},
|
|
"keys": sorted(str(key) for key in details.keys()),
|
|
}
|
|
return _redact_audit_value(details)
|
|
|
|
|
|
def _redact_audit_value(value: Any) -> Any:
|
|
if isinstance(value, dict):
|
|
redacted: dict[str, Any] = {}
|
|
for key, item in value.items():
|
|
if str(key).lower() in AUDIT_DETAIL_REDACTION_KEYS:
|
|
redacted[key] = "<redacted>"
|
|
else:
|
|
redacted[key] = _redact_audit_value(item)
|
|
return redacted
|
|
if isinstance(value, list):
|
|
return [_redact_audit_value(item) for item in value]
|
|
return value
|
|
|
|
|
|
def _is_before_cutoff(value: datetime | None, cutoff: datetime | None) -> bool:
|
|
if value is None or cutoff is None:
|
|
return False
|
|
candidate = value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value
|
|
return candidate < cutoff
|
|
|
|
|
|
def _system_cutoffs(policy: PrivacyRetentionPolicy, *, now: datetime) -> dict[str, datetime | None]:
|
|
return {
|
|
"raw_campaign_json": _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now),
|
|
"generated_eml": _cutoff(policy.generated_eml_retention_days, now=now),
|
|
"stored_report_detail": _cutoff(policy.stored_report_detail_retention_days, now=now),
|
|
"mock_mailbox": _cutoff(policy.mock_mailbox_retention_days, now=now),
|
|
"audit_detail": _cutoff(policy.audit_detail_retention_days, now=now),
|
|
}
|
|
|
|
|
|
def apply_retention_policy(session: Session, *, dry_run: bool = True) -> dict[str, Any]:
|
|
now = _utcnow()
|
|
policy = privacy_policy_from_session(session)
|
|
cutoffs = _system_cutoffs(policy, now=now)
|
|
policy_cache: dict[str, PrivacyRetentionPolicy] = {}
|
|
campaign_retention = _campaign_retention_provider()
|
|
campaign_counts = {
|
|
"raw_campaign_json": {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0},
|
|
"generated_eml": {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0},
|
|
"stored_report_detail": {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0},
|
|
}
|
|
if campaign_retention is not None:
|
|
campaign_counts.update(
|
|
{
|
|
key: dict(value)
|
|
for key, value in campaign_retention.apply_retention(
|
|
session,
|
|
dry_run=dry_run,
|
|
now=now,
|
|
policy_for_campaign_id=lambda campaign_id: _campaign_policy_for_id(session, campaign_id, policy_cache),
|
|
).items()
|
|
}
|
|
)
|
|
counts = {
|
|
"raw_campaign_json": campaign_counts["raw_campaign_json"],
|
|
"generated_eml": campaign_counts["generated_eml"],
|
|
"stored_report_detail": campaign_counts["stored_report_detail"],
|
|
"mock_mailbox": _apply_mock_mailbox_retention(cutoffs["mock_mailbox"], dry_run=dry_run),
|
|
"audit_detail": _apply_audit_detail_retention(session, dry_run=dry_run, now=now),
|
|
}
|
|
return {
|
|
"dry_run": dry_run,
|
|
"policy": policy.model_dump(mode="json"),
|
|
"cutoffs": {key: value.isoformat() if value else None for key, value in cutoffs.items()},
|
|
"effective_policy_scope": "per-object",
|
|
"counts": counts,
|
|
}
|
|
|
|
|
|
def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy:
|
|
if not campaign_id:
|
|
return privacy_policy_from_session(session)
|
|
if campaign_id not in cache:
|
|
cache[campaign_id] = effective_privacy_policy(session, campaign_id=campaign_id)
|
|
return cache[campaign_id]
|
|
|
|
|
|
def _apply_mock_mailbox_retention(cutoff: datetime | None, *, dry_run: bool) -> dict[str, int]:
|
|
result = {"eligible_records": 0, "json_deleted": 0, "eml_deleted": 0}
|
|
if cutoff is None:
|
|
return result
|
|
message_dir = Path(settings.mock_mailbox_dir) / "messages"
|
|
if not message_dir.exists():
|
|
return result
|
|
for path in message_dir.glob("*.json"):
|
|
try:
|
|
record = json.loads(path.read_text(encoding="utf-8"))
|
|
except json.JSONDecodeError:
|
|
continue
|
|
created_at = _parse_datetime(record.get("created_at"))
|
|
if created_at and created_at >= cutoff:
|
|
continue
|
|
result["eligible_records"] += 1
|
|
if dry_run:
|
|
continue
|
|
raw_filename = record.get("raw_filename")
|
|
if raw_filename:
|
|
raw_path = message_dir / str(raw_filename)
|
|
if raw_path.exists():
|
|
raw_path.unlink()
|
|
result["eml_deleted"] += 1
|
|
if path.exists():
|
|
path.unlink()
|
|
result["json_deleted"] += 1
|
|
return result
|
|
|
|
|
|
def _parse_datetime(value: Any) -> datetime | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
return parsed
|
|
|
|
|
|
def _privacy_policy_for_audit_item(session: Session, item: AuditLog, campaign_cache: dict[str, PrivacyRetentionPolicy], tenant_cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy:
|
|
if item.object_type == "campaign" and item.object_id:
|
|
provider = _campaign_policy_provider()
|
|
if provider is not None:
|
|
context = provider.get_campaign_policy_context(session, campaign_id=str(item.object_id))
|
|
if context is not None:
|
|
return _campaign_policy_for_id(session, context.id, campaign_cache)
|
|
if item.tenant_id:
|
|
if item.tenant_id not in tenant_cache:
|
|
tenant_cache[item.tenant_id] = effective_privacy_policy(session, tenant_id=item.tenant_id)
|
|
return tenant_cache[item.tenant_id]
|
|
return privacy_policy_from_session(session)
|
|
|
|
|
|
def _apply_audit_detail_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]:
|
|
result = {"eligible": 0, "redacted": 0, "already_redacted": 0}
|
|
campaign_cache: dict[str, PrivacyRetentionPolicy] = {}
|
|
tenant_cache: dict[str, PrivacyRetentionPolicy] = {}
|
|
items = (
|
|
session.query(AuditLog)
|
|
.order_by(AuditLog.created_at.asc())
|
|
.all()
|
|
)
|
|
for item in items:
|
|
policy = _privacy_policy_for_audit_item(session, item, campaign_cache, tenant_cache)
|
|
cutoff = _cutoff(policy.audit_detail_retention_days, now=now)
|
|
if not _is_before_cutoff(item.created_at, cutoff):
|
|
continue
|
|
details = item.details if isinstance(item.details, dict) else {}
|
|
if details.get("_retention", {}).get("audit_detail_redacted"):
|
|
result["already_redacted"] += 1
|
|
continue
|
|
result["eligible"] += 1
|
|
if dry_run:
|
|
continue
|
|
item.details = {
|
|
"_retention": {
|
|
"audit_detail_redacted": True,
|
|
"redacted_at": now.isoformat(),
|
|
"original_detail_sha256": _json_sha256(details),
|
|
}
|
|
}
|
|
session.add(item)
|
|
result["redacted"] += 1
|
|
return result
|