Prepare GovOPlaN self-hosted release workflow
This commit is contained in:
@@ -1,29 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
"""Compatibility facade for privacy retention policy.
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from sqlalchemy.orm import Session
|
||||
Policy-owned behavior is provided by ``govoplan-policy`` through the
|
||||
``policy.privacyRetention`` capability. This module keeps legacy imports stable
|
||||
without importing policy implementation code from core.
|
||||
"""
|
||||
|
||||
from govoplan_core.admin.settings import get_system_settings
|
||||
from govoplan_core.core.campaigns import (
|
||||
CAPABILITY_CAMPAIGNS_POLICY_CONTEXT,
|
||||
CAPABILITY_CAMPAIGNS_RETENTION,
|
||||
CampaignPolicyContext,
|
||||
CampaignPolicyContextProvider,
|
||||
CampaignRetentionProvider,
|
||||
)
|
||||
from govoplan_core.core.policy import policy_source_step
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping
|
||||
|
||||
from govoplan_core.core.policy import CAPABILITY_POLICY_PRIVACY_RETENTION, PrivacyRetentionService
|
||||
from govoplan_core.core.runtime import get_registry
|
||||
from govoplan_access.backend.db.models import Group, User
|
||||
from govoplan_audit.backend.db.models import AuditLog
|
||||
from govoplan_core.admin.models import SystemSettings
|
||||
from govoplan_core.settings import settings
|
||||
from govoplan_core.tenancy.scope import Tenant
|
||||
|
||||
|
||||
PRIVACY_POLICY_SETTINGS_KEY = "privacy_retention_policy"
|
||||
RETENTION_DAY_KEYS = (
|
||||
@@ -38,17 +27,109 @@ RETENTION_POLICY_FIELD_KEYS = (
|
||||
*RETENTION_DAY_KEYS,
|
||||
"audit_detail_level",
|
||||
)
|
||||
AUDIT_DETAIL_LEVEL_ORDER = {"full": 0, "redacted": 1, "minimal": 2}
|
||||
|
||||
_DEFAULT_POLICY = {
|
||||
"store_raw_campaign_json": True,
|
||||
"raw_campaign_json_retention_days": None,
|
||||
"generated_eml_retention_days": None,
|
||||
"stored_report_detail_retention_days": None,
|
||||
"mock_mailbox_retention_days": None,
|
||||
"audit_detail_retention_days": None,
|
||||
"audit_detail_level": "full",
|
||||
}
|
||||
|
||||
_EXPORTS = (
|
||||
"PRIVACY_POLICY_SETTINGS_KEY",
|
||||
"RETENTION_DAY_KEYS",
|
||||
"RETENTION_POLICY_FIELD_KEYS",
|
||||
"PrivacyRetentionPolicy",
|
||||
"PrivacyRetentionPolicyPatch",
|
||||
"PrivacyPolicyError",
|
||||
"default_allow_lower_level_limits",
|
||||
"privacy_policy_from_settings",
|
||||
"privacy_policy_from_session",
|
||||
"set_privacy_policy",
|
||||
"effective_privacy_policy",
|
||||
"parent_privacy_policy",
|
||||
"effective_privacy_policy_sources",
|
||||
"parent_privacy_policy_sources",
|
||||
"get_privacy_policy_for_scope",
|
||||
"set_privacy_policy_for_scope",
|
||||
"sanitize_audit_details_for_policy",
|
||||
"apply_retention_policy",
|
||||
)
|
||||
|
||||
|
||||
def default_allow_lower_level_limits() -> dict[str, bool]:
|
||||
return {key: True for key in RETENTION_POLICY_FIELD_KEYS}
|
||||
class PrivacyPolicyError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class _UnavailablePolicyModel:
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
del args, kwargs
|
||||
_raise_unavailable()
|
||||
|
||||
@classmethod
|
||||
def model_validate(cls, value: Any) -> Any:
|
||||
del value
|
||||
_raise_unavailable()
|
||||
|
||||
|
||||
PrivacyRetentionPolicy = _UnavailablePolicyModel
|
||||
PrivacyRetentionPolicyPatch = _UnavailablePolicyModel
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SystemPrivacyPolicy:
|
||||
payload: Mapping[str, Any]
|
||||
|
||||
def model_dump(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
|
||||
del args, kwargs
|
||||
return dict(self.payload)
|
||||
|
||||
|
||||
def _raise_unavailable() -> None:
|
||||
raise PrivacyPolicyError("Privacy retention policy requires the active govoplan-policy module.")
|
||||
|
||||
|
||||
def _runtime_service() -> PrivacyRetentionService | None:
|
||||
registry = get_registry()
|
||||
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_POLICY_PRIVACY_RETENTION):
|
||||
return None
|
||||
capability = registry.require_capability(CAPABILITY_POLICY_PRIVACY_RETENTION)
|
||||
if not isinstance(capability, PrivacyRetentionService):
|
||||
raise PrivacyPolicyError("Privacy retention policy capability is invalid")
|
||||
return capability
|
||||
|
||||
|
||||
def _service() -> PrivacyRetentionService:
|
||||
service = _runtime_service()
|
||||
if service is None:
|
||||
_raise_unavailable()
|
||||
return service
|
||||
|
||||
|
||||
def _settings_payload(item: object) -> dict[str, Any]:
|
||||
settings = getattr(item, "settings", None)
|
||||
return dict(settings or {}) if isinstance(settings, Mapping) else {}
|
||||
|
||||
|
||||
def _policy_data(settings_payload: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||
raw = settings_payload.get(PRIVACY_POLICY_SETTINGS_KEY) if isinstance(settings_payload, Mapping) else None
|
||||
if not isinstance(raw, Mapping):
|
||||
raw = {}
|
||||
payload = dict(_DEFAULT_POLICY)
|
||||
for key in RETENTION_POLICY_FIELD_KEYS:
|
||||
if key in raw:
|
||||
payload[key] = raw[key]
|
||||
payload["allow_lower_level_limits"] = _normalize_allow_lower_level_limits(raw.get("allow_lower_level_limits"), fill_defaults=True)
|
||||
return payload
|
||||
|
||||
|
||||
def _normalize_allow_lower_level_limits(value: Any, *, fill_defaults: bool) -> dict[str, bool] | None:
|
||||
if value in (None, ""):
|
||||
return default_allow_lower_level_limits() if fill_defaults else None
|
||||
if not isinstance(value, dict):
|
||||
if not isinstance(value, Mapping):
|
||||
raise ValueError("allow_lower_level_limits must be an object")
|
||||
normalized = default_allow_lower_level_limits() if fill_defaults else {}
|
||||
for key, allowed in value.items():
|
||||
@@ -58,612 +139,67 @@ def _normalize_allow_lower_level_limits(value: Any, *, fill_defaults: bool) -> d
|
||||
normalized[clean_key] = bool(allowed)
|
||||
return normalized
|
||||
|
||||
AUDIT_DETAIL_REDACTION_KEYS = {
|
||||
"attachments",
|
||||
"bcc",
|
||||
"body",
|
||||
"campaign_json",
|
||||
"cc",
|
||||
"content",
|
||||
"entries",
|
||||
"html",
|
||||
"imap",
|
||||
"message",
|
||||
"password",
|
||||
"raw_eml",
|
||||
"raw_json",
|
||||
"recipients",
|
||||
"secret",
|
||||
"smtp",
|
||||
"template",
|
||||
"text",
|
||||
"token",
|
||||
"to",
|
||||
}
|
||||
|
||||
def default_allow_lower_level_limits() -> dict[str, bool]:
|
||||
return {key: True for key in RETENTION_POLICY_FIELD_KEYS}
|
||||
|
||||
class PrivacyRetentionPolicy(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
store_raw_campaign_json: bool = True
|
||||
raw_campaign_json_retention_days: int | None = Field(default=None, ge=0)
|
||||
generated_eml_retention_days: int | None = Field(default=None, ge=0)
|
||||
stored_report_detail_retention_days: int | None = Field(default=None, ge=0)
|
||||
mock_mailbox_retention_days: int | None = Field(default=None, ge=0)
|
||||
audit_detail_retention_days: int | None = Field(default=None, ge=0)
|
||||
audit_detail_level: Literal["full", "redacted", "minimal"] = "full"
|
||||
allow_lower_level_limits: dict[str, bool] = Field(default_factory=default_allow_lower_level_limits)
|
||||
def privacy_policy_from_settings(item: object) -> Any:
|
||||
service = _runtime_service()
|
||||
if service is not None:
|
||||
return service.privacy_policy_from_settings(item)
|
||||
return _SystemPrivacyPolicy(_policy_data(_settings_payload(item)))
|
||||
|
||||
@field_validator(
|
||||
"raw_campaign_json_retention_days",
|
||||
"generated_eml_retention_days",
|
||||
"stored_report_detail_retention_days",
|
||||
"mock_mailbox_retention_days",
|
||||
"audit_detail_retention_days",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
def _blank_string_is_none(cls, value: Any) -> Any:
|
||||
if value == "":
|
||||
return None
|
||||
return value
|
||||
|
||||
@field_validator("allow_lower_level_limits", mode="before")
|
||||
@classmethod
|
||||
def _normalize_allow_lower_level_limits(cls, value: Any) -> Any:
|
||||
return _normalize_allow_lower_level_limits(value, fill_defaults=True)
|
||||
def privacy_policy_from_session(*args: Any, **kwargs: Any) -> Any:
|
||||
return _service().privacy_policy_from_session(*args, **kwargs)
|
||||
|
||||
|
||||
class PrivacyRetentionPolicyPatch(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
def set_privacy_policy(item: object, policy: Any) -> Any:
|
||||
service = _runtime_service()
|
||||
if service is not None:
|
||||
return service.set_privacy_policy(item, policy)
|
||||
value = policy.model_dump(mode="json") if hasattr(policy, "model_dump") else dict(policy or {})
|
||||
next_settings = _settings_payload(item)
|
||||
next_settings[PRIVACY_POLICY_SETTINGS_KEY] = _policy_data({PRIVACY_POLICY_SETTINGS_KEY: value})
|
||||
setattr(item, "settings", next_settings)
|
||||
return _SystemPrivacyPolicy(next_settings[PRIVACY_POLICY_SETTINGS_KEY])
|
||||
|
||||
store_raw_campaign_json: bool | None = None
|
||||
raw_campaign_json_retention_days: int | None = Field(default=None, ge=0)
|
||||
generated_eml_retention_days: int | None = Field(default=None, ge=0)
|
||||
stored_report_detail_retention_days: int | None = Field(default=None, ge=0)
|
||||
mock_mailbox_retention_days: int | None = Field(default=None, ge=0)
|
||||
audit_detail_retention_days: int | None = Field(default=None, ge=0)
|
||||
audit_detail_level: Literal["full", "redacted", "minimal"] | None = None
|
||||
allow_lower_level_limits: dict[str, bool] | None = None
|
||||
|
||||
@field_validator(*RETENTION_DAY_KEYS, mode="before")
|
||||
@classmethod
|
||||
def _blank_string_is_none(cls, value: Any) -> Any:
|
||||
if value == "":
|
||||
return None
|
||||
return value
|
||||
def effective_privacy_policy(*args: Any, **kwargs: Any) -> Any:
|
||||
return _service().effective_privacy_policy(*args, **kwargs)
|
||||
|
||||
@field_validator("allow_lower_level_limits", mode="before")
|
||||
@classmethod
|
||||
def _normalize_allow_lower_level_limits(cls, value: Any) -> Any:
|
||||
return _normalize_allow_lower_level_limits(value, fill_defaults=False)
|
||||
|
||||
def parent_privacy_policy(*args: Any, **kwargs: Any) -> Any:
|
||||
return _service().parent_privacy_policy(*args, **kwargs)
|
||||
|
||||
class PrivacyPolicyError(RuntimeError):
|
||||
pass
|
||||
|
||||
def effective_privacy_policy_sources(*args: Any, **kwargs: Any) -> Any:
|
||||
return _service().effective_privacy_policy_sources(*args, **kwargs)
|
||||
|
||||
def _campaign_policy_provider() -> CampaignPolicyContextProvider | None:
|
||||
registry = get_registry()
|
||||
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT):
|
||||
return None
|
||||
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT)
|
||||
if not isinstance(capability, CampaignPolicyContextProvider):
|
||||
raise PrivacyPolicyError("Campaign policy context capability is invalid")
|
||||
return capability
|
||||
|
||||
def parent_privacy_policy_sources(*args: Any, **kwargs: Any) -> Any:
|
||||
return _service().parent_privacy_policy_sources(*args, **kwargs)
|
||||
|
||||
def _campaign_retention_provider() -> CampaignRetentionProvider | None:
|
||||
registry = get_registry()
|
||||
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_RETENTION):
|
||||
return None
|
||||
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_RETENTION)
|
||||
if not isinstance(capability, CampaignRetentionProvider):
|
||||
raise PrivacyPolicyError("Campaign retention capability is invalid")
|
||||
return capability
|
||||
|
||||
def get_privacy_policy_for_scope(*args: Any, **kwargs: Any) -> Any:
|
||||
return _service().get_privacy_policy_for_scope(*args, **kwargs)
|
||||
|
||||
def _campaign_policy_context(session: Session, *, campaign_id: str, tenant_id: str | None = None) -> CampaignPolicyContext:
|
||||
provider = _campaign_policy_provider()
|
||||
if provider is None:
|
||||
raise PrivacyPolicyError("Campaign module is not installed")
|
||||
context = provider.get_campaign_policy_context(session, tenant_id=tenant_id, campaign_id=campaign_id)
|
||||
if context is None:
|
||||
raise PrivacyPolicyError("Campaign not found for privacy policy")
|
||||
return context
|
||||
|
||||
def set_privacy_policy_for_scope(*args: Any, **kwargs: Any) -> Any:
|
||||
return _service().set_privacy_policy_for_scope(*args, **kwargs)
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
def sanitize_audit_details_for_policy(*args: Any, **kwargs: Any) -> dict[str, Any]:
|
||||
service = _runtime_service()
|
||||
if service is not None:
|
||||
return dict(service.sanitize_audit_details_for_policy(*args, **kwargs))
|
||||
details = args[1] if len(args) > 1 else kwargs.get("details")
|
||||
return dict(details or {})
|
||||
|
||||
def _cutoff(days: int | None, *, now: datetime) -> datetime | None:
|
||||
if days is None:
|
||||
return None
|
||||
return now - timedelta(days=days)
|
||||
|
||||
def apply_retention_policy(*args: Any, **kwargs: Any) -> Any:
|
||||
return _service().apply_retention_policy(*args, **kwargs)
|
||||
|
||||
def _json_sha256(value: Any) -> str:
|
||||
payload = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8")
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _privacy_policy_data(settings_payload: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if not isinstance(settings_payload, dict):
|
||||
return {}
|
||||
data = settings_payload.get(PRIVACY_POLICY_SETTINGS_KEY, {})
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _privacy_policy_patch_from_settings(settings_payload: dict[str, Any] | None) -> dict[str, Any]:
|
||||
data = _privacy_policy_data(settings_payload)
|
||||
if not data:
|
||||
return {}
|
||||
return PrivacyRetentionPolicyPatch.model_validate(data).model_dump(mode="json", exclude_none=True)
|
||||
|
||||
|
||||
def _merge_privacy_policy(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> PrivacyRetentionPolicy:
|
||||
payload = parent.model_dump(mode="json")
|
||||
parent_allow = {**default_allow_lower_level_limits(), **(payload.get("allow_lower_level_limits") or {})}
|
||||
if parent_allow["store_raw_campaign_json"] and patch.get("store_raw_campaign_json") is False:
|
||||
payload["store_raw_campaign_json"] = False
|
||||
for key in RETENTION_DAY_KEYS:
|
||||
value = patch.get(key)
|
||||
if value is None or not parent_allow[key]:
|
||||
continue
|
||||
current = payload.get(key)
|
||||
payload[key] = int(value) if current is None else min(int(current), int(value))
|
||||
detail_level = patch.get("audit_detail_level")
|
||||
if parent_allow["audit_detail_level"] and detail_level and AUDIT_DETAIL_LEVEL_ORDER[detail_level] > AUDIT_DETAIL_LEVEL_ORDER[payload["audit_detail_level"]]:
|
||||
payload["audit_detail_level"] = detail_level
|
||||
|
||||
patch_allow = patch.get("allow_lower_level_limits") or {}
|
||||
payload["allow_lower_level_limits"] = {
|
||||
key: parent_allow[key] and bool(patch_allow.get(key, parent_allow[key]))
|
||||
for key in RETENTION_POLICY_FIELD_KEYS
|
||||
}
|
||||
return PrivacyRetentionPolicy.model_validate(payload)
|
||||
|
||||
|
||||
def _validate_privacy_patch_against_parent(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> None:
|
||||
parent_payload = parent.model_dump(mode="json")
|
||||
parent_allow = {**default_allow_lower_level_limits(), **(parent_payload.get("allow_lower_level_limits") or {})}
|
||||
for key in RETENTION_POLICY_FIELD_KEYS:
|
||||
if key in patch and patch.get(key) is not None and not parent_allow[key]:
|
||||
raise PrivacyPolicyError(f"{key} is locked by the parent retention policy.")
|
||||
patch_allow = patch.get("allow_lower_level_limits") or {}
|
||||
for key, allowed in patch_allow.items():
|
||||
if allowed and not parent_allow[key]:
|
||||
raise PrivacyPolicyError(f"{key} limiting cannot be re-enabled below a parent retention policy lock.")
|
||||
if patch.get("store_raw_campaign_json") is True and not parent.store_raw_campaign_json:
|
||||
raise PrivacyPolicyError("Raw campaign JSON storage cannot be re-enabled below a parent policy that disables it.")
|
||||
for key in RETENTION_DAY_KEYS:
|
||||
value = patch.get(key)
|
||||
parent_value = parent_payload.get(key)
|
||||
if value is not None and parent_value is not None and int(value) > int(parent_value):
|
||||
raise PrivacyPolicyError(f"{key} cannot be less restrictive than the parent retention policy.")
|
||||
detail_level = patch.get("audit_detail_level")
|
||||
if detail_level and AUDIT_DETAIL_LEVEL_ORDER[detail_level] < AUDIT_DETAIL_LEVEL_ORDER[parent.audit_detail_level]:
|
||||
raise PrivacyPolicyError("Audit detail level cannot be less restrictive than the parent retention policy.")
|
||||
|
||||
|
||||
def _set_settings_privacy_policy(settings_payload: dict[str, Any] | None, policy: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = dict(settings_payload or {})
|
||||
payload[PRIVACY_POLICY_SETTINGS_KEY] = policy
|
||||
return payload
|
||||
|
||||
|
||||
def privacy_policy_from_settings(item: SystemSettings) -> PrivacyRetentionPolicy:
|
||||
return PrivacyRetentionPolicy.model_validate(_privacy_policy_data(item.settings or {}))
|
||||
|
||||
|
||||
def privacy_policy_from_session(session: Session) -> PrivacyRetentionPolicy:
|
||||
return privacy_policy_from_settings(get_system_settings(session))
|
||||
|
||||
|
||||
def set_privacy_policy(item: SystemSettings, policy: PrivacyRetentionPolicy | dict[str, Any]) -> PrivacyRetentionPolicy:
|
||||
validated = policy if isinstance(policy, PrivacyRetentionPolicy) else PrivacyRetentionPolicy.model_validate(policy)
|
||||
item.settings = _set_settings_privacy_policy(item.settings, validated.model_dump(mode="json"))
|
||||
return validated
|
||||
|
||||
|
||||
def effective_privacy_policy(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str | None = None,
|
||||
owner_user_id: str | None = None,
|
||||
owner_group_id: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
) -> PrivacyRetentionPolicy:
|
||||
policy = privacy_policy_from_session(session)
|
||||
campaign: CampaignPolicyContext | None = None
|
||||
if campaign_id:
|
||||
campaign = _campaign_policy_context(session, campaign_id=campaign_id)
|
||||
tenant_id = campaign.tenant_id
|
||||
owner_user_id = campaign.owner_user_id
|
||||
owner_group_id = campaign.owner_group_id
|
||||
if tenant_id:
|
||||
tenant = session.get(Tenant, tenant_id)
|
||||
if tenant is None:
|
||||
raise PrivacyPolicyError("Tenant not found for privacy policy")
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {}))
|
||||
if owner_user_id:
|
||||
user = session.get(User, owner_user_id)
|
||||
if user and (not tenant_id or user.tenant_id == tenant_id):
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user.settings or {}))
|
||||
if owner_group_id:
|
||||
group = session.get(Group, owner_group_id)
|
||||
if group and (not tenant_id or group.tenant_id == tenant_id):
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group.settings or {}))
|
||||
if campaign is not None:
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(dict(campaign.settings or {})))
|
||||
return policy
|
||||
|
||||
|
||||
def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> PrivacyRetentionPolicy:
|
||||
clean_scope = scope_type.strip().casefold()
|
||||
policy = privacy_policy_from_session(session)
|
||||
if clean_scope == "tenant":
|
||||
return policy
|
||||
tenant = session.get(Tenant, tenant_id)
|
||||
if tenant is None:
|
||||
raise PrivacyPolicyError("Tenant not found for privacy policy")
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {}))
|
||||
if clean_scope in {"user", "group"}:
|
||||
return policy
|
||||
if clean_scope != "campaign" or not scope_id:
|
||||
return policy
|
||||
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
|
||||
if campaign.owner_user_id:
|
||||
user = session.get(User, campaign.owner_user_id)
|
||||
if user and user.tenant_id == tenant_id:
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user.settings or {}))
|
||||
if campaign.owner_group_id:
|
||||
group = session.get(Group, campaign.owner_group_id)
|
||||
if group and group.tenant_id == tenant_id:
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group.settings or {}))
|
||||
return policy
|
||||
|
||||
|
||||
|
||||
def _retention_policy_source_fields(patch: dict[str, Any], *, baseline: bool = False) -> list[str]:
|
||||
fields: list[str] = []
|
||||
for key in RETENTION_POLICY_FIELD_KEYS:
|
||||
if key in patch and patch.get(key) is not None:
|
||||
fields.append(key)
|
||||
if isinstance(patch.get("allow_lower_level_limits"), dict) and patch["allow_lower_level_limits"]:
|
||||
fields.append("allow_lower_level_limits")
|
||||
if baseline and not fields:
|
||||
fields.append("defaults")
|
||||
return fields
|
||||
|
||||
|
||||
def _retention_policy_source_step(scope_type: str, label: str, scope_id: str | None, patch: dict[str, Any], *, baseline: bool = False) -> dict[str, Any]:
|
||||
source_policy = PrivacyRetentionPolicy.model_validate(patch).model_dump(mode="json") if baseline else dict(patch)
|
||||
return policy_source_step(
|
||||
scope_type,
|
||||
label,
|
||||
scope_id,
|
||||
applied_fields=_retention_policy_source_fields(patch, baseline=baseline),
|
||||
policy=source_policy,
|
||||
)
|
||||
|
||||
|
||||
def effective_privacy_policy_sources(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str | None = None,
|
||||
owner_user_id: str | None = None,
|
||||
owner_group_id: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
system_settings = get_system_settings(session)
|
||||
sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)]
|
||||
campaign: CampaignPolicyContext | None = None
|
||||
if campaign_id:
|
||||
campaign = _campaign_policy_context(session, campaign_id=campaign_id)
|
||||
tenant_id = campaign.tenant_id
|
||||
owner_user_id = campaign.owner_user_id
|
||||
owner_group_id = campaign.owner_group_id
|
||||
if tenant_id:
|
||||
tenant = session.get(Tenant, tenant_id)
|
||||
if tenant is None:
|
||||
raise PrivacyPolicyError("Tenant not found for privacy policy")
|
||||
sources.append(_retention_policy_source_step("tenant", "Tenant", tenant.id, _privacy_policy_patch_from_settings(tenant.settings or {})))
|
||||
if owner_user_id:
|
||||
user = session.get(User, owner_user_id)
|
||||
if user and (not tenant_id or user.tenant_id == tenant_id):
|
||||
sources.append(_retention_policy_source_step("user", "Owner user", user.id, _privacy_policy_patch_from_settings(user.settings or {})))
|
||||
if owner_group_id:
|
||||
group = session.get(Group, owner_group_id)
|
||||
if group and (not tenant_id or group.tenant_id == tenant_id):
|
||||
sources.append(_retention_policy_source_step("group", "Owner group", group.id, _privacy_policy_patch_from_settings(group.settings or {})))
|
||||
if campaign is not None:
|
||||
sources.append(_retention_policy_source_step("campaign", "Campaign", campaign.id, _privacy_policy_patch_from_settings(dict(campaign.settings or {}))))
|
||||
return sources
|
||||
|
||||
|
||||
def parent_privacy_policy_sources(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> list[dict[str, Any]]:
|
||||
clean_scope = scope_type.strip().casefold()
|
||||
if clean_scope == "system":
|
||||
return []
|
||||
system_settings = get_system_settings(session)
|
||||
sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)]
|
||||
if clean_scope == "tenant":
|
||||
return sources
|
||||
tenant = session.get(Tenant, tenant_id)
|
||||
if tenant is None:
|
||||
raise PrivacyPolicyError("Tenant not found for privacy policy")
|
||||
sources.append(_retention_policy_source_step("tenant", "Tenant", tenant.id, _privacy_policy_patch_from_settings(tenant.settings or {})))
|
||||
if clean_scope in {"user", "group"}:
|
||||
return sources
|
||||
if clean_scope != "campaign" or not scope_id:
|
||||
return sources
|
||||
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
|
||||
if campaign.owner_user_id:
|
||||
user = session.get(User, campaign.owner_user_id)
|
||||
if user and user.tenant_id == tenant_id:
|
||||
sources.append(_retention_policy_source_step("user", "Owner user", user.id, _privacy_policy_patch_from_settings(user.settings or {})))
|
||||
if campaign.owner_group_id:
|
||||
group = session.get(Group, campaign.owner_group_id)
|
||||
if group and group.tenant_id == tenant_id:
|
||||
sources.append(_retention_policy_source_step("group", "Owner group", group.id, _privacy_policy_patch_from_settings(group.settings or {})))
|
||||
return sources
|
||||
|
||||
def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> dict[str, Any]:
|
||||
clean_scope = scope_type.strip().casefold()
|
||||
if clean_scope == "system":
|
||||
return privacy_policy_from_session(session).model_dump(mode="json")
|
||||
if clean_scope == "tenant":
|
||||
tenant = session.get(Tenant, scope_id or tenant_id)
|
||||
if tenant is None or tenant.id != tenant_id:
|
||||
raise PrivacyPolicyError("Tenant privacy policy not found")
|
||||
return _privacy_policy_patch_from_settings(tenant.settings or {})
|
||||
if not scope_id:
|
||||
raise PrivacyPolicyError(f"{clean_scope.capitalize()} privacy policy requires scope_id")
|
||||
if clean_scope == "user":
|
||||
user = session.get(User, scope_id)
|
||||
if user is None or user.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("User privacy policy not found")
|
||||
return _privacy_policy_patch_from_settings(user.settings or {})
|
||||
if clean_scope == "group":
|
||||
group = session.get(Group, scope_id)
|
||||
if group is None or group.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("Group privacy policy not found")
|
||||
return _privacy_policy_patch_from_settings(group.settings or {})
|
||||
if clean_scope == "campaign":
|
||||
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
|
||||
return _privacy_policy_patch_from_settings(dict(campaign.settings or {}))
|
||||
raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign")
|
||||
|
||||
|
||||
def set_privacy_policy_for_scope(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
scope_type: str,
|
||||
scope_id: str | None = None,
|
||||
policy: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
clean_scope = scope_type.strip().casefold()
|
||||
if clean_scope == "system":
|
||||
item = get_system_settings(session)
|
||||
validated = set_privacy_policy(item, PrivacyRetentionPolicy.model_validate(policy or {}))
|
||||
session.add(item)
|
||||
return validated.model_dump(mode="json")
|
||||
patch = PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(mode="json", exclude_none=True)
|
||||
_validate_privacy_patch_against_parent(parent_privacy_policy(session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id or (tenant_id if clean_scope == "tenant" else None)), patch)
|
||||
if clean_scope == "tenant":
|
||||
tenant = session.get(Tenant, scope_id or tenant_id)
|
||||
if tenant is None or tenant.id != tenant_id:
|
||||
raise PrivacyPolicyError("Tenant privacy policy not found")
|
||||
tenant.settings = _set_settings_privacy_policy(tenant.settings, patch)
|
||||
session.add(tenant)
|
||||
return patch
|
||||
if not scope_id:
|
||||
raise PrivacyPolicyError(f"{clean_scope.capitalize()} privacy policy requires scope_id")
|
||||
if clean_scope == "user":
|
||||
user = session.get(User, scope_id)
|
||||
if user is None or user.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("User privacy policy not found")
|
||||
user.settings = _set_settings_privacy_policy(user.settings, patch)
|
||||
session.add(user)
|
||||
return patch
|
||||
if clean_scope == "group":
|
||||
group = session.get(Group, scope_id)
|
||||
if group is None or group.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("Group privacy policy not found")
|
||||
group.settings = _set_settings_privacy_policy(group.settings, patch)
|
||||
session.add(group)
|
||||
return patch
|
||||
if clean_scope == "campaign":
|
||||
provider = _campaign_policy_provider()
|
||||
if provider is None:
|
||||
raise PrivacyPolicyError("Campaign module is not installed")
|
||||
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
|
||||
settings_payload = _set_settings_privacy_policy(dict(campaign.settings or {}), patch)
|
||||
try:
|
||||
provider.set_campaign_settings(session, tenant_id=tenant_id, campaign_id=scope_id, settings=settings_payload)
|
||||
except ValueError as exc:
|
||||
raise PrivacyPolicyError("Campaign privacy policy not found") from exc
|
||||
return patch
|
||||
raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign")
|
||||
|
||||
|
||||
def sanitize_audit_details_for_policy(session: Session, details: dict[str, Any]) -> dict[str, Any]:
|
||||
policy = privacy_policy_from_session(session)
|
||||
if policy.audit_detail_level == "full":
|
||||
return details
|
||||
if policy.audit_detail_level == "minimal":
|
||||
return {
|
||||
"_privacy": {"detail_level": "minimal"},
|
||||
"keys": sorted(str(key) for key in details.keys()),
|
||||
}
|
||||
return _redact_audit_value(details)
|
||||
|
||||
|
||||
def _redact_audit_value(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
redacted: dict[str, Any] = {}
|
||||
for key, item in value.items():
|
||||
if str(key).lower() in AUDIT_DETAIL_REDACTION_KEYS:
|
||||
redacted[key] = "<redacted>"
|
||||
else:
|
||||
redacted[key] = _redact_audit_value(item)
|
||||
return redacted
|
||||
if isinstance(value, list):
|
||||
return [_redact_audit_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _is_before_cutoff(value: datetime | None, cutoff: datetime | None) -> bool:
|
||||
if value is None or cutoff is None:
|
||||
return False
|
||||
candidate = value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value
|
||||
return candidate < cutoff
|
||||
|
||||
|
||||
def _system_cutoffs(policy: PrivacyRetentionPolicy, *, now: datetime) -> dict[str, datetime | None]:
|
||||
return {
|
||||
"raw_campaign_json": _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now),
|
||||
"generated_eml": _cutoff(policy.generated_eml_retention_days, now=now),
|
||||
"stored_report_detail": _cutoff(policy.stored_report_detail_retention_days, now=now),
|
||||
"mock_mailbox": _cutoff(policy.mock_mailbox_retention_days, now=now),
|
||||
"audit_detail": _cutoff(policy.audit_detail_retention_days, now=now),
|
||||
}
|
||||
|
||||
|
||||
def apply_retention_policy(session: Session, *, dry_run: bool = True) -> dict[str, Any]:
|
||||
now = _utcnow()
|
||||
policy = privacy_policy_from_session(session)
|
||||
cutoffs = _system_cutoffs(policy, now=now)
|
||||
policy_cache: dict[str, PrivacyRetentionPolicy] = {}
|
||||
campaign_retention = _campaign_retention_provider()
|
||||
campaign_counts = {
|
||||
"raw_campaign_json": {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0},
|
||||
"generated_eml": {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0},
|
||||
"stored_report_detail": {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0},
|
||||
}
|
||||
if campaign_retention is not None:
|
||||
campaign_counts.update(
|
||||
{
|
||||
key: dict(value)
|
||||
for key, value in campaign_retention.apply_retention(
|
||||
session,
|
||||
dry_run=dry_run,
|
||||
now=now,
|
||||
policy_for_campaign_id=lambda campaign_id: _campaign_policy_for_id(session, campaign_id, policy_cache),
|
||||
).items()
|
||||
}
|
||||
)
|
||||
counts = {
|
||||
"raw_campaign_json": campaign_counts["raw_campaign_json"],
|
||||
"generated_eml": campaign_counts["generated_eml"],
|
||||
"stored_report_detail": campaign_counts["stored_report_detail"],
|
||||
"mock_mailbox": _apply_mock_mailbox_retention(cutoffs["mock_mailbox"], dry_run=dry_run),
|
||||
"audit_detail": _apply_audit_detail_retention(session, dry_run=dry_run, now=now),
|
||||
}
|
||||
return {
|
||||
"dry_run": dry_run,
|
||||
"policy": policy.model_dump(mode="json"),
|
||||
"cutoffs": {key: value.isoformat() if value else None for key, value in cutoffs.items()},
|
||||
"effective_policy_scope": "per-object",
|
||||
"counts": counts,
|
||||
}
|
||||
|
||||
|
||||
def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy:
|
||||
if not campaign_id:
|
||||
return privacy_policy_from_session(session)
|
||||
if campaign_id not in cache:
|
||||
cache[campaign_id] = effective_privacy_policy(session, campaign_id=campaign_id)
|
||||
return cache[campaign_id]
|
||||
|
||||
|
||||
def _apply_mock_mailbox_retention(cutoff: datetime | None, *, dry_run: bool) -> dict[str, int]:
|
||||
result = {"eligible_records": 0, "json_deleted": 0, "eml_deleted": 0}
|
||||
if cutoff is None:
|
||||
return result
|
||||
message_dir = Path(settings.mock_mailbox_dir) / "messages"
|
||||
if not message_dir.exists():
|
||||
return result
|
||||
for path in message_dir.glob("*.json"):
|
||||
try:
|
||||
record = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
created_at = _parse_datetime(record.get("created_at"))
|
||||
if created_at and created_at >= cutoff:
|
||||
continue
|
||||
result["eligible_records"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
raw_filename = record.get("raw_filename")
|
||||
if raw_filename:
|
||||
raw_path = message_dir / str(raw_filename)
|
||||
if raw_path.exists():
|
||||
raw_path.unlink()
|
||||
result["eml_deleted"] += 1
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
result["json_deleted"] += 1
|
||||
return result
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed
|
||||
|
||||
|
||||
def _privacy_policy_for_audit_item(session: Session, item: AuditLog, campaign_cache: dict[str, PrivacyRetentionPolicy], tenant_cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy:
|
||||
if item.object_type == "campaign" and item.object_id:
|
||||
provider = _campaign_policy_provider()
|
||||
if provider is not None:
|
||||
context = provider.get_campaign_policy_context(session, campaign_id=str(item.object_id))
|
||||
if context is not None:
|
||||
return _campaign_policy_for_id(session, context.id, campaign_cache)
|
||||
if item.tenant_id:
|
||||
if item.tenant_id not in tenant_cache:
|
||||
tenant_cache[item.tenant_id] = effective_privacy_policy(session, tenant_id=item.tenant_id)
|
||||
return tenant_cache[item.tenant_id]
|
||||
return privacy_policy_from_session(session)
|
||||
|
||||
|
||||
def _apply_audit_detail_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]:
|
||||
result = {"eligible": 0, "redacted": 0, "already_redacted": 0}
|
||||
campaign_cache: dict[str, PrivacyRetentionPolicy] = {}
|
||||
tenant_cache: dict[str, PrivacyRetentionPolicy] = {}
|
||||
items = (
|
||||
session.query(AuditLog)
|
||||
.order_by(AuditLog.created_at.asc())
|
||||
.all()
|
||||
)
|
||||
for item in items:
|
||||
policy = _privacy_policy_for_audit_item(session, item, campaign_cache, tenant_cache)
|
||||
cutoff = _cutoff(policy.audit_detail_retention_days, now=now)
|
||||
if not _is_before_cutoff(item.created_at, cutoff):
|
||||
continue
|
||||
details = item.details if isinstance(item.details, dict) else {}
|
||||
if details.get("_retention", {}).get("audit_detail_redacted"):
|
||||
result["already_redacted"] += 1
|
||||
continue
|
||||
result["eligible"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
item.details = {
|
||||
"_retention": {
|
||||
"audit_detail_redacted": True,
|
||||
"redacted_at": now.isoformat(),
|
||||
"original_detail_sha256": _json_sha256(details),
|
||||
}
|
||||
}
|
||||
session.add(item)
|
||||
result["redacted"] += 1
|
||||
return result
|
||||
__all__ = list(_EXPORTS)
|
||||
|
||||
Reference in New Issue
Block a user