initial commit after split
This commit is contained in:
664
src/govoplan_core/privacy/retention.py
Normal file
664
src/govoplan_core/privacy/retention.py
Normal file
@@ -0,0 +1,664 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.admin.governance import get_system_settings
|
||||
from govoplan_core.db.models import AuditLog, Group, SystemSettings, Tenant, User
|
||||
from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus, JobSendStatus
|
||||
from govoplan_core.settings import settings
|
||||
|
||||
PRIVACY_POLICY_SETTINGS_KEY = "privacy_retention_policy"
|
||||
RETENTION_DAY_KEYS = (
|
||||
"raw_campaign_json_retention_days",
|
||||
"generated_eml_retention_days",
|
||||
"stored_report_detail_retention_days",
|
||||
"mock_mailbox_retention_days",
|
||||
"audit_detail_retention_days",
|
||||
)
|
||||
RETENTION_POLICY_FIELD_KEYS = (
|
||||
"store_raw_campaign_json",
|
||||
*RETENTION_DAY_KEYS,
|
||||
"audit_detail_level",
|
||||
)
|
||||
AUDIT_DETAIL_LEVEL_ORDER = {"full": 0, "redacted": 1, "minimal": 2}
|
||||
|
||||
|
||||
def default_allow_lower_level_limits() -> dict[str, bool]:
|
||||
return {key: True for key in RETENTION_POLICY_FIELD_KEYS}
|
||||
|
||||
|
||||
def _normalize_allow_lower_level_limits(value: Any, *, fill_defaults: bool) -> dict[str, bool] | None:
|
||||
if value in (None, ""):
|
||||
return default_allow_lower_level_limits() if fill_defaults else None
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError("allow_lower_level_limits must be an object")
|
||||
normalized = default_allow_lower_level_limits() if fill_defaults else {}
|
||||
for key, allowed in value.items():
|
||||
clean_key = str(key)
|
||||
if clean_key not in RETENTION_POLICY_FIELD_KEYS:
|
||||
raise ValueError(f"Unknown retention policy field: {clean_key}")
|
||||
normalized[clean_key] = bool(allowed)
|
||||
return normalized
|
||||
|
||||
FINAL_VERSION_STATES = {
|
||||
"completed",
|
||||
"partially_completed",
|
||||
"outcome_unknown",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"archived",
|
||||
}
|
||||
FINAL_EML_SEND_STATUSES = {
|
||||
JobSendStatus.SMTP_ACCEPTED.value,
|
||||
JobSendStatus.SENT.value,
|
||||
JobSendStatus.FAILED_PERMANENT.value,
|
||||
JobSendStatus.CANCELLED.value,
|
||||
JobSendStatus.OUTCOME_UNKNOWN.value,
|
||||
}
|
||||
AUDIT_DETAIL_REDACTION_KEYS = {
|
||||
"attachments",
|
||||
"bcc",
|
||||
"body",
|
||||
"campaign_json",
|
||||
"cc",
|
||||
"content",
|
||||
"entries",
|
||||
"html",
|
||||
"imap",
|
||||
"message",
|
||||
"password",
|
||||
"raw_eml",
|
||||
"raw_json",
|
||||
"recipients",
|
||||
"secret",
|
||||
"smtp",
|
||||
"template",
|
||||
"text",
|
||||
"token",
|
||||
"to",
|
||||
}
|
||||
|
||||
|
||||
class PrivacyRetentionPolicy(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
store_raw_campaign_json: bool = True
|
||||
raw_campaign_json_retention_days: int | None = Field(default=None, ge=0)
|
||||
generated_eml_retention_days: int | None = Field(default=None, ge=0)
|
||||
stored_report_detail_retention_days: int | None = Field(default=None, ge=0)
|
||||
mock_mailbox_retention_days: int | None = Field(default=None, ge=0)
|
||||
audit_detail_retention_days: int | None = Field(default=None, ge=0)
|
||||
audit_detail_level: Literal["full", "redacted", "minimal"] = "full"
|
||||
allow_lower_level_limits: dict[str, bool] = Field(default_factory=default_allow_lower_level_limits)
|
||||
|
||||
@field_validator(
|
||||
"raw_campaign_json_retention_days",
|
||||
"generated_eml_retention_days",
|
||||
"stored_report_detail_retention_days",
|
||||
"mock_mailbox_retention_days",
|
||||
"audit_detail_retention_days",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
def _blank_string_is_none(cls, value: Any) -> Any:
|
||||
if value == "":
|
||||
return None
|
||||
return value
|
||||
|
||||
@field_validator("allow_lower_level_limits", mode="before")
|
||||
@classmethod
|
||||
def _normalize_allow_lower_level_limits(cls, value: Any) -> Any:
|
||||
return _normalize_allow_lower_level_limits(value, fill_defaults=True)
|
||||
|
||||
|
||||
class PrivacyRetentionPolicyPatch(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
store_raw_campaign_json: bool | None = None
|
||||
raw_campaign_json_retention_days: int | None = Field(default=None, ge=0)
|
||||
generated_eml_retention_days: int | None = Field(default=None, ge=0)
|
||||
stored_report_detail_retention_days: int | None = Field(default=None, ge=0)
|
||||
mock_mailbox_retention_days: int | None = Field(default=None, ge=0)
|
||||
audit_detail_retention_days: int | None = Field(default=None, ge=0)
|
||||
audit_detail_level: Literal["full", "redacted", "minimal"] | None = None
|
||||
allow_lower_level_limits: dict[str, bool] | None = None
|
||||
|
||||
@field_validator(*RETENTION_DAY_KEYS, mode="before")
|
||||
@classmethod
|
||||
def _blank_string_is_none(cls, value: Any) -> Any:
|
||||
if value == "":
|
||||
return None
|
||||
return value
|
||||
|
||||
@field_validator("allow_lower_level_limits", mode="before")
|
||||
@classmethod
|
||||
def _normalize_allow_lower_level_limits(cls, value: Any) -> Any:
|
||||
return _normalize_allow_lower_level_limits(value, fill_defaults=False)
|
||||
|
||||
|
||||
class PrivacyPolicyError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _cutoff(days: int | None, *, now: datetime) -> datetime | None:
|
||||
if days is None:
|
||||
return None
|
||||
return now - timedelta(days=days)
|
||||
|
||||
|
||||
def _json_sha256(value: Any) -> str:
|
||||
payload = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8")
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _privacy_policy_data(settings_payload: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if not isinstance(settings_payload, dict):
|
||||
return {}
|
||||
data = settings_payload.get(PRIVACY_POLICY_SETTINGS_KEY, {})
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _privacy_policy_patch_from_settings(settings_payload: dict[str, Any] | None) -> dict[str, Any]:
|
||||
data = _privacy_policy_data(settings_payload)
|
||||
if not data:
|
||||
return {}
|
||||
return PrivacyRetentionPolicyPatch.model_validate(data).model_dump(mode="json", exclude_none=True)
|
||||
|
||||
|
||||
def _merge_privacy_policy(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> PrivacyRetentionPolicy:
|
||||
payload = parent.model_dump(mode="json")
|
||||
parent_allow = {**default_allow_lower_level_limits(), **(payload.get("allow_lower_level_limits") or {})}
|
||||
if parent_allow["store_raw_campaign_json"] and patch.get("store_raw_campaign_json") is False:
|
||||
payload["store_raw_campaign_json"] = False
|
||||
for key in RETENTION_DAY_KEYS:
|
||||
value = patch.get(key)
|
||||
if value is None or not parent_allow[key]:
|
||||
continue
|
||||
current = payload.get(key)
|
||||
payload[key] = int(value) if current is None else min(int(current), int(value))
|
||||
detail_level = patch.get("audit_detail_level")
|
||||
if parent_allow["audit_detail_level"] and detail_level and AUDIT_DETAIL_LEVEL_ORDER[detail_level] > AUDIT_DETAIL_LEVEL_ORDER[payload["audit_detail_level"]]:
|
||||
payload["audit_detail_level"] = detail_level
|
||||
|
||||
patch_allow = patch.get("allow_lower_level_limits") or {}
|
||||
payload["allow_lower_level_limits"] = {
|
||||
key: parent_allow[key] and bool(patch_allow.get(key, parent_allow[key]))
|
||||
for key in RETENTION_POLICY_FIELD_KEYS
|
||||
}
|
||||
return PrivacyRetentionPolicy.model_validate(payload)
|
||||
|
||||
|
||||
def _validate_privacy_patch_against_parent(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> None:
|
||||
parent_payload = parent.model_dump(mode="json")
|
||||
parent_allow = {**default_allow_lower_level_limits(), **(parent_payload.get("allow_lower_level_limits") or {})}
|
||||
for key in RETENTION_POLICY_FIELD_KEYS:
|
||||
if key in patch and patch.get(key) is not None and not parent_allow[key]:
|
||||
raise PrivacyPolicyError(f"{key} is locked by the parent retention policy.")
|
||||
patch_allow = patch.get("allow_lower_level_limits") or {}
|
||||
for key, allowed in patch_allow.items():
|
||||
if allowed and not parent_allow[key]:
|
||||
raise PrivacyPolicyError(f"{key} limiting cannot be re-enabled below a parent retention policy lock.")
|
||||
if patch.get("store_raw_campaign_json") is True and not parent.store_raw_campaign_json:
|
||||
raise PrivacyPolicyError("Raw campaign JSON storage cannot be re-enabled below a parent policy that disables it.")
|
||||
for key in RETENTION_DAY_KEYS:
|
||||
value = patch.get(key)
|
||||
parent_value = parent_payload.get(key)
|
||||
if value is not None and parent_value is not None and int(value) > int(parent_value):
|
||||
raise PrivacyPolicyError(f"{key} cannot be less restrictive than the parent retention policy.")
|
||||
detail_level = patch.get("audit_detail_level")
|
||||
if detail_level and AUDIT_DETAIL_LEVEL_ORDER[detail_level] < AUDIT_DETAIL_LEVEL_ORDER[parent.audit_detail_level]:
|
||||
raise PrivacyPolicyError("Audit detail level cannot be less restrictive than the parent retention policy.")
|
||||
|
||||
|
||||
def _set_settings_privacy_policy(settings_payload: dict[str, Any] | None, policy: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = dict(settings_payload or {})
|
||||
payload[PRIVACY_POLICY_SETTINGS_KEY] = policy
|
||||
return payload
|
||||
|
||||
|
||||
def privacy_policy_from_settings(item: SystemSettings) -> PrivacyRetentionPolicy:
|
||||
return PrivacyRetentionPolicy.model_validate(_privacy_policy_data(item.settings or {}))
|
||||
|
||||
|
||||
def privacy_policy_from_session(session: Session) -> PrivacyRetentionPolicy:
|
||||
return privacy_policy_from_settings(get_system_settings(session))
|
||||
|
||||
|
||||
def set_privacy_policy(item: SystemSettings, policy: PrivacyRetentionPolicy | dict[str, Any]) -> PrivacyRetentionPolicy:
|
||||
validated = policy if isinstance(policy, PrivacyRetentionPolicy) else PrivacyRetentionPolicy.model_validate(policy)
|
||||
item.settings = _set_settings_privacy_policy(item.settings, validated.model_dump(mode="json"))
|
||||
return validated
|
||||
|
||||
|
||||
def effective_privacy_policy(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str | None = None,
|
||||
owner_user_id: str | None = None,
|
||||
owner_group_id: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
) -> PrivacyRetentionPolicy:
|
||||
policy = privacy_policy_from_session(session)
|
||||
campaign: Campaign | None = None
|
||||
if campaign_id:
|
||||
campaign = session.get(Campaign, campaign_id)
|
||||
if campaign is None:
|
||||
raise PrivacyPolicyError("Campaign not found for privacy policy")
|
||||
tenant_id = campaign.tenant_id
|
||||
owner_user_id = campaign.owner_user_id
|
||||
owner_group_id = campaign.owner_group_id
|
||||
if tenant_id:
|
||||
tenant = session.get(Tenant, tenant_id)
|
||||
if tenant is None:
|
||||
raise PrivacyPolicyError("Tenant not found for privacy policy")
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {}))
|
||||
if owner_user_id:
|
||||
user = session.get(User, owner_user_id)
|
||||
if user and (not tenant_id or user.tenant_id == tenant_id):
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user.settings or {}))
|
||||
if owner_group_id:
|
||||
group = session.get(Group, owner_group_id)
|
||||
if group and (not tenant_id or group.tenant_id == tenant_id):
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group.settings or {}))
|
||||
if campaign is not None:
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(campaign.settings or {}))
|
||||
return policy
|
||||
|
||||
|
||||
def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> PrivacyRetentionPolicy:
|
||||
clean_scope = scope_type.strip().casefold()
|
||||
policy = privacy_policy_from_session(session)
|
||||
if clean_scope == "tenant":
|
||||
return policy
|
||||
tenant = session.get(Tenant, tenant_id)
|
||||
if tenant is None:
|
||||
raise PrivacyPolicyError("Tenant not found for privacy policy")
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {}))
|
||||
if clean_scope in {"user", "group"}:
|
||||
return policy
|
||||
if clean_scope != "campaign" or not scope_id:
|
||||
return policy
|
||||
campaign = session.get(Campaign, scope_id)
|
||||
if campaign is None or campaign.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("Campaign not found for privacy policy")
|
||||
if campaign.owner_user_id:
|
||||
user = session.get(User, campaign.owner_user_id)
|
||||
if user and user.tenant_id == tenant_id:
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user.settings or {}))
|
||||
if campaign.owner_group_id:
|
||||
group = session.get(Group, campaign.owner_group_id)
|
||||
if group and group.tenant_id == tenant_id:
|
||||
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group.settings or {}))
|
||||
return policy
|
||||
|
||||
|
||||
def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> dict[str, Any]:
|
||||
clean_scope = scope_type.strip().casefold()
|
||||
if clean_scope == "system":
|
||||
return privacy_policy_from_session(session).model_dump(mode="json")
|
||||
if clean_scope == "tenant":
|
||||
tenant = session.get(Tenant, scope_id or tenant_id)
|
||||
if tenant is None or tenant.id != tenant_id:
|
||||
raise PrivacyPolicyError("Tenant privacy policy not found")
|
||||
return _privacy_policy_patch_from_settings(tenant.settings or {})
|
||||
if not scope_id:
|
||||
raise PrivacyPolicyError(f"{clean_scope.capitalize()} privacy policy requires scope_id")
|
||||
if clean_scope == "user":
|
||||
user = session.get(User, scope_id)
|
||||
if user is None or user.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("User privacy policy not found")
|
||||
return _privacy_policy_patch_from_settings(user.settings or {})
|
||||
if clean_scope == "group":
|
||||
group = session.get(Group, scope_id)
|
||||
if group is None or group.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("Group privacy policy not found")
|
||||
return _privacy_policy_patch_from_settings(group.settings or {})
|
||||
if clean_scope == "campaign":
|
||||
campaign = session.get(Campaign, scope_id)
|
||||
if campaign is None or campaign.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("Campaign privacy policy not found")
|
||||
return _privacy_policy_patch_from_settings(campaign.settings or {})
|
||||
raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign")
|
||||
|
||||
|
||||
def set_privacy_policy_for_scope(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
scope_type: str,
|
||||
scope_id: str | None = None,
|
||||
policy: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
clean_scope = scope_type.strip().casefold()
|
||||
if clean_scope == "system":
|
||||
item = get_system_settings(session)
|
||||
validated = set_privacy_policy(item, PrivacyRetentionPolicy.model_validate(policy or {}))
|
||||
session.add(item)
|
||||
return validated.model_dump(mode="json")
|
||||
patch = PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(mode="json", exclude_none=True)
|
||||
_validate_privacy_patch_against_parent(parent_privacy_policy(session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id or (tenant_id if clean_scope == "tenant" else None)), patch)
|
||||
if clean_scope == "tenant":
|
||||
tenant = session.get(Tenant, scope_id or tenant_id)
|
||||
if tenant is None or tenant.id != tenant_id:
|
||||
raise PrivacyPolicyError("Tenant privacy policy not found")
|
||||
tenant.settings = _set_settings_privacy_policy(tenant.settings, patch)
|
||||
session.add(tenant)
|
||||
return patch
|
||||
if not scope_id:
|
||||
raise PrivacyPolicyError(f"{clean_scope.capitalize()} privacy policy requires scope_id")
|
||||
if clean_scope == "user":
|
||||
user = session.get(User, scope_id)
|
||||
if user is None or user.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("User privacy policy not found")
|
||||
user.settings = _set_settings_privacy_policy(user.settings, patch)
|
||||
session.add(user)
|
||||
return patch
|
||||
if clean_scope == "group":
|
||||
group = session.get(Group, scope_id)
|
||||
if group is None or group.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("Group privacy policy not found")
|
||||
group.settings = _set_settings_privacy_policy(group.settings, patch)
|
||||
session.add(group)
|
||||
return patch
|
||||
if clean_scope == "campaign":
|
||||
campaign = session.get(Campaign, scope_id)
|
||||
if campaign is None or campaign.tenant_id != tenant_id:
|
||||
raise PrivacyPolicyError("Campaign privacy policy not found")
|
||||
campaign.settings = _set_settings_privacy_policy(campaign.settings, patch)
|
||||
session.add(campaign)
|
||||
return patch
|
||||
raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign")
|
||||
|
||||
|
||||
def sanitize_audit_details_for_policy(session: Session, details: dict[str, Any]) -> dict[str, Any]:
|
||||
policy = privacy_policy_from_session(session)
|
||||
if policy.audit_detail_level == "full":
|
||||
return details
|
||||
if policy.audit_detail_level == "minimal":
|
||||
return {
|
||||
"_privacy": {"detail_level": "minimal"},
|
||||
"keys": sorted(str(key) for key in details.keys()),
|
||||
}
|
||||
return _redact_audit_value(details)
|
||||
|
||||
|
||||
def _redact_audit_value(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
redacted: dict[str, Any] = {}
|
||||
for key, item in value.items():
|
||||
if str(key).lower() in AUDIT_DETAIL_REDACTION_KEYS:
|
||||
redacted[key] = "<redacted>"
|
||||
else:
|
||||
redacted[key] = _redact_audit_value(item)
|
||||
return redacted
|
||||
if isinstance(value, list):
|
||||
return [_redact_audit_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _is_before_cutoff(value: datetime | None, cutoff: datetime | None) -> bool:
|
||||
if value is None or cutoff is None:
|
||||
return False
|
||||
candidate = value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value
|
||||
return candidate < cutoff
|
||||
|
||||
|
||||
def _system_cutoffs(policy: PrivacyRetentionPolicy, *, now: datetime) -> dict[str, datetime | None]:
|
||||
return {
|
||||
"raw_campaign_json": _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now),
|
||||
"generated_eml": _cutoff(policy.generated_eml_retention_days, now=now),
|
||||
"stored_report_detail": _cutoff(policy.stored_report_detail_retention_days, now=now),
|
||||
"mock_mailbox": _cutoff(policy.mock_mailbox_retention_days, now=now),
|
||||
"audit_detail": _cutoff(policy.audit_detail_retention_days, now=now),
|
||||
}
|
||||
|
||||
|
||||
def apply_retention_policy(session: Session, *, dry_run: bool = True) -> dict[str, Any]:
|
||||
now = _utcnow()
|
||||
policy = privacy_policy_from_session(session)
|
||||
cutoffs = _system_cutoffs(policy, now=now)
|
||||
counts = {
|
||||
"raw_campaign_json": _apply_raw_json_retention(session, dry_run=dry_run, now=now),
|
||||
"generated_eml": _apply_eml_retention(session, dry_run=dry_run, now=now),
|
||||
"stored_report_detail": _apply_report_detail_retention(session, dry_run=dry_run, now=now),
|
||||
"mock_mailbox": _apply_mock_mailbox_retention(cutoffs["mock_mailbox"], dry_run=dry_run),
|
||||
"audit_detail": _apply_audit_detail_retention(session, dry_run=dry_run, now=now),
|
||||
}
|
||||
return {
|
||||
"dry_run": dry_run,
|
||||
"policy": policy.model_dump(mode="json"),
|
||||
"cutoffs": {key: value.isoformat() if value else None for key, value in cutoffs.items()},
|
||||
"effective_policy_scope": "per-object",
|
||||
"counts": counts,
|
||||
}
|
||||
|
||||
|
||||
def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy:
|
||||
if not campaign_id:
|
||||
return privacy_policy_from_session(session)
|
||||
if campaign_id not in cache:
|
||||
cache[campaign_id] = effective_privacy_policy(session, campaign_id=campaign_id)
|
||||
return cache[campaign_id]
|
||||
|
||||
|
||||
def _apply_raw_json_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]:
|
||||
result = {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0}
|
||||
policy_cache: dict[str, PrivacyRetentionPolicy] = {}
|
||||
versions = (
|
||||
session.query(CampaignVersion)
|
||||
.order_by(CampaignVersion.updated_at.asc())
|
||||
.all()
|
||||
)
|
||||
for version in versions:
|
||||
policy = _campaign_policy_for_id(session, version.campaign_id, policy_cache)
|
||||
cutoff = _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now)
|
||||
if not _is_before_cutoff(version.updated_at, cutoff):
|
||||
continue
|
||||
raw_json = version.raw_json if isinstance(version.raw_json, dict) else {}
|
||||
if raw_json.get("_retention", {}).get("raw_json_redacted"):
|
||||
result["already_redacted"] += 1
|
||||
continue
|
||||
if version.workflow_state not in FINAL_VERSION_STATES:
|
||||
result["skipped_not_final"] += 1
|
||||
continue
|
||||
result["eligible"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
snapshot = version.execution_snapshot if isinstance(version.execution_snapshot, dict) else {}
|
||||
version.raw_json = {
|
||||
"version": version.schema_version or "1.0",
|
||||
"_retention": {
|
||||
"raw_json_redacted": True,
|
||||
"redacted_at": now.isoformat(),
|
||||
"original_sha256": snapshot.get("campaign_json_sha256") or _json_sha256(raw_json),
|
||||
},
|
||||
}
|
||||
session.add(version)
|
||||
result["redacted"] += 1
|
||||
return result
|
||||
|
||||
|
||||
def _apply_eml_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]:
|
||||
result = {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0}
|
||||
policy_cache: dict[str, PrivacyRetentionPolicy] = {}
|
||||
jobs = (
|
||||
session.query(CampaignJob)
|
||||
.filter((CampaignJob.eml_local_path.is_not(None)) | (CampaignJob.eml_storage_key.is_not(None)))
|
||||
.order_by(CampaignJob.updated_at.asc())
|
||||
.all()
|
||||
)
|
||||
for job in jobs:
|
||||
policy = _campaign_policy_for_id(session, job.campaign_id, policy_cache)
|
||||
cutoff = _cutoff(policy.generated_eml_retention_days, now=now)
|
||||
if not _is_before_cutoff(job.updated_at, cutoff):
|
||||
continue
|
||||
if job.queue_status in {JobQueueStatus.QUEUED.value, JobQueueStatus.SENDING.value} or job.send_status not in FINAL_EML_SEND_STATUSES:
|
||||
result["skipped_not_final"] += 1
|
||||
continue
|
||||
if job.imap_status == JobImapStatus.PENDING.value:
|
||||
result["skipped_not_final"] += 1
|
||||
continue
|
||||
result["eligible"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
if job.eml_local_path:
|
||||
path = Path(job.eml_local_path)
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
result["files_deleted"] += 1
|
||||
else:
|
||||
result["files_missing"] += 1
|
||||
job.eml_local_path = None
|
||||
job.eml_storage_key = None
|
||||
session.add(job)
|
||||
result["metadata_cleared"] += 1
|
||||
return result
|
||||
|
||||
|
||||
def _apply_report_detail_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]:
|
||||
result = {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0}
|
||||
policy_cache: dict[str, PrivacyRetentionPolicy] = {}
|
||||
versions = (
|
||||
session.query(CampaignVersion)
|
||||
.order_by(CampaignVersion.updated_at.asc())
|
||||
.all()
|
||||
)
|
||||
for version in versions:
|
||||
policy = _campaign_policy_for_id(session, version.campaign_id, policy_cache)
|
||||
cutoff = _cutoff(policy.stored_report_detail_retention_days, now=now)
|
||||
if not _is_before_cutoff(version.updated_at, cutoff):
|
||||
continue
|
||||
next_validation, validation_changed = _redacted_report_summary(version.validation_summary, now=now)
|
||||
next_build, build_changed = _redacted_report_summary(version.build_summary, now=now)
|
||||
if not validation_changed and not build_changed:
|
||||
if _summary_was_redacted(version.validation_summary) or _summary_was_redacted(version.build_summary):
|
||||
result["already_redacted"] += 1
|
||||
continue
|
||||
result["eligible_versions"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
version.validation_summary = next_validation
|
||||
version.build_summary = next_build
|
||||
session.add(version)
|
||||
result["summaries_redacted"] += int(validation_changed) + int(build_changed)
|
||||
return result
|
||||
|
||||
|
||||
def _summary_was_redacted(summary: Any) -> bool:
|
||||
return isinstance(summary, dict) and bool(summary.get("_retention", {}).get("report_detail_redacted"))
|
||||
|
||||
|
||||
def _redacted_report_summary(summary: Any, *, now: datetime) -> tuple[dict[str, Any] | None, bool]:
|
||||
if not isinstance(summary, dict):
|
||||
return summary, False
|
||||
if _summary_was_redacted(summary):
|
||||
return summary, False
|
||||
detail_keys = {"messages", "issues", "recent_failures", "jobs"}
|
||||
if not any(key in summary for key in detail_keys):
|
||||
return summary, False
|
||||
next_summary = copy.deepcopy(summary)
|
||||
for key in detail_keys:
|
||||
next_summary.pop(key, None)
|
||||
retention = dict(next_summary.get("_retention") or {})
|
||||
retention.update({"report_detail_redacted": True, "redacted_at": now.isoformat()})
|
||||
next_summary["_retention"] = retention
|
||||
return next_summary, True
|
||||
|
||||
|
||||
def _apply_mock_mailbox_retention(cutoff: datetime | None, *, dry_run: bool) -> dict[str, int]:
|
||||
result = {"eligible_records": 0, "json_deleted": 0, "eml_deleted": 0}
|
||||
if cutoff is None:
|
||||
return result
|
||||
message_dir = Path(settings.mock_mailbox_dir) / "messages"
|
||||
if not message_dir.exists():
|
||||
return result
|
||||
for path in message_dir.glob("*.json"):
|
||||
try:
|
||||
record = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
created_at = _parse_datetime(record.get("created_at"))
|
||||
if created_at and created_at >= cutoff:
|
||||
continue
|
||||
result["eligible_records"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
raw_filename = record.get("raw_filename")
|
||||
if raw_filename:
|
||||
raw_path = message_dir / str(raw_filename)
|
||||
if raw_path.exists():
|
||||
raw_path.unlink()
|
||||
result["eml_deleted"] += 1
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
result["json_deleted"] += 1
|
||||
return result
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed
|
||||
|
||||
|
||||
def _privacy_policy_for_audit_item(session: Session, item: AuditLog, campaign_cache: dict[str, PrivacyRetentionPolicy], tenant_cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy:
|
||||
if item.object_type == "campaign" and item.object_id:
|
||||
campaign = session.get(Campaign, str(item.object_id))
|
||||
if campaign is not None:
|
||||
return _campaign_policy_for_id(session, campaign.id, campaign_cache)
|
||||
if item.tenant_id:
|
||||
if item.tenant_id not in tenant_cache:
|
||||
tenant_cache[item.tenant_id] = effective_privacy_policy(session, tenant_id=item.tenant_id)
|
||||
return tenant_cache[item.tenant_id]
|
||||
return privacy_policy_from_session(session)
|
||||
|
||||
|
||||
def _apply_audit_detail_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]:
|
||||
result = {"eligible": 0, "redacted": 0, "already_redacted": 0}
|
||||
campaign_cache: dict[str, PrivacyRetentionPolicy] = {}
|
||||
tenant_cache: dict[str, PrivacyRetentionPolicy] = {}
|
||||
items = (
|
||||
session.query(AuditLog)
|
||||
.order_by(AuditLog.created_at.asc())
|
||||
.all()
|
||||
)
|
||||
for item in items:
|
||||
policy = _privacy_policy_for_audit_item(session, item, campaign_cache, tenant_cache)
|
||||
cutoff = _cutoff(policy.audit_detail_retention_days, now=now)
|
||||
if not _is_before_cutoff(item.created_at, cutoff):
|
||||
continue
|
||||
details = item.details if isinstance(item.details, dict) else {}
|
||||
if details.get("_retention", {}).get("audit_detail_redacted"):
|
||||
result["already_redacted"] += 1
|
||||
continue
|
||||
result["eligible"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
item.details = {
|
||||
"_retention": {
|
||||
"audit_detail_redacted": True,
|
||||
"redacted_at": now.isoformat(),
|
||||
"original_detail_sha256": _json_sha256(details),
|
||||
}
|
||||
}
|
||||
session.add(item)
|
||||
result["redacted"] += 1
|
||||
return result
|
||||
Reference in New Issue
Block a user