add-ideas/govoplan-core
1
0
Fork 0
Code Issues 45 Pull Requests Actions Packages Projects Releases Wiki Activity

Release v0.1.6

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-07 16:00:38 +02:00
parent a2053518d1
commit 150b720f12
149 changed files with 14311 additions and 8005 deletions
Download Patch File Download Diff File Expand all files Collapse all files

294
src/govoplan_core/privacy/retention.py
View File

@@ -1,6 +1,5 @@
from __future__ import annotations
import copy
import hashlib
import json
from datetime import datetime, timedelta, timezone
@@ -10,10 +9,20 @@ from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
from sqlalchemy.orm import Session
from govoplan_core.admin.governance import get_system_settings
from govoplan_core.core.optional import reraise_unless_missing_package
from govoplan_core.db.models import AuditLog, Group, SystemSettings, Tenant, User
from govoplan_core.admin.settings import get_system_settings
from govoplan_core.core.campaigns import (
CAPABILITY_CAMPAIGNS_POLICY_CONTEXT,
CAPABILITY_CAMPAIGNS_RETENTION,
CampaignPolicyContext,
CampaignPolicyContextProvider,
CampaignRetentionProvider,
)
from govoplan_core.core.runtime import get_registry
from govoplan_access.backend.db.models import Group, User
from govoplan_audit.backend.db.models import AuditLog
from govoplan_core.admin.models import SystemSettings
from govoplan_core.settings import settings
from govoplan_tenancy.backend.db.models import Tenant
PRIVACY_POLICY_SETTINGS_KEY = "privacy_retention_policy"
RETENTION_DAY_KEYS = (
@@ -48,21 +57,6 @@ def _normalize_allow_lower_level_limits(value: Any, *, fill_defaults: bool) -> d
normalized[clean_key] = bool(allowed)
return normalized
FINAL_VERSION_STATES = {
"completed",
"partially_completed",
"outcome_unknown",
"failed",
"cancelled",
"archived",
}
FINAL_EML_SEND_STATUSES = {
"smtp_accepted",
"sent",
"failed_permanent",
"cancelled",
"outcome_unknown",
}
AUDIT_DETAIL_REDACTION_KEYS = {
"attachments",
"bcc",
@@ -148,20 +142,34 @@ class PrivacyPolicyError(RuntimeError):
pass
def _campaign_models():
try:
from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus
except ModuleNotFoundError as exc:
reraise_unless_missing_package(exc, "govoplan_campaign")
raise PrivacyPolicyError("Campaign module is not installed") from exc
return Campaign, CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus
def _campaign_models_or_none():
try:
return _campaign_models()
except PrivacyPolicyError:
def _campaign_policy_provider() -> CampaignPolicyContextProvider | None:
registry = get_registry()
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT):
return None
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT)
if not isinstance(capability, CampaignPolicyContextProvider):
raise PrivacyPolicyError("Campaign policy context capability is invalid")
return capability
def _campaign_retention_provider() -> CampaignRetentionProvider | None:
registry = get_registry()
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_RETENTION):
return None
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_RETENTION)
if not isinstance(capability, CampaignRetentionProvider):
raise PrivacyPolicyError("Campaign retention capability is invalid")
return capability
def _campaign_policy_context(session: Session, *, campaign_id: str, tenant_id: str | None = None) -> CampaignPolicyContext:
provider = _campaign_policy_provider()
if provider is None:
raise PrivacyPolicyError("Campaign module is not installed")
context = provider.get_campaign_policy_context(session, tenant_id=tenant_id, campaign_id=campaign_id)
if context is None:
raise PrivacyPolicyError("Campaign not found for privacy policy")
return context
def _utcnow() -> datetime:
@@ -267,12 +275,9 @@ def effective_privacy_policy(
campaign_id: str | None = None,
) -> PrivacyRetentionPolicy:
policy = privacy_policy_from_session(session)
campaign: Any | None = None
campaign: CampaignPolicyContext | None = None
if campaign_id:
Campaign, _, _, _, _ = _campaign_models()
campaign = session.get(Campaign, campaign_id)
if campaign is None:
raise PrivacyPolicyError("Campaign not found for privacy policy")
campaign = _campaign_policy_context(session, campaign_id=campaign_id)
tenant_id = campaign.tenant_id
owner_user_id = campaign.owner_user_id
owner_group_id = campaign.owner_group_id
@@ -290,7 +295,7 @@ def effective_privacy_policy(
if group and (not tenant_id or group.tenant_id == tenant_id):
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group.settings or {}))
if campaign is not None:
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(campaign.settings or {}))
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(dict(campaign.settings or {})))
return policy
@@ -307,10 +312,7 @@ def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str,
return policy
if clean_scope != "campaign" or not scope_id:
return policy
Campaign, _, _, _, _ = _campaign_models()
campaign = session.get(Campaign, scope_id)
if campaign is None or campaign.tenant_id != tenant_id:
raise PrivacyPolicyError("Campaign not found for privacy policy")
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
if campaign.owner_user_id:
user = session.get(User, campaign.owner_user_id)
if user and user.tenant_id == tenant_id:
@@ -356,12 +358,9 @@ def effective_privacy_policy_sources(
) -> list[dict[str, Any]]:
system_settings = get_system_settings(session)
sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)]
campaign: Any | None = None
campaign: CampaignPolicyContext | None = None
if campaign_id:
Campaign, _, _, _, _ = _campaign_models()
campaign = session.get(Campaign, campaign_id)
if campaign is None:
raise PrivacyPolicyError("Campaign not found for privacy policy")
campaign = _campaign_policy_context(session, campaign_id=campaign_id)
tenant_id = campaign.tenant_id
owner_user_id = campaign.owner_user_id
owner_group_id = campaign.owner_group_id
@@ -379,7 +378,7 @@ def effective_privacy_policy_sources(
if group and (not tenant_id or group.tenant_id == tenant_id):
sources.append(_retention_policy_source_step("group", "Owner group", group.id, _privacy_policy_patch_from_settings(group.settings or {})))
if campaign is not None:
sources.append(_retention_policy_source_step("campaign", "Campaign", campaign.id, _privacy_policy_patch_from_settings(campaign.settings or {})))
sources.append(_retention_policy_source_step("campaign", "Campaign", campaign.id, _privacy_policy_patch_from_settings(dict(campaign.settings or {}))))
return sources
@@ -399,10 +398,7 @@ def parent_privacy_policy_sources(session: Session, *, tenant_id: str, scope_typ
return sources
if clean_scope != "campaign" or not scope_id:
return sources
Campaign, _, _, _, _ = _campaign_models()
campaign = session.get(Campaign, scope_id)
if campaign is None or campaign.tenant_id != tenant_id:
raise PrivacyPolicyError("Campaign not found for privacy policy")
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
if campaign.owner_user_id:
user = session.get(User, campaign.owner_user_id)
if user and user.tenant_id == tenant_id:
@@ -435,11 +431,8 @@ def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type
raise PrivacyPolicyError("Group privacy policy not found")
return _privacy_policy_patch_from_settings(group.settings or {})
if clean_scope == "campaign":
Campaign, _, _, _, _ = _campaign_models()
campaign = session.get(Campaign, scope_id)
if campaign is None or campaign.tenant_id != tenant_id:
raise PrivacyPolicyError("Campaign privacy policy not found")
return _privacy_policy_patch_from_settings(campaign.settings or {})
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
return _privacy_policy_patch_from_settings(dict(campaign.settings or {}))
raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign")
@@ -483,12 +476,15 @@ def set_privacy_policy_for_scope(
session.add(group)
return patch
if clean_scope == "campaign":
Campaign, _, _, _, _ = _campaign_models()
campaign = session.get(Campaign, scope_id)
if campaign is None or campaign.tenant_id != tenant_id:
raise PrivacyPolicyError("Campaign privacy policy not found")
campaign.settings = _set_settings_privacy_policy(campaign.settings, patch)
session.add(campaign)
provider = _campaign_policy_provider()
if provider is None:
raise PrivacyPolicyError("Campaign module is not installed")
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
settings_payload = _set_settings_privacy_policy(dict(campaign.settings or {}), patch)
try:
provider.set_campaign_settings(session, tenant_id=tenant_id, campaign_id=scope_id, settings=settings_payload)
except ValueError as exc:
raise PrivacyPolicyError("Campaign privacy policy not found") from exc
return patch
raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign")
@@ -540,10 +536,29 @@ def apply_retention_policy(session: Session, *, dry_run: bool = True) -> dict[st
now = _utcnow()
policy = privacy_policy_from_session(session)
cutoffs = _system_cutoffs(policy, now=now)
policy_cache: dict[str, PrivacyRetentionPolicy] = {}
campaign_retention = _campaign_retention_provider()
campaign_counts = {
"raw_campaign_json": {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0},
"generated_eml": {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0},
"stored_report_detail": {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0},
}
if campaign_retention is not None:
campaign_counts.update(
{
key: dict(value)
for key, value in campaign_retention.apply_retention(
session,
dry_run=dry_run,
now=now,
policy_for_campaign_id=lambda campaign_id: _campaign_policy_for_id(session, campaign_id, policy_cache),
).items()
}
)
counts = {
"raw_campaign_json": _apply_raw_json_retention(session, dry_run=dry_run, now=now),
"generated_eml": _apply_eml_retention(session, dry_run=dry_run, now=now),
"stored_report_detail": _apply_report_detail_retention(session, dry_run=dry_run, now=now),
"raw_campaign_json": campaign_counts["raw_campaign_json"],
"generated_eml": campaign_counts["generated_eml"],
"stored_report_detail": campaign_counts["stored_report_detail"],
"mock_mailbox": _apply_mock_mailbox_retention(cutoffs["mock_mailbox"], dry_run=dry_run),
"audit_detail": _apply_audit_detail_retention(session, dry_run=dry_run, now=now),
}
@@ -564,142 +579,6 @@ def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: di
return cache[campaign_id]
def _apply_raw_json_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]:
result = {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0}
models = _campaign_models_or_none()
if models is None:
return result
_, _, CampaignVersion, _, _ = models
policy_cache: dict[str, PrivacyRetentionPolicy] = {}
versions = (
session.query(CampaignVersion)
.order_by(CampaignVersion.updated_at.asc())
.all()
)
for version in versions:
policy = _campaign_policy_for_id(session, version.campaign_id, policy_cache)
cutoff = _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now)
if not _is_before_cutoff(version.updated_at, cutoff):
continue
raw_json = version.raw_json if isinstance(version.raw_json, dict) else {}
if raw_json.get("_retention", {}).get("raw_json_redacted"):
result["already_redacted"] += 1
continue
if version.workflow_state not in FINAL_VERSION_STATES:
result["skipped_not_final"] += 1
continue
result["eligible"] += 1
if dry_run:
continue
snapshot = version.execution_snapshot if isinstance(version.execution_snapshot, dict) else {}
version.raw_json = {
"version": version.schema_version or "1.0",
"_retention": {
"raw_json_redacted": True,
"redacted_at": now.isoformat(),
"original_sha256": snapshot.get("campaign_json_sha256") or _json_sha256(raw_json),
},
}
session.add(version)
result["redacted"] += 1
return result
def _apply_eml_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]:
result = {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0}
models = _campaign_models_or_none()
if models is None:
return result
_, CampaignJob, _, JobImapStatus, JobQueueStatus = models
policy_cache: dict[str, PrivacyRetentionPolicy] = {}
jobs = (
session.query(CampaignJob)
.filter((CampaignJob.eml_local_path.is_not(None)) | (CampaignJob.eml_storage_key.is_not(None)))
.order_by(CampaignJob.updated_at.asc())
.all()
)
for job in jobs:
policy = _campaign_policy_for_id(session, job.campaign_id, policy_cache)
cutoff = _cutoff(policy.generated_eml_retention_days, now=now)
if not _is_before_cutoff(job.updated_at, cutoff):
continue
if job.queue_status in {JobQueueStatus.QUEUED.value, JobQueueStatus.SENDING.value} or job.send_status not in FINAL_EML_SEND_STATUSES:
result["skipped_not_final"] += 1
continue
if job.imap_status == JobImapStatus.PENDING.value:
result["skipped_not_final"] += 1
continue
result["eligible"] += 1
if dry_run:
continue
if job.eml_local_path:
path = Path(job.eml_local_path)
if path.exists():
path.unlink()
result["files_deleted"] += 1
else:
result["files_missing"] += 1
job.eml_local_path = None
job.eml_storage_key = None
session.add(job)
result["metadata_cleared"] += 1
return result
def _apply_report_detail_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]:
result = {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0}
models = _campaign_models_or_none()
if models is None:
return result
_, _, CampaignVersion, _, _ = models
policy_cache: dict[str, PrivacyRetentionPolicy] = {}
versions = (
session.query(CampaignVersion)
.order_by(CampaignVersion.updated_at.asc())
.all()
)
for version in versions:
policy = _campaign_policy_for_id(session, version.campaign_id, policy_cache)
cutoff = _cutoff(policy.stored_report_detail_retention_days, now=now)
if not _is_before_cutoff(version.updated_at, cutoff):
continue
next_validation, validation_changed = _redacted_report_summary(version.validation_summary, now=now)
next_build, build_changed = _redacted_report_summary(version.build_summary, now=now)
if not validation_changed and not build_changed:
if _summary_was_redacted(version.validation_summary) or _summary_was_redacted(version.build_summary):
result["already_redacted"] += 1
continue
result["eligible_versions"] += 1
if dry_run:
continue
version.validation_summary = next_validation
version.build_summary = next_build
session.add(version)
result["summaries_redacted"] += int(validation_changed) + int(build_changed)
return result
def _summary_was_redacted(summary: Any) -> bool:
return isinstance(summary, dict) and bool(summary.get("_retention", {}).get("report_detail_redacted"))
def _redacted_report_summary(summary: Any, *, now: datetime) -> tuple[dict[str, Any] | None, bool]:
if not isinstance(summary, dict):
return summary, False
if _summary_was_redacted(summary):
return summary, False
detail_keys = {"messages", "issues", "recent_failures", "jobs"}
if not any(key in summary for key in detail_keys):
return summary, False
next_summary = copy.deepcopy(summary)
for key in detail_keys:
next_summary.pop(key, None)
retention = dict(next_summary.get("_retention") or {})
retention.update({"report_detail_redacted": True, "redacted_at": now.isoformat()})
next_summary["_retention"] = retention
return next_summary, True
def _apply_mock_mailbox_retention(cutoff: datetime | None, *, dry_run: bool) -> dict[str, int]:
result = {"eligible_records": 0, "json_deleted": 0, "eml_deleted": 0}
if cutoff is None:
@@ -744,12 +623,11 @@ def _parse_datetime(value: Any) -> datetime | None:
def _privacy_policy_for_audit_item(session: Session, item: AuditLog, campaign_cache: dict[str, PrivacyRetentionPolicy], tenant_cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy:
if item.object_type == "campaign" and item.object_id:
models = _campaign_models_or_none()
if models is not None:
Campaign, _, _, _, _ = models
campaign = session.get(Campaign, str(item.object_id))
if campaign is not None:
return _campaign_policy_for_id(session, campaign.id, campaign_cache)
provider = _campaign_policy_provider()
if provider is not None:
context = provider.get_campaign_policy_context(session, campaign_id=str(item.object_id))
if context is not None:
return _campaign_policy_for_id(session, context.id, campaign_cache)
if item.tenant_id:
if item.tenant_id not in tenant_cache:
tenant_cache[item.tenant_id] = effective_privacy_policy(session, tenant_id=item.tenant_id)