GovOPlaN/govoplan-policy
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: govern reporting privacy and retention

Browse Source
This commit is contained in:
zemion
2026-08-02 05:29:42 +02:00
parent 3aaa842ee6
commit 344e15dea4
6 changed files with 947 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+8
View File
@@ -36,3 +36,11 @@ actions across system, tenant, group, and user scopes. Templates cannot run or
be automated. Derived definitions retain ancestor ceilings, and every
decision includes the ordered Policy source path and effective limits so a UI
can explain why an action is available or blocked.
Cross-module reports use `policy.reporting_governance`. System policy defines
the export, retention, privacy-transform, and high re-identification-risk
ceiling; tenant policy may only tighten it, and malformed explicit policy fails
closed. The shared privacy-retention run calls the optional
`reporting.retention` capability to clear expired provider-report payloads
without importing Reporting models, while Reporting keeps hashes and bounded
provenance as audit evidence.
src/govoplan_policy/backend/manifest.py
+46 -4
View File
@@ -21,6 +21,7 @@ from govoplan_core.core.policy import (
CAPABILITY_POLICY_VIEW_GOVERNANCE,
)
from govoplan_core.core.modules import (
CapabilityDocumentation,
DocumentationTopic,
FrontendModule,
MigrationSpec,
@@ -28,6 +29,7 @@ from govoplan_core.core.modules import (
ModuleInterfaceProvider,
ModuleManifest,
)
from govoplan_core.core.reporting import CAPABILITY_POLICY_REPORTING_GOVERNANCE
from govoplan_core.core.provider_governance import declared_module_architecture
from govoplan_core.core.views import ViewSurface
from govoplan_core.db.base import Base
@@ -93,6 +95,15 @@ def _distribution_channel_policy(context: ModuleContext) -> object:
return DistributionChannelPolicyProvider()
def _reporting_governance_policy(context: ModuleContext) -> object:
del context
from govoplan_policy.backend.reporting_governance import (
ReportingGovernancePolicyProvider,
)
return ReportingGovernancePolicyProvider()
manifest = ModuleManifest(
id="policy",
name="Policy",
@@ -118,6 +129,10 @@ manifest = ModuleManifest(
name=CAPABILITY_POLICY_DISTRIBUTION_CHANNELS,
version="1.0.0",
),
ModuleInterfaceProvider(
name=CAPABILITY_POLICY_REPORTING_GOVERNANCE,
version="1.0.0",
),
),
route_factory=_route_factory,
documentation=(
@@ -134,10 +149,18 @@ manifest = ModuleManifest(
id="policy.hierarchy-overrides-and-retention",
title="Administer policy hierarchy and overrides",
summary="Policy evaluates versioned system, tenant, group, and user rules for retention and module-owned governed actions.",
body="Administrators can simulate effective retention before saving a lower-level change. Explicit overrides retain source and provenance information and are evaluated through typed capabilities for definitions, Views, function assignments, distribution channels, scheduling privacy, and retention. Templates and inherited definitions keep their upstream ceilings when reused or derived.",
body="Administrators can simulate effective retention before saving a lower-level change. Explicit overrides retain source and provenance information and are evaluated through typed capabilities for definitions, Views, function assignments, distribution channels, scheduling privacy, cross-module reporting, and retention. Templates and inherited definitions keep their upstream ceilings when reused or derived.",
documentation_types=("admin",),
audience=("policy_admin", "tenant_admin", "system_admin"),
related_modules=("dataflow", "workflow_engine", "views", "idm", "dist_lists", "scheduling"),
related_modules=(
"dataflow",
"workflow_engine",
"views",
"idm",
"dist_lists",
"scheduling",
"reporting",
),
metadata={"kind": "reference"},
),
),
@@ -202,17 +225,36 @@ manifest = ModuleManifest(
_function_assignment_governance_policy
),
CAPABILITY_POLICY_DISTRIBUTION_CHANNELS: _distribution_channel_policy,
CAPABILITY_POLICY_REPORTING_GOVERNANCE: _reporting_governance_policy,
CAPABILITY_POLICY_PRIVACY_RETENTION: _privacy_retention_service,
CAPABILITY_POLICY_SCHEDULING_PARTICIPANT_PRIVACY: _scheduling_participant_privacy_policy,
},
capability_documentation={
CAPABILITY_POLICY_REPORTING_GOVERNANCE: CapabilityDocumentation(
label="Reporting privacy governance",
summary=(
"Tightens report minimization, retention, export formats, and "
"re-identification-risk decisions across system and tenant scopes."
),
contract_version="1.0",
documentation_types=("admin",),
audience=("policy_admin", "privacy_officer", "system_admin"),
),
},
architecture=declared_module_architecture(
layer="governance_accountability",
kind="governance",
maturity="vertical_slice",
documentation_ref="docs/POLICY_DECISION_PROVENANCE.md",
test_ref="tests/test_policy_hierarchy.py",
known_limits=("The current policy families do not yet form a universal expression or enforcement engine.",),
owned_concepts=("policy definition", "policy override", "policy decision provenance"),
known_limits=(
"The current policy families do not yet form a universal expression or enforcement engine.",
),
owned_concepts=(
"policy definition",
"policy override",
"policy decision provenance",
),
non_owned_concepts=("application permission", "domain record", "audit record"),
recovery_docs=("docs/POLICY_DECISION_PROVENANCE.md",),
security_docs=("docs/POLICY_DECISION_PROVENANCE.md",),
src/govoplan_policy/backend/reporting_governance.py
+263
View File
@@ -0,0 +1,263 @@
"""Hierarchical privacy and export policy for cross-module reports."""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from sqlalchemy.orm import Session
from govoplan_core.admin.settings import get_system_settings
from govoplan_core.core.reporting import (
ReportingGovernanceDecision,
ReportingGovernanceRequest,
)
from govoplan_core.tenancy.scope import Tenant
from govoplan_policy.backend.retention import effective_privacy_policy
REPORTING_POLICY_SETTINGS_KEY = "reporting_governance_policy"
_ALLOWED_KEYS = {
"allow_exports",
"allowed_export_formats",
"allow_high_reidentification_risk",
"max_retention_days",
"required_privacy_transforms",
}
@dataclass(frozen=True, slots=True)
class _Policy:
allow_exports: bool = True
allowed_export_formats: tuple[str, ...] = ("json", "csv")
allow_high_reidentification_risk: bool = False
max_retention_days: int | None = 30
required_privacy_transforms: tuple[str, ...] = ()
class ReportingGovernancePolicyProvider:
def decide_reporting_action(
self,
session: object,
principal: object,
*,
request: ReportingGovernanceRequest,
) -> ReportingGovernanceDecision:
if not isinstance(session, Session):
raise TypeError("Reporting governance requires a SQLAlchemy Session")
try:
policy, sources = _effective_policy(
session,
tenant_id=request.tenant_id,
)
retention = effective_privacy_policy(
session,
tenant_id=request.tenant_id,
).stored_report_detail_retention_days
except (LookupError, TypeError, ValueError) as exc:
return ReportingGovernanceDecision(
allowed=False,
reason="Reporting governance Policy is malformed or unavailable.",
retention_days=0,
export_formats=(),
required_privacy_transforms=request.declared_privacy_transforms,
provenance={
"provider": "policy.reporting_governance",
"version": "1",
"decision": "fail_closed",
"error_type": type(exc).__name__,
},
)
del principal
retention_days = _minimum_optional(
policy.max_retention_days,
retention,
)
allowed = True
reason = None
if (
request.reidentification_risk == "high"
and not policy.allow_high_reidentification_risk
):
allowed = False
reason = "Policy blocks reports with high re-identification risk."
if request.action == "export":
if not policy.allow_exports:
allowed = False
reason = "Policy disables cross-module report exports."
elif request.export_format not in policy.allowed_export_formats:
allowed = False
reason = "Policy does not allow the requested report export format."
required = tuple(
sorted(
set(request.declared_privacy_transforms)
| set(policy.required_privacy_transforms)
)
)
return ReportingGovernanceDecision(
allowed=allowed,
reason=reason,
retention_days=retention_days,
export_formats=(
policy.allowed_export_formats if policy.allow_exports else ()
),
required_privacy_transforms=required,
provenance={
"provider": "policy.reporting_governance",
"version": "1",
"sources": sources,
"effective": {
"allow_exports": policy.allow_exports,
"allowed_export_formats": list(policy.allowed_export_formats),
"allow_high_reidentification_risk": (
policy.allow_high_reidentification_risk
),
"max_retention_days": policy.max_retention_days,
"privacy_retention_days": retention,
"required_privacy_transforms": list(required),
},
},
)
def _effective_policy(
session: Session,
*,
tenant_id: str,
) -> tuple[_Policy, list[dict[str, object]]]:
system_settings = get_system_settings(session).settings or {}
system_patch = _patch(system_settings, "system")
policy = _apply_patch(_Policy(), system_patch)
sources = [
{
"scope_type": "system",
"scope_id": None,
"applied_fields": sorted(system_patch),
}
]
tenant = session.get(Tenant, tenant_id)
if tenant is None:
raise LookupError("Tenant not found for Reporting Policy")
tenant_patch = _patch(tenant.settings or {}, "tenant")
policy = _tighten(policy, tenant_patch)
sources.append(
{
"scope_type": "tenant",
"scope_id": tenant_id,
"applied_fields": sorted(tenant_patch),
}
)
return policy, sources
def _patch(settings: Mapping[str, object], source: str) -> dict[str, object]:
raw = settings.get(REPORTING_POLICY_SETTINGS_KEY)
if raw in (None, ""):
return {}
if not isinstance(raw, Mapping):
raise ValueError(f"{source} Reporting Policy must be an object")
unknown = set(raw) - _ALLOWED_KEYS
if unknown:
raise ValueError(
f"{source} Reporting Policy has unknown fields: "
+ ", ".join(sorted(str(item) for item in unknown))
)
return dict(raw)
def _apply_patch(parent: _Policy, patch: Mapping[str, object]) -> _Policy:
return _Policy(
allow_exports=_boolean(patch, "allow_exports", parent.allow_exports),
allowed_export_formats=_formats(
patch.get("allowed_export_formats"),
default=parent.allowed_export_formats,
),
allow_high_reidentification_risk=_boolean(
patch,
"allow_high_reidentification_risk",
parent.allow_high_reidentification_risk,
),
max_retention_days=_days(
patch.get("max_retention_days"),
default=parent.max_retention_days,
),
required_privacy_transforms=_transforms(
patch.get("required_privacy_transforms"),
default=parent.required_privacy_transforms,
),
)
def _tighten(parent: _Policy, patch: Mapping[str, object]) -> _Policy:
child = _apply_patch(parent, patch)
return _Policy(
allow_exports=parent.allow_exports and child.allow_exports,
allowed_export_formats=tuple(
item
for item in parent.allowed_export_formats
if item in child.allowed_export_formats
),
allow_high_reidentification_risk=(
parent.allow_high_reidentification_risk
and child.allow_high_reidentification_risk
),
max_retention_days=_minimum_optional(
parent.max_retention_days,
child.max_retention_days,
),
required_privacy_transforms=tuple(
sorted(
set(parent.required_privacy_transforms)
| set(child.required_privacy_transforms)
)
),
)
def _boolean(
value: Mapping[str, object],
key: str,
default: bool,
) -> bool:
raw = value.get(key)
if raw is None:
return default
if not isinstance(raw, bool):
raise ValueError(f"Reporting Policy {key} must be boolean")
return raw
def _formats(value: object, *, default: tuple[str, ...]) -> tuple[str, ...]:
if value is None:
return default
if not isinstance(value, list) or any(
item not in {"json", "csv"} for item in value
):
raise ValueError("Reporting Policy export formats must contain json or csv")
return tuple(dict.fromkeys(str(item) for item in value))
def _transforms(value: object, *, default: tuple[str, ...]) -> tuple[str, ...]:
if value is None:
return default
if not isinstance(value, list) or any(
not isinstance(item, str) or not item.strip() for item in value
):
raise ValueError("Reporting Policy privacy transforms must be strings")
return tuple(sorted(set(item.strip() for item in value)))
def _days(value: object, *, default: int | None) -> int | None:
if value is None:
return default
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise ValueError("Reporting Policy retention days must be a positive integer")
return value
def _minimum_optional(left: int | None, right: int | None) -> int | None:
values = [item for item in (left, right) if item is not None]
return min(values) if values else None
__all__ = ["ReportingGovernancePolicyProvider"]
src/govoplan_policy/backend/retention.py
+490 -117
View File
@@ -24,6 +24,10 @@ from govoplan_core.core.access import (
AuditRetentionProvider,
)
from govoplan_core.core.policy import PolicySourceStep, policy_source_step
from govoplan_core.core.reporting import (
CAPABILITY_REPORTING_RETENTION,
ReportingRetentionProvider,
)
from govoplan_core.core.runtime import get_registry
from govoplan_core.admin.models import SystemSettings
from govoplan_core.privacy.schemas import (
@@ -110,7 +114,11 @@ class PrivacyPolicyError(RuntimeError):
def _campaign_policy_provider() -> CampaignPolicyContextProvider | None:
registry = get_registry()
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT):
if (
registry is None
or not hasattr(registry, "has_capability")
or not registry.has_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT)
):
return None
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT)
if not isinstance(capability, CampaignPolicyContextProvider):
@@ -120,7 +128,11 @@ def _campaign_policy_provider() -> CampaignPolicyContextProvider | None:
def _campaign_retention_provider() -> CampaignRetentionProvider | None:
registry = get_registry()
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_RETENTION):
if (
registry is None
or not hasattr(registry, "has_capability")
or not registry.has_capability(CAPABILITY_CAMPAIGNS_RETENTION)
):
return None
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_RETENTION)
if not isinstance(capability, CampaignRetentionProvider):
@@ -130,7 +142,11 @@ def _campaign_retention_provider() -> CampaignRetentionProvider | None:
def _access_administration() -> AccessAdministration | None:
registry = get_registry()
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_ACCESS_ADMINISTRATION):
if (
registry is None
or not hasattr(registry, "has_capability")
or not registry.has_capability(CAPABILITY_ACCESS_ADMINISTRATION)
):
return None
capability = registry.require_capability(CAPABILITY_ACCESS_ADMINISTRATION)
if not isinstance(capability, AccessAdministration):
@@ -140,7 +156,11 @@ def _access_administration() -> AccessAdministration | None:
def _audit_retention_provider() -> AuditRetentionProvider | None:
registry = get_registry()
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_AUDIT_RETENTION):
if (
registry is None
or not hasattr(registry, "has_capability")
or not registry.has_capability(CAPABILITY_AUDIT_RETENTION)
):
return None
capability = registry.require_capability(CAPABILITY_AUDIT_RETENTION)
@@ -149,7 +169,23 @@ def _audit_retention_provider() -> AuditRetentionProvider | None:
return capability
def _user_settings(session: Session, *, user_id: str, tenant_id: str) -> dict[str, Any] | None:
def _reporting_retention_provider() -> ReportingRetentionProvider | None:
registry = get_registry()
if (
registry is None
or not hasattr(registry, "has_capability")
or not registry.has_capability(CAPABILITY_REPORTING_RETENTION)
):
return None
capability = registry.require_capability(CAPABILITY_REPORTING_RETENTION)
if not isinstance(capability, ReportingRetentionProvider):
raise PrivacyPolicyError("Reporting retention capability is invalid")
return capability
def _user_settings(
session: Session, *, user_id: str, tenant_id: str
) -> dict[str, Any] | None:
provider = _access_administration()
if provider is None:
return None
@@ -157,15 +193,21 @@ def _user_settings(session: Session, *, user_id: str, tenant_id: str) -> dict[st
return dict(payload) if payload is not None else None
def _set_user_settings(session: Session, *, user_id: str, tenant_id: str, settings_payload: dict[str, Any]) -> dict[str, Any] | None:
def _set_user_settings(
session: Session, *, user_id: str, tenant_id: str, settings_payload: dict[str, Any]
) -> dict[str, Any] | None:
provider = _access_administration()
if provider is None:
return None
payload = provider.set_user_settings(session, user_id, tenant_id=tenant_id, settings=settings_payload)
payload = provider.set_user_settings(
session, user_id, tenant_id=tenant_id, settings=settings_payload
)
return dict(payload) if payload is not None else None
def _group_settings(session: Session, *, group_id: str, tenant_id: str) -> dict[str, Any] | None:
def _group_settings(
session: Session, *, group_id: str, tenant_id: str
) -> dict[str, Any] | None:
provider = _access_administration()
if provider is None:
return None
@@ -173,19 +215,27 @@ def _group_settings(session: Session, *, group_id: str, tenant_id: str) -> dict[
return dict(payload) if payload is not None else None
def _set_group_settings(session: Session, *, group_id: str, tenant_id: str, settings_payload: dict[str, Any]) -> dict[str, Any] | None:
def _set_group_settings(
session: Session, *, group_id: str, tenant_id: str, settings_payload: dict[str, Any]
) -> dict[str, Any] | None:
provider = _access_administration()
if provider is None:
return None
payload = provider.set_group_settings(session, group_id, tenant_id=tenant_id, settings=settings_payload)
payload = provider.set_group_settings(
session, group_id, tenant_id=tenant_id, settings=settings_payload
)
return dict(payload) if payload is not None else None
def _campaign_policy_context(session: Session, *, campaign_id: str, tenant_id: str | None = None) -> CampaignPolicyContext:
def _campaign_policy_context(
session: Session, *, campaign_id: str, tenant_id: str | None = None
) -> CampaignPolicyContext:
provider = _campaign_policy_provider()
if provider is None:
raise PrivacyPolicyError("Campaign module is not installed")
context = provider.get_campaign_policy_context(session, tenant_id=tenant_id, campaign_id=campaign_id)
context = provider.get_campaign_policy_context(
session, tenant_id=tenant_id, campaign_id=campaign_id
)
if context is None:
raise PrivacyPolicyError("Campaign not found for privacy policy")
return context
@@ -208,17 +258,29 @@ def _privacy_policy_data(settings_payload: dict[str, Any] | None) -> dict[str, A
return data if isinstance(data, dict) else {}
def _privacy_policy_patch_from_settings(settings_payload: dict[str, Any] | None) -> dict[str, Any]:
def _privacy_policy_patch_from_settings(
settings_payload: dict[str, Any] | None,
) -> dict[str, Any]:
data = _privacy_policy_data(settings_payload)
if not data:
return {}
return PrivacyRetentionPolicyPatch.model_validate(data).model_dump(mode="json", exclude_none=True)
return PrivacyRetentionPolicyPatch.model_validate(data).model_dump(
mode="json", exclude_none=True
)
def _merge_privacy_policy(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> PrivacyRetentionPolicy:
def _merge_privacy_policy(
parent: PrivacyRetentionPolicy, patch: dict[str, Any]
) -> PrivacyRetentionPolicy:
payload = parent.model_dump(mode="json")
parent_allow = {**default_allow_lower_level_limits(), **(payload.get("allow_lower_level_limits") or {})}
if parent_allow["store_raw_campaign_json"] and patch.get("store_raw_campaign_json") is False:
parent_allow = {
**default_allow_lower_level_limits(),
**(payload.get("allow_lower_level_limits") or {}),
}
if (
parent_allow["store_raw_campaign_json"]
and patch.get("store_raw_campaign_json") is False
):
payload["store_raw_campaign_json"] = False
for key in RETENTION_DAY_KEYS:
value = patch.get(key)
@@ -227,7 +289,12 @@ def _merge_privacy_policy(parent: PrivacyRetentionPolicy, patch: dict[str, Any])
current = payload.get(key)
payload[key] = int(value) if current is None else min(int(current), int(value))
detail_level = patch.get("audit_detail_level")
if parent_allow["audit_detail_level"] and detail_level and AUDIT_DETAIL_LEVEL_ORDER[detail_level] > AUDIT_DETAIL_LEVEL_ORDER[payload["audit_detail_level"]]:
if (
parent_allow["audit_detail_level"]
and detail_level
and AUDIT_DETAIL_LEVEL_ORDER[detail_level]
> AUDIT_DETAIL_LEVEL_ORDER[payload["audit_detail_level"]]
):
payload["audit_detail_level"] = detail_level
patch_allow = patch.get("allow_lower_level_limits") or {}
@@ -239,64 +306,94 @@ def _merge_privacy_policy(parent: PrivacyRetentionPolicy, patch: dict[str, Any])
def _parent_allow_lower_level_limits(parent_payload: dict[str, Any]) -> dict[str, bool]:
return {**default_allow_lower_level_limits(), **(parent_payload.get("allow_lower_level_limits") or {})}
return {
**default_allow_lower_level_limits(),
**(parent_payload.get("allow_lower_level_limits") or {}),
}
def _privacy_restriction_rules() -> tuple[PolicyRestrictionRule, ...]:
return (
PolicyRestrictionRule(
field="store_raw_campaign_json",
is_more_restrictive_or_equal=lambda parent_value, requested_value: not (requested_value is True and parent_value is False),
is_more_restrictive_or_equal=lambda parent_value, requested_value: (
not (requested_value is True and parent_value is False)
),
less_restrictive_message="Raw campaign JSON storage cannot be re-enabled below a parent policy that disables it.",
),
*(
PolicyRestrictionRule(
field=key,
is_more_restrictive_or_equal=lambda parent_value, requested_value: parent_value is None or int(requested_value) <= int(parent_value),
is_more_restrictive_or_equal=lambda parent_value, requested_value: (
parent_value is None or int(requested_value) <= int(parent_value)
),
less_restrictive_message=f"{key} cannot be less restrictive than the parent retention policy.",
)
for key in RETENTION_DAY_KEYS
),
PolicyRestrictionRule(
field="audit_detail_level",
is_more_restrictive_or_equal=lambda parent_value, requested_value: AUDIT_DETAIL_LEVEL_ORDER[str(requested_value)] >= AUDIT_DETAIL_LEVEL_ORDER[str(parent_value or "full")],
is_more_restrictive_or_equal=lambda parent_value, requested_value: (
AUDIT_DETAIL_LEVEL_ORDER[str(requested_value)]
>= AUDIT_DETAIL_LEVEL_ORDER[str(parent_value or "full")]
),
less_restrictive_message="Audit detail level cannot be less restrictive than the parent retention policy.",
),
)
def _validate_privacy_patch_against_parent(parent: PrivacyRetentionPolicy, patch: dict[str, Any]) -> None:
def _validate_privacy_patch_against_parent(
parent: PrivacyRetentionPolicy, patch: dict[str, Any]
) -> None:
parent_payload = parent.model_dump(mode="json")
issues = validate_hierarchical_policy_patch(
parent_policy=parent_payload,
patch=patch,
field_keys=RETENTION_POLICY_FIELD_KEYS,
parent_allow_lower_level_limits=_parent_allow_lower_level_limits(parent_payload),
parent_allow_lower_level_limits=_parent_allow_lower_level_limits(
parent_payload
),
restriction_rules=_privacy_restriction_rules(),
locked_field_message=lambda key: f"{key} is locked by the parent retention policy.",
relock_message=lambda key: f"{key} limiting cannot be re-enabled below a parent retention policy lock.",
locked_field_message=lambda key: (
f"{key} is locked by the parent retention policy."
),
relock_message=lambda key: (
f"{key} limiting cannot be re-enabled below a parent retention policy lock."
),
)
if issues:
raise PrivacyPolicyError(issues[0].message)
def _set_settings_privacy_policy(settings_payload: dict[str, Any] | None, policy: dict[str, Any]) -> dict[str, Any]:
def _set_settings_privacy_policy(
settings_payload: dict[str, Any] | None, policy: dict[str, Any]
) -> dict[str, Any]:
payload = dict(settings_payload or {})
payload[PRIVACY_POLICY_SETTINGS_KEY] = policy
return payload
def privacy_policy_from_settings(item: SystemSettings) -> PrivacyRetentionPolicy:
return PrivacyRetentionPolicy.model_validate(_privacy_policy_data(item.settings or {}))
return PrivacyRetentionPolicy.model_validate(
_privacy_policy_data(item.settings or {})
)
def privacy_policy_from_session(session: Session) -> PrivacyRetentionPolicy:
return privacy_policy_from_settings(get_system_settings(session))
def set_privacy_policy(item: SystemSettings, policy: PrivacyRetentionPolicy | dict[str, Any]) -> PrivacyRetentionPolicy:
validated = policy if isinstance(policy, PrivacyRetentionPolicy) else PrivacyRetentionPolicy.model_validate(policy)
item.settings = _set_settings_privacy_policy(item.settings, validated.model_dump(mode="json"))
def set_privacy_policy(
item: SystemSettings, policy: PrivacyRetentionPolicy | dict[str, Any]
) -> PrivacyRetentionPolicy:
validated = (
policy
if isinstance(policy, PrivacyRetentionPolicy)
else PrivacyRetentionPolicy.model_validate(policy)
)
item.settings = _set_settings_privacy_policy(
item.settings, validated.model_dump(mode="json")
)
return validated
@@ -319,21 +416,39 @@ def effective_privacy_policy(
tenant = session.get(Tenant, tenant_id)
if tenant is None:
raise PrivacyPolicyError("Tenant not found for privacy policy")
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {}))
policy = _merge_privacy_policy(
policy, _privacy_policy_patch_from_settings(tenant.settings or {})
)
if owner_user_id:
user_settings = _user_settings(session, user_id=owner_user_id, tenant_id=tenant_id) if tenant_id else None
user_settings = (
_user_settings(session, user_id=owner_user_id, tenant_id=tenant_id)
if tenant_id
else None
)
if user_settings is not None:
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user_settings))
policy = _merge_privacy_policy(
policy, _privacy_policy_patch_from_settings(user_settings)
)
if owner_group_id:
group_settings = _group_settings(session, group_id=owner_group_id, tenant_id=tenant_id) if tenant_id else None
group_settings = (
_group_settings(session, group_id=owner_group_id, tenant_id=tenant_id)
if tenant_id
else None
)
if group_settings is not None:
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group_settings))
policy = _merge_privacy_policy(
policy, _privacy_policy_patch_from_settings(group_settings)
)
if campaign is not None:
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(dict(campaign.settings or {})))
policy = _merge_privacy_policy(
policy, _privacy_policy_patch_from_settings(dict(campaign.settings or {}))
)
return policy
def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> PrivacyRetentionPolicy:
def parent_privacy_policy(
session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None
) -> PrivacyRetentionPolicy:
clean_scope = scope_type.strip().casefold()
policy = privacy_policy_from_session(session)
if clean_scope == "tenant":
@@ -341,38 +456,65 @@ def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str,
tenant = session.get(Tenant, tenant_id)
if tenant is None:
raise PrivacyPolicyError("Tenant not found for privacy policy")
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(tenant.settings or {}))
policy = _merge_privacy_policy(
policy, _privacy_policy_patch_from_settings(tenant.settings or {})
)
if clean_scope in {"user", "group"}:
return policy
if clean_scope != "campaign" or not scope_id:
return policy
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
campaign = _campaign_policy_context(
session, tenant_id=tenant_id, campaign_id=scope_id
)
if campaign.owner_user_id:
user_settings = _user_settings(session, user_id=campaign.owner_user_id, tenant_id=tenant_id)
user_settings = _user_settings(
session, user_id=campaign.owner_user_id, tenant_id=tenant_id
)
if user_settings is not None:
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(user_settings))
policy = _merge_privacy_policy(
policy, _privacy_policy_patch_from_settings(user_settings)
)
if campaign.owner_group_id:
group_settings = _group_settings(session, group_id=campaign.owner_group_id, tenant_id=tenant_id)
group_settings = _group_settings(
session, group_id=campaign.owner_group_id, tenant_id=tenant_id
)
if group_settings is not None:
policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group_settings))
policy = _merge_privacy_policy(
policy, _privacy_policy_patch_from_settings(group_settings)
)
return policy
def _retention_policy_source_fields(patch: dict[str, Any], *, baseline: bool = False) -> list[str]:
def _retention_policy_source_fields(
patch: dict[str, Any], *, baseline: bool = False
) -> list[str]:
fields: list[str] = []
for key in RETENTION_POLICY_FIELD_KEYS:
if key in patch and patch.get(key) is not None:
fields.append(key)
if isinstance(patch.get("allow_lower_level_limits"), dict) and patch["allow_lower_level_limits"]:
if (
isinstance(patch.get("allow_lower_level_limits"), dict)
and patch["allow_lower_level_limits"]
):
fields.append("allow_lower_level_limits")
if baseline and not fields:
fields.append("defaults")
return fields
def _retention_policy_source_step(scope_type: str, label: str, scope_id: str | None, patch: dict[str, Any], *, baseline: bool = False) -> dict[str, Any]:
source_policy = PrivacyRetentionPolicy.model_validate(patch).model_dump(mode="json") if baseline else dict(patch)
def _retention_policy_source_step(
scope_type: str,
label: str,
scope_id: str | None,
patch: dict[str, Any],
*,
baseline: bool = False,
) -> dict[str, Any]:
source_policy = (
PrivacyRetentionPolicy.model_validate(patch).model_dump(mode="json")
if baseline
else dict(patch)
)
return policy_source_step(
scope_type,
label,
@@ -391,7 +533,15 @@ def effective_privacy_policy_sources(
campaign_id: str | None = None,
) -> list[dict[str, Any]]:
system_settings = get_system_settings(session)
sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)]
sources = [
_retention_policy_source_step(
"system",
"System",
None,
_privacy_policy_patch_from_settings(system_settings.settings or {}),
baseline=True,
)
]
campaign: CampaignPolicyContext | None = None
if campaign_id:
campaign = _campaign_policy_context(session, campaign_id=campaign_id)
@@ -402,48 +552,124 @@ def effective_privacy_policy_sources(
tenant = session.get(Tenant, tenant_id)
if tenant is None:
raise PrivacyPolicyError("Tenant not found for privacy policy")
sources.append(_retention_policy_source_step("tenant", "Tenant", tenant.id, _privacy_policy_patch_from_settings(tenant.settings or {})))
sources.append(
_retention_policy_source_step(
"tenant",
"Tenant",
tenant.id,
_privacy_policy_patch_from_settings(tenant.settings or {}),
)
)
if owner_user_id:
user_settings = _user_settings(session, user_id=owner_user_id, tenant_id=tenant_id) if tenant_id else None
user_settings = (
_user_settings(session, user_id=owner_user_id, tenant_id=tenant_id)
if tenant_id
else None
)
if user_settings is not None:
sources.append(_retention_policy_source_step("user", "Owner user", owner_user_id, _privacy_policy_patch_from_settings(user_settings)))
sources.append(
_retention_policy_source_step(
"user",
"Owner user",
owner_user_id,
_privacy_policy_patch_from_settings(user_settings),
)
)
if owner_group_id:
group_settings = _group_settings(session, group_id=owner_group_id, tenant_id=tenant_id) if tenant_id else None
group_settings = (
_group_settings(session, group_id=owner_group_id, tenant_id=tenant_id)
if tenant_id
else None
)
if group_settings is not None:
sources.append(_retention_policy_source_step("group", "Owner group", owner_group_id, _privacy_policy_patch_from_settings(group_settings)))
sources.append(
_retention_policy_source_step(
"group",
"Owner group",
owner_group_id,
_privacy_policy_patch_from_settings(group_settings),
)
)
if campaign is not None:
sources.append(_retention_policy_source_step("campaign", "Campaign", campaign.id, _privacy_policy_patch_from_settings(dict(campaign.settings or {}))))
sources.append(
_retention_policy_source_step(
"campaign",
"Campaign",
campaign.id,
_privacy_policy_patch_from_settings(dict(campaign.settings or {})),
)
)
return sources
def parent_privacy_policy_sources(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> list[dict[str, Any]]:
def parent_privacy_policy_sources(
session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None
) -> list[dict[str, Any]]:
clean_scope = scope_type.strip().casefold()
if clean_scope == "system":
return []
system_settings = get_system_settings(session)
sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)]
sources = [
_retention_policy_source_step(
"system",
"System",
None,
_privacy_policy_patch_from_settings(system_settings.settings or {}),
baseline=True,
)
]
if clean_scope == "tenant":
return sources
tenant = session.get(Tenant, tenant_id)
if tenant is None:
raise PrivacyPolicyError("Tenant not found for privacy policy")
sources.append(_retention_policy_source_step("tenant", "Tenant", tenant.id, _privacy_policy_patch_from_settings(tenant.settings or {})))
sources.append(
_retention_policy_source_step(
"tenant",
"Tenant",
tenant.id,
_privacy_policy_patch_from_settings(tenant.settings or {}),
)
)
if clean_scope in {"user", "group"}:
return sources
if clean_scope != "campaign" or not scope_id:
return sources
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
campaign = _campaign_policy_context(
session, tenant_id=tenant_id, campaign_id=scope_id
)
if campaign.owner_user_id:
user_settings = _user_settings(session, user_id=campaign.owner_user_id, tenant_id=tenant_id)
user_settings = _user_settings(
session, user_id=campaign.owner_user_id, tenant_id=tenant_id
)
if user_settings is not None:
sources.append(_retention_policy_source_step("user", "Owner user", campaign.owner_user_id, _privacy_policy_patch_from_settings(user_settings)))
sources.append(
_retention_policy_source_step(
"user",
"Owner user",
campaign.owner_user_id,
_privacy_policy_patch_from_settings(user_settings),
)
)
if campaign.owner_group_id:
group_settings = _group_settings(session, group_id=campaign.owner_group_id, tenant_id=tenant_id)
group_settings = _group_settings(
session, group_id=campaign.owner_group_id, tenant_id=tenant_id
)
if group_settings is not None:
sources.append(_retention_policy_source_step("group", "Owner group", campaign.owner_group_id, _privacy_policy_patch_from_settings(group_settings)))
sources.append(
_retention_policy_source_step(
"group",
"Owner group",
campaign.owner_group_id,
_privacy_policy_patch_from_settings(group_settings),
)
)
return sources
def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None) -> dict[str, Any]:
def get_privacy_policy_for_scope(
session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None = None
) -> dict[str, Any]:
clean_scope = scope_type.strip().casefold()
if clean_scope == "system":
return privacy_policy_from_session(session).model_dump(mode="json")
@@ -453,21 +679,29 @@ def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type
raise PrivacyPolicyError("Tenant privacy policy not found")
return _privacy_policy_patch_from_settings(tenant.settings or {})
if not scope_id:
raise PrivacyPolicyError(f"{clean_scope.capitalize()} privacy policy requires scope_id")
raise PrivacyPolicyError(
f"{clean_scope.capitalize()} privacy policy requires scope_id"
)
if clean_scope == "user":
user_settings = _user_settings(session, user_id=scope_id, tenant_id=tenant_id)
if user_settings is None:
raise PrivacyPolicyError("User privacy policy not found")
return _privacy_policy_patch_from_settings(user_settings)
if clean_scope == "group":
group_settings = _group_settings(session, group_id=scope_id, tenant_id=tenant_id)
group_settings = _group_settings(
session, group_id=scope_id, tenant_id=tenant_id
)
if group_settings is None:
raise PrivacyPolicyError("Group privacy policy not found")
return _privacy_policy_patch_from_settings(group_settings)
if clean_scope == "campaign":
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id)
campaign = _campaign_policy_context(
session, tenant_id=tenant_id, campaign_id=scope_id
)
return _privacy_policy_patch_from_settings(dict(campaign.settings or {}))
raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign")
raise PrivacyPolicyError(
"Privacy policy scope must be system, tenant, user, group or campaign"
)
def set_privacy_policy_for_scope(
@@ -498,15 +732,21 @@ def set_privacy_policy_for_scope(
)
def _set_system_privacy_policy(session: Session, policy: dict[str, Any] | None) -> dict[str, Any]:
def _set_system_privacy_policy(
session: Session, policy: dict[str, Any] | None
) -> dict[str, Any]:
item = get_system_settings(session)
validated = set_privacy_policy(item, PrivacyRetentionPolicy.model_validate(policy or {}))
validated = set_privacy_policy(
item, PrivacyRetentionPolicy.model_validate(policy or {})
)
session.add(item)
return validated.model_dump(mode="json")
def _privacy_policy_patch_payload(policy: dict[str, Any] | None) -> dict[str, Any]:
return PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(mode="json", exclude_none=True)
return PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(
mode="json", exclude_none=True
)
def _validate_privacy_policy_patch_for_scope(
@@ -517,12 +757,18 @@ def _validate_privacy_policy_patch_for_scope(
scope_id: str | None,
patch: dict[str, Any],
) -> None:
parent_scope_id = _parent_privacy_policy_scope_id(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
parent = parent_privacy_policy(session, tenant_id=tenant_id, scope_type=scope_type, scope_id=parent_scope_id)
parent_scope_id = _parent_privacy_policy_scope_id(
tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id
)
parent = parent_privacy_policy(
session, tenant_id=tenant_id, scope_type=scope_type, scope_id=parent_scope_id
)
_validate_privacy_patch_against_parent(parent, patch)
def _parent_privacy_policy_scope_id(*, tenant_id: str, scope_type: str, scope_id: str | None) -> str | None:
def _parent_privacy_policy_scope_id(
*, tenant_id: str, scope_type: str, scope_id: str | None
) -> str | None:
if scope_id:
return scope_id
if scope_type == "tenant":
@@ -539,21 +785,33 @@ def _set_scoped_privacy_policy(
patch: dict[str, Any],
) -> dict[str, Any]:
if scope_type == "tenant":
return _set_tenant_privacy_policy(session, tenant_id=tenant_id, scope_id=scope_id, patch=patch)
return _set_tenant_privacy_policy(
session, tenant_id=tenant_id, scope_id=scope_id, patch=patch
)
clean_scope_id = _required_privacy_policy_scope_id(scope_type, scope_id)
if scope_type == "user":
return _set_user_privacy_policy(session, tenant_id=tenant_id, user_id=clean_scope_id, patch=patch)
return _set_user_privacy_policy(
session, tenant_id=tenant_id, user_id=clean_scope_id, patch=patch
)
if scope_type == "group":
return _set_group_privacy_policy(session, tenant_id=tenant_id, group_id=clean_scope_id, patch=patch)
return _set_group_privacy_policy(
session, tenant_id=tenant_id, group_id=clean_scope_id, patch=patch
)
if scope_type == "campaign":
return _set_campaign_privacy_policy(session, tenant_id=tenant_id, campaign_id=clean_scope_id, patch=patch)
raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign")
return _set_campaign_privacy_policy(
session, tenant_id=tenant_id, campaign_id=clean_scope_id, patch=patch
)
raise PrivacyPolicyError(
"Privacy policy scope must be system, tenant, user, group or campaign"
)
def _required_privacy_policy_scope_id(scope_type: str, scope_id: str | None) -> str:
if scope_id:
return scope_id
raise PrivacyPolicyError(f"{scope_type.capitalize()} privacy policy requires scope_id")
raise PrivacyPolicyError(
f"{scope_type.capitalize()} privacy policy requires scope_id"
)
def _set_tenant_privacy_policy(
@@ -571,34 +829,65 @@ def _set_tenant_privacy_policy(
return patch
def _set_user_privacy_policy(session: Session, *, tenant_id: str, user_id: str, patch: dict[str, Any]) -> dict[str, Any]:
def _set_user_privacy_policy(
session: Session, *, tenant_id: str, user_id: str, patch: dict[str, Any]
) -> dict[str, Any]:
current_settings = _user_settings(session, user_id=user_id, tenant_id=tenant_id)
if current_settings is None:
raise PrivacyPolicyError("User privacy policy not found")
settings_payload = _set_settings_privacy_policy(current_settings, patch)
if _set_user_settings(session, user_id=user_id, tenant_id=tenant_id, settings_payload=settings_payload) is None:
if (
_set_user_settings(
session,
user_id=user_id,
tenant_id=tenant_id,
settings_payload=settings_payload,
)
is None
):
raise PrivacyPolicyError("User privacy policy not found")
return patch
def _set_group_privacy_policy(session: Session, *, tenant_id: str, group_id: str, patch: dict[str, Any]) -> dict[str, Any]:
def _set_group_privacy_policy(
session: Session, *, tenant_id: str, group_id: str, patch: dict[str, Any]
) -> dict[str, Any]:
current_settings = _group_settings(session, group_id=group_id, tenant_id=tenant_id)
if current_settings is None:
raise PrivacyPolicyError("Group privacy policy not found")
settings_payload = _set_settings_privacy_policy(current_settings, patch)
if _set_group_settings(session, group_id=group_id, tenant_id=tenant_id, settings_payload=settings_payload) is None:
if (
_set_group_settings(
session,
group_id=group_id,
tenant_id=tenant_id,
settings_payload=settings_payload,
)
is None
):
raise PrivacyPolicyError("Group privacy policy not found")
return patch
def _set_campaign_privacy_policy(session: Session, *, tenant_id: str, campaign_id: str, patch: dict[str, Any]) -> dict[str, Any]:
def _set_campaign_privacy_policy(
session: Session, *, tenant_id: str, campaign_id: str, patch: dict[str, Any]
) -> dict[str, Any]:
provider = _campaign_policy_provider()
if provider is None:
raise PrivacyPolicyError("Campaign module is not installed")
campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=campaign_id)
settings_payload = _set_settings_privacy_policy(dict(campaign.settings or {}), patch)
campaign = _campaign_policy_context(
session, tenant_id=tenant_id, campaign_id=campaign_id
)
settings_payload = _set_settings_privacy_policy(
dict(campaign.settings or {}), patch
)
try:
provider.set_campaign_settings(session, tenant_id=tenant_id, campaign_id=campaign_id, settings=settings_payload)
provider.set_campaign_settings(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
settings=settings_payload,
)
except ValueError as exc:
raise PrivacyPolicyError("Campaign privacy policy not found") from exc
return patch
@@ -613,10 +902,16 @@ def simulate_privacy_policy_change(
policy: dict[str, Any] | None = None,
) -> dict[str, Any]:
clean_scope = scope_type.strip().casefold()
patch = PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(mode="json", exclude_none=True)
current = get_privacy_policy_for_scope(session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id)
patch = PrivacyRetentionPolicyPatch.model_validate(policy or {}).model_dump(
mode="json", exclude_none=True
)
current = get_privacy_policy_for_scope(
session, tenant_id=tenant_id, scope_type=clean_scope, scope_id=scope_id
)
if clean_scope == "system":
parent_payload = PrivacyRetentionPolicy.model_validate(current).model_dump(mode="json")
parent_payload = PrivacyRetentionPolicy.model_validate(current).model_dump(
mode="json"
)
parent_allow = default_allow_lower_level_limits()
parent_sources = effective_privacy_policy_sources(session)
else:
@@ -641,15 +936,25 @@ def simulate_privacy_policy_change(
patch=patch,
field_keys=RETENTION_POLICY_FIELD_KEYS,
parent_allow_lower_level_limits=parent_allow,
restriction_rules=() if clean_scope == "system" else _privacy_restriction_rules(),
source_path=[PolicySourceStep.from_mapping(source) for source in parent_sources],
locked_field_message=lambda key: f"{key} is locked by the parent retention policy.",
relock_message=lambda key: f"{key} limiting cannot be re-enabled below a parent retention policy lock.",
restriction_rules=()
if clean_scope == "system"
else _privacy_restriction_rules(),
source_path=[
PolicySourceStep.from_mapping(source) for source in parent_sources
],
locked_field_message=lambda key: (
f"{key} is locked by the parent retention policy."
),
relock_message=lambda key: (
f"{key} limiting cannot be re-enabled below a parent retention policy lock."
),
)
return simulation.to_dict()
def sanitize_audit_details_for_policy(session: Session, details: dict[str, Any]) -> dict[str, Any]:
def sanitize_audit_details_for_policy(
session: Session, details: dict[str, Any]
) -> dict[str, Any]:
policy = privacy_policy_from_session(session)
if policy.audit_detail_level == "full":
return details
@@ -675,11 +980,20 @@ def _redact_audit_value(value: Any) -> Any:
return value
def _system_cutoffs(policy: PrivacyRetentionPolicy, *, now: datetime) -> dict[str, datetime | None]:
def _system_cutoffs(
policy: PrivacyRetentionPolicy, *, now: datetime
) -> dict[str, datetime | None]:
return {
"raw_campaign_json": _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now),
"raw_campaign_json": _cutoff(
0
if not policy.store_raw_campaign_json
else policy.raw_campaign_json_retention_days,
now=now,
),
"generated_eml": _cutoff(policy.generated_eml_retention_days, now=now),
"stored_report_detail": _cutoff(policy.stored_report_detail_retention_days, now=now),
"stored_report_detail": _cutoff(
policy.stored_report_detail_retention_days, now=now
),
"mock_mailbox": _cutoff(policy.mock_mailbox_retention_days, now=now),
"audit_detail": _cutoff(policy.audit_detail_retention_days, now=now),
}
@@ -692,9 +1006,24 @@ def apply_retention_policy(session: Session, *, dry_run: bool = True) -> dict[st
policy_cache: dict[str, PrivacyRetentionPolicy] = {}
campaign_retention = _campaign_retention_provider()
campaign_counts = {
"raw_campaign_json": {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0},
"generated_eml": {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0},
"stored_report_detail": {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0},
"raw_campaign_json": {
"eligible": 0,
"redacted": 0,
"skipped_not_final": 0,
"already_redacted": 0,
},
"generated_eml": {
"eligible": 0,
"metadata_cleared": 0,
"files_deleted": 0,
"files_missing": 0,
"skipped_not_final": 0,
},
"stored_report_detail": {
"eligible_versions": 0,
"summaries_redacted": 0,
"already_redacted": 0,
},
}
if campaign_retention is not None:
campaign_counts.update(
@@ -704,27 +1033,54 @@ def apply_retention_policy(session: Session, *, dry_run: bool = True) -> dict[st
session,
dry_run=dry_run,
now=now,
policy_for_campaign_id=lambda campaign_id: _campaign_policy_for_id(session, campaign_id, policy_cache),
policy_for_campaign_id=lambda campaign_id: _campaign_policy_for_id(
session, campaign_id, policy_cache
),
).items()
}
)
reporting_retention = _reporting_retention_provider()
reporting_counts = {
"eligible": 0,
"redacted": 0,
"remaining_in_batch": 0,
}
if reporting_retention is not None:
reporting_counts.update(
reporting_retention.apply_retention(
session,
dry_run=dry_run,
now=now,
)
)
counts = {
"raw_campaign_json": campaign_counts["raw_campaign_json"],
"generated_eml": campaign_counts["generated_eml"],
"stored_report_detail": campaign_counts["stored_report_detail"],
"mock_mailbox": _apply_mock_mailbox_retention(cutoffs["mock_mailbox"], dry_run=dry_run),
"audit_detail": _apply_audit_detail_retention(session, dry_run=dry_run, now=now),
"stored_report_detail": {
**campaign_counts["stored_report_detail"],
"provider_reports": reporting_counts,
},
"mock_mailbox": _apply_mock_mailbox_retention(
cutoffs["mock_mailbox"], dry_run=dry_run
),
"audit_detail": _apply_audit_detail_retention(
session, dry_run=dry_run, now=now
),
}
return {
"dry_run": dry_run,
"policy": policy.model_dump(mode="json"),
"cutoffs": {key: value.isoformat() if value else None for key, value in cutoffs.items()},
"cutoffs": {
key: value.isoformat() if value else None for key, value in cutoffs.items()
},
"effective_policy_scope": "per-object",
"counts": counts,
}
def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy:
def _campaign_policy_for_id(
session: Session, campaign_id: str | None, cache: dict[str, PrivacyRetentionPolicy]
) -> PrivacyRetentionPolicy:
if not campaign_id:
return privacy_policy_from_session(session)
if campaign_id not in cache:
@@ -732,7 +1088,9 @@ def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: di
return cache[campaign_id]
def _apply_mock_mailbox_retention(cutoff: datetime | None, *, dry_run: bool) -> dict[str, int]:
def _apply_mock_mailbox_retention(
cutoff: datetime | None, *, dry_run: bool
) -> dict[str, int]:
result = {"eligible_records": 0, "json_deleted": 0, "eml_deleted": 0}
if cutoff is None:
return result
@@ -774,32 +1132,47 @@ def _parse_datetime(value: Any) -> datetime | None:
return parsed
def _privacy_policy_for_audit_item(session: Session, item: AuditRecordRef, campaign_cache: dict[str, PrivacyRetentionPolicy], tenant_cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy:
def _privacy_policy_for_audit_item(
session: Session,
item: AuditRecordRef,
campaign_cache: dict[str, PrivacyRetentionPolicy],
tenant_cache: dict[str, PrivacyRetentionPolicy],
) -> PrivacyRetentionPolicy:
if item.object_type == "campaign" and item.object_id:
provider = _campaign_policy_provider()
if provider is not None:
context = provider.get_campaign_policy_context(session, campaign_id=str(item.object_id))
context = provider.get_campaign_policy_context(
session, campaign_id=str(item.object_id)
)
if context is not None:
return _campaign_policy_for_id(session, context.id, campaign_cache)
if item.tenant_id:
if item.tenant_id not in tenant_cache:
tenant_cache[item.tenant_id] = effective_privacy_policy(session, tenant_id=item.tenant_id)
tenant_cache[item.tenant_id] = effective_privacy_policy(
session, tenant_id=item.tenant_id
)
return tenant_cache[item.tenant_id]
return privacy_policy_from_session(session)
def _apply_audit_detail_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]:
def _apply_audit_detail_retention(
session: Session, *, dry_run: bool, now: datetime
) -> dict[str, int]:
provider = _audit_retention_provider()
if provider is None:
return {"eligible": 0, "redacted": 0, "already_redacted": 0}
campaign_cache: dict[str, PrivacyRetentionPolicy] = {}
tenant_cache: dict[str, PrivacyRetentionPolicy] = {}
return dict(provider.apply_detail_retention(
return dict(
provider.apply_detail_retention(
session,
dry_run=dry_run,
now=now,
policy_for_record=lambda item: _privacy_policy_for_audit_item(session, item, campaign_cache, tenant_cache),
))
policy_for_record=lambda item: _privacy_policy_for_audit_item(
session, item, campaign_cache, tenant_cache
),
)
)
class SqlPrivacyRetentionService:
tests/test_policy_module_contract.py
+2
View File
@@ -14,6 +14,7 @@ from govoplan_core.core.policy import (
from govoplan_core.core.distribution_lists import (
CAPABILITY_POLICY_DISTRIBUTION_CHANNELS,
)
from govoplan_core.core.reporting import CAPABILITY_POLICY_REPORTING_GOVERNANCE
from govoplan_policy.backend.manifest import manifest
@@ -50,6 +51,7 @@ class PolicyModuleContractTests(unittest.TestCase):
CAPABILITY_POLICY_DISTRIBUTION_CHANNELS,
CAPABILITY_POLICY_FUNCTION_ASSIGNMENT_GOVERNANCE,
CAPABILITY_POLICY_PRIVACY_RETENTION,
CAPABILITY_POLICY_REPORTING_GOVERNANCE,
CAPABILITY_POLICY_SCHEDULING_PARTICIPANT_PRIVACY,
CAPABILITY_POLICY_VIEW_GOVERNANCE,
},
tests/test_reporting_governance.py
+135
View File
@@ -0,0 +1,135 @@
from __future__ import annotations
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from govoplan_core.core.reporting import ReportingGovernanceRequest
from govoplan_core.db.base import Base
from govoplan_core.tenancy.scope import Tenant, create_scope_tables
from govoplan_policy.backend.reporting_governance import (
ReportingGovernancePolicyProvider,
)
from govoplan_policy.backend import retention as retention_module
def _request(action: str, *, export_format: str | None = None):
return ReportingGovernanceRequest(
action=action, # type: ignore[arg-type]
tenant_id="tenant-1",
provider_id="campaigns",
report_id="delivery-outcomes",
purpose="Operational review",
audience_scope={"scope_type": "tenant", "scope_id": "tenant-1"},
retention_class="stored_report_detail",
export_format=export_format,
reidentification_risk="low",
declared_privacy_transforms=("small_cell_suppression",),
applied_privacy_transforms=("small_cell_suppression",),
)
def test_tenant_reporting_policy_can_tighten_export_and_retention() -> None:
engine = create_engine("sqlite+pysqlite:///:memory:")
create_scope_tables(engine)
Base.metadata.create_all(engine)
with Session(engine) as session:
session.add(
Tenant(
id="tenant-1",
slug="tenant-1",
name="Tenant 1",
settings={
"privacy_retention_policy": {
"stored_report_detail_retention_days": 10
},
"reporting_governance_policy": {
"allow_exports": False,
"required_privacy_transforms": ["explicit_denominator"],
},
},
)
)
session.commit()
provider = ReportingGovernancePolicyProvider()
execute = provider.decide_reporting_action(
session,
object(),
request=_request("execute"),
)
export = provider.decide_reporting_action(
session,
object(),
request=_request("export", export_format="json"),
)
assert execute.allowed is True
assert execute.retention_days == 10
assert set(execute.required_privacy_transforms) == {
"small_cell_suppression",
"explicit_denominator",
}
assert export.allowed is False
assert export.export_formats == ()
engine.dispose()
def test_malformed_reporting_policy_fails_closed() -> None:
engine = create_engine("sqlite+pysqlite:///:memory:")
create_scope_tables(engine)
Base.metadata.create_all(engine)
with Session(engine) as session:
session.add(
Tenant(
id="tenant-1",
slug="tenant-1",
name="Tenant 1",
settings={"reporting_governance_policy": {"allow_exports": "yes"}},
)
)
session.commit()
decision = ReportingGovernancePolicyProvider().decide_reporting_action(
session,
object(),
request=_request("execute"),
)
assert decision.allowed is False
assert decision.provenance["decision"] == "fail_closed"
engine.dispose()
def test_shared_retention_run_invokes_reporting_without_model_imports(
monkeypatch,
) -> None:
class _ReportingRetention:
def apply_retention(self, session, *, dry_run, now, limit=500):
del session, now, limit
return {
"eligible": 2,
"redacted": 0 if dry_run else 2,
"remaining_in_batch": 0,
}
class _Registry:
def has_capability(self, name):
return name == "reporting.retention"
def require_capability(self, name):
assert name == "reporting.retention"
return _ReportingRetention()
monkeypatch.setattr(retention_module, "get_registry", lambda: _Registry())
engine = create_engine("sqlite+pysqlite:///:memory:")
create_scope_tables(engine)
Base.metadata.create_all(engine)
with Session(engine) as session:
result = retention_module.apply_retention_policy(session, dry_run=False)
assert result["counts"]["stored_report_detail"]["provider_reports"] == {
"eligible": 2,
"redacted": 2,
"remaining_in_batch": 0,
}
engine.dispose()