GovOPlaN/govoplan-campaign
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: govern shared campaign artifact storage

Browse Source
This commit is contained in:
zemion
2026-08-01 17:48:24 +02:00
parent 82ddc0c34c
commit cec3d17bff
15 changed files with 518 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_campaign/backend/persistence/campaigns.py
+218 -98
View File
@@ -2,18 +2,24 @@ from __future__ import annotations
import copy
import hashlib
import json
import os
from dataclasses import dataclass
from datetime import UTC, datetime
from email import policy
from email.parser import BytesParser
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any
from uuid import uuid4
from sqlalchemy import func
from sqlalchemy.orm import Session
from govoplan_core.core.object_storage import (
StorageBackend,
StorageBackendError,
configured_storage_backend,
)
from govoplan_core.settings import settings as core_settings
from govoplan_campaign.backend.db.models import (
Campaign,
CampaignIssue,
@@ -52,20 +58,19 @@ from govoplan_campaign.backend.integrations import (
postbox_integration,
)
from govoplan_campaign.backend.path_security import assert_server_safe_campaign_paths
RUNTIME_DIR = Path(__file__).resolve().parents[3] / "runtime"
CAMPAIGN_SNAPSHOT_DIR = RUNTIME_DIR / "campaign_snapshots"
BUILD_OUTPUT_DIR = RUNTIME_DIR / "generated_eml"
from govoplan_campaign.backend.runtime import get_settings
class CampaignPersistenceError(RuntimeError):
pass
def _ensure_dirs() -> None:
for directory in (CAMPAIGN_SNAPSHOT_DIR, BUILD_OUTPUT_DIR):
directory.mkdir(parents=True, mode=0o700, exist_ok=True)
directory.chmod(0o700)
@dataclass(frozen=True, slots=True)
class _StoredEmlArtifact:
storage_key: str
size_bytes: int
sha256: str
message_id_header: str | None
@@ -107,22 +112,82 @@ def load_campaign_config_from_json(
return CampaignConfig.model_validate(materialized)
def _write_campaign_snapshot(version: CampaignVersion) -> Path:
_ensure_dirs()
path = CAMPAIGN_SNAPSHOT_DIR / f"{version.id}.json"
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
if hasattr(os, "O_NOFOLLOW"):
flags |= os.O_NOFOLLOW
descriptor = os.open(path, flags, 0o600)
def _campaign_reference_path(version: CampaignVersion) -> Path:
"""Return a path anchor without persisting a second configuration copy."""
source_base_path = str(version.source_base_path or "").strip()
base = Path(source_base_path).expanduser().resolve() if source_base_path else Path.cwd()
return base / "campaign.json"
def _object_storage() -> StorageBackend:
return configured_storage_backend(get_settings() or core_settings)
def _persist_built_eml_artifacts(
*,
storage: StorageBackend,
tenant_id: str,
campaign_id: str,
version_id: str,
build_id: str,
built_messages: list[Any],
) -> dict[int, _StoredEmlArtifact]:
artifacts: dict[int, _StoredEmlArtifact] = {}
try:
os.fchmod(descriptor, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
descriptor = -1
json.dump(version.raw_json, stream, ensure_ascii=False, indent=2)
finally:
if descriptor >= 0:
os.close(descriptor)
return path
for built in built_messages:
message = built.draft
if not message.eml_path:
continue
path = Path(message.eml_path)
if not path.is_file():
raise CampaignPersistenceError(
"Generated EML evidence is missing for entry "
f"{message.entry_index}"
)
payload = path.read_bytes()
digest = hashlib.sha256(payload).hexdigest()
parsed = BytesParser(policy=policy.default).parsebytes(payload)
message_id = parsed.get("Message-ID")
storage_key = (
f"campaign-artifacts/{tenant_id}/{campaign_id}/{version_id}/"
f"{build_id}/{message.entry_index:08d}-{digest}.eml"
)
try:
storage.put_bytes(
storage_key,
payload,
content_type="message/rfc822",
)
except StorageBackendError as exc:
raise CampaignPersistenceError(
f"Generated EML evidence could not be persisted: {exc}"
) from exc
artifacts[message.entry_index] = _StoredEmlArtifact(
storage_key=storage_key,
size_bytes=len(payload),
sha256=digest,
message_id_header=str(message_id) if message_id else None,
)
message.eml_path = None
message.eml_size_bytes = len(payload)
except Exception:
_delete_storage_keys(
storage,
[artifact.storage_key for artifact in artifacts.values()],
)
raise
return artifacts
def _delete_storage_keys(storage: StorageBackend, keys: list[str]) -> None:
for key in keys:
try:
storage.delete(key)
except StorageBackendError:
# A committed build remains authoritative. Reconciliation can
# remove an orphaned superseded object later.
continue
def _next_version_number(session: Session, campaign_id: str) -> int:
@@ -143,11 +208,11 @@ def normalize_campaign_paths(raw_json: dict[str, Any], source_base_path: str | P
"""Resolve paths for an explicitly trusted, file-oriented import.
The CLI naturally resolves relative paths against the campaign.json file.
Once the campaign is stored in the database, the JSON snapshot lives in
app/mailer/runtime/campaign_snapshots. To keep existing file-based
campaigns working, relative file paths are normalized to absolute paths at
import time when a source_base_path is known. HTTP/API callers are rejected
by ``assert_server_safe_campaign_paths`` before they can reach this helper.
Once the campaign is stored in the database, its JSON is authoritative.
To keep existing file-based campaigns working, relative file paths are
normalized to absolute paths at import time when a source_base_path is
known. HTTP/API callers are rejected by
``assert_server_safe_campaign_paths`` before they can reach this helper.
"""
base = Path(source_base_path).expanduser().resolve() if source_base_path else None
data = copy.deepcopy(raw_json)
@@ -232,7 +297,6 @@ def create_campaign_version_from_json(
campaign.current_version_id = version.id
session.add(campaign)
if commit:
_write_campaign_snapshot(version)
session.commit()
else:
session.flush()
@@ -299,7 +363,7 @@ def load_version_config(session: Session, version_id: str):
raw_json,
managed_files_available=files_integration().available,
)
path = _write_campaign_snapshot(version)
path = _campaign_reference_path(version)
return version, path, load_campaign_config_from_json(session, tenant_id=campaign.tenant_id, raw_json=raw_json, campaign_id=campaign.id)
@@ -391,8 +455,6 @@ def validate_campaign_version(
version.workflow_state = CampaignVersionWorkflowState.APPROVED.value
version.is_complete = True
if lock_on_success and version.locked_at is None:
from datetime import UTC, datetime
version.locked_at = datetime.now(UTC)
version.locked_by_user_id = user_id
else:
@@ -426,9 +488,13 @@ def _job_from_message(
version_id: str,
message: MessageDraft,
resolved_postbox_targets: list[dict[str, Any]] | None = None,
stored_eml: _StoredEmlArtifact | None = None,
) -> CampaignJob:
recipient_email = message.to[0].email if message.to else None
eml_sha256, message_id_header = _eml_evidence(message.eml_path)
if stored_eml is not None:
eml_sha256 = stored_eml.sha256
message_id_header = stored_eml.message_id_header
return CampaignJob(
tenant_id=tenant_id,
campaign_id=campaign_id,
@@ -438,8 +504,9 @@ def _job_from_message(
recipient_email=recipient_email,
subject=message.subject,
message_id_header=message_id_header,
eml_local_path=message.eml_path,
eml_size_bytes=message.eml_size_bytes,
eml_storage_key=stored_eml.storage_key if stored_eml else None,
eml_local_path=message.eml_path if stored_eml is None else None,
eml_size_bytes=stored_eml.size_bytes if stored_eml else message.eml_size_bytes,
eml_sha256=eml_sha256,
build_status=message.build_status.value if hasattr(message.build_status, "value") else str(message.build_status),
validation_status=_job_validation_status(message.validation_status.value),
@@ -531,7 +598,18 @@ def _replace_version_jobs(
version_id: str,
built_messages: list[Any],
postbox_targets_by_index: dict[int, list[dict[str, Any]]],
) -> list[tuple[CampaignJob, MessageDraft]]:
stored_eml_by_index: dict[int, _StoredEmlArtifact],
) -> tuple[list[tuple[CampaignJob, MessageDraft]], list[str]]:
old_storage_keys = [
str(key)
for (key,) in session.query(CampaignJob.eml_storage_key)
.filter(
CampaignJob.campaign_version_id == version_id,
CampaignJob.eml_storage_key.is_not(None),
)
.all()
if key
]
session.query(CampaignIssue).filter(
CampaignIssue.campaign_version_id == version_id,
CampaignIssue.job_id.is_not(None),
@@ -547,11 +625,12 @@ def _replace_version_jobs(
version_id=version_id,
message=built.draft,
resolved_postbox_targets=postbox_targets_by_index.get(built.draft.entry_index, []),
stored_eml=stored_eml_by_index.get(built.draft.entry_index),
)
session.add(job)
pairs.append((job, built.draft))
session.flush()
return pairs
return pairs, old_storage_keys
def _mail_execution_profile(
@@ -664,71 +743,112 @@ def build_campaign_version(
raise CampaignPersistenceError("Campaign version must be successfully validated before messages are built")
_ensure_version_validated_and_locked(version)
output_dir = BUILD_OUTPUT_DIR / campaign.id / version.id
files = files_integration()
with files.prepared_campaign_snapshot(
session,
tenant_id=tenant_id,
campaign_id=campaign.id,
raw_json=version.raw_json if isinstance(version.raw_json, dict) else {},
include_bytes=True,
prefix="govoplan-managed-build-",
) as prepared:
managed_raw = load_campaign_json(prepared.path)
managed_config = load_campaign_config_from_json(session, tenant_id=tenant_id, raw_json=managed_raw, campaign_id=campaign.id)
result = build_campaign_messages(managed_config, campaign_file=prepared.path, output_dir=output_dir, write_eml=write_eml)
files.annotate_built_messages_with_managed_files(result.built_messages, prepared.managed_files_by_local_path)
entries_by_index = {
index: entry
for index, entry in enumerate(
load_campaign_entries(
managed_config,
campaign_file=prepared.path,
),
start=1,
)
}
resolved_postbox_targets_by_index = _resolve_built_postbox_targets(
storage = _object_storage()
build_id = uuid4().hex
with TemporaryDirectory(prefix="govoplan-campaign-build-") as output_directory:
with files.prepared_campaign_snapshot(
session,
tenant_id=tenant_id,
config=managed_config,
campaign_id=campaign.id,
raw_json=version.raw_json if isinstance(version.raw_json, dict) else {},
include_bytes=True,
prefix="govoplan-managed-build-",
) as prepared:
managed_raw = load_campaign_json(prepared.path)
managed_config = load_campaign_config_from_json(
session,
tenant_id=tenant_id,
raw_json=managed_raw,
campaign_id=campaign.id,
)
result = build_campaign_messages(
managed_config,
campaign_file=prepared.path,
output_dir=Path(output_directory),
write_eml=write_eml,
)
files.annotate_built_messages_with_managed_files(
result.built_messages,
prepared.managed_files_by_local_path,
)
entries_by_index = {
index: entry
for index, entry in enumerate(
load_campaign_entries(
managed_config,
campaign_file=prepared.path,
),
start=1,
)
}
resolved_postbox_targets_by_index = _resolve_built_postbox_targets(
session,
tenant_id=tenant_id,
config=managed_config,
built_messages=result.built_messages,
entries_by_index=entries_by_index,
)
stored_eml_by_index = _persist_built_eml_artifacts(
storage=storage,
tenant_id=tenant_id,
campaign_id=campaign.id,
version_id=version.id,
build_id=build_id,
built_messages=result.built_messages,
entries_by_index=entries_by_index,
)
report_json = _campaign_build_report(result, files)
report_json["built_by_user_id"] = user_id
version.build_summary = report_json
editor_state = copy.deepcopy(version.editor_state or {})
editor_state.pop("review_send", None)
version.editor_state = editor_state
new_storage_keys = [item.storage_key for item in stored_eml_by_index.values()]
try:
report_json = _campaign_build_report(result, files)
report_json["built_by_user_id"] = user_id
version.build_summary = report_json
editor_state = copy.deepcopy(version.editor_state or {})
editor_state.pop("review_send", None)
version.editor_state = editor_state
job_build_pairs = _replace_version_jobs(
session,
tenant_id=tenant_id,
campaign_id=campaign.id,
version_id=version.id,
built_messages=result.built_messages,
postbox_targets_by_index=resolved_postbox_targets_by_index,
)
jobs = [job for job, _message in job_build_pairs]
files.record_campaign_attachment_uses_for_jobs(session, jobs, stage="built")
_store_execution_snapshot(session, version=version, config=managed_config, jobs=jobs, build_summary=report_json)
_store_job_issues(
session,
tenant_id=tenant_id,
campaign_id=campaign.id,
version_id=version.id,
job_build_pairs=job_build_pairs,
)
_apply_campaign_build_state(
campaign,
version,
needs_review_count=result.report.needs_review_count,
blocked_count=result.report.blocked_count,
queueable_count=result.report.queueable_count,
)
job_build_pairs, old_storage_keys = _replace_version_jobs(
session,
tenant_id=tenant_id,
campaign_id=campaign.id,
version_id=version.id,
built_messages=result.built_messages,
postbox_targets_by_index=resolved_postbox_targets_by_index,
stored_eml_by_index=stored_eml_by_index,
)
jobs = [job for job, _message in job_build_pairs]
files.record_campaign_attachment_uses_for_jobs(
session,
jobs,
stage="built",
)
_store_execution_snapshot(
session,
version=version,
config=managed_config,
jobs=jobs,
build_summary=report_json,
)
_store_job_issues(
session,
tenant_id=tenant_id,
campaign_id=campaign.id,
version_id=version.id,
job_build_pairs=job_build_pairs,
)
_apply_campaign_build_state(
campaign,
version,
needs_review_count=result.report.needs_review_count,
blocked_count=result.report.blocked_count,
queueable_count=result.report.queueable_count,
)
session.add(version)
session.add(campaign)
session.commit()
session.add(version)
session.add(campaign)
session.commit()
except Exception:
session.rollback()
_delete_storage_keys(storage, new_storage_keys)
raise
_delete_storage_keys(storage, old_storage_keys)
return report_json