feat: govern shared campaign artifact storage
This commit is contained in:
@@ -28,6 +28,7 @@ from govoplan_core.core.modules import (
|
||||
RoleTemplate,
|
||||
)
|
||||
from govoplan_core.core.operations import OperationalCheckProviderRegistration
|
||||
from govoplan_core.core.provider_governance import declared_module_architecture
|
||||
from govoplan_core.core.views import ViewSurface
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.core.postbox import (
|
||||
@@ -718,6 +719,37 @@ manifest = ModuleManifest(
|
||||
],
|
||||
},
|
||||
),
|
||||
DocumentationTopic(
|
||||
id="campaigns.reference.shared-build-artifacts",
|
||||
title="Operate Campaign build artifacts across workers",
|
||||
summary="Generated messages use shared object storage and are verified before delivery.",
|
||||
body="Campaign stores generated EML under opaque shared object keys and records expected size, SHA-256 digest, and Message-ID in each job. A worker may run on another node and verifies that evidence before delivery. Failed builds compensate newly written objects; retention retains metadata when deletion fails. Never copy or edit runtime object keys as business data.",
|
||||
layer="evidence",
|
||||
documentation_types=("admin",),
|
||||
audience=("campaign_operator", "platform_operator", "release_reviewer"),
|
||||
order=53,
|
||||
conditions=(
|
||||
DocumentationCondition(
|
||||
required_modules=("campaigns",),
|
||||
any_scopes=("campaigns:diagnostic:read", "system:settings:read"),
|
||||
),
|
||||
),
|
||||
links=(
|
||||
DocumentationLink(label="Campaign operator queue", href="/campaigns/queue", kind="runtime"),
|
||||
DocumentationLink(label="Campaign delivery runbook", href="govoplan-campaign/docs/CAMPAIGN_DELIVERY_RUNBOOK.md", kind="repository"),
|
||||
),
|
||||
related_modules=("files", "ops", "mail"),
|
||||
metadata={
|
||||
"kind": "reference",
|
||||
"route": "/campaigns/queue",
|
||||
"screen": "Campaign operator queue",
|
||||
"verification": "Build on one replica, deliver from another, verify object digest/size evidence, and exercise storage failure during build and retention.",
|
||||
"limitations": [
|
||||
"A hard process loss between object creation and database commit can leave an orphan object until an inventory reconciler removes it.",
|
||||
"Database, object storage, and encryption keys require a coordinated deployment backup and restore procedure.",
|
||||
],
|
||||
},
|
||||
),
|
||||
),
|
||||
documentation_providers=(documentation_topics,),
|
||||
ownership_providers=(
|
||||
@@ -762,6 +794,19 @@ manifest = ModuleManifest(
|
||||
cache_seconds=60,
|
||||
),
|
||||
),
|
||||
architecture=declared_module_architecture(
|
||||
layer="communication_participation",
|
||||
kind="domain",
|
||||
maturity="vertical_slice",
|
||||
documentation_ref="docs/CAMPAIGN_HANDBOOK.md",
|
||||
test_ref="tests/test_execution_snapshot_integrity.py",
|
||||
known_limits=("Target-environment recovery and accessibility evidence is incomplete for reference-ready maturity.",),
|
||||
owned_concepts=("campaign", "campaign version", "recipient snapshot", "delivery job", "delivery report"),
|
||||
non_owned_concepts=("mail transport", "postbox", "durable address directory", "file storage"),
|
||||
recovery_docs=("docs/CAMPAIGN_DELIVERY_RUNBOOK.md",),
|
||||
security_docs=("docs/ACCESS_EXPLANATION_COVERAGE.md",),
|
||||
operations_docs=("docs/CAMPAIGN_DELIVERY_RUNBOOK.md",),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,55 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
from govoplan_core.core.object_storage import (
|
||||
StorageBackendError,
|
||||
configured_storage_backend,
|
||||
)
|
||||
from govoplan_core.core.operations import OperationalCheck
|
||||
from govoplan_campaign.backend.persistence.campaigns import BUILD_OUTPUT_DIR
|
||||
from govoplan_core.settings import settings as core_settings
|
||||
from govoplan_campaign.backend.runtime import get_settings
|
||||
|
||||
|
||||
def generated_eml_storage_check() -> OperationalCheck:
|
||||
"""Verify the current EML evidence path and report its node-local boundary."""
|
||||
"""Verify Campaign evidence against the deployment object-store boundary."""
|
||||
|
||||
root = Path(BUILD_OUTPUT_DIR)
|
||||
probe = root / ".govoplan-health" / f"{secrets.token_hex(16)}.probe"
|
||||
storage = configured_storage_backend(get_settings() or core_settings)
|
||||
probe = f"campaign-artifacts/.health/{secrets.token_hex(16)}.probe"
|
||||
payload = secrets.token_bytes(64)
|
||||
try:
|
||||
probe.parent.mkdir(parents=True, mode=0o700, exist_ok=True)
|
||||
with probe.open("xb") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
if probe.read_bytes() != payload:
|
||||
storage.put_bytes(probe, payload, content_type="application/octet-stream")
|
||||
if storage.get_bytes(probe) != payload:
|
||||
raise OSError("generated EML persistence returned different bytes")
|
||||
except OSError as exc:
|
||||
except (OSError, StorageBackendError) as exc:
|
||||
return OperationalCheck(
|
||||
id="campaign.generated_eml_storage",
|
||||
label="Generated Campaign EML evidence",
|
||||
state="error",
|
||||
detail=(
|
||||
"The generated EML evidence path failed a bounded durable-write probe "
|
||||
"The generated EML object store failed a bounded write/read probe "
|
||||
f"({type(exc).__name__})."
|
||||
),
|
||||
readiness_critical=True,
|
||||
metrics={"backend": "node_local_filesystem"},
|
||||
metrics={"backend": storage.name},
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
probe.unlink(missing_ok=True)
|
||||
probe.parent.rmdir()
|
||||
except OSError:
|
||||
storage.delete(probe)
|
||||
except StorageBackendError:
|
||||
pass
|
||||
|
||||
node_local = storage.name == "local"
|
||||
return OperationalCheck(
|
||||
id="campaign.generated_eml_storage",
|
||||
label="Generated Campaign EML evidence",
|
||||
state="warning",
|
||||
state="warning" if node_local else "ok",
|
||||
detail=(
|
||||
"Generated EML passed a local write/fsync/read probe, but remains node-local. "
|
||||
"Use one shared runtime volume and include it in coordinated backups until "
|
||||
"Campaign evidence is migrated to managed object storage."
|
||||
"Generated EML passed the object-store write/read/delete probe. "
|
||||
+ (
|
||||
"The configured backend is node-local and is suitable only for a single-node profile."
|
||||
if node_local
|
||||
else "The configured backend is shared across application and worker nodes."
|
||||
)
|
||||
),
|
||||
metrics={"backend": "node_local_filesystem"},
|
||||
metrics={"backend": storage.name},
|
||||
)
|
||||
|
||||
|
||||
@@ -2,18 +2,24 @@ from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from email import policy
|
||||
from email.parser import BytesParser
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.object_storage import (
|
||||
StorageBackend,
|
||||
StorageBackendError,
|
||||
configured_storage_backend,
|
||||
)
|
||||
from govoplan_core.settings import settings as core_settings
|
||||
from govoplan_campaign.backend.db.models import (
|
||||
Campaign,
|
||||
CampaignIssue,
|
||||
@@ -52,20 +58,19 @@ from govoplan_campaign.backend.integrations import (
|
||||
postbox_integration,
|
||||
)
|
||||
from govoplan_campaign.backend.path_security import assert_server_safe_campaign_paths
|
||||
|
||||
RUNTIME_DIR = Path(__file__).resolve().parents[3] / "runtime"
|
||||
CAMPAIGN_SNAPSHOT_DIR = RUNTIME_DIR / "campaign_snapshots"
|
||||
BUILD_OUTPUT_DIR = RUNTIME_DIR / "generated_eml"
|
||||
from govoplan_campaign.backend.runtime import get_settings
|
||||
|
||||
|
||||
class CampaignPersistenceError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _ensure_dirs() -> None:
|
||||
for directory in (CAMPAIGN_SNAPSHOT_DIR, BUILD_OUTPUT_DIR):
|
||||
directory.mkdir(parents=True, mode=0o700, exist_ok=True)
|
||||
directory.chmod(0o700)
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _StoredEmlArtifact:
|
||||
storage_key: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
message_id_header: str | None
|
||||
|
||||
|
||||
|
||||
@@ -107,22 +112,82 @@ def load_campaign_config_from_json(
|
||||
return CampaignConfig.model_validate(materialized)
|
||||
|
||||
|
||||
def _write_campaign_snapshot(version: CampaignVersion) -> Path:
|
||||
_ensure_dirs()
|
||||
path = CAMPAIGN_SNAPSHOT_DIR / f"{version.id}.json"
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
flags |= os.O_NOFOLLOW
|
||||
descriptor = os.open(path, flags, 0o600)
|
||||
def _campaign_reference_path(version: CampaignVersion) -> Path:
|
||||
"""Return a path anchor without persisting a second configuration copy."""
|
||||
|
||||
source_base_path = str(version.source_base_path or "").strip()
|
||||
base = Path(source_base_path).expanduser().resolve() if source_base_path else Path.cwd()
|
||||
return base / "campaign.json"
|
||||
|
||||
|
||||
def _object_storage() -> StorageBackend:
|
||||
return configured_storage_backend(get_settings() or core_settings)
|
||||
|
||||
|
||||
def _persist_built_eml_artifacts(
|
||||
*,
|
||||
storage: StorageBackend,
|
||||
tenant_id: str,
|
||||
campaign_id: str,
|
||||
version_id: str,
|
||||
build_id: str,
|
||||
built_messages: list[Any],
|
||||
) -> dict[int, _StoredEmlArtifact]:
|
||||
artifacts: dict[int, _StoredEmlArtifact] = {}
|
||||
try:
|
||||
os.fchmod(descriptor, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
|
||||
descriptor = -1
|
||||
json.dump(version.raw_json, stream, ensure_ascii=False, indent=2)
|
||||
finally:
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
return path
|
||||
for built in built_messages:
|
||||
message = built.draft
|
||||
if not message.eml_path:
|
||||
continue
|
||||
path = Path(message.eml_path)
|
||||
if not path.is_file():
|
||||
raise CampaignPersistenceError(
|
||||
"Generated EML evidence is missing for entry "
|
||||
f"{message.entry_index}"
|
||||
)
|
||||
payload = path.read_bytes()
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
parsed = BytesParser(policy=policy.default).parsebytes(payload)
|
||||
message_id = parsed.get("Message-ID")
|
||||
storage_key = (
|
||||
f"campaign-artifacts/{tenant_id}/{campaign_id}/{version_id}/"
|
||||
f"{build_id}/{message.entry_index:08d}-{digest}.eml"
|
||||
)
|
||||
try:
|
||||
storage.put_bytes(
|
||||
storage_key,
|
||||
payload,
|
||||
content_type="message/rfc822",
|
||||
)
|
||||
except StorageBackendError as exc:
|
||||
raise CampaignPersistenceError(
|
||||
f"Generated EML evidence could not be persisted: {exc}"
|
||||
) from exc
|
||||
artifacts[message.entry_index] = _StoredEmlArtifact(
|
||||
storage_key=storage_key,
|
||||
size_bytes=len(payload),
|
||||
sha256=digest,
|
||||
message_id_header=str(message_id) if message_id else None,
|
||||
)
|
||||
message.eml_path = None
|
||||
message.eml_size_bytes = len(payload)
|
||||
except Exception:
|
||||
_delete_storage_keys(
|
||||
storage,
|
||||
[artifact.storage_key for artifact in artifacts.values()],
|
||||
)
|
||||
raise
|
||||
return artifacts
|
||||
|
||||
|
||||
def _delete_storage_keys(storage: StorageBackend, keys: list[str]) -> None:
|
||||
for key in keys:
|
||||
try:
|
||||
storage.delete(key)
|
||||
except StorageBackendError:
|
||||
# A committed build remains authoritative. Reconciliation can
|
||||
# remove an orphaned superseded object later.
|
||||
continue
|
||||
|
||||
|
||||
def _next_version_number(session: Session, campaign_id: str) -> int:
|
||||
@@ -143,11 +208,11 @@ def normalize_campaign_paths(raw_json: dict[str, Any], source_base_path: str | P
|
||||
"""Resolve paths for an explicitly trusted, file-oriented import.
|
||||
|
||||
The CLI naturally resolves relative paths against the campaign.json file.
|
||||
Once the campaign is stored in the database, the JSON snapshot lives in
|
||||
app/mailer/runtime/campaign_snapshots. To keep existing file-based
|
||||
campaigns working, relative file paths are normalized to absolute paths at
|
||||
import time when a source_base_path is known. HTTP/API callers are rejected
|
||||
by ``assert_server_safe_campaign_paths`` before they can reach this helper.
|
||||
Once the campaign is stored in the database, its JSON is authoritative.
|
||||
To keep existing file-based campaigns working, relative file paths are
|
||||
normalized to absolute paths at import time when a source_base_path is
|
||||
known. HTTP/API callers are rejected by
|
||||
``assert_server_safe_campaign_paths`` before they can reach this helper.
|
||||
"""
|
||||
base = Path(source_base_path).expanduser().resolve() if source_base_path else None
|
||||
data = copy.deepcopy(raw_json)
|
||||
@@ -232,7 +297,6 @@ def create_campaign_version_from_json(
|
||||
campaign.current_version_id = version.id
|
||||
session.add(campaign)
|
||||
if commit:
|
||||
_write_campaign_snapshot(version)
|
||||
session.commit()
|
||||
else:
|
||||
session.flush()
|
||||
@@ -299,7 +363,7 @@ def load_version_config(session: Session, version_id: str):
|
||||
raw_json,
|
||||
managed_files_available=files_integration().available,
|
||||
)
|
||||
path = _write_campaign_snapshot(version)
|
||||
path = _campaign_reference_path(version)
|
||||
return version, path, load_campaign_config_from_json(session, tenant_id=campaign.tenant_id, raw_json=raw_json, campaign_id=campaign.id)
|
||||
|
||||
|
||||
@@ -391,8 +455,6 @@ def validate_campaign_version(
|
||||
version.workflow_state = CampaignVersionWorkflowState.APPROVED.value
|
||||
version.is_complete = True
|
||||
if lock_on_success and version.locked_at is None:
|
||||
from datetime import UTC, datetime
|
||||
|
||||
version.locked_at = datetime.now(UTC)
|
||||
version.locked_by_user_id = user_id
|
||||
else:
|
||||
@@ -426,9 +488,13 @@ def _job_from_message(
|
||||
version_id: str,
|
||||
message: MessageDraft,
|
||||
resolved_postbox_targets: list[dict[str, Any]] | None = None,
|
||||
stored_eml: _StoredEmlArtifact | None = None,
|
||||
) -> CampaignJob:
|
||||
recipient_email = message.to[0].email if message.to else None
|
||||
eml_sha256, message_id_header = _eml_evidence(message.eml_path)
|
||||
if stored_eml is not None:
|
||||
eml_sha256 = stored_eml.sha256
|
||||
message_id_header = stored_eml.message_id_header
|
||||
return CampaignJob(
|
||||
tenant_id=tenant_id,
|
||||
campaign_id=campaign_id,
|
||||
@@ -438,8 +504,9 @@ def _job_from_message(
|
||||
recipient_email=recipient_email,
|
||||
subject=message.subject,
|
||||
message_id_header=message_id_header,
|
||||
eml_local_path=message.eml_path,
|
||||
eml_size_bytes=message.eml_size_bytes,
|
||||
eml_storage_key=stored_eml.storage_key if stored_eml else None,
|
||||
eml_local_path=message.eml_path if stored_eml is None else None,
|
||||
eml_size_bytes=stored_eml.size_bytes if stored_eml else message.eml_size_bytes,
|
||||
eml_sha256=eml_sha256,
|
||||
build_status=message.build_status.value if hasattr(message.build_status, "value") else str(message.build_status),
|
||||
validation_status=_job_validation_status(message.validation_status.value),
|
||||
@@ -531,7 +598,18 @@ def _replace_version_jobs(
|
||||
version_id: str,
|
||||
built_messages: list[Any],
|
||||
postbox_targets_by_index: dict[int, list[dict[str, Any]]],
|
||||
) -> list[tuple[CampaignJob, MessageDraft]]:
|
||||
stored_eml_by_index: dict[int, _StoredEmlArtifact],
|
||||
) -> tuple[list[tuple[CampaignJob, MessageDraft]], list[str]]:
|
||||
old_storage_keys = [
|
||||
str(key)
|
||||
for (key,) in session.query(CampaignJob.eml_storage_key)
|
||||
.filter(
|
||||
CampaignJob.campaign_version_id == version_id,
|
||||
CampaignJob.eml_storage_key.is_not(None),
|
||||
)
|
||||
.all()
|
||||
if key
|
||||
]
|
||||
session.query(CampaignIssue).filter(
|
||||
CampaignIssue.campaign_version_id == version_id,
|
||||
CampaignIssue.job_id.is_not(None),
|
||||
@@ -547,11 +625,12 @@ def _replace_version_jobs(
|
||||
version_id=version_id,
|
||||
message=built.draft,
|
||||
resolved_postbox_targets=postbox_targets_by_index.get(built.draft.entry_index, []),
|
||||
stored_eml=stored_eml_by_index.get(built.draft.entry_index),
|
||||
)
|
||||
session.add(job)
|
||||
pairs.append((job, built.draft))
|
||||
session.flush()
|
||||
return pairs
|
||||
return pairs, old_storage_keys
|
||||
|
||||
|
||||
def _mail_execution_profile(
|
||||
@@ -664,71 +743,112 @@ def build_campaign_version(
|
||||
raise CampaignPersistenceError("Campaign version must be successfully validated before messages are built")
|
||||
_ensure_version_validated_and_locked(version)
|
||||
|
||||
output_dir = BUILD_OUTPUT_DIR / campaign.id / version.id
|
||||
files = files_integration()
|
||||
with files.prepared_campaign_snapshot(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
campaign_id=campaign.id,
|
||||
raw_json=version.raw_json if isinstance(version.raw_json, dict) else {},
|
||||
include_bytes=True,
|
||||
prefix="govoplan-managed-build-",
|
||||
) as prepared:
|
||||
managed_raw = load_campaign_json(prepared.path)
|
||||
managed_config = load_campaign_config_from_json(session, tenant_id=tenant_id, raw_json=managed_raw, campaign_id=campaign.id)
|
||||
result = build_campaign_messages(managed_config, campaign_file=prepared.path, output_dir=output_dir, write_eml=write_eml)
|
||||
files.annotate_built_messages_with_managed_files(result.built_messages, prepared.managed_files_by_local_path)
|
||||
entries_by_index = {
|
||||
index: entry
|
||||
for index, entry in enumerate(
|
||||
load_campaign_entries(
|
||||
managed_config,
|
||||
campaign_file=prepared.path,
|
||||
),
|
||||
start=1,
|
||||
)
|
||||
}
|
||||
resolved_postbox_targets_by_index = _resolve_built_postbox_targets(
|
||||
storage = _object_storage()
|
||||
build_id = uuid4().hex
|
||||
with TemporaryDirectory(prefix="govoplan-campaign-build-") as output_directory:
|
||||
with files.prepared_campaign_snapshot(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
config=managed_config,
|
||||
campaign_id=campaign.id,
|
||||
raw_json=version.raw_json if isinstance(version.raw_json, dict) else {},
|
||||
include_bytes=True,
|
||||
prefix="govoplan-managed-build-",
|
||||
) as prepared:
|
||||
managed_raw = load_campaign_json(prepared.path)
|
||||
managed_config = load_campaign_config_from_json(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
raw_json=managed_raw,
|
||||
campaign_id=campaign.id,
|
||||
)
|
||||
result = build_campaign_messages(
|
||||
managed_config,
|
||||
campaign_file=prepared.path,
|
||||
output_dir=Path(output_directory),
|
||||
write_eml=write_eml,
|
||||
)
|
||||
files.annotate_built_messages_with_managed_files(
|
||||
result.built_messages,
|
||||
prepared.managed_files_by_local_path,
|
||||
)
|
||||
entries_by_index = {
|
||||
index: entry
|
||||
for index, entry in enumerate(
|
||||
load_campaign_entries(
|
||||
managed_config,
|
||||
campaign_file=prepared.path,
|
||||
),
|
||||
start=1,
|
||||
)
|
||||
}
|
||||
resolved_postbox_targets_by_index = _resolve_built_postbox_targets(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
config=managed_config,
|
||||
built_messages=result.built_messages,
|
||||
entries_by_index=entries_by_index,
|
||||
)
|
||||
stored_eml_by_index = _persist_built_eml_artifacts(
|
||||
storage=storage,
|
||||
tenant_id=tenant_id,
|
||||
campaign_id=campaign.id,
|
||||
version_id=version.id,
|
||||
build_id=build_id,
|
||||
built_messages=result.built_messages,
|
||||
entries_by_index=entries_by_index,
|
||||
)
|
||||
report_json = _campaign_build_report(result, files)
|
||||
report_json["built_by_user_id"] = user_id
|
||||
version.build_summary = report_json
|
||||
editor_state = copy.deepcopy(version.editor_state or {})
|
||||
editor_state.pop("review_send", None)
|
||||
version.editor_state = editor_state
|
||||
new_storage_keys = [item.storage_key for item in stored_eml_by_index.values()]
|
||||
try:
|
||||
report_json = _campaign_build_report(result, files)
|
||||
report_json["built_by_user_id"] = user_id
|
||||
version.build_summary = report_json
|
||||
editor_state = copy.deepcopy(version.editor_state or {})
|
||||
editor_state.pop("review_send", None)
|
||||
version.editor_state = editor_state
|
||||
|
||||
job_build_pairs = _replace_version_jobs(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
campaign_id=campaign.id,
|
||||
version_id=version.id,
|
||||
built_messages=result.built_messages,
|
||||
postbox_targets_by_index=resolved_postbox_targets_by_index,
|
||||
)
|
||||
jobs = [job for job, _message in job_build_pairs]
|
||||
files.record_campaign_attachment_uses_for_jobs(session, jobs, stage="built")
|
||||
_store_execution_snapshot(session, version=version, config=managed_config, jobs=jobs, build_summary=report_json)
|
||||
_store_job_issues(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
campaign_id=campaign.id,
|
||||
version_id=version.id,
|
||||
job_build_pairs=job_build_pairs,
|
||||
)
|
||||
_apply_campaign_build_state(
|
||||
campaign,
|
||||
version,
|
||||
needs_review_count=result.report.needs_review_count,
|
||||
blocked_count=result.report.blocked_count,
|
||||
queueable_count=result.report.queueable_count,
|
||||
)
|
||||
job_build_pairs, old_storage_keys = _replace_version_jobs(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
campaign_id=campaign.id,
|
||||
version_id=version.id,
|
||||
built_messages=result.built_messages,
|
||||
postbox_targets_by_index=resolved_postbox_targets_by_index,
|
||||
stored_eml_by_index=stored_eml_by_index,
|
||||
)
|
||||
jobs = [job for job, _message in job_build_pairs]
|
||||
files.record_campaign_attachment_uses_for_jobs(
|
||||
session,
|
||||
jobs,
|
||||
stage="built",
|
||||
)
|
||||
_store_execution_snapshot(
|
||||
session,
|
||||
version=version,
|
||||
config=managed_config,
|
||||
jobs=jobs,
|
||||
build_summary=report_json,
|
||||
)
|
||||
_store_job_issues(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
campaign_id=campaign.id,
|
||||
version_id=version.id,
|
||||
job_build_pairs=job_build_pairs,
|
||||
)
|
||||
_apply_campaign_build_state(
|
||||
campaign,
|
||||
version,
|
||||
needs_review_count=result.report.needs_review_count,
|
||||
blocked_count=result.report.blocked_count,
|
||||
queueable_count=result.report.queueable_count,
|
||||
)
|
||||
|
||||
session.add(version)
|
||||
session.add(campaign)
|
||||
session.commit()
|
||||
session.add(version)
|
||||
session.add(campaign)
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
_delete_storage_keys(storage, new_storage_keys)
|
||||
raise
|
||||
_delete_storage_keys(storage, old_storage_keys)
|
||||
return report_json
|
||||
|
||||
@@ -38,7 +38,6 @@ from govoplan_campaign.backend.integrations import files_integration, mail_integ
|
||||
from govoplan_campaign.backend.persistence.campaigns import (
|
||||
CampaignPersistenceError,
|
||||
_next_version_number,
|
||||
_write_campaign_snapshot,
|
||||
normalize_campaign_paths,
|
||||
)
|
||||
from govoplan_campaign.backend.path_security import assert_server_safe_campaign_paths
|
||||
@@ -250,7 +249,6 @@ def create_minimal_campaign(
|
||||
campaign.current_version_id = version.id
|
||||
session.add(campaign)
|
||||
if commit:
|
||||
_write_campaign_snapshot(version)
|
||||
session.commit()
|
||||
else:
|
||||
session.flush()
|
||||
@@ -435,7 +433,6 @@ def _persist_forked_version(
|
||||
campaign.status = CampaignStatus.DRAFT.value
|
||||
session.add(campaign)
|
||||
if commit:
|
||||
_write_campaign_snapshot(version)
|
||||
session.commit()
|
||||
else:
|
||||
session.flush()
|
||||
@@ -705,7 +702,6 @@ def _persist_updated_version(
|
||||
session.add(campaign)
|
||||
session.flush()
|
||||
if commit:
|
||||
_write_campaign_snapshot(version)
|
||||
session.commit()
|
||||
else:
|
||||
session.flush()
|
||||
|
||||
@@ -9,7 +9,15 @@ from typing import Any, Callable
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.object_storage import (
|
||||
StorageBackend,
|
||||
StorageBackendError,
|
||||
StorageObjectMissing,
|
||||
configured_storage_backend,
|
||||
)
|
||||
from govoplan_core.settings import settings as core_settings
|
||||
from govoplan_campaign.backend.db.models import CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus
|
||||
from govoplan_campaign.backend.runtime import get_settings
|
||||
|
||||
FINAL_VERSION_STATES = {
|
||||
"completed",
|
||||
@@ -111,8 +119,16 @@ def _apply_eml_retention(
|
||||
dry_run: bool,
|
||||
now: datetime,
|
||||
policy_for_campaign_id: Callable[[str | None], object],
|
||||
storage: StorageBackend | None = None,
|
||||
) -> dict[str, int]:
|
||||
result = {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0}
|
||||
result = {
|
||||
"eligible": 0,
|
||||
"metadata_cleared": 0,
|
||||
"files_deleted": 0,
|
||||
"files_missing": 0,
|
||||
"delete_failed": 0,
|
||||
"skipped_not_final": 0,
|
||||
}
|
||||
jobs = (
|
||||
session.query(CampaignJob)
|
||||
.filter((CampaignJob.eml_local_path.is_not(None)) | (CampaignJob.eml_storage_key.is_not(None)))
|
||||
@@ -137,6 +153,21 @@ def _apply_eml_retention(
|
||||
result["eligible"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
if job.eml_storage_key:
|
||||
active_storage = storage or configured_storage_backend(
|
||||
get_settings() or core_settings
|
||||
)
|
||||
try:
|
||||
if active_storage.exists(job.eml_storage_key):
|
||||
active_storage.delete(job.eml_storage_key)
|
||||
result["files_deleted"] += 1
|
||||
else:
|
||||
result["files_missing"] += 1
|
||||
except StorageObjectMissing:
|
||||
result["files_missing"] += 1
|
||||
except StorageBackendError:
|
||||
result["delete_failed"] += 1
|
||||
continue
|
||||
if job.eml_local_path:
|
||||
path = Path(job.eml_local_path)
|
||||
if path.exists():
|
||||
|
||||
@@ -645,17 +645,9 @@ def _clear_current_version_mail_profile_for_owner_transfer(
|
||||
|
||||
|
||||
def _write_current_version_snapshot_if_available(version: CampaignVersion) -> None:
|
||||
try:
|
||||
from govoplan_campaign.backend.persistence.campaigns import (
|
||||
_write_campaign_snapshot,
|
||||
)
|
||||
|
||||
_write_campaign_snapshot(version)
|
||||
except Exception:
|
||||
# The database state is authoritative for the WebUI. Snapshot writing is
|
||||
# best-effort here because ownership changes should not fail due to an
|
||||
# unavailable local runtime directory.
|
||||
return
|
||||
# Kept as a compatibility no-op for callers outside this package. Campaign
|
||||
# JSON is database-authoritative and no longer mirrored onto an API node.
|
||||
del version
|
||||
|
||||
|
||||
def bounded_query_rows(query, *, limit: int, label: str):
|
||||
|
||||
@@ -16,6 +16,10 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.notifications import NotificationDispatchRequest, notification_dispatch_provider
|
||||
from govoplan_core.core.object_storage import (
|
||||
StorageBackendError,
|
||||
configured_storage_backend,
|
||||
)
|
||||
from govoplan_core.audit.logging import audit_event
|
||||
from govoplan_core.security.redaction import redact_secret_values
|
||||
from govoplan_core.settings import settings as core_settings
|
||||
@@ -53,7 +57,7 @@ from govoplan_campaign.backend.sending.postbox_delivery import (
|
||||
PostboxChannelOutcome,
|
||||
deliver_campaign_job_to_postboxes,
|
||||
)
|
||||
from govoplan_campaign.backend.runtime import get_registry
|
||||
from govoplan_campaign.backend.runtime import get_registry, get_settings
|
||||
from govoplan_campaign.backend.integrations import (
|
||||
ImapAppendError,
|
||||
ImapConfigurationError,
|
||||
@@ -2389,6 +2393,17 @@ def _verify_eml_evidence(job: CampaignJob, payload: bytes) -> None:
|
||||
|
||||
|
||||
def _load_eml_bytes_for_job(job: CampaignJob) -> bytes:
|
||||
if job.eml_storage_key:
|
||||
try:
|
||||
payload = configured_storage_backend(
|
||||
get_settings() or core_settings
|
||||
).get_bytes(job.eml_storage_key)
|
||||
except StorageBackendError as exc:
|
||||
raise SendJobError(
|
||||
f"Generated EML object could not be read for job {job.id}: {exc}"
|
||||
) from exc
|
||||
_verify_eml_evidence(job, payload)
|
||||
return payload
|
||||
if job.eml_local_path:
|
||||
path = Path(job.eml_local_path)
|
||||
if not path.exists():
|
||||
@@ -2396,7 +2411,7 @@ def _load_eml_bytes_for_job(job: CampaignJob) -> bytes:
|
||||
payload = path.read_bytes()
|
||||
_verify_eml_evidence(job, payload)
|
||||
return payload
|
||||
raise SendJobError("Only local EML paths are supported for sending in this implementation step")
|
||||
raise SendJobError("Generated EML evidence is not available for this job")
|
||||
|
||||
|
||||
def _load_eml_for_job(job: CampaignJob):
|
||||
|
||||
Reference in New Issue
Block a user