attachment backend use

This commit is contained in:
2026-06-13 04:14:10 +02:00
parent 36e9211ee6
commit fe5ac084b7
11 changed files with 696 additions and 145 deletions

View File

@@ -1,7 +1,6 @@
from __future__ import annotations from __future__ import annotations
import copy import copy
import re
from fastapi import APIRouter, Depends, HTTPException, Response, status from fastapi import APIRouter, Depends, HTTPException, Response, status
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -38,7 +37,10 @@ from app.mailer.persistence.campaigns import (
create_campaign_version_from_json, create_campaign_version_from_json,
validate_campaign_version, validate_campaign_version,
) )
from app.storage.services import list_assets_for_user, resolve_patterns from app.storage.files import current_version_and_blob
from app.storage.campaign_attachments import managed_match_payloads, prepared_campaign_snapshot
from app.mailer.campaign.loader import load_campaign_config
from app.mailer.attachments.resolver import resolve_campaign_attachments
from app.mailer.persistence.versions import ( from app.mailer.persistence.versions import (
LockedCampaignVersionError, LockedCampaignVersionError,
create_minimal_campaign, create_minimal_campaign,
@@ -969,6 +971,7 @@ def append_sent(
class CampaignAttachmentPreviewRequest(BaseModel): class CampaignAttachmentPreviewRequest(BaseModel):
include_unmatched: bool = True include_unmatched: bool = True
campaign_json: dict[str, object] | None = None
class CampaignAttachmentPreviewResponse(BaseModel): class CampaignAttachmentPreviewResponse(BaseModel):
@@ -979,44 +982,19 @@ class CampaignAttachmentPreviewResponse(BaseModel):
unused_shared_files: list[dict[str, object]] = Field(default_factory=list) unused_shared_files: list[dict[str, object]] = Field(default_factory=list)
_BRACE_PLACEHOLDER_RE = re.compile(r"(?<!\\)\{\{\s*(.*?)\s*\}\}") def _file_preview(session: Session, asset) -> dict[str, object]:
_DOLLAR_PLACEHOLDER_RE = re.compile(r"(?<!\\)\$\{(.*?)(?<!\\)\}") version, blob = current_version_and_blob(session, asset)
def _preview_render_template(template: str, *, global_values: dict[str, object], entry_fields: dict[str, object]) -> str:
def value_for(raw_key: str) -> str:
key = raw_key.strip()
if key.startswith("local:"):
value = entry_fields.get(key.removeprefix("local:"), "")
elif key.startswith("local."):
value = entry_fields.get(key.removeprefix("local."), "")
elif key.startswith("global:"):
value = global_values.get(key.removeprefix("global:"), "")
elif key.startswith("global."):
value = global_values.get(key.removeprefix("global."), "")
else:
value = entry_fields.get(key, global_values.get(key, ""))
return "" if value is None else str(value)
rendered = _BRACE_PLACEHOLDER_RE.sub(lambda match: value_for(match.group(1)), template)
return _DOLLAR_PLACEHOLDER_RE.sub(lambda match: value_for(match.group(1)), rendered)
def _rule_pattern(rule: dict[str, object], base_path_names: set[str], *, global_values: dict[str, object], entry_fields: dict[str, object]) -> str:
base_dir = _preview_render_template(str(rule.get("base_dir") or "."), global_values=global_values, entry_fields=entry_fields).strip().strip("/")
file_filter = _preview_render_template(str(rule.get("file_filter") or "*"), global_values=global_values, entry_fields=entry_fields).strip() or "*"
if not base_dir or base_dir == "." or base_dir in base_path_names:
return file_filter
return f"{base_dir}/{file_filter}"
def _file_preview(asset) -> dict[str, object]:
return { return {
"id": asset.id, "id": asset.id,
"version_id": version.id,
"blob_id": blob.id,
"display_path": asset.display_path, "display_path": asset.display_path,
"filename": asset.filename, "filename": asset.filename,
"owner_type": asset.owner_type, "owner_type": asset.owner_type,
"owner_id": asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id, "owner_id": asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id,
"checksum_sha256": blob.checksum_sha256,
"size_bytes": blob.size_bytes,
"content_type": blob.content_type,
} }
@@ -1033,61 +1011,77 @@ def preview_campaign_attachments(
if version.campaign_id != campaign.id: if version.campaign_id != campaign.id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Campaign version not found") raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Campaign version not found")
raw = version.raw_json if isinstance(version.raw_json, dict) else {} payload = payload or CampaignAttachmentPreviewRequest()
attachments = raw.get("attachments") if isinstance(raw.get("attachments"), dict) else {} raw = payload.campaign_json if isinstance(payload.campaign_json, dict) else version.raw_json
base_paths = attachments.get("base_paths") if isinstance(attachments.get("base_paths"), list) else [] raw = raw if isinstance(raw, dict) else {}
base_path_names = {str(item.get("name")) for item in base_paths if isinstance(item, dict) and item.get("name")}
global_values = raw.get("global_values") if isinstance(raw.get("global_values"), dict) else {}
rules: list[dict[str, object]] = []
global_rules = attachments.get("global") if isinstance(attachments.get("global"), list) else [] with prepared_campaign_snapshot(
for index, rule in enumerate(global_rules):
if not isinstance(rule, dict):
continue
rules.append({
"source": "global",
"index": index,
"label": rule.get("label"),
"required": bool(rule.get("required", True)),
"pattern": _rule_pattern(rule, base_path_names, global_values=global_values, entry_fields={}),
})
entries = raw.get("entries") if isinstance(raw.get("entries"), dict) else {}
inline_entries = entries.get("inline") if isinstance(entries.get("inline"), list) else []
for entry_index, entry in enumerate(inline_entries, start=1):
if not isinstance(entry, dict):
continue
entry_fields = entry.get("fields") if isinstance(entry.get("fields"), dict) else {}
entry_rules = entry.get("attachments") if isinstance(entry.get("attachments"), list) else []
for rule_index, rule in enumerate(entry_rules):
if not isinstance(rule, dict):
continue
rules.append({
"source": "entry",
"entry_index": entry_index,
"entry_id": entry.get("id"),
"index": rule_index,
"label": rule.get("label"),
"required": bool(rule.get("required", True)),
"pattern": _rule_pattern(rule, base_path_names, global_values=global_values, entry_fields=entry_fields),
})
shared_assets = list_assets_for_user(
session, session,
tenant_id=principal.tenant_id, tenant_id=principal.tenant_id,
user_id=principal.user.id,
campaign_id=campaign.id, campaign_id=campaign.id,
is_admin=principal.user.is_tenant_admin or "*" in set(principal.scopes or []), raw_json=raw,
) include_bytes=False,
resolved, unmatched = resolve_patterns(shared_assets, [str(rule["pattern"]) for rule in rules]) prefix="multimailer-managed-preview-",
for rule, result in zip(rules, resolved, strict=False): ) as prepared:
rule["matches"] = [_file_preview(asset) for asset in result.matches] config = load_campaign_config(prepared.path)
rule["match_count"] = len(result.matches) report = resolve_campaign_attachments(config, campaign_file=prepared.path)
rules: list[dict[str, object]] = []
matched_asset_ids: set[str] = set()
for entry in report.entries:
for attachment in entry.attachments:
managed_matches = managed_match_payloads(attachment.matches, prepared.managed_files_by_local_path)
matched_asset_ids.update(str(item["asset_id"]) for item in managed_matches)
matches: list[dict[str, object]] = [
{
"id": item["asset_id"],
"version_id": item["version_id"],
"blob_id": item["blob_id"],
"display_path": item["display_path"],
"filename": item["filename"],
"owner_type": item["owner_type"],
"owner_id": item["owner_id"],
"checksum_sha256": item["checksum_sha256"],
"size_bytes": item["size_bytes"],
"content_type": item["content_type"],
}
for item in managed_matches
]
if not matches:
matches = [
{
"id": "",
"display_path": match,
"filename": match.rsplit("/", 1)[-1].rsplit("\\", 1)[-1],
"owner_type": "legacy",
"owner_id": "",
}
for match in attachment.matches
]
rules.append({
"source": attachment.scope.value,
"entry_index": entry.entry_index,
"entry_id": entry.entry_id,
"index": attachment.index,
"attachment_id": attachment.attachment_id,
"label": attachment.label,
"required": attachment.required,
"pattern": attachment.file_filter,
"base_path_name": attachment.base_path_name,
"base_path": attachment.base_path,
"status": attachment.status.value,
"behavior": attachment.behavior.value if attachment.behavior else None,
"matches": matches,
"match_count": len(matches),
"issues": [issue.model_dump(mode="json") for issue in attachment.issues],
})
unused = [asset for asset in prepared.shared_assets if asset.id not in matched_asset_ids]
return CampaignAttachmentPreviewResponse( return CampaignAttachmentPreviewResponse(
campaign_id=campaign.id, campaign_id=campaign.id,
version_id=version.id, version_id=version.id,
shared_file_count=len(shared_assets), shared_file_count=len(prepared.shared_assets),
rules=rules, rules=rules,
unused_shared_files=[_file_preview(asset) for asset in unmatched] if payload is None or payload.include_unmatched else [], unused_shared_files=[_file_preview(session, asset) for asset in unused] if payload.include_unmatched else [],
) )

View File

@@ -211,16 +211,29 @@ def _base_path_by_path(config: CampaignConfig, rendered_base_dir: str) -> Attach
return None return None
def _base_path_by_id(config: CampaignConfig, base_path_id: str | None) -> AttachmentBasePathConfig | None:
if not base_path_id:
return None
return next((base_path for base_path in config.attachments.base_paths if base_path.id == base_path_id), None)
def _default_base_path(config: CampaignConfig) -> AttachmentBasePathConfig: def _default_base_path(config: CampaignConfig) -> AttachmentBasePathConfig:
return config.attachments.base_paths[0] return config.attachments.base_paths[0]
def _selected_base_path(config: CampaignConfig, rendered_base_dir: str) -> AttachmentBasePathConfig | None: def _selected_base_path(
if config.attachments.base_paths: config: CampaignConfig,
attachment_config: AttachmentConfig,
rendered_base_dir: str,
) -> AttachmentBasePathConfig | None:
if not config.attachments.base_paths:
return None
selected_by_id = _base_path_by_id(config, attachment_config.base_path_id)
if selected_by_id is not None:
return selected_by_id
if rendered_base_dir in {"", "."}: if rendered_base_dir in {"", "."}:
return _default_base_path(config) return _default_base_path(config)
return _base_path_by_path(config, rendered_base_dir) return _base_path_by_path(config, rendered_base_dir)
return None
def _rule_allows_multiple(config: AttachmentConfig, rendered_file_filter: str) -> bool: def _rule_allows_multiple(config: AttachmentConfig, rendered_file_filter: str) -> bool:
@@ -248,9 +261,9 @@ def _ambiguous_behavior(campaign_config: CampaignConfig, config: AttachmentConfi
def _entry_attachment_allowed(config: CampaignConfig, attachment_config: AttachmentConfig, values: dict[str, Any]) -> bool: def _entry_attachment_allowed(config: CampaignConfig, attachment_config: AttachmentConfig, values: dict[str, Any]) -> bool:
rendered_base_dir = _rendered_base_dir(attachment_config, values) rendered_base_dir = _rendered_base_dir(attachment_config, values)
individual_paths = config.attachments.individual_base_path_values selected = _selected_base_path(config, attachment_config, rendered_base_dir)
if individual_paths: if config.attachments.base_paths:
return rendered_base_dir in individual_paths return bool(selected and selected.allow_individual)
return config.attachments.allow_individual return config.attachments.allow_individual
@@ -271,6 +284,7 @@ def _resolve_attachment_directory(
*, *,
campaign_file: str | Path, campaign_file: str | Path,
campaign_config: CampaignConfig, campaign_config: CampaignConfig,
attachment_config: AttachmentConfig,
rendered_base_dir: str, rendered_base_dir: str,
) -> tuple[Path, AttachmentBasePathConfig | None]: ) -> tuple[Path, AttachmentBasePathConfig | None]:
"""Resolve the directory for an attachment rule. """Resolve the directory for an attachment rule.
@@ -281,7 +295,7 @@ def _resolve_attachment_directory(
e.g. attachments/base_path + base_dir twice. e.g. attachments/base_path + base_dir twice.
""" """
selected_base_path = _selected_base_path(campaign_config, rendered_base_dir) selected_base_path = _selected_base_path(campaign_config, attachment_config, rendered_base_dir)
if selected_base_path is not None: if selected_base_path is not None:
return _resolve_path(campaign_file, selected_base_path.path), selected_base_path return _resolve_path(campaign_file, selected_base_path.path), selected_base_path
@@ -337,6 +351,7 @@ def _resolve_one_config(
directory, selected_base_path = _resolve_attachment_directory( directory, selected_base_path = _resolve_attachment_directory(
campaign_file=campaign_file, campaign_file=campaign_file,
campaign_config=campaign_config, campaign_config=campaign_config,
attachment_config=config,
rendered_base_dir=rendered_base_dir, rendered_base_dir=rendered_base_dir,
) )
matches = _match_files(directory, rendered_file_filter, config.include_subdirs) matches = _match_files(directory, rendered_file_filter, config.include_subdirs)

View File

@@ -223,6 +223,7 @@ class AttachmentBasePathConfig(StrictModel):
class AttachmentConfig(StrictModel): class AttachmentConfig(StrictModel):
id: str | None = None id: str | None = None
label: str | None = None label: str | None = None
base_path_id: str | None = None
# Legacy UI helper. Current attachment resolution ignores this value and # Legacy UI helper. Current attachment resolution ignores this value and
# treats direct files as plain file_filter patterns without wildcards. # treats direct files as plain file_filter patterns without wildcards.
# Keep accepting it so existing drafts with {"type": ""}, "direct" # Keep accepting it so existing drafts with {"type": ""}, "direct"

View File

@@ -1,7 +1,5 @@
from __future__ import annotations from __future__ import annotations
import json
import tempfile
from email import policy from email import policy
from email.message import EmailMessage from email.message import EmailMessage
from pathlib import Path from pathlib import Path
@@ -14,6 +12,10 @@ from app.mailer.campaign.loader import load_campaign_config
from app.mailer.campaign.validation import validate_campaign_config from app.mailer.campaign.validation import validate_campaign_config
from app.mailer.messages.builder import build_campaign_messages from app.mailer.messages.builder import build_campaign_messages
from app.mailer.messages.models import MessageAddress, MessageDraft, MessageValidationStatus from app.mailer.messages.models import MessageAddress, MessageDraft, MessageValidationStatus
from app.storage.campaign_attachments import (
annotate_built_messages_with_managed_files,
prepared_campaign_snapshot,
)
from app.mailer.dev.mock_mailbox import ( from app.mailer.dev.mock_mailbox import (
clear_records, clear_records,
consume_fail_next_imap, consume_fail_next_imap,
@@ -147,13 +149,18 @@ def run_mock_campaign_send(
if clear_mailbox: if clear_mailbox:
clear_records() clear_records()
with tempfile.TemporaryDirectory(prefix="multimailer-mock-send-") as temp_dir: with prepared_campaign_snapshot(
temp_path = Path(temp_dir) session,
campaign_path = temp_path / f"campaign-{version.id}.json" tenant_id=tenant_id,
campaign_path.write_text(json.dumps(version.raw_json, ensure_ascii=False, indent=2), encoding="utf-8") campaign_id=campaign.id,
config = load_campaign_config(campaign_path) raw_json=version.raw_json if isinstance(version.raw_json, dict) else {},
validation_report = validate_campaign_config(config, campaign_file=campaign_path, check_files=check_files) include_bytes=True,
build_result = build_campaign_messages(config, campaign_file=campaign_path, write_eml=False) prefix="multimailer-mock-send-",
) as prepared:
config = load_campaign_config(prepared.path)
validation_report = validate_campaign_config(config, campaign_file=prepared.path, check_files=check_files)
build_result = build_campaign_messages(config, campaign_file=prepared.path, write_eml=False)
annotate_built_messages_with_managed_files(build_result.built_messages, prepared.managed_files_by_local_path)
send_results: list[dict[str, Any]] = [] send_results: list[dict[str, Any]] = []
sent_count = 0 sent_count = 0

View File

@@ -57,6 +57,7 @@ class MessageAttachmentSummary(BaseModel):
file_filter: str file_filter: str
directory: str directory: str
matches: list[str] = Field(default_factory=list) matches: list[str] = Field(default_factory=list)
managed_matches: list[dict[str, object]] = Field(default_factory=list)
class MessageDraft(BaseModel): class MessageDraft(BaseModel):

View File

@@ -26,6 +26,10 @@ from app.mailer.campaign.validation import Severity, validate_campaign_config
from app.mailer.messages.builder import build_campaign_messages from app.mailer.messages.builder import build_campaign_messages
from app.mailer.messages.models import MessageDraft from app.mailer.messages.models import MessageDraft
from app.storage.services import record_campaign_attachment_uses_for_job from app.storage.services import record_campaign_attachment_uses_for_job
from app.storage.campaign_attachments import (
annotate_built_messages_with_managed_files,
prepared_campaign_snapshot,
)
RUNTIME_DIR = Path(__file__).resolve().parents[3] / "runtime" RUNTIME_DIR = Path(__file__).resolve().parents[3] / "runtime"
CAMPAIGN_SNAPSHOT_DIR = RUNTIME_DIR / "campaign_snapshots" CAMPAIGN_SNAPSHOT_DIR = RUNTIME_DIR / "campaign_snapshots"
@@ -195,7 +199,19 @@ def validate_campaign_version(
}: }:
raise CampaignPersistenceError("Audit-safe/final campaign versions cannot be validated. Create an editable copy instead.") raise CampaignPersistenceError("Audit-safe/final campaign versions cannot be validated. Create an editable copy instead.")
report = validate_campaign_config(config, campaign_file=snapshot_path, check_files=check_files) if check_files:
with prepared_campaign_snapshot(
session,
tenant_id=tenant_id,
campaign_id=campaign.id,
raw_json=version.raw_json if isinstance(version.raw_json, dict) else {},
include_bytes=False,
prefix="multimailer-managed-validate-",
) as prepared:
managed_config = load_campaign_config(prepared.path)
report = validate_campaign_config(managed_config, campaign_file=prepared.path, check_files=True)
else:
report = validate_campaign_config(config, campaign_file=snapshot_path, check_files=False)
report_json = report.model_dump(mode="json") report_json = report.model_dump(mode="json")
report_json.update({"ok": report.ok, "error_count": report.error_count, "warning_count": report.warning_count}) report_json.update({"ok": report.ok, "error_count": report.error_count, "warning_count": report.warning_count})
version.validation_summary = report_json version.validation_summary = report_json
@@ -298,7 +314,17 @@ def build_campaign_version(
_ensure_version_validated_and_locked(version) _ensure_version_validated_and_locked(version)
output_dir = BUILD_OUTPUT_DIR / campaign.id / version.id output_dir = BUILD_OUTPUT_DIR / campaign.id / version.id
result = build_campaign_messages(config, campaign_file=snapshot_path, output_dir=output_dir, write_eml=write_eml) with prepared_campaign_snapshot(
session,
tenant_id=tenant_id,
campaign_id=campaign.id,
raw_json=version.raw_json if isinstance(version.raw_json, dict) else {},
include_bytes=True,
prefix="multimailer-managed-build-",
) as prepared:
managed_config = load_campaign_config(prepared.path)
result = build_campaign_messages(managed_config, campaign_file=prepared.path, output_dir=output_dir, write_eml=write_eml)
annotate_built_messages_with_managed_files(result.built_messages, prepared.managed_files_by_local_path)
report_json = result.report.model_dump(mode="json", by_alias=True) report_json = result.report.model_dump(mode="json", by_alias=True)
report_json.update({ report_json.update({
"built_count": result.report.built_count, "built_count": result.report.built_count,

View File

@@ -648,6 +648,13 @@
"label": { "label": {
"type": "string" "type": "string"
}, },
"base_path_id": {
"type": [
"string",
"null"
],
"description": "Stable reference to attachments.base_paths[].id. Preferred over path matching when multiple managed spaces use the same logical path."
},
"base_dir": { "base_dir": {
"type": "string", "type": "string",
"description": "Selected attachment base path for current WebUI campaigns, or a directory relative to attachments.base_path for legacy campaigns." "description": "Selected attachment base path for current WebUI campaigns, or a directory relative to attachments.base_path for legacy campaigns."

View File

@@ -0,0 +1,293 @@
from __future__ import annotations
import copy
import json
import shutil
import tempfile
from contextlib import contextmanager
from dataclasses import asdict, dataclass
from pathlib import Path, PurePosixPath
from typing import Any, Iterator
from sqlalchemy.orm import Session
from app.db.models import FileAsset
from app.storage.files import current_version_and_blob, list_assets_for_user, read_asset_bytes
from app.storage.paths import normalize_folder, normalize_logical_path, safe_storage_component
MANAGED_SOURCE_PREFIX = "managed:"
@dataclass(frozen=True, slots=True)
class ManagedAttachmentFile:
local_path: str
asset_id: str
version_id: str
blob_id: str
display_path: str
relative_path: str
filename: str
owner_type: str
owner_id: str
checksum_sha256: str
size_bytes: int
content_type: str | None
def as_dict(self) -> dict[str, Any]:
payload = asdict(self)
payload.pop("local_path", None)
return payload
@dataclass(slots=True)
class PreparedCampaignSnapshot:
path: Path
raw_json: dict[str, Any]
managed_files_by_local_path: dict[str, ManagedAttachmentFile]
shared_assets: list[FileAsset]
def parse_managed_source(value: object) -> tuple[str, str] | None:
if not isinstance(value, str) or not value.startswith(MANAGED_SOURCE_PREFIX):
return None
parts = value.split(":", 2)
if len(parts) != 3 or parts[1] not in {"user", "group"} or not parts[2].strip():
return None
return parts[1], parts[2].strip()
def _asset_owner_id(asset: FileAsset) -> str | None:
if asset.owner_type == "user":
return asset.owner_user_id
if asset.owner_type == "group":
return asset.owner_group_id
return None
def _relative_asset_path(asset: FileAsset, logical_root: str) -> str | None:
display_path = normalize_logical_path(asset.display_path)
root = normalize_folder(logical_root)
if not root:
return display_path
prefix = f"{root}/"
if not display_path.startswith(prefix):
return None
return display_path[len(prefix) :]
def _safe_local_target(root: Path, relative_path: str) -> Path:
parts = PurePosixPath(normalize_logical_path(relative_path)).parts
if not parts or any(part in {"", ".", ".."} for part in parts):
raise ValueError(f"Unsafe managed attachment path: {relative_path!r}")
target = root.joinpath(*parts).resolve()
resolved_root = root.resolve()
if not target.is_relative_to(resolved_root):
raise ValueError(f"Managed attachment path escapes materialization root: {relative_path!r}")
return target
def _iter_rule_dicts(attachments: dict[str, Any], raw_json: dict[str, Any]):
global_rules = attachments.get("global")
if isinstance(global_rules, list):
for rule in global_rules:
if isinstance(rule, dict):
yield rule
entries = raw_json.get("entries")
inline = entries.get("inline") if isinstance(entries, dict) else None
if isinstance(inline, list):
for entry in inline:
if not isinstance(entry, dict):
continue
rules = entry.get("attachments")
if isinstance(rules, list):
for rule in rules:
if isinstance(rule, dict):
yield rule
def _selected_base_path(
rule: dict[str, Any],
prepared_by_id: dict[str, tuple[str, str]],
prepared_by_old_path: dict[str, list[tuple[str, str]]],
first_prepared: tuple[str, str] | None,
) -> tuple[str, str] | None:
base_path_id = str(rule.get("base_path_id") or "").strip()
if base_path_id and base_path_id in prepared_by_id:
return prepared_by_id[base_path_id]
base_dir = str(rule.get("base_dir") or ".").strip() or "."
candidates = prepared_by_old_path.get(base_dir)
if candidates:
return candidates[0]
if base_dir in {"", "."}:
return first_prepared
return None
def prepare_campaign_snapshot(
session: Session,
*,
tenant_id: str,
campaign_id: str,
raw_json: dict[str, Any],
destination: Path,
include_bytes: bool,
) -> PreparedCampaignSnapshot:
"""Create a temporary file-oriented campaign snapshot for managed attachments.
The existing mailer resolver deliberately remains file-oriented. Managed
campaign-shared file versions are materialized into an isolated tree and a
copied campaign JSON is rewritten to point to those directories. The
returned manifest preserves exact asset/version/blob identity so build and
audit code never has to guess by filename.
"""
destination = destination.expanduser().resolve()
destination.mkdir(parents=True, exist_ok=True)
materialized_root = destination / "managed-attachments"
materialized_root.mkdir(parents=True, exist_ok=True)
prepared_json = copy.deepcopy(raw_json if isinstance(raw_json, dict) else {})
attachments = prepared_json.get("attachments")
if not isinstance(attachments, dict):
attachments = {}
prepared_json["attachments"] = attachments
base_paths = attachments.get("base_paths")
if not isinstance(base_paths, list):
base_paths = []
shared_assets = list_assets_for_user(
session,
tenant_id=tenant_id,
user_id="",
campaign_id=campaign_id,
is_admin=True,
)
manifest: dict[str, ManagedAttachmentFile] = {}
prepared_by_id: dict[str, tuple[str, str]] = {}
prepared_by_old_path: dict[str, list[tuple[str, str]]] = {}
first_prepared: tuple[str, str] | None = None
for index, item in enumerate(base_paths):
if not isinstance(item, dict):
continue
parsed_source = parse_managed_source(item.get("source"))
if parsed_source is None:
continue
owner_type, owner_id = parsed_source
old_path = str(item.get("path") or ".").strip() or "."
logical_root = "" if old_path in {"", ".", "/"} else normalize_folder(old_path)
base_path_id = str(item.get("id") or f"base-path-{index + 1}")
local_root = materialized_root / f"{index + 1:03d}-{safe_storage_component(base_path_id)}"
local_root.mkdir(parents=True, exist_ok=True)
local_root_string = str(local_root.resolve())
prepared = (base_path_id, local_root_string)
prepared_by_id[base_path_id] = prepared
prepared_by_old_path.setdefault(old_path, []).append(prepared)
if first_prepared is None:
first_prepared = prepared
item["path"] = local_root_string
for asset in shared_assets:
if asset.owner_type != owner_type or _asset_owner_id(asset) != owner_id:
continue
relative_path = _relative_asset_path(asset, logical_root)
if not relative_path:
continue
target = _safe_local_target(local_root, relative_path)
target.parent.mkdir(parents=True, exist_ok=True)
if include_bytes:
data, version, blob = read_asset_bytes(session, asset)
target.write_bytes(data)
else:
version, blob = current_version_and_blob(session, asset)
target.touch()
local_key = str(target.resolve())
manifest[local_key] = ManagedAttachmentFile(
local_path=local_key,
asset_id=asset.id,
version_id=version.id,
blob_id=blob.id,
display_path=asset.display_path,
relative_path=normalize_logical_path(relative_path),
filename=asset.filename,
owner_type=asset.owner_type,
owner_id=owner_id,
checksum_sha256=blob.checksum_sha256,
size_bytes=blob.size_bytes,
content_type=blob.content_type,
)
for rule in _iter_rule_dicts(attachments, prepared_json):
selected = _selected_base_path(rule, prepared_by_id, prepared_by_old_path, first_prepared)
if selected is None:
continue
base_path_id, local_root_string = selected
rule["base_path_id"] = base_path_id
rule["base_dir"] = local_root_string
if first_prepared is not None:
attachments["base_path"] = first_prepared[1]
snapshot_path = destination / "campaign.json"
snapshot_path.write_text(json.dumps(prepared_json, ensure_ascii=False, indent=2), encoding="utf-8")
return PreparedCampaignSnapshot(
path=snapshot_path,
raw_json=prepared_json,
managed_files_by_local_path=manifest,
shared_assets=shared_assets,
)
@contextmanager
def prepared_campaign_snapshot(
session: Session,
*,
tenant_id: str,
campaign_id: str,
raw_json: dict[str, Any],
include_bytes: bool,
prefix: str = "multimailer-managed-campaign-",
) -> Iterator[PreparedCampaignSnapshot]:
temp_dir = Path(tempfile.mkdtemp(prefix=prefix))
try:
yield prepare_campaign_snapshot(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
raw_json=raw_json,
destination=temp_dir,
include_bytes=include_bytes,
)
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
def managed_match_payloads(
matches: list[str],
manifest: dict[str, ManagedAttachmentFile],
) -> list[dict[str, Any]]:
payloads: list[dict[str, Any]] = []
for match in matches:
item = manifest.get(str(Path(match).resolve()))
if item is not None:
payloads.append(item.as_dict())
return payloads
def annotate_built_messages_with_managed_files(
built_messages: list[Any],
manifest: dict[str, ManagedAttachmentFile],
) -> None:
"""Attach exact managed-file identities to built attachment summaries."""
for built in built_messages:
draft = getattr(built, "draft", None)
for attachment in getattr(draft, "attachments", []) if draft is not None else []:
matches = list(getattr(attachment, "matches", []) or [])
attachment.managed_matches = managed_match_payloads(matches, manifest)

View File

@@ -4,7 +4,7 @@ from pathlib import PurePosixPath
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.db.models import CampaignAttachmentUse, CampaignJob, FileAsset from app.db.models import CampaignAttachmentUse, CampaignJob, FileAsset, FileBlob, FileVersion
from app.storage.common import utcnow from app.storage.common import utcnow
from app.storage.files import current_version_and_blob, list_assets_for_user from app.storage.files import current_version_and_blob, list_assets_for_user
@@ -17,51 +17,28 @@ def _candidate_match_keys(raw_match: str) -> set[str]:
return {item for item in result if item} return {item for item in result if item}
def record_campaign_attachment_uses_for_job(session: Session, job: CampaignJob, *, stage: str = "built") -> None: def _add_use(
"""Create best-effort immutable file-use records for matched managed files. session: Session,
job: CampaignJob,
Existing attachment resolution is still filesystem/path based. This bridge *,
records uses when a resolved attachment match can be tied to a managed file asset: FileAsset,
by logical path or filename among files shared with the campaign. version: FileVersion,
""" blob: FileBlob,
filename_used: str,
attachments = job.resolved_attachments or [] stage: str,
if not isinstance(attachments, list): ) -> None:
return
assets = list_assets_for_user(
session,
tenant_id=job.tenant_id,
user_id="",
campaign_id=job.campaign_id,
is_admin=True,
)
by_key: dict[str, FileAsset] = {}
for asset in assets:
by_key[asset.display_path.strip("/")] = asset
by_key[asset.filename] = asset
for attachment in attachments:
if not isinstance(attachment, dict):
continue
matches = attachment.get("matches") if isinstance(attachment.get("matches"), list) else []
for raw in matches:
if not isinstance(raw, str):
continue
asset = next((by_key[key] for key in _candidate_match_keys(raw) if key in by_key), None)
if not asset:
continue
version, blob = current_version_and_blob(session, asset)
exists = ( exists = (
session.query(CampaignAttachmentUse) session.query(CampaignAttachmentUse)
.filter( .filter(
CampaignAttachmentUse.campaign_job_id == job.id, CampaignAttachmentUse.campaign_job_id == job.id,
CampaignAttachmentUse.file_version_id == version.id, CampaignAttachmentUse.file_version_id == version.id,
CampaignAttachmentUse.filename_used == asset.filename, CampaignAttachmentUse.filename_used == filename_used,
CampaignAttachmentUse.use_stage == stage, CampaignAttachmentUse.use_stage == stage,
) )
.one_or_none() .one_or_none()
) )
if exists: if exists:
continue return
session.add( session.add(
CampaignAttachmentUse( CampaignAttachmentUse(
tenant_id=job.tenant_id, tenant_id=job.tenant_id,
@@ -73,7 +50,7 @@ def record_campaign_attachment_uses_for_job(session: Session, job: CampaignJob,
file_asset_id=asset.id, file_asset_id=asset.id,
file_version_id=version.id, file_version_id=version.id,
file_blob_id=blob.id, file_blob_id=blob.id,
filename_used=asset.filename, filename_used=filename_used,
checksum_sha256=blob.checksum_sha256, checksum_sha256=blob.checksum_sha256,
size_bytes=blob.size_bytes, size_bytes=blob.size_bytes,
content_type=blob.content_type, content_type=blob.content_type,
@@ -82,6 +59,79 @@ def record_campaign_attachment_uses_for_job(session: Session, job: CampaignJob,
) )
def record_campaign_attachment_uses_for_job(session: Session, job: CampaignJob, *, stage: str = "built") -> None:
"""Record immutable managed file versions used by a built/sent job.
New builds carry exact managed asset/version/blob IDs. Filename matching is
retained only as a compatibility fallback for jobs created before managed
attachment materialization was introduced.
"""
attachments = job.resolved_attachments or []
if not isinstance(attachments, list):
return
assets = list_assets_for_user(
session,
tenant_id=job.tenant_id,
user_id="",
campaign_id=job.campaign_id,
is_admin=True,
)
assets_by_id = {asset.id: asset for asset in assets}
for attachment in attachments:
if not isinstance(attachment, dict):
continue
managed_matches = attachment.get("managed_matches")
if isinstance(managed_matches, list):
for item in managed_matches:
if not isinstance(item, dict):
continue
asset = assets_by_id.get(str(item.get("asset_id") or ""))
version = session.get(FileVersion, str(item.get("version_id") or ""))
blob = session.get(FileBlob, str(item.get("blob_id") or ""))
if not asset or not version or not blob:
continue
if version.file_asset_id != asset.id or version.blob_id != blob.id:
continue
_add_use(
session,
job,
asset=asset,
version=version,
blob=blob,
filename_used=str(item.get("filename") or asset.filename),
stage=stage,
)
# Compatibility fallback for older job snapshots without managed_matches.
by_key: dict[str, FileAsset] = {}
for asset in assets:
by_key[asset.display_path.strip("/")] = asset
by_key.setdefault(asset.filename, asset)
for attachment in attachments:
if not isinstance(attachment, dict) or attachment.get("managed_matches"):
continue
matches = attachment.get("matches") if isinstance(attachment.get("matches"), list) else []
for raw in matches:
if not isinstance(raw, str):
continue
asset = next((by_key[key] for key in _candidate_match_keys(raw) if key in by_key), None)
if not asset:
continue
version, blob = current_version_and_blob(session, asset)
_add_use(
session,
job,
asset=asset,
version=version,
blob=blob,
filename_used=asset.filename,
stage=stage,
)
def mark_job_attachment_uses_sent(session: Session, job: CampaignJob) -> None: def mark_job_attachment_uses_sent(session: Session, job: CampaignJob) -> None:
record_campaign_attachment_uses_for_job(session, job, stage="built") record_campaign_attachment_uses_for_job(session, job, stage="built")
now = utcnow() now = utcnow()

Binary file not shown.

View File

@@ -252,6 +252,163 @@ class ApiSmokeTests(unittest.TestCase):
self.assertEqual(result["send"]["imap_appended_count"], 1) self.assertEqual(result["send"]["imap_appended_count"], 1)
self.assertEqual(result["send"]["imap_failed_count"], 0) self.assertEqual(result["send"]["imap_failed_count"], 0)
def test_managed_attachment_patterns_preview_build_and_mock_send(self) -> None:
headers, login = self._login()
user_id = login["user"]["id"]
campaign_json = {
"version": "1.0",
"campaign": {"id": "managed-attachments", "name": "Managed attachments", "mode": "test"},
"fields": [{"name": "invoice_number", "type": "string", "required": True}],
"global_values": {},
"server": {
"smtp": {
"host": "smtp.example.invalid",
"port": 587,
"username": "sender@example.org",
"password": "test-secret",
"security": "starttls",
}
},
"recipients": {
"from": {"email": "sender@example.org", "name": "Sender", "type": "to"},
"allow_individual_to": True,
},
"template": {"subject": "Invoice", "text": "Please see the attached files."},
"attachments": {
"base_path": "invoices",
"base_paths": [
{
"id": "personal-invoices",
"name": "My invoices",
"source": f"managed:user:{user_id}",
"path": "invoices",
"allow_individual": True,
}
],
"global": [],
"allow_individual": True,
},
"entries": {
"inline": [
{
"id": "recipient-1",
"to": [{"email": "recipient@example.org", "name": "Recipient", "type": "to"}],
"fields": {"invoice_number": "202605-010001"},
"attachments": [
{
"id": "nested-pattern",
"label": "Nested workbook",
"base_path_id": "personal-invoices",
"base_dir": "invoices",
"file_filter": "**/{{local:invoice_number}}-report.XLSX",
"required": True,
},
{
"id": "exact-pattern",
"label": "Exact workbook",
"base_path_id": "personal-invoices",
"base_dir": "invoices",
"file_filter": "{{local:invoice_number}}-90100010-9601741.XLSX",
"required": True,
},
],
}
]
},
"validation_policy": {
"missing_email": "block",
"template_error": "block",
"missing_required_attachment": "block",
},
"delivery": {"imap_append_sent": {"enabled": False}},
"status_tracking": {"enabled": True},
}
created = self.client.post("/api/v1/campaigns", headers=headers, json={"config": campaign_json})
self.assertEqual(created.status_code, 200, created.text)
campaign_id = created.json()["campaign"]["id"]
version_id = created.json()["version"]["id"]
for path, filename, content in [
("invoices/archive", "202605-010001-report.XLSX", b"nested workbook"),
("invoices", "202605-010001-90100010-9601741.XLSX", b"exact workbook"),
]:
uploaded = self.client.post(
"/api/v1/files/upload",
headers=headers,
data={
"owner_type": "user",
"owner_id": user_id,
"path": path,
"campaign_id": campaign_id,
},
files=[("files", (filename, content, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"))],
)
self.assertEqual(uploaded.status_code, 200, uploaded.text)
preview = self.client.post(
f"/api/v1/campaigns/{campaign_id}/versions/{version_id}/attachments/preview",
headers=headers,
json={"include_unmatched": True},
)
self.assertEqual(preview.status_code, 200, preview.text)
preview_payload = preview.json()
self.assertEqual(len(preview_payload["rules"]), 2)
self.assertEqual([rule["match_count"] for rule in preview_payload["rules"]], [1, 1])
self.assertEqual(
{rule["matches"][0]["filename"] for rule in preview_payload["rules"]},
{"202605-010001-report.XLSX", "202605-010001-90100010-9601741.XLSX"},
)
self.assertEqual(preview_payload["unused_shared_files"], [])
validated = self.client.post(
f"/api/v1/campaigns/versions/{version_id}/validate",
headers=headers,
json={"check_files": True},
)
self.assertEqual(validated.status_code, 200, validated.text)
self.assertTrue(validated.json()["ok"], validated.text)
built = self.client.post(
f"/api/v1/campaigns/versions/{version_id}/build",
headers=headers,
json={"write_eml": False},
)
self.assertEqual(built.status_code, 200, built.text)
self.assertEqual(built.json()["built_count"], 1)
self.assertEqual(built.json()["messages"][0]["attachment_count"], 2)
self.assertEqual(
sum(len(item["managed_matches"]) for item in built.json()["messages"][0]["attachments"]),
2,
)
mocked = self.client.post(
f"/api/v1/campaigns/{campaign_id}/mock-send",
headers=headers,
json={
"version_id": version_id,
"send": True,
"append_sent": False,
"clear_mailbox": True,
"check_files": False,
},
)
self.assertEqual(mocked.status_code, 200, mocked.text)
result = mocked.json()["result"]
self.assertTrue(result["validation"]["ok"], mocked.text)
self.assertEqual(result["send"]["sent_count"], 1)
self.assertEqual(result["send"]["results"][0]["attachments"][0]["status"], "ok")
from app.db.models import CampaignAttachmentUse
with SessionLocal() as session:
uses = session.query(CampaignAttachmentUse).filter(CampaignAttachmentUse.campaign_id == campaign_id).all()
self.assertEqual(len(uses), 2)
self.assertEqual({use.filename_used for use in uses}, {
"202605-010001-report.XLSX",
"202605-010001-90100010-9601741.XLSX",
})
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()