intermittent commit
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any, Iterable
|
||||
|
||||
@@ -24,6 +25,15 @@ def _candidate_match_keys(raw_match: str) -> set[str]:
|
||||
AttachmentUseKey = tuple[str, str, str, str]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _AttachmentBatchRefs:
|
||||
managed_by_job: dict[str, list[dict[str, object]]] = field(default_factory=lambda: defaultdict(list))
|
||||
fallback_attachments_by_job: dict[str, list[dict[str, object]]] = field(default_factory=lambda: defaultdict(list))
|
||||
asset_ids: set[str] = field(default_factory=set)
|
||||
version_ids: set[str] = field(default_factory=set)
|
||||
blob_ids: set[str] = field(default_factory=set)
|
||||
|
||||
|
||||
def _attachment_use_key(*, job_id: str, file_version_id: str, filename_used: str, stage: str) -> AttachmentUseKey:
|
||||
return job_id, file_version_id, filename_used, stage
|
||||
|
||||
@@ -134,62 +144,91 @@ def record_campaign_attachment_uses_for_jobs(
|
||||
if not job_list:
|
||||
return
|
||||
|
||||
managed_by_job: dict[str, list[dict[str, object]]] = defaultdict(list)
|
||||
fallback_attachments_by_job: dict[str, list[dict[str, object]]] = defaultdict(list)
|
||||
refs = _collect_attachment_batch_refs(job_list)
|
||||
job_by_id = {job.id: job for job in job_list}
|
||||
asset_ids: set[str] = set()
|
||||
version_ids: set[str] = set()
|
||||
blob_ids: set[str] = set()
|
||||
known_keys = _known_use_keys_for_jobs(session, job_list, stage=stage)
|
||||
assets_by_id, versions_by_id, blobs_by_id = _load_managed_attachment_entities(session, refs)
|
||||
|
||||
for job in job_list:
|
||||
_record_managed_attachment_uses(
|
||||
session,
|
||||
job_by_id=job_by_id,
|
||||
managed_by_job=refs.managed_by_job,
|
||||
assets_by_id=assets_by_id,
|
||||
versions_by_id=versions_by_id,
|
||||
blobs_by_id=blobs_by_id,
|
||||
stage=stage,
|
||||
known_keys=known_keys,
|
||||
)
|
||||
_record_fallback_attachment_uses(
|
||||
session,
|
||||
job_by_id=job_by_id,
|
||||
fallback_attachments_by_job=refs.fallback_attachments_by_job,
|
||||
stage=stage,
|
||||
known_keys=known_keys,
|
||||
)
|
||||
|
||||
|
||||
def _collect_attachment_batch_refs(jobs: list[CampaignJobLike]) -> _AttachmentBatchRefs:
|
||||
refs = _AttachmentBatchRefs()
|
||||
for job in jobs:
|
||||
attachments = job.resolved_attachments or []
|
||||
if not isinstance(attachments, list):
|
||||
continue
|
||||
for attachment in attachments:
|
||||
if not isinstance(attachment, dict):
|
||||
continue
|
||||
managed_matches = attachment.get("managed_matches")
|
||||
if isinstance(managed_matches, list) and managed_matches:
|
||||
for item in managed_matches:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
asset_id = str(item.get("asset_id") or "")
|
||||
version_id = str(item.get("version_id") or "")
|
||||
blob_id = str(item.get("blob_id") or "")
|
||||
if not asset_id or not version_id or not blob_id:
|
||||
continue
|
||||
managed_by_job[job.id].append(item)
|
||||
asset_ids.add(asset_id)
|
||||
version_ids.add(version_id)
|
||||
blob_ids.add(blob_id)
|
||||
else:
|
||||
fallback_attachments_by_job[job.id].append(attachment)
|
||||
if not _collect_managed_attachment_refs(refs, job.id, attachment):
|
||||
refs.fallback_attachments_by_job[job.id].append(attachment)
|
||||
return refs
|
||||
|
||||
assets_by_id = {
|
||||
item.id: item
|
||||
for item in session.query(FileAsset).filter(FileAsset.id.in_(asset_ids)).all()
|
||||
} if asset_ids else {}
|
||||
versions_by_id = {
|
||||
item.id: item
|
||||
for item in session.query(FileVersion).filter(FileVersion.id.in_(version_ids)).all()
|
||||
} if version_ids else {}
|
||||
blobs_by_id = {
|
||||
item.id: item
|
||||
for item in session.query(FileBlob).filter(FileBlob.id.in_(blob_ids)).all()
|
||||
} if blob_ids else {}
|
||||
known_keys = _known_use_keys_for_jobs(session, job_list, stage=stage)
|
||||
|
||||
def _collect_managed_attachment_refs(refs: _AttachmentBatchRefs, job_id: str, attachment: dict[str, object]) -> bool:
|
||||
managed_matches = attachment.get("managed_matches")
|
||||
if not isinstance(managed_matches, list) or not managed_matches:
|
||||
return False
|
||||
for item in managed_matches:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
asset_id = str(item.get("asset_id") or "")
|
||||
version_id = str(item.get("version_id") or "")
|
||||
blob_id = str(item.get("blob_id") or "")
|
||||
if not asset_id or not version_id or not blob_id:
|
||||
continue
|
||||
refs.managed_by_job[job_id].append(item)
|
||||
refs.asset_ids.add(asset_id)
|
||||
refs.version_ids.add(version_id)
|
||||
refs.blob_ids.add(blob_id)
|
||||
return True
|
||||
|
||||
|
||||
def _load_managed_attachment_entities(
|
||||
session: Session,
|
||||
refs: _AttachmentBatchRefs,
|
||||
) -> tuple[dict[str, FileAsset], dict[str, FileVersion], dict[str, FileBlob]]:
|
||||
assets_by_id = {item.id: item for item in session.query(FileAsset).filter(FileAsset.id.in_(refs.asset_ids)).all()} if refs.asset_ids else {}
|
||||
versions_by_id = {item.id: item for item in session.query(FileVersion).filter(FileVersion.id.in_(refs.version_ids)).all()} if refs.version_ids else {}
|
||||
blobs_by_id = {item.id: item for item in session.query(FileBlob).filter(FileBlob.id.in_(refs.blob_ids)).all()} if refs.blob_ids else {}
|
||||
return assets_by_id, versions_by_id, blobs_by_id
|
||||
|
||||
|
||||
def _record_managed_attachment_uses(
|
||||
session: Session,
|
||||
*,
|
||||
job_by_id: dict[str, CampaignJobLike],
|
||||
managed_by_job: dict[str, list[dict[str, object]]],
|
||||
assets_by_id: dict[str, FileAsset],
|
||||
versions_by_id: dict[str, FileVersion],
|
||||
blobs_by_id: dict[str, FileBlob],
|
||||
stage: str,
|
||||
known_keys: set[AttachmentUseKey],
|
||||
) -> None:
|
||||
for job_id, items in managed_by_job.items():
|
||||
job = job_by_id[job_id]
|
||||
for item in items:
|
||||
asset = assets_by_id.get(str(item.get("asset_id") or ""))
|
||||
version = versions_by_id.get(str(item.get("version_id") or ""))
|
||||
blob = blobs_by_id.get(str(item.get("blob_id") or ""))
|
||||
if not asset or not version or not blob:
|
||||
continue
|
||||
if asset.tenant_id != job.tenant_id or version.tenant_id != job.tenant_id or blob.tenant_id != job.tenant_id:
|
||||
continue
|
||||
if version.file_asset_id != asset.id or version.blob_id != blob.id:
|
||||
if not _managed_attachment_entities_match_job(job, asset, version, blob):
|
||||
continue
|
||||
_add_use(
|
||||
session,
|
||||
@@ -202,50 +241,100 @@ def record_campaign_attachment_uses_for_jobs(
|
||||
known_keys=known_keys,
|
||||
)
|
||||
|
||||
|
||||
def _managed_attachment_entities_match_job(
|
||||
job: CampaignJobLike,
|
||||
asset: FileAsset | None,
|
||||
version: FileVersion | None,
|
||||
blob: FileBlob | None,
|
||||
) -> bool:
|
||||
if not asset or not version or not blob:
|
||||
return False
|
||||
if asset.tenant_id != job.tenant_id or version.tenant_id != job.tenant_id or blob.tenant_id != job.tenant_id:
|
||||
return False
|
||||
return version.file_asset_id == asset.id and version.blob_id == blob.id
|
||||
|
||||
|
||||
def _record_fallback_attachment_uses(
|
||||
session: Session,
|
||||
*,
|
||||
job_by_id: dict[str, CampaignJobLike],
|
||||
fallback_attachments_by_job: dict[str, list[dict[str, object]]],
|
||||
stage: str,
|
||||
known_keys: set[AttachmentUseKey],
|
||||
) -> None:
|
||||
assets_by_campaign: dict[tuple[str, str], dict[str, FileAsset]] = {}
|
||||
version_blobs_by_campaign: dict[tuple[str, str], dict[str, tuple[FileVersion, FileBlob]]] = {}
|
||||
for job_id, attachments in fallback_attachments_by_job.items():
|
||||
job = job_by_id[job_id]
|
||||
campaign_key = (job.tenant_id, job.campaign_id)
|
||||
by_key = assets_by_campaign.get(campaign_key)
|
||||
if by_key is None:
|
||||
assets = list_assets_for_user(
|
||||
session,
|
||||
tenant_id=job.tenant_id,
|
||||
user_id="",
|
||||
campaign_id=job.campaign_id,
|
||||
is_admin=True,
|
||||
)
|
||||
by_key = {}
|
||||
for asset in assets:
|
||||
by_key[asset.display_path.strip("/")] = asset
|
||||
by_key.setdefault(asset.filename, asset)
|
||||
assets_by_campaign[campaign_key] = by_key
|
||||
version_blobs_by_campaign[campaign_key] = current_versions_and_blobs(session, assets)
|
||||
version_blobs = version_blobs_by_campaign[campaign_key]
|
||||
|
||||
by_key, version_blobs = _fallback_campaign_assets(
|
||||
session,
|
||||
job,
|
||||
campaign_key=campaign_key,
|
||||
assets_by_campaign=assets_by_campaign,
|
||||
version_blobs_by_campaign=version_blobs_by_campaign,
|
||||
)
|
||||
for attachment in attachments:
|
||||
matches = attachment.get("matches") if isinstance(attachment.get("matches"), list) else []
|
||||
for raw in matches:
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
asset = next((by_key[key] for key in _candidate_match_keys(raw) if key in by_key), None)
|
||||
if not asset:
|
||||
continue
|
||||
version_blob = version_blobs.get(asset.id)
|
||||
if not version_blob:
|
||||
continue
|
||||
version, blob = version_blob
|
||||
_add_use(
|
||||
session,
|
||||
job,
|
||||
asset=asset,
|
||||
version=version,
|
||||
blob=blob,
|
||||
filename_used=asset.filename,
|
||||
stage=stage,
|
||||
known_keys=known_keys,
|
||||
)
|
||||
_record_fallback_attachment(session, job, attachment, by_key=by_key, version_blobs=version_blobs, stage=stage, known_keys=known_keys)
|
||||
|
||||
|
||||
def _fallback_campaign_assets(
|
||||
session: Session,
|
||||
job: CampaignJobLike,
|
||||
*,
|
||||
campaign_key: tuple[str, str],
|
||||
assets_by_campaign: dict[tuple[str, str], dict[str, FileAsset]],
|
||||
version_blobs_by_campaign: dict[tuple[str, str], dict[str, tuple[FileVersion, FileBlob]]],
|
||||
) -> tuple[dict[str, FileAsset], dict[str, tuple[FileVersion, FileBlob]]]:
|
||||
by_key = assets_by_campaign.get(campaign_key)
|
||||
if by_key is None:
|
||||
assets = list_assets_for_user(session, tenant_id=job.tenant_id, user_id="", campaign_id=job.campaign_id, is_admin=True)
|
||||
by_key = _asset_lookup_by_path_and_filename(assets)
|
||||
assets_by_campaign[campaign_key] = by_key
|
||||
version_blobs_by_campaign[campaign_key] = current_versions_and_blobs(session, assets)
|
||||
return by_key, version_blobs_by_campaign[campaign_key]
|
||||
|
||||
|
||||
def _asset_lookup_by_path_and_filename(assets: list[FileAsset]) -> dict[str, FileAsset]:
|
||||
by_key: dict[str, FileAsset] = {}
|
||||
for asset in assets:
|
||||
by_key[asset.display_path.strip("/")] = asset
|
||||
by_key.setdefault(asset.filename, asset)
|
||||
return by_key
|
||||
|
||||
|
||||
def _record_fallback_attachment(
|
||||
session: Session,
|
||||
job: CampaignJobLike,
|
||||
attachment: dict[str, object],
|
||||
*,
|
||||
by_key: dict[str, FileAsset],
|
||||
version_blobs: dict[str, tuple[FileVersion, FileBlob]],
|
||||
stage: str,
|
||||
known_keys: set[AttachmentUseKey],
|
||||
) -> None:
|
||||
matches = attachment.get("matches") if isinstance(attachment.get("matches"), list) else []
|
||||
for raw in matches:
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
asset = next((by_key[key] for key in _candidate_match_keys(raw) if key in by_key), None)
|
||||
if not asset:
|
||||
continue
|
||||
version_blob = version_blobs.get(asset.id)
|
||||
if not version_blob:
|
||||
continue
|
||||
version, blob = version_blob
|
||||
_add_use(
|
||||
session,
|
||||
job,
|
||||
asset=asset,
|
||||
version=version,
|
||||
blob=blob,
|
||||
filename_used=asset.filename,
|
||||
stage=stage,
|
||||
known_keys=known_keys,
|
||||
)
|
||||
|
||||
|
||||
def record_campaign_attachment_uses_for_job(session: Session, job: CampaignJobLike, *, stage: str = "built") -> None:
|
||||
|
||||
Reference in New Issue
Block a user