perf(campaign): share linear collision-safe attachment naming
Release v0.1.29. Coordinated integrity review: GovOPlaN/govoplan-core#298.
This commit is contained in:
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@govoplan/campaign-webui",
|
"name": "@govoplan/campaign-webui",
|
||||||
"version": "0.1.28",
|
"version": "0.1.29",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "webui/src/index.ts",
|
"main": "webui/src/index.ts",
|
||||||
|
|||||||
+2
-2
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "govoplan-campaign"
|
name = "govoplan-campaign"
|
||||||
version = "0.1.28"
|
version = "0.1.29"
|
||||||
description = "GovOPlaN campaigns module with backend and WebUI integration."
|
description = "GovOPlaN campaigns module with backend and WebUI integration."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
license = { file = "LICENSE" }
|
license = { file = "LICENSE" }
|
||||||
authors = [{ name = "GovOPlaN" }]
|
authors = [{ name = "GovOPlaN" }]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"govoplan-core>=0.1.45",
|
"govoplan-core>=0.1.46",
|
||||||
"jsonschema>=4,<5",
|
"jsonschema>=4,<5",
|
||||||
"pydantic>=2,<3",
|
"pydantic>=2,<3",
|
||||||
"SQLAlchemy>=2,<3",
|
"SQLAlchemy>=2,<3",
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import replace
|
|
||||||
from typing import Iterable
|
from typing import Iterable
|
||||||
|
|
||||||
from govoplan_core.core.modules import DocumentationTopic
|
from govoplan_core.core.modules import DocumentationTopic, localize_documentation_topics as _localize_topics
|
||||||
|
|
||||||
|
|
||||||
_TRANSLATIONS = {
|
_TRANSLATIONS = {
|
||||||
@@ -258,15 +257,4 @@ _TRANSLATIONS = {
|
|||||||
def localize_documentation_topics(
|
def localize_documentation_topics(
|
||||||
topics: Iterable[DocumentationTopic],
|
topics: Iterable[DocumentationTopic],
|
||||||
) -> tuple[DocumentationTopic, ...]:
|
) -> tuple[DocumentationTopic, ...]:
|
||||||
localized: list[DocumentationTopic] = []
|
return _localize_topics(topics, locale="de", translations=_TRANSLATIONS)
|
||||||
for topic in topics:
|
|
||||||
german = _TRANSLATIONS.get(topic.id)
|
|
||||||
if german is None:
|
|
||||||
localized.append(topic)
|
|
||||||
continue
|
|
||||||
translations = {
|
|
||||||
locale: dict(value) for locale, value in topic.translations.items()
|
|
||||||
}
|
|
||||||
translations["de"] = {**translations.get("de", {}), **german}
|
|
||||||
localized.append(replace(topic, translations=translations))
|
|
||||||
return tuple(localized)
|
|
||||||
|
|||||||
@@ -463,7 +463,7 @@ def _campaigns_router(context: ModuleContext):
|
|||||||
manifest = ModuleManifest(
|
manifest = ModuleManifest(
|
||||||
id="campaigns",
|
id="campaigns",
|
||||||
name="Campaigns",
|
name="Campaigns",
|
||||||
version="0.1.28",
|
version="0.1.29",
|
||||||
workflow_definitions=campaign_workflow_definitions(module_version="0.1.28"),
|
workflow_definitions=campaign_workflow_definitions(module_version="0.1.28"),
|
||||||
required_capabilities=(
|
required_capabilities=(
|
||||||
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
||||||
@@ -834,6 +834,20 @@ manifest = ModuleManifest(
|
|||||||
),
|
),
|
||||||
),
|
),
|
||||||
documentation=localize_documentation_topics((
|
documentation=localize_documentation_topics((
|
||||||
|
DocumentationTopic(
|
||||||
|
id="campaigns.attachment-filename-fidelity",
|
||||||
|
title="Deterministic attachment and ZIP names",
|
||||||
|
summary="Resolve repeated names efficiently without dropping or reordering attachments.",
|
||||||
|
body="Message attachments and ZIP members share a first-free suffix allocator. Repeated names retain the established case-insensitive collision rule and exact numbered suffixes, including names already containing suffixes, Unicode case folding and multiple extensions. Each message/archive has independent allocation state. Large groups of identical requested names no longer restart every suffix search from two. This changes naming work only: intended attachment bytes, recipients, order, existing duplicate-file review policy and ZIP encryption remain unchanged.",
|
||||||
|
layer="available", documentation_types=("user", "admin"), audience=("campaign_manager", "campaign_admin"), order=18,
|
||||||
|
conditions=(DocumentationCondition(required_modules=("campaigns",), any_scopes=("campaigns:campaign:read", "campaigns:campaign:write")),),
|
||||||
|
links=(DocumentationLink(label="Campaigns", href="/campaigns", kind="runtime"),),
|
||||||
|
translations={"de": {
|
||||||
|
"title": "Deterministische Namen für Anhänge und ZIP-Einträge",
|
||||||
|
"summary": "Wiederholte Namen effizient auflösen, ohne Anhänge auszulassen oder umzuordnen.",
|
||||||
|
"body": "Nachrichtenanhänge und ZIP-Einträge verwenden dieselbe Vergabe des ersten freien nummerierten Suffixes. Die bisherige groß-/kleinschreibungsunabhängige Kollisionsregel und exakte Nummerierung bleiben erhalten, auch bei vorhandenen Nummernsuffixen, Unicode-Groß-/Kleinschreibung und mehrfachen Erweiterungen. Jede Nachricht und jedes Archiv besitzt einen getrennten Vergabezustand. Große Gruppen gleicher gewünschter Namen beginnen die Suffixsuche nicht mehr jeweils bei zwei. Nur der Suchaufwand ändert sich: vorgesehene Bytes, Empfänger, Reihenfolge, bestehende Prüfung mehrfach verwendeter Dateien und ZIP-Verschlüsselung bleiben unverändert.",
|
||||||
|
}},
|
||||||
|
),
|
||||||
*CAMPAIGN_USER_DOCUMENTATION,
|
*CAMPAIGN_USER_DOCUMENTATION,
|
||||||
DocumentationTopic(
|
DocumentationTopic(
|
||||||
id="campaigns.workflow.link-exact-campaign-to-case",
|
id="campaigns.workflow.link-exact-campaign-to-case",
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from govoplan_campaign.backend.services.filenames import FilenameAllocator
|
||||||
|
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import re
|
import re
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -303,15 +305,8 @@ def _archive_filename(archive: ZipArchiveConfig, values: dict[str, Any], entry_i
|
|||||||
return filename if filename.lower().endswith(".zip") else f"{filename}.zip"
|
return filename if filename.lower().endswith(".zip") else f"{filename}.zip"
|
||||||
|
|
||||||
|
|
||||||
def _unique_attachment_filename(filename: str, used: set[str]) -> str:
|
def _unique_attachment_filename(filename: str, used: FilenameAllocator) -> str:
|
||||||
candidate = filename
|
return used.allocate(filename)
|
||||||
path = Path(filename)
|
|
||||||
counter = 2
|
|
||||||
while candidate.casefold() in used:
|
|
||||||
candidate = f"{path.stem} ({counter}){path.suffix}"
|
|
||||||
counter += 1
|
|
||||||
used.add(candidate.casefold())
|
|
||||||
return candidate
|
|
||||||
|
|
||||||
|
|
||||||
def _deduplicated_archive_members(members: list[tuple[Path, str]]) -> list[tuple[Path, str]]:
|
def _deduplicated_archive_members(members: list[tuple[Path, str]]) -> list[tuple[Path, str]]:
|
||||||
@@ -381,8 +376,8 @@ def _attach_files(
|
|||||||
evidence: list[dict[str, object]] = []
|
evidence: list[dict[str, object]] = []
|
||||||
archive_members: dict[str, list[tuple[Path, str]]] = {}
|
archive_members: dict[str, list[tuple[Path, str]]] = {}
|
||||||
archive_attachments: dict[str, list[ResolvedAttachment]] = {}
|
archive_attachments: dict[str, list[ResolvedAttachment]] = {}
|
||||||
used_message_filenames: set[str] = set()
|
used_message_filenames = FilenameAllocator()
|
||||||
used_zip_member_filenames: dict[str, set[str]] = {}
|
used_zip_member_filenames: dict[str, FilenameAllocator] = {}
|
||||||
|
|
||||||
for attachment in resolution.attachments:
|
for attachment in resolution.attachments:
|
||||||
attachment.message_filenames = []
|
attachment.message_filenames = []
|
||||||
@@ -394,7 +389,7 @@ def _attach_files(
|
|||||||
continue
|
continue
|
||||||
match_paths = [Path(match) for match in attachment.matches]
|
match_paths = [Path(match) for match in attachment.matches]
|
||||||
if attachment.zip_enabled and attachment.zip_archive_id:
|
if attachment.zip_enabled and attachment.zip_archive_id:
|
||||||
used_archive_names = used_zip_member_filenames.setdefault(attachment.zip_archive_id, set())
|
used_archive_names = used_zip_member_filenames.setdefault(attachment.zip_archive_id, FilenameAllocator())
|
||||||
for position, path in enumerate(match_paths, start=1):
|
for position, path in enumerate(match_paths, start=1):
|
||||||
requested = _render_attachment_filename(
|
requested = _render_attachment_filename(
|
||||||
template=attachment.zip_entry_name_template,
|
template=attachment.zip_entry_name_template,
|
||||||
|
|||||||
+23
@@ -0,0 +1,23 @@
|
|||||||
|
"""Deterministic collision naming shared by message and ZIP construction."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class FilenameAllocator:
|
||||||
|
"""Append-only names with the original first-free, casefolded suffix rule."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.used: set[str] = set()
|
||||||
|
self._next: dict[str, int] = {}
|
||||||
|
|
||||||
|
def allocate(self, filename: str) -> str:
|
||||||
|
key = filename.casefold()
|
||||||
|
candidate = filename
|
||||||
|
path = Path(filename)
|
||||||
|
counter = self._next.get(key, 2)
|
||||||
|
while candidate.casefold() in self.used:
|
||||||
|
candidate = f"{path.stem} ({counter}){path.suffix}"
|
||||||
|
counter += 1
|
||||||
|
self.used.add(candidate.casefold())
|
||||||
|
self._next[key] = counter
|
||||||
|
return candidate
|
||||||
@@ -12,6 +12,8 @@ from pathlib import Path
|
|||||||
from typing import Iterable
|
from typing import Iterable
|
||||||
import zlib
|
import zlib
|
||||||
|
|
||||||
|
from govoplan_campaign.backend.services.filenames import FilenameAllocator
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import pyzipper
|
import pyzipper
|
||||||
except ImportError: # pragma: no cover
|
except ImportError: # pragma: no cover
|
||||||
@@ -24,18 +26,11 @@ ZIP_METHOD_STANDARD = "zip_standard"
|
|||||||
|
|
||||||
def _normalized_members(files: Iterable[Path | ArchiveMember]) -> list[ArchiveMember]:
|
def _normalized_members(files: Iterable[Path | ArchiveMember]) -> list[ArchiveMember]:
|
||||||
members: list[ArchiveMember] = []
|
members: list[ArchiveMember] = []
|
||||||
used_names: set[str] = set()
|
names = FilenameAllocator()
|
||||||
for item in files:
|
for item in files:
|
||||||
path, requested_name = item if isinstance(item, tuple) else (item, item.name)
|
path, requested_name = item if isinstance(item, tuple) else (item, item.name)
|
||||||
requested = Path(requested_name).name or path.name
|
requested = Path(requested_name).name or path.name
|
||||||
stem = Path(requested).stem
|
candidate = names.allocate(requested)
|
||||||
suffix = Path(requested).suffix
|
|
||||||
candidate = requested
|
|
||||||
counter = 2
|
|
||||||
while candidate.casefold() in used_names:
|
|
||||||
candidate = f"{stem} ({counter}){suffix}"
|
|
||||||
counter += 1
|
|
||||||
used_names.add(candidate.casefold())
|
|
||||||
members.append((path, candidate))
|
members.append((path, candidate))
|
||||||
return members
|
return members
|
||||||
|
|
||||||
|
|||||||
Executable
+75
@@ -0,0 +1,75 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
import random
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from govoplan_campaign.backend.services.filenames import FilenameAllocator
|
||||||
|
from govoplan_campaign.backend.services.zip_service import _normalized_members
|
||||||
|
|
||||||
|
|
||||||
|
def legacy_names(names):
|
||||||
|
used = set()
|
||||||
|
result = []
|
||||||
|
for name in names:
|
||||||
|
path = Path(name)
|
||||||
|
candidate = name
|
||||||
|
counter = 2
|
||||||
|
while candidate.casefold() in used:
|
||||||
|
candidate = f"{path.stem} ({counter}){path.suffix}"
|
||||||
|
counter += 1
|
||||||
|
used.add(candidate.casefold())
|
||||||
|
result.append(candidate)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class FilenameAllocatorTests(unittest.TestCase):
|
||||||
|
def test_exact_equivalence_with_colliding_suffixes_case_unicode_and_multiple_dots(
|
||||||
|
self,
|
||||||
|
):
|
||||||
|
choices = [
|
||||||
|
"report.pdf",
|
||||||
|
"REPORT.PDF",
|
||||||
|
"report (2).pdf",
|
||||||
|
"report (3).pdf",
|
||||||
|
"report (2) (2).pdf",
|
||||||
|
".hidden",
|
||||||
|
"a.tar.gz",
|
||||||
|
"A.TAR.GZ",
|
||||||
|
"Straße.txt",
|
||||||
|
"STRASSE.txt",
|
||||||
|
"readme",
|
||||||
|
]
|
||||||
|
rng = random.Random(42)
|
||||||
|
for _ in range(30):
|
||||||
|
names = [rng.choice(choices) for _ in range(200)]
|
||||||
|
allocator = FilenameAllocator()
|
||||||
|
self.assertEqual(
|
||||||
|
legacy_names(names), [allocator.allocate(name) for name in names]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_zip_and_message_names_preserve_every_input_and_sequence(self):
|
||||||
|
names = ["a.pdf", "A.pdf", "a (2).pdf", "a.pdf"]
|
||||||
|
paths = [Path(f"/synthetic/{index}/source") for index in range(len(names))]
|
||||||
|
members = _normalized_members(list(zip(paths, names)))
|
||||||
|
self.assertEqual(paths, [path for path, _ in members])
|
||||||
|
self.assertEqual(legacy_names(names), [name for _, name in members])
|
||||||
|
self.assertEqual("a.pdf", FilenameAllocator().allocate("a.pdf"))
|
||||||
|
|
||||||
|
def test_many_collisions_have_linear_membership_work(self):
|
||||||
|
class CountingSet(set):
|
||||||
|
probes = 0
|
||||||
|
|
||||||
|
def __contains__(self, item):
|
||||||
|
self.probes += 1
|
||||||
|
return super().__contains__(item)
|
||||||
|
|
||||||
|
allocator = FilenameAllocator()
|
||||||
|
allocator.used = CountingSet()
|
||||||
|
for _ in range(10000):
|
||||||
|
last = allocator.allocate("report.pdf")
|
||||||
|
self.assertEqual("report (10000).pdf", last)
|
||||||
|
self.assertEqual(10000, len(allocator.used))
|
||||||
|
self.assertEqual(19999, allocator.used.probes)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@govoplan/campaign-webui",
|
"name": "@govoplan/campaign-webui",
|
||||||
"version": "0.1.28",
|
"version": "0.1.29",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "src/index.ts",
|
"main": "src/index.ts",
|
||||||
|
|||||||
Reference in New Issue
Block a user