Release govoplan-files v0.1.26: speed archive workflows and unify file tools
Module Package Release / publish-packages (push) Successful in 12s

This commit is contained in:
2026-09-08 01:32:41 +02:00
parent 2baa8f2657
commit ff84812f7f
35 changed files with 4331 additions and 275 deletions
+238
View File
@@ -0,0 +1,238 @@
"""Bounded, private, transient archive work shared by workers on one host.
This is not a detached job queue: confirmation still owns its transaction.
Progress never contains filenames, passwords, or extracted document content.
"""
from __future__ import annotations
from contextlib import contextmanager
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import tempfile
import time
from uuid import UUID, uuid4
from govoplan_files.backend.storage.common import FileStorageError
class ArchiveWorkExpired(FileStorageError):
pass
def _uuid(value: str) -> str:
try:
result = str(UUID(value))
except (ValueError, AttributeError, TypeError) as exc:
raise FileStorageError("Invalid archive operation identifier") from exc
if result != value:
raise FileStorageError("Invalid archive operation identifier")
return result
def _actor(tenant_id: str, user_id: str) -> str:
return hashlib.sha256(json.dumps([tenant_id, user_id]).encode()).hexdigest()
def _root(settings) -> Path:
configured = getattr(settings, "file_archive_work_root", None)
identity = hashlib.sha256(str(Path(getattr(settings, "file_storage_local_root", "runtime/files")).absolute()).encode()).hexdigest()[:16]
base = Path(configured) if configured else Path(tempfile.gettempdir()) / f"govoplan-archive-work-{os.getuid()}-{identity}"
root = base.absolute() / "v1"
# All internal paths are fixed subdirectories or server-generated UUIDs.
root.mkdir(mode=0o700, parents=True, exist_ok=True)
if root.is_symlink() or root.stat().st_mode & 0o077:
raise FileStorageError("Archive work directory must be private (mode 0700)")
return root
@contextmanager
def _locked(root: Path):
# POSIX advisory locks release automatically if a worker exits. GovOPlaN's
# server deployment targets POSIX; no credentials are passed to a process.
import fcntl
descriptor = os.open(root / ".lock", os.O_CREAT | os.O_RDWR, 0o600)
try:
fcntl.flock(descriptor, fcntl.LOCK_EX)
yield
finally:
os.close(descriptor)
def _ttl(settings) -> int:
return settings.file_archive_preview_ttl_seconds
def _lease_active(path: Path) -> bool:
import fcntl
try:
descriptor = os.open(path, os.O_RDWR | os.O_NOFOLLOW)
except FileNotFoundError:
return False
try:
try:
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
return True
return False
finally:
os.close(descriptor)
def _cleanup(root: Path, ttl: int) -> None:
cutoff = time.time() - ttl
for entry in root.iterdir():
if not re.fullmatch(r"[a-f0-9]{64}-[a-f0-9-]{36}\.(archive|progress|lease)", entry.name) or entry.is_symlink():
continue
try:
if entry.stat().st_mtime >= cutoff:
continue
lease = entry.with_suffix(".lease")
if entry.suffix in {".archive", ".lease"} and _lease_active(lease):
continue
entry.unlink(missing_ok=True)
except FileNotFoundError:
pass
def stage_upload(settings, source: str, *, tenant_id: str, user_id: str) -> str:
root = _root(settings)
actor = _actor(tenant_id, user_id)
stage_id = str(uuid4())
target = root / f"{actor}-{stage_id}.archive"
size = Path(source).stat().st_size
with _locked(root):
_cleanup(root, _ttl(settings))
stages = [entry for entry in root.glob("*.archive") if not entry.is_symlink()]
own = sorted((entry for entry in stages if entry.name.startswith(actor + "-")), key=lambda item: item.stat().st_mtime)
limit = getattr(settings, "file_archive_staged_per_actor", 4)
while len(own) >= limit:
victim = next((entry for entry in own if not _lease_active(entry.with_suffix(".lease"))), None)
if victim is None:
raise FileStorageError("Too many archive imports are active; wait for one to finish")
victim.unlink(missing_ok=True)
own.remove(victim)
stages.remove(victim)
maximum = getattr(settings, "file_archive_staged_max_bytes", 2 * 1024 ** 3)
if size + sum(entry.stat().st_size for entry in stages) > maximum:
raise FileStorageError("Temporary archive storage is full; retry after existing previews expire")
descriptor = os.open(target, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
try:
with os.fdopen(descriptor, "wb") as output, open(source, "rb") as stream:
shutil.copyfileobj(stream, output, length=1024 * 1024)
except BaseException:
target.unlink(missing_ok=True)
raise
return stage_id
@contextmanager
def use_staged_upload(settings, stage_id: str, *, tenant_id: str, user_id: str):
import fcntl
root = _root(settings)
path = root / f"{_actor(tenant_id, user_id)}-{_uuid(stage_id)}.archive"
lease = path.with_suffix(".lease")
with _locked(root):
_cleanup(root, _ttl(settings))
if not path.is_file() or path.is_symlink() or path.stat().st_mtime < time.time() - _ttl(settings):
raise ArchiveWorkExpired("Archive preview expired; select the archive and preview it again")
descriptor = os.open(lease, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
try:
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as exc:
os.close(descriptor)
raise FileStorageError("This archive is already being processed") from exc
try:
yield str(path)
finally:
try:
with _locked(root):
lease.unlink(missing_ok=True)
except OSError:
pass
finally:
os.close(descriptor)
def discard_staged_upload(settings, stage_id: str, *, tenant_id: str, user_id: str) -> None:
root = _root(settings)
path = root / f"{_actor(tenant_id, user_id)}-{_uuid(stage_id)}.archive"
with _locked(root):
if not _lease_active(path.with_suffix(".lease")):
path.unlink(missing_ok=True)
class ArchiveProgress:
def __init__(self, settings, operation_id: str | None, *, tenant_id: str, user_id: str):
self.path: Path | None = None
self.value = {"phase": "inspecting", "completed_files": 0, "total_files": 0,
"completed_bytes": 0, "total_bytes": 0, "status": "running"}
self.last_write = 0.0
if not isinstance(operation_id, str):
return
root = _root(settings)
self.path = root / f"{_actor(tenant_id, user_id)}-{_uuid(operation_id)}.progress"
with _locked(root):
_cleanup(root, _ttl(settings))
# Bounded receipts as well as bounded file bytes; keep newest256 per
# actor, never remove a live progress record to admit another one.
own = sorted(root.glob(f"{_actor(tenant_id, user_id)}-*.progress"), key=lambda entry: entry.stat().st_mtime)
for old in own[:-255]:
try:
if json.loads(old.read_text()).get("status") != "running":
old.unlink(missing_ok=True)
except (OSError, ValueError):
pass
if len(list(root.glob(f"{_actor(tenant_id, user_id)}-*.progress"))) >= 256:
raise FileStorageError("Too many active archive operations")
try:
descriptor = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
except FileExistsError as exc:
raise FileStorageError("Archive operation identifier has already been used; do not resubmit it") from exc
with os.fdopen(descriptor, "w") as target:
json.dump(self.value, target)
def __call__(self, phase: str, completed_files: int, total_files: int, completed_bytes: int, total_bytes: int):
changed_phase = self.value["phase"] != phase
self.value.update(phase=phase, completed_files=completed_files, total_files=total_files,
completed_bytes=completed_bytes, total_bytes=total_bytes)
if changed_phase or time.monotonic() - self.last_write >= .2:
self._write()
def finish(self, success: bool):
self.value.update(status="complete" if success else "failed", phase="complete" if success else "failed")
self._write()
def _write(self):
if self.path is None:
return
temporary = None
try:
descriptor, temporary = tempfile.mkstemp(prefix=".progress-", dir=self.path.parent)
with os.fdopen(descriptor, "w") as target:
json.dump(self.value, target)
os.replace(temporary, self.path)
self.last_write = time.monotonic()
except OSError:
# Observability cannot turn a committed import into a failed
# response or cause the browser to retry a write.
pass
finally:
if temporary:
try:
Path(temporary).unlink(missing_ok=True)
except OSError:
pass
def read_progress(settings, operation_id: str, *, tenant_id: str, user_id: str) -> dict:
path = _root(settings) / f"{_actor(tenant_id, user_id)}-{_uuid(operation_id)}.progress"
try:
if path.is_symlink() or path.stat().st_mtime < time.time() - _ttl(settings):
raise ArchiveWorkExpired("Archive progress is not available")
return json.loads(path.read_text())
except (FileNotFoundError, json.JSONDecodeError) as exc:
raise ArchiveWorkExpired("Archive progress is not available") from exc
+27 -2
View File
@@ -179,8 +179,32 @@ def _archive_topic(
f"The deployment accepts ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ requests up to {request_limit}, limits actual expanded data to {expanded_limit}, "
f"each member to {member_limit}, the archive to {max_entries:,} entries, and expansion to {max_expansion_ratio}:1. "
f"The server-issued preview expires after {preview_minutes} minutes. Password-protected ZIP archives are supported; passwords remain request-only. "
"Unsafe paths and special filesystem entries are rejected. Actual extracted bytes are counted instead of trusting archive headers."
"The UI uploads the archive once into bounded private temporary storage; password verification and confirmation reuse that staged upload without another transfer. "
"Repreviewing does not extend its original lifetime. Cancel, replacing the archive and successful confirmation release the stage; an expired or missing stage requires a new upload. "
"A requested preview may safely reupload the selected file if its stage expired; confirmation is never retried automatically. "
"Unsafe paths and special filesystem entries are rejected. Actual extracted bytes are counted instead of trusting archive headers. "
"Archive paths are limited to 4096 UTF-8 bytes and 128 components; derived parent directories count toward the entry limit. TAR inspection checks each header's size and ratio before advancing across that member's payload. "
"The shared blurred dialog overlay separates upload transfer, inspection, extraction/storage and final commit, without an envelope animation. "
"Selected file/byte totals appear immediately and extraction progress uses actual server counters. Unknown progress is indeterminate; completion is shown only after the transaction succeeds. "
"Keep the dialog open while processing; errors release the overlay and preserve the selection for review."
),
translations={"de": {
"title": "Ein Archiv prüfen und entpacken",
"summary": f"Bis zu {max_entries:,} Einträge aus ZIP- oder TAR-Archiven prüfen und gezielt entpacken, bevor verwaltete Dateien entstehen.",
"body": (
f"Diese Installation erlaubt ZIP, TAR, TAR.GZ, TAR.BZ2 und TAR.XZ bis {request_limit}, höchstens {expanded_limit} tatsächlich entpackte Daten, "
f"{member_limit} pro Datei, {max_entries:,} Einträge und ein Expansionsverhältnis von {max_expansion_ratio}:1. "
f"Die serverseitige Vorschau läuft nach {preview_minutes} Minuten ab. Passwortgeschützte ZIP-Archive werden unterstützt; Passwörter bleiben auf die Anfrage beschränkt. "
"Die Oberfläche überträgt das Archiv einmal in begrenzten privaten Zwischenspeicher; Passwortprüfung und Bestätigung verwenden diese temporäre Kopie ohne erneute Übertragung. "
"Eine erneute Vorschau verlängert die ursprüngliche Laufzeit nicht. Abbrechen, Archivwechsel und erfolgreicher Abschluss geben die Kopie frei; fehlt sie oder ist sie abgelaufen, muss das Archiv erneut hochgeladen werden. "
"Bei einer angeforderten Vorschau darf die Oberfläche die ausgewählte Datei nach Ablauf erneut übertragen; die Bestätigung wird niemals automatisch wiederholt. "
"Unsichere Pfade und besondere Dateisystemeinträge werden abgelehnt; tatsächlich gelesene Bytes werden gezählt. "
"Archivpfade sind auf 4096 UTF-8-Bytes und 128 Komponenten begrenzt; abgeleitete übergeordnete Ordner zählen zur Eintragsgrenze. Die TAR-Prüfung kontrolliert Größe und Expansionsverhältnis bei jedem Header, bevor dessen Nutzdaten übersprungen werden. "
"Die gemeinsame Überlagerung des weichgezeichneten Dialogs unterscheidet Übertragung, Prüfung, Entpacken/Speichern und abschließende Transaktionsbestätigung, ohne Briefumschlaganimation. "
"Ausgewählte Datei- und Bytezahlen erscheinen sofort; der Entpackfortschritt nutzt tatsächliche Serverzähler. Unbekannter Fortschritt bleibt unbestimmt, der Abschluss wird erst nach erfolgreicher Transaktion angezeigt. "
"Den Dialog während der Verarbeitung geöffnet lassen; Fehler entfernen die Überlagerung und erhalten die Auswahl zur Prüfung."
),
}},
layer="configured",
documentation_types=("user",),
audience=("file_user", "file_manager", "process_participant"),
@@ -238,7 +262,7 @@ def _archive_topic(
{
"id": "archive-entry-count",
"label": "Maximum entry count",
"description": f"An archive may contain at most {max_entries:,} declared entries.",
"description": f"An archive may contain at most {max_entries:,} entries, including derived parent directories.",
"values": [f"{max_entries:,} entries"],
},
{
@@ -252,6 +276,7 @@ def _archive_topic(
"files.workflow.upload-managed-files",
"files.workflow.organize-managed-files",
"files.workflow.find-and-download-files",
"files.reference.shared-storage-profile",
],
},
)
@@ -7,6 +7,24 @@ from govoplan_core.core.modules import DocumentationTopic
_TRANSLATIONS = {
"files.workflow.unpack-managed-archive": {
"title": "Ein bereits hochgeladenes Archiv entpacken",
"summary": "Ein verwaltetes ZIP- oder TAR-Archiv auswählen und denselben Vorschau-, Passwort- und Auswahlablauf wie beim Hochladen nutzen, ohne es erneut zu übertragen.",
"body": (
"Wählen Sie genau eine ZIP-, TAR-, TAR.GZ-, TAR.BZ2- oder TAR.XZ-Datei in einem verwalteten Bereich und anschließend Archiv entpacken in der Aktionsleiste oder im Kontextmenü. "
"Wählen Sie einen zugänglichen persönlichen oder Gruppen-Zielbereich und fordern Sie die Archivvorschau an; dabei entstehen noch keine Dateien. Im gemeinsamen Archivdialog prüfen Sie Einträge und gegebenenfalls das ZIP-Passwort und wählen Dateien oder Ordner aus. "
"Beim Bestätigen liest der Server das verwaltete Quellarchiv und erstellt neue Zieldateien. Quellarchiv, Metadaten und Version bleiben unverändert; Zielkonflikte werden abgelehnt und überschreiben niemals das Quellarchiv. "
"Lese-, Download- und Upload-Berechtigung sind gemeinsam erforderlich. Eigentum, aktuelle Freigaben, Mandant, Quellversion, Connector-Richtlinien, Integrität, Quarantäne und gegebenenfalls die Verfügbarkeit der Hüllenentschlüsselung werden beim Bestätigen erneut geprüft. "
"Die kurzlebige Vorschau ist an Quellarchiv, Version, Person und Ziel gebunden. Geänderte oder unzugängliche Quellen verlangen eine neue Vorschau; die Version wird niemals still gewechselt. "
"Dieselben Grenzen für Archivgröße, tatsächliche entpackte Gesamtgröße, einzelne Dateien, Anzahl der Einträge, Expansionsverhältnis und sichere Pfade wie beim Archiv-Upload gelten. Passwörter bleiben auf die Anfrage beschränkt. "
"Wie beim Archiv-Upload entstehen gewöhnliche Zieldateien; der Passwortschutz des Archivs oder seine Speicher-Verschlüsselungshülle wird nicht automatisch auf Einträge übertragen. "
"Während Prüfung und Entpacken überlagert die gemeinsame Ladeanzeige den weichgezeichneten Dialog, ohne Briefumschlaganimation. "
"Die Anzahl und Gesamtgröße ausgewählter Dateien sind sofort sichtbar; der laufende Fortschritt zeigt gemessene Serverzähler statt zeitbasierter Schätzungen. "
"Übertragung, Prüfung, Entpacken/Speichern und abschließende Transaktionsbestätigung sind getrennte Phasen; übertragene oder verarbeitete Bytes allein bedeuten noch keinen erfolgreichen Abschluss. "
"Unbekannter Fortschritt bleibt unbestimmt. Schließen, Auswahl und Zieländerung sind während der Verarbeitung gesperrt; Fehler geben den unveränderten Prüfdialog wieder frei. "
"Die Verarbeitung läuft synchron: Lassen Sie den beschäftigten Dialog bis zum Abschluss geöffnet. Dateien aus entfernten Connector-Bereichen müssen zuerst in den verwalteten Speicher synchronisiert werden."
),
},
"files.tabular-content": {
"title": "Verwaltete CSV- und XLSX-Versionen als gesteuerte Datenquellen verwenden",
"summary": "Exakte autorisierte Dateiversionen für Connectors bereitstellen, ohne Files-Kontrollen zu umgehen.",
@@ -39,7 +57,10 @@ _TRANSLATIONS = {
"title": "Verwaltete Dateien und Ordner organisieren",
"summary": "Ordner anlegen und zugängliche Inhalte mit ausdrücklicher Konfliktbehandlung umbenennen, verschieben oder kopieren.",
"body": (
"Die Organisation bleibt in gesteuerten persönlichen oder Gruppenbereichen. Verschieben erhält die Asset-Identität; Kopieren erzeugt neue Assets und Versionen, die unveränderliche Blob-Bytes wiederverwenden. Jeder Zielkonflikt muss ausdrücklich abgelehnt, durch Umbenennen gelöst, überschrieben oder übersprungen werden."
"Die Organisation bleibt in gesteuerten persönlichen oder Gruppenbereichen. Verschieben erhält die Asset-Identität; Kopieren erzeugt neue Assets und Versionen, die unveränderliche Blob-Bytes wiederverwenden. Jeder Zielkonflikt muss ausdrücklich abgelehnt, durch Umbenennen gelöst, überschrieben oder übersprungen werden. "
"Ordner erstellen und Hochladen bleiben neben Neu laden im Kopf des Arbeitsbereichs. Nach der Auswahl von Dateien oder Ordnern öffnet Auswahl verwalten die Aktionen Verschieben, Kopieren, Umbenennen, Freigaben und Zugriffserklärungen. "
"Löschen ist von der Organisation getrennt und benötigt weiterhin die vorhandene Bestätigung. Herunterladen und Archiv entpacken bleiben für passende Auswahlen direkt erreichbar. "
"Verbindungen und Importe enthält die selteneren Import-, Synchronisierungs- und Connector-Bereichswerkzeuge; allein das Öffnen dieser Werkzeugauswahl importiert oder synchronisiert nichts."
),
},
"files.workflow.find-and-download-files": {
@@ -81,6 +102,7 @@ _TRANSLATIONS = {
"title": "Files-Integrität, Recovery und ausfallsichere Connector-Transporte betreiben",
"summary": "Datenbanknachweise, Blob-Chiffretexte und Encryption-Verwahrung als eine Recovery-Einheit sichern und jeden SDK-verwalteten Gegenpunkt binden.",
"body": (
"Die Archivprüfung setzt TAR-Eintrags-, Größen- und Expansionsgrenzen bei jedem Header durch, bevor die nächsten Nutzdaten durchlaufen werden. Pfade sind auf 4096 UTF-8-Bytes und 128 Komponenten begrenzt; abgeleitete Elternordner zählen zur konfigurierten Eintragsgrenze. Zu große Archive müssen vor einem erneuten Versuch verkleinert oder aufgeteilt werden; die Richtlinie wird nicht automatisch gelockert. "
"Dauerhafter lokaler Speicher ist die betriebliche Basis. Files wird aus einem abgestimmten Datenbank-/Blob-Snapshot mit passenden Encryption-Tabellen und ursprünglichem Deployment-Hauptschlüssel wiederhergestellt; anschließend ist der begrenzte fortsetzbare Integritätsscan auszuführen und geschützter sowie ungeschützter Zugriff stichprobenartig zu prüfen. Scan- und Befundaktionen benötigen die angezeigte Revision. Geschützte Scans prüfen erst den Chiffretext, dann nach Entschlüsselung den semantischen Klartextnachweis. Unter PostgreSQL sichern lease-gebundene Core-Recovery-Absichten Objektauswirkungen ab; Abweichungen werden quarantänisiert und bleiben in Ops sichtbar. SQLite verwendet für Entwicklung eine prozesslokale Sperre und ist kein Produktions-Recovery-Profil; nach hartem Prozessverlust ist ein Integritätsscan erforderlich. Fehlende oder abweichende Blobs werden quarantänisiert, verwaiste Objekte vor einer ausdrücklich autorisierten Bereinigung zunächst nur gemeldet. Physische Vernichtung und Blob-Garbage-Collection prüfen ihre irreversiblen Wirkungen getrennt. S3-Schreibvorgänge nutzen bedingte Effekte und digestbasierte Vorwärts-Recovery; S3- und SMB-Transporte binden Wiederholungen, Umleitungen, Aliasse und erkannte Endpunkte und wenden die Richtlinie für private Netze vor jeder Verbindung erneut an. Fehlt eine verifizierbare Transportnaht, wird geschlossen abgebrochen. Destruktive Modulstilllegung entfernt Datenbanktabellen, nicht jedoch Blob-Objekte im Backend."
),
},
@@ -99,10 +121,14 @@ _TRANSLATIONS = {
),
},
"files.reference.shared-storage-profile": {
"title": "Files mit einem gemeinsamen Speicherprofil betreiben",
"summary": "Lokalen, hostweit gemeinsamen oder S3-basierten Speicher passend zur Laufzeittopologie wählen.",
"title": "Files-Speicher und temporäre Archivverarbeitung betreiben",
"summary": "Dauerhaften Speicher, privaten Archivzwischenspeicher und gemessenen Fortschritt auf die Worker-Topologie abstimmen.",
"body": (
"Core stellt das gemeinsame lokale/S3-Objektspeicher-Backend bereit; Files verantwortet Dateimetadaten und Objektschlüssel. Lokaler Speicher eignet sich für einen Laufzeitprozess, ein gemeinsames Host-Volume für Replikate auf demselben Host. Unabhängige Hosts benötigen einen ausdrücklich vertrauenswürdigen HTTPS-S3-kompatiblen Endpunkt. PostgreSQL, Objekte und Hauptschlüssel müssen auf denselben abgestimmten Recovery-Zeitpunkt wiederhergestellt werden."
"Core stellt das gemeinsame lokale/S3-Objektspeicher-Backend bereit; Files verantwortet Dateimetadaten und Objektschlüssel. Lokaler Speicher eignet sich für einen Laufzeitprozess, ein gemeinsames Host-Volume für Replikate auf demselben Host. Unabhängige Hosts benötigen einen ausdrücklich vertrauenswürdigen HTTPS-S3-kompatiblen Endpunkt. PostgreSQL, Objekte und Hauptschlüssel müssen auf denselben abgestimmten Recovery-Zeitpunkt wiederhergestellt werden. "
"Die Archivoberfläche fordert ausdrücklich einmalige Übertragung mit Zwischenspeicherung an: Erneute Passwortprüfung und Bestätigung verwenden eine an Mandant und Person gebundene Kopie, speichern jedoch niemals Passwörter. Temporäre Archiv- und Fortschrittsdateien haben Modus 0600 unter einem privaten Verzeichnis mit Modus 0700. FILE_ARCHIVE_WORK_ROOT ersetzt den Standard im temporären Betriebssystemverzeichnis, dessen Name von Dienst-UID und Hash des konfigurierten Speicherwurzelpfads abgeleitet wird. Alle Worker desselben Hosts müssen darauf zugreifen; unabhängige Hosts benötigen gemeinsamen POSIX-Speicher mit funktionierenden flock-Sperren oder feste Host-Zuordnung für Archiv- und Fortschrittsanfragen, auch bei dauerhaftem S3-Blob-Speicher. "
"Standardgrenzen sind 2 GiB zwischengespeicherte Archive pro Arbeitsverzeichnis, vier Kopien pro Mandant/Person und 1.800 Sekunden ab der ersten Speicherung ohne Verlängerung. Abbruch- und Wechselanfragen sowie erfolgreicher Abschluss geben Kopien frei; verlassene abgelaufene Kopien werden bei späterer Archivverarbeitung bereinigt, nicht garantiert zu einem exakten Zeitpunkt ohne weitere Aktivität. Aktive POSIX-Leases schützen benutzte Kopien auch nach Ablauf vor Verdrängung. Kopien und Fortschrittsdaten sind keine fortsetzbaren Hintergrundaufträge. "
"Eine optionale, aktuell gehaltene Systembibliothek libarchive beschleunigt klassisches ZIPCrypto mit gespeicherten oder Deflate-komprimierten Einträgen und ASCII- oder ausdrücklich als UTF-8 markierten Namen. Fehlende Bibliotheken, AES und nicht unterstützte Kodierungen/Formate wählen vor dem Dekodieren den bisherigen Python-Leser; native Fehler führen zum sicheren Abbruch. Vollständige Header-, Pfad- und Grenzprüfungen bleiben erhalten; der native Pfad prüft CRC und Größe zusätzlich unabhängig und entpackt niemals direkt ins Dateisystem. "
"Die Verarbeitung speichert jeweils einen geprüften Eintrag. Wiederverwendung des Speicher-Clients innerhalb eines Archivs und gebündelte Antwortmetadaten vermeiden Zusatzarbeit, ohne Autorisierung zwischenzuspeichern oder dauerhafte Recovery-, Integritäts- und Commit-Prüfungen pro Datei zu entfernen. Fortschritt zeigt tatsächliche Datei-/Bytezähler und eine eigene Abschlussphase statt Zeitschätzungen oder vorzeitiger Erfolgsmeldung. Ein schnellerer Decoder verspricht keine entsprechende Beschleunigung des gesamten Imports oder von S3."
),
},
"files.reference.generated-artifact-store": {
@@ -5,6 +5,19 @@ from __future__ import annotations
from typing import Any
_MANAGED_ARCHIVE_GERMAN = {
"prerequisites": ["Das ausgewählte verwaltete Archiv ist lesbar und herunterladbar; Sie dürfen in den gewählten Zielbereich hochladen."],
"steps": [
"Wählen Sie ein unterstütztes verwaltetes Archiv und Archiv entpacken in der Aktionsleiste oder im Kontextmenü.",
"Wählen Sie Zielbereich und Zielordner und fordern Sie die Archivvorschau an.",
"Prüfen Sie Einträge, geben Sie gegebenenfalls das ZIP-Passwort ein und verifizieren Sie es; wählen Sie anschließend Dateien oder Ordner zum Entpacken aus.",
"Bestätigen Sie die ausgewählten Einträge und lassen Sie den Dialog bis zum Abschluss geöffnet. Wählen Sie bei vorhandenen Zieldateien ein anderes Ziel.",
],
"outcome": "Ausgewählte Einträge sind neue verwaltete Dateien; das Quellarchiv bleibt unverändert.",
"verification": "Prüfen Sie Zielpfade und Anzahl sowie die unveränderte Version des Quellarchivs.",
}
GERMAN_STRUCTURED_TRANSLATIONS: dict[str, dict[str, Any]] = {'files.assurance.process-and-release-readiness': {'outcome': 'Der Prozess oder die '
'Veröffentlichung hat eine explizite '
'Genehmigungsaufzeichnung, die an '
@@ -355,10 +368,10 @@ GERMAN_STRUCTURED_TRANSLATIONS: dict[str, dict[str, Any]] = {'files.assurance.pr
'steps': ['Öffnen Sie Dateien und wählen Sie den '
'persönlichen oder Gruppenraum zum '
'Organisieren aus.',
'Erstellen Sie die erforderlichen Zielordner '
'oder wählen Sie die Dateien und Ordner aus, '
'um sie umzubenennen, zu verschieben oder zu '
'kopieren.',
'Verwenden Sie Ordner erstellen im Kopf des '
'Arbeitsbereichs oder wählen Sie Dateien und '
'Ordner aus und öffnen Sie Auswahl verwalten '
'zum Umbenennen, Verschieben oder Kopieren.',
'Wählen Sie das Ziel aus und lösen Sie jeden '
'Zielkonflikt explizit.',
'Wenden Sie die Operation an und öffnen Sie '
@@ -456,3 +469,5 @@ GERMAN_STRUCTURED_TRANSLATIONS: dict[str, dict[str, Any]] = {'files.assurance.pr
'widerrufen Sie die Gewährung und stellen '
'Sie sicher, dass nur noch unabhängige '
'Zugangspfade verbleiben.'}}
GERMAN_STRUCTURED_TRANSLATIONS["files.workflow.unpack-managed-archive"] = _MANAGED_ARCHIVE_GERMAN
+72 -8
View File
@@ -461,7 +461,7 @@ def _dsar_provider(context: ModuleContext) -> object:
manifest = ModuleManifest(
id="files",
name="Files",
version="0.1.25",
version="0.1.26",
required_capabilities=(
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
@@ -804,7 +804,11 @@ manifest = ModuleManifest(
"Accounts with upload permission can launch the full Files upload dialog into a freshly reauthorized managed space. Opening "
"either path causes Files to re-run its own provider, space, folder, object, and scope checks; completion and cancellation are "
"explicit, and the full Files workspace remains available as a deep-link fallback. View and rail settings may recommend or "
"focus this tool, but do not bypass file ownership, shares, connector policy, integrity gates, or purpose-aware access."
"focus this tool, but do not bypass file ownership, shares, connector policy, integrity gates, or purpose-aware access. "
"The full workspace keeps Reload, Create folder, and Upload at the top right, with Upload as the primary action. "
"Reload refreshes the current folder and active filters; it never imports or synchronizes remote content. Failed reloads retain loaded data and report the failure. "
"Connections and imports groups explicit synchronization and linked-space tools. The selection bar keeps Download and, for an archive, Unpack archive; "
"Manage selection groups organization, sharing, access explanations, and a separate confirmed deletion action. Permission and connector restrictions remain enforced per action."
),
layer="configured",
documentation_types=("user", "admin"),
@@ -835,7 +839,11 @@ manifest = ModuleManifest(
"Seite und kann einen exakten Dateiversionsverweis zurückgeben. Mit Upload-Berechtigung lässt sich der vollständige "
"Upload-Dialog in einem erneut berechtigungsgeprüften verwalteten Bereich öffnen. Files prüft Anbieter, Bereich, Ordner, "
"Objekt und Berechtigung bei jedem Pfad erneut. Abschluss und Abbruch sind ausdrücklich; Eigentum, Freigaben, Connector-"
"Richtlinien, Integritätsprüfungen und zweckgebundener Zugriff bleiben maßgeblich."
"Richtlinien, Integritätsprüfungen und zweckgebundener Zugriff bleiben maßgeblich. "
"Der vollständige Arbeitsbereich zeigt oben rechts Neu laden, Ordner erstellen und die Hauptaktion Hochladen. "
"Neu laden aktualisiert den aktuellen Ordner mit aktiven Filtern; es importiert oder synchronisiert niemals entfernte Inhalte. Bei Fehlern bleiben geladene Daten mit Fehlermeldung erhalten. "
"Verbindungen und Importe bündelt ausdrückliche Synchronisierung und verknüpfte Dateibereiche. Bei der Auswahl bleiben Herunterladen und für Archive Archiv entpacken direkt sichtbar. "
"Auswahl verwalten bündelt Organisation, Freigaben, Zugriffserklärungen und getrennt bestätigtes Löschen. Berechtigungen und Connector-Beschränkungen gelten weiterhin je Aktion."
),
}
},
@@ -986,7 +994,10 @@ manifest = ModuleManifest(
summary="Create folders and rename, move, or copy accessible managed content with explicit conflict handling.",
body=(
"Organization stays inside governed personal or group spaces. Moves preserve the asset identity, while copies create new assets and versions that reuse immutable blob bytes. "
"Every target conflict must be rejected, renamed, overwritten, or skipped explicitly."
"Every target conflict must be rejected, renamed, overwritten, or skipped explicitly. "
"Create folder and Upload remain beside Reload in the workspace header. Select files or folders and open Manage selection for Move, Copy, Rename, sharing, or access explanations. "
"Deletion is separated from organization and still requires its existing confirmation. Download and Unpack archive stay directly available for applicable selections. "
"Connections and imports holds the less frequent import, synchronization, and connector-space tools; opening this tool chooser performs no import or synchronization."
),
layer="configured",
documentation_types=("user",),
@@ -1023,7 +1034,7 @@ manifest = ModuleManifest(
],
"steps": [
"Open Files and select the personal or group space to organize.",
"Create the required destination folders or select the files and folders to rename, move, or copy.",
"Use Create folder in the workspace header, or select files and folders and open Manage selection to rename, move, or copy them.",
"Choose the destination and resolve each target conflict explicitly.",
"Apply the operation and reopen the destination folder.",
],
@@ -1036,6 +1047,48 @@ manifest = ModuleManifest(
],
},
),
DocumentationTopic(
id="files.workflow.unpack-managed-archive",
title="Unpack an already uploaded archive",
summary="Select a managed ZIP or TAR archive and reuse the upload preview, password and selective extraction workflow without downloading and uploading it again.",
body=(
"Select exactly one ZIP, TAR, TAR.GZ, TAR.BZ2 or TAR.XZ file in a managed space, then choose Unpack archive in the action bar or context menu. "
"Choose an accessible personal or group destination and request a preview; no files are created yet. The ordinary archive dialog reviews entries, verifies any ZIP password and selects files or folders. "
"Confirmation reads the managed source on the server and creates new destination files. The source archive, metadata and version remain unchanged; destination collisions are rejected and never overwrite the source. "
"Read, download and upload permissions are all required. Ownership, current shares, tenant, source version, connector restrictions, integrity, quarantine and optional envelope-decryption availability are checked again at confirmation. "
"The expiring preview is bound to the exact source/version, actor and destination. Changed or inaccessible sources require a fresh preview; the server never silently switches to another version. "
"The same compressed-size, expanded-size, per-member, entry-count, expansion-ratio and safe-path limits apply as for uploaded archives. Passwords remain request-only. "
"Extraction creates ordinary destination files, just as archive upload does; source archive password protection or a source storage envelope is not automatically copied to members. "
"The shared blurred loading overlay covers the dialog during inspection and extraction, without the envelope animation. "
"Selected file and byte totals are shown immediately; live progress displays measured server counters, not an estimated timer. "
"Upload transfer, inspection, extraction/storage and final commit are distinct phases; transferred or processed bytes alone never mean a completed transaction. "
"An unknown phase is indeterminate. Close, selection and destination controls remain blocked while work runs; errors restore the unchanged review dialog for inspection. "
"Extraction is synchronous: leave the busy dialog open until completion. Remote connector-space entries must first be synchronized to managed storage."
),
layer="configured",
documentation_types=("user", "admin"),
audience=("file_user", "file_manager", "tenant_admin", "system_admin"),
order=42,
conditions=(DocumentationCondition(required_modules=("files",), required_scopes=("files:file:read", "files:file:download", "files:file:upload")),),
links=(
DocumentationLink(label="Files", href="/files", kind="runtime"),
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
),
metadata={
"kind": "workflow", "route": "/files", "screen": "Files",
"help_contexts": ["files.list", "files.archive-extract"],
"prerequisites": ["The selected managed archive is readable and downloadable, and you may upload into the selected destination."],
"steps": [
"Select one supported managed archive and choose Unpack archive in the action bar or context menu.",
"Choose the destination space and folder, then request Preview archive.",
"Review entries, enter and verify the ZIP password if required, and select the files or folders to extract.",
"Confirm Import selected and keep the dialog open until extraction finishes; choose a different destination if files already exist.",
],
"outcome": "Selected members are new managed files while the source archive remains unchanged.",
"verification": "Check destination paths and counts, and confirm the source archive still has the same version.",
"related_topic_ids": ["files.workflow.upload-and-unpack-zip", "files.workflow.find-and-download-files", "files.reference.shared-storage-profile"],
},
),
DocumentationTopic(
id="files.workflow.find-and-download-files",
title="Find and download managed files",
@@ -1469,6 +1522,7 @@ manifest = ModuleManifest(
title="Operate Files integrity, recovery, and connector transport safety",
summary="Back up database evidence, blob ciphertext, and Encryption custody as one recovery unit, and pin every SDK-managed connector peer.",
body=(
"Archive inspection enforces per-header TAR entry, size and expansion-ratio limits before traversing the next payload. Paths are capped at 4096 UTF-8 bytes and 128 components; derived parent folders count toward the configured entry limit. Reduce or split oversized archives before retrying; no automatic policy relaxation is performed. "
"Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the matching Encryption tables and original deployment master key, then run the bounded resumable integrity scan from Administration and verify representative protected and unprotected access paths. Each scan batch and finding action requires the revision shown to the operator, so a stale screen cannot recheck or delete after concurrent reconciliation. Protected scans verify stored ciphertext before decryption and then verify plaintext semantic evidence. On PostgreSQL, managed blob creation/repair and applied orphan cleanup commit lease-fenced Core recovery intent before object effects; success, compensation, and forward completion require independent database and object checks, while mismatch is quarantined and unresolved work remains visible in Ops. Development SQLite instead records blob intent in the caller transaction to avoid a second-writer deadlock, uses a process-local fence, verifies after commit, and reconstructs durable compensation evidence after handled rollback. A hard process loss before the SQLite caller commits can therefore leave an unrecorded object; SQLite is not a production recovery profile and operators must run an integrity scan after such a loss. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. "
"Hard purge records irreversible intent before deleting database evidence, and blob garbage collection separately rechecks references under the shared blob lease before deleting a managed object. S3 connector write-back records digest-only forward-recovery intent before a conditional provider effect; request and content markers prove success, while mismatches remain visible in Ops and fence later writers. S3 connector pools pin every retry, redirect, discovered endpoint, and provider alias while retaining the configured TLS authority; outbound proxies and ambient credential discovery are disabled. SMB initial connections, reconnects, aliases, and DFS referrals use a Files-owned pinned transport and cache. Both apply the deployment private-network policy immediately before each socket opens and fail closed if an SDK no longer exposes the verified transport seam. Installer-owned Garage storage is supported only at the exact deployment service endpoint with its explicit trust marker. Destructive module retirement drops database tables but does not remove backend blob objects."
),
@@ -1715,9 +1769,15 @@ manifest = ModuleManifest(
),
DocumentationTopic(
id="files.reference.shared-storage-profile",
title="Operate Files with shared object storage",
summary="Choose local, host-shared, or S3-backed storage consistently with the runtime topology.",
body="Core supplies the common local/S3 object-storage backend while Files owns file metadata and object-key semantics. Local storage is valid for one runtime process; a shared host volume supports same-host replicas; independent hosts require an explicitly trusted HTTPS S3-compatible endpoint. Restore PostgreSQL, objects, and the master key to one coordinated recovery point.",
title="Operate Files storage and temporary archive work",
summary="Align durable storage, private archive staging and measured progress with the worker topology.",
body=(
"Core supplies the common local/S3 object-storage backend while Files owns file metadata and object-key semantics. Local storage is valid for one runtime process; a shared host volume supports same-host replicas; independent hosts require an explicitly trusted HTTPS S3-compatible endpoint. Restore PostgreSQL, objects, and the master key to one coordinated recovery point. "
"The archive UI opts into upload-once staging: password repreview and confirmation reuse a tenant/user-bound copy, but never persist passwords. Temporary archive and progress files use mode 0600 below a private 0700 directory. FILE_ARCHIVE_WORK_ROOT overrides the default OS-temporary directory derived from the service uid and configured storage-root hash. All same-host workers must share it; independent hosts need shared POSIX storage with working flock locks or sticky routing for archive and progress requests, even when durable blobs use S3. "
"Defaults are 2 GiB of staged archives per work root, four stages per tenant/user, and 1,800 seconds from initial staging without renewal. Cancel/replacement requests and successful confirmation release copies; abandoned expired copies are cleaned opportunistically by later archive work, not on an exact idle deadline. POSIX active leases prevent eviction of an in-use copy even after its TTL; stages and progress are not resumable background jobs. "
"A maintained optional system libarchive accelerates classic stored/deflated ZIPCrypto decoding for ASCII or UTF-8-flagged names. Missing libraries, AES and unsupported encodings/formats select the original Python reader before decoding; native errors fail closed. Both paths retain full header/path/limit checks, and the native path independently verifies CRC and size without filesystem extraction. "
"Extraction stores one verified member at a time. Archive-local backend-client reuse and batched response metadata reduce avoidable work without caching authorization or removing durable per-file recovery, integrity or commit verification. Progress reports actual file/byte counters and a separate finalization phase, never a timed estimate or an early success. Native decoder speed does not promise an equivalent whole-import or S3 improvement."
),
layer="configured",
documentation_types=("admin",),
audience=("file_admin", "operator", "system_admin"),
@@ -1745,6 +1805,10 @@ manifest = ModuleManifest(
"FILE_STORAGE_S3_ENDPOINT_URL",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
"FILE_ARCHIVE_WORK_ROOT",
"FILE_ARCHIVE_STAGED_MAX_BYTES",
"FILE_ARCHIVE_STAGED_PER_ACTOR",
"FILE_ARCHIVE_PREVIEW_TTL_SECONDS",
),
metadata={
"kind": "reference",
@@ -1150,6 +1150,10 @@ def _loaded_asset_response(
created_at=asset.created_at.isoformat(),
updated_at=asset.updated_at.isoformat(),
deleted_at=asset.deleted_at.isoformat() if asset.deleted_at else None,
retained_until=asset.retained_until.isoformat() if asset.retained_until else None,
legal_hold=asset.legal_hold,
lifecycle_revision=asset.lifecycle_revision,
lifecycle_reason=asset.lifecycle_reason,
audit_relevant=asset.id in sent_asset_ids,
metadata=metadata,
source_provenance=source_provenance_from_metadata(metadata),
+2
View File
@@ -19,6 +19,7 @@ from govoplan_files.backend.routes.shares import router as shares_router
from govoplan_files.backend.routes.spaces import router as spaces_router
from govoplan_files.backend.routes.transfers import router as transfers_router
from govoplan_files.backend.routes.uploads import router as uploads_router
from govoplan_files.backend.routes.managed_archives import router as managed_archives_router
router = APIRouter()
@@ -30,6 +31,7 @@ for workflow_router in (
lifecycle_router,
listing_router,
uploads_router,
managed_archives_router,
connector_settings_router,
connector_io_router,
connector_profiles_router,
@@ -0,0 +1,263 @@
"""Extract an existing managed archive through the ordinary preview pipeline."""
from __future__ import annotations
from io import BytesIO
from collections.abc import Mapping
import json
from typing import Literal
from fastapi import APIRouter, Depends, HTTPException, UploadFile
from pydantic import BaseModel, ConfigDict, Field
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, has_scope, require_scope
from govoplan_core.db.session import get_session
from govoplan_core.security.secrets import (
TransientPayloadError,
open_transient_payload,
seal_transient_payload,
)
from govoplan_files.backend.route_support import (
_connector_policy_error,
_http_error,
_is_admin,
)
from govoplan_files.backend.runtime import settings
from govoplan_files.backend.schemas import ArchivePreviewResponse, FileUploadResponse
from govoplan_files.backend.storage.access import ensure_owner_access
from govoplan_files.backend.storage.archives import archive_format_for_filename
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_policy import (
ConnectorAccessRequest,
ConnectorPolicyDenied,
ensure_connector_policy_allows,
)
from govoplan_files.backend.storage.connector_policy_store import (
effective_connector_policy_sources,
)
from govoplan_files.backend.storage.files import (
current_version_and_blob,
get_asset_for_user,
read_asset_bytes,
)
from govoplan_files.backend.storage.paths import normalize_folder
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata
from govoplan_files.backend.routes.uploads import (
confirm_archive_upload,
preview_archive_upload,
)
router = APIRouter(prefix="/files", tags=["files"])
_PURPOSE = "files.managed-archive-preview.v1"
class ManagedArchivePreviewRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
source_version_id: str = Field(min_length=1, max_length=128)
owner_type: Literal["user", "group"] = "user"
owner_id: str | None = Field(default=None, max_length=128)
path: str = Field(default="", max_length=4096)
password: str | None = Field(default=None, max_length=4096)
class ManagedArchiveConfirmRequest(ManagedArchivePreviewRequest):
preview_token: str = Field(min_length=1, max_length=32768)
selected_paths: list[str] = Field(min_length=1, max_length=10_000)
operation_id: str | None = Field(default=None, min_length=36, max_length=36)
def _resolve_source(
session: Session,
principal: ApiPrincipal,
file_id: str,
payload: ManagedArchivePreviewRequest,
):
for scope in ("files:file:read", "files:file:download"):
if not has_scope(principal, scope):
raise HTTPException(status_code=403, detail=f"Missing scope: {scope}")
target_owner = payload.owner_id or principal.user.id
ensure_owner_access(
session,
tenant_id=principal.tenant_id,
owner_type=payload.owner_type,
owner_id=target_owner,
user_id=principal.user.id,
is_admin=_is_admin(principal),
)
asset = get_asset_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
is_admin=_is_admin(principal),
)
if asset.current_version_id != payload.source_version_id:
raise FileStorageError(
"The managed archive version changed; reload the file and preview it again"
)
archive_format_for_filename(asset.filename)
version, blob = current_version_and_blob(session, asset)
if (
version.file_asset_id != asset.id
or version.tenant_id != principal.tenant_id
or blob.tenant_id != principal.tenant_id
):
raise FileStorageError(
"Managed archive version or content does not belong to this source"
)
if blob.size_bytes > settings.file_upload_zip_max_bytes:
raise FileStorageError(
"Managed archive exceeds the configured archive size limit"
)
provenance = source_provenance_from_metadata(asset.metadata_ or {})
if provenance:
source_owner = (
asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id
)
sources = []
for scope_type, scope_id in dict.fromkeys(
(
(asset.owner_type, source_owner),
(payload.owner_type, target_owner),
("user", principal.user.id),
)
):
sources.extend(
effective_connector_policy_sources(
session,
tenant_id=principal.tenant_id,
scope_type=scope_type,
scope_id=scope_id,
)
)
ensure_connector_policy_allows(
ConnectorAccessRequest.from_provenance(provenance, operation="import"),
sources,
)
# This is the same integrity/quarantine and envelope-decryption path used
# by authorized managed-file downloads; never read raw backend bytes here.
data, version, _blob = read_asset_bytes(session, asset)
if version.id != payload.source_version_id:
raise FileStorageError(
"The managed archive version changed during the verified read; reload and preview again"
)
if len(data) > settings.file_upload_zip_max_bytes:
raise FileStorageError(
"Managed archive exceeds the configured archive size limit"
)
return asset, version, data, target_owner, provenance
@router.post("/{file_id}/archive-preview", response_model=ArchivePreviewResponse)
def preview_managed_archive(
file_id: str,
payload: ManagedArchivePreviewRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
try:
asset, version, data, owner, _provenance = _resolve_source(
session, principal, file_id, payload
)
with BytesIO(data) as source:
preview = preview_archive_upload(
file=UploadFile(file=source, filename=asset.filename),
owner_type=payload.owner_type,
owner_id=owner,
path=payload.path,
campaign_id=None,
password=payload.password,
principal=principal,
)
preview.preview_token = seal_transient_payload(
{
"purpose": _PURPOSE,
"file_id": asset.id,
"version_id": version.id,
"tenant_id": principal.tenant_id,
"user_id": principal.user.id,
"owner_type": payload.owner_type,
"owner_id": owner,
"path": normalize_folder(payload.path),
"upload_preview_token": preview.preview_token,
}
)
return preview
except ConnectorPolicyDenied as exc:
raise _connector_policy_error(exc) from exc
except (FileStorageError, ValueError) as exc:
raise _http_error(exc) from exc
@router.post("/{file_id}/archive-confirm", response_model=FileUploadResponse)
def confirm_managed_archive(
file_id: str,
payload: ManagedArchiveConfirmRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
try:
token = open_transient_payload(
payload.preview_token, ttl_seconds=settings.file_archive_preview_ttl_seconds
)
expected = {
"purpose": _PURPOSE,
"file_id": file_id,
"version_id": payload.source_version_id,
"tenant_id": principal.tenant_id,
"user_id": principal.user.id,
"owner_type": payload.owner_type,
"owner_id": payload.owner_id or principal.user.id,
"path": normalize_folder(payload.path),
}
if any(
token.get(key) != value for key, value in expected.items()
) or not isinstance(token.get("upload_preview_token"), str):
raise FileStorageError(
"Managed archive preview does not match this source, actor, or destination; preview again"
)
asset, version, data, owner, provenance = _resolve_source(
session, principal, file_id, payload
)
source_provenance = dict(
provenance or {"source_type": "managed_archive", "external_id": asset.id}
)
provenance_metadata = source_provenance.get("metadata")
source_provenance["metadata"] = {
**(
dict(provenance_metadata)
if isinstance(provenance_metadata, Mapping)
else {}
),
"archive_source_file_id": asset.id,
"archive_source_version_id": version.id,
}
with BytesIO(data) as source:
# Reject collisions, including an archive member targeting its own
# source. The original file/version can never be overwritten here.
return confirm_archive_upload(
file=UploadFile(file=source, filename=asset.filename),
preview_token=token["upload_preview_token"],
selected_paths_json=json.dumps(payload.selected_paths),
owner_type=payload.owner_type,
owner_id=owner,
path=payload.path,
campaign_id=None,
password=payload.password,
conflict_strategy="reject",
conflict_resolutions_json=None,
source_provenance_json=json.dumps(source_provenance),
source_revision=version.id,
connector_policy_json=None,
encryption_vault_id=None,
operation_id=payload.operation_id,
session=session,
principal=principal,
)
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (FileStorageError, TransientPayloadError, ValueError) as exc:
session.rollback()
raise _http_error(exc) from exc
+118 -41
View File
@@ -3,9 +3,11 @@ from __future__ import annotations
import hashlib
import hmac
import json
from contextlib import ExitStack, contextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Literal
from fastapi import APIRouter, Depends, File as FastAPIFile, Form, UploadFile
from fastapi import APIRouter, Depends, File as FastAPIFile, Form, HTTPException, UploadFile
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_scope
@@ -24,6 +26,10 @@ from govoplan_files.backend.schemas import (
from govoplan_files.backend.db.models import FileAsset
from govoplan_core.db.session import get_session
from govoplan_files.backend.runtime import settings
from govoplan_files.backend.archive_work import (
ArchiveProgress, ArchiveWorkExpired, discard_staged_upload, read_progress,
stage_upload, use_staged_upload,
)
from govoplan_files.backend.storage.paths import (
UnsafeFilePathError,
normalize_folder,
@@ -44,7 +50,7 @@ from govoplan_files.backend.storage.files import (
from govoplan_files.backend.route_support import (
_asset_response,
_asset_list_response,
_audit_connector_imports,
_cleanup_temp_file,
_connector_policy_error,
@@ -60,6 +66,57 @@ router = APIRouter(prefix="/files", tags=["files"])
_ARCHIVE_PREVIEW_PURPOSE = "files.archive-preview.v1"
@contextmanager
def _archive_source(file, staged_upload_id, preview_token, principal):
if isinstance(staged_upload_id, str):
if file is not None and hasattr(file, "file"):
raise FileStorageError("Choose either the staged archive or a new upload")
if not isinstance(preview_token, str) or not preview_token:
raise FileStorageError("A valid preview token is required for a staged archive")
payload = open_transient_payload(preview_token, ttl_seconds=settings.file_archive_preview_ttl_seconds)
expected = {"purpose": _ARCHIVE_PREVIEW_PURPOSE, "tenant_id": principal.tenant_id,
"user_id": principal.user.id, "staged_upload_id": staged_upload_id}
if any(payload.get(key) != value for key, value in expected.items()) or not isinstance(payload.get("filename"), str):
raise FileStorageError("Archive preview does not match this staged upload")
with use_staged_upload(settings, staged_upload_id, tenant_id=principal.tenant_id, user_id=principal.user.id) as path:
token_digest = payload.get("archive_sha256")
if not isinstance(token_digest, str) or not hmac.compare_digest(_archive_sha256(path), token_digest):
raise FileStorageError("Archive contents changed after preview; preview it again")
yield path, payload["filename"]
return
if file is None or not hasattr(file, "file"):
raise FileStorageError("Select an archive to upload")
filename = file.filename or "archive"
path = _spool_limited_upload_to_temp(file, max_bytes=settings.file_upload_zip_max_bytes, suffix=_archive_suffix(filename))
try:
yield path, filename
finally:
try:
_cleanup_temp_file(path)
except OSError:
pass
@router.get("/archive-progress/{operation_id}")
async def archive_operation_progress(operation_id: str, principal: ApiPrincipal = Depends(require_scope("files:file:upload"))):
# Small local receipt read stays available even while sync upload workers
# are busy. The request owns the transaction; this is not a job retry API.
try:
return read_progress(settings, operation_id, tenant_id=principal.tenant_id, user_id=principal.user.id)
except ArchiveWorkExpired as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except FileStorageError as exc:
raise _http_error(exc) from exc
@router.delete("/archive-staging/{stage_id}", status_code=204)
def release_staged_archive(stage_id: str, principal: ApiPrincipal = Depends(require_scope("files:file:upload"))):
try:
discard_staged_upload(settings, stage_id, tenant_id=principal.tenant_id, user_id=principal.user.id)
except FileStorageError as exc:
raise _http_error(exc) from exc
def _archive_suffix(filename: str) -> str:
lowered = filename.casefold()
for suffix in (".tar.bz2", ".tar.gz", ".tar.xz", ".tbz2", ".tgz", ".txz", ".tar", ".zip"):
@@ -135,24 +192,22 @@ def _validate_archive_preview_token(
@router.post("/archive-preview", response_model=ArchivePreviewResponse)
def preview_archive_upload(
file: UploadFile = FastAPIFile(...),
file: UploadFile | None = FastAPIFile(default=None),
owner_type: Literal["user", "group"] = Form(default="user"),
owner_id: str | None = Form(default=None),
path: str = Form(default=""),
campaign_id: str | None = Form(default=None),
password: str | None = Form(default=None),
retain_upload: bool = Form(default=False),
staged_upload_id: str | None = Form(default=None),
preview_token: str | None = Form(default=None),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
target_owner = owner_id or principal.user.id
archive_path: str | None = None
filename = file.filename or "archive"
sources = ExitStack()
try:
archive_path, filename = sources.enter_context(_archive_source(file, staged_upload_id, preview_token, principal))
archive_format = archive_format_for_filename(filename)
archive_path = _spool_limited_upload_to_temp(
file,
max_bytes=settings.file_upload_zip_max_bytes,
suffix=_archive_suffix(filename),
)
inspection = inspect_archive(
archive_path,
filename=filename,
@@ -163,6 +218,9 @@ def preview_archive_upload(
)
digest = _archive_sha256(archive_path)
normalized_path = normalize_folder(path)
stage_id = staged_upload_id if isinstance(staged_upload_id, str) else None
if retain_upload is True and stage_id is None:
stage_id = stage_upload(settings, archive_path, tenant_id=principal.tenant_id, user_id=principal.user.id)
preview_token = seal_transient_payload(
{
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
@@ -174,13 +232,21 @@ def preview_archive_upload(
"campaign_id": campaign_id or "",
"archive_format": archive_format,
"archive_sha256": digest,
**({"staged_upload_id": stage_id, "filename": filename} if stage_id else {}),
}
)
expires_at = datetime.now(UTC) + timedelta(
seconds=settings.file_archive_preview_ttl_seconds
)
if isinstance(staged_upload_id, str):
# Repreviewing (for example with a password) does not renew the
# underlying private stage's lifetime.
expires_at = min(expires_at, datetime.fromtimestamp(
Path(archive_path).stat().st_mtime, UTC
) + timedelta(seconds=settings.file_archive_preview_ttl_seconds))
return ArchivePreviewResponse(
preview_token=preview_token,
staged_upload_id=stage_id,
archive_format=inspection.archive_format,
entries=[
ArchiveEntryResponse(
@@ -200,16 +266,17 @@ def preview_archive_upload(
password_verified=inspection.password_verified,
expires_at=expires_at.isoformat(),
)
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
except ArchiveWorkExpired as exc:
raise HTTPException(status_code=410, detail=str(exc)) from exc
except (FileStorageError, TransientPayloadError, UnsafeFilePathError, ValueError) as exc:
raise _http_error(exc) from exc
finally:
if archive_path:
_cleanup_temp_file(archive_path)
sources.close()
@router.post("/archive-confirm", response_model=FileUploadResponse)
def confirm_archive_upload(
file: UploadFile = FastAPIFile(...),
file: UploadFile | None = FastAPIFile(default=None),
preview_token: str = Form(...),
selected_paths_json: str = Form(...),
owner_type: Literal["user", "group"] = Form(default="user"),
@@ -225,13 +292,17 @@ def confirm_archive_upload(
source_revision: str | None = Form(default=None),
connector_policy_json: str | None = Form(default=None),
encryption_vault_id: str | None = Form(default=None),
staged_upload_id: str | None = Form(default=None),
operation_id: str | None = Form(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
target_owner = owner_id or principal.user.id
archive_path: str | None = None
filename = file.filename or "archive"
sources = ExitStack()
progress = None
committed = False
try:
archive_path, filename = sources.enter_context(_archive_source(file, staged_upload_id, preview_token, principal))
raw_resolutions = (
json.loads(conflict_resolutions_json)
if conflict_resolutions_json
@@ -251,11 +322,6 @@ def confirm_archive_upload(
)
archive_format = archive_format_for_filename(filename)
normalized_path = normalize_folder(path)
archive_path = _spool_limited_upload_to_temp(
file,
max_bytes=settings.file_upload_zip_max_bytes,
suffix=_archive_suffix(filename),
)
digest = _archive_sha256(archive_path)
_validate_archive_preview_token(
preview_token,
@@ -268,6 +334,7 @@ def confirm_archive_upload(
archive_format=archive_format,
archive_sha256=digest,
)
progress = ArchiveProgress(settings, operation_id, tenant_id=principal.tenant_id, user_id=principal.user.id)
extracted = extract_archive_upload(
session,
tenant_id=principal.tenant_id,
@@ -289,16 +356,20 @@ def confirm_archive_upload(
max_file_bytes=settings.file_upload_max_bytes,
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
progress=progress,
)
uploaded_assets = [item.asset for item in extracted]
_audit_connector_imports(session, principal, uploaded_assets)
progress("finalizing", len(extracted), len(extracted), progress.value["completed_bytes"], progress.value["total_bytes"])
session.flush()
response = FileUploadResponse(files=_asset_list_response(session, uploaded_assets, include_shares=True))
session.commit()
return FileUploadResponse(
files=[
_asset_response(session, asset, include_shares=True)
for asset in uploaded_assets
]
)
committed = True
progress.finish(True)
return response
except ArchiveWorkExpired as exc:
session.rollback()
raise HTTPException(status_code=410, detail=str(exc)) from exc
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
@@ -311,9 +382,21 @@ def confirm_archive_upload(
) as exc:
session.rollback()
raise _http_error(exc) from exc
except Exception:
session.rollback()
raise
finally:
if archive_path:
_cleanup_temp_file(archive_path)
if progress and not committed:
progress.finish(False)
try:
sources.close()
except OSError:
pass
if committed and isinstance(staged_upload_id, str):
try:
discard_staged_upload(settings, staged_upload_id, tenant_id=principal.tenant_id, user_id=principal.user.id)
except (OSError, FileStorageError):
pass
@router.post("/upload", response_model=FileUploadResponse)
@@ -402,6 +485,8 @@ def upload_files(
)
uploaded_assets.append(stored.asset)
_audit_connector_imports(session, principal, uploaded_assets)
session.flush()
response = FileUploadResponse(files=_asset_list_response(session, uploaded_assets, include_shares=True))
session.commit()
except ConnectorPolicyDenied as exc:
session.rollback()
@@ -409,12 +494,7 @@ def upload_files(
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
session.rollback()
raise _http_error(exc) from exc
return FileUploadResponse(
files=[
_asset_response(session, asset, include_shares=True)
for asset in uploaded_assets
]
)
return response
@router.post("/upload-zip", response_model=FileUploadResponse)
@@ -469,6 +549,8 @@ def upload_zip(
max_total_bytes=settings.file_upload_zip_max_bytes,
)
_audit_connector_imports(session, principal, [item.asset for item in extracted])
session.flush()
response = FileUploadResponse(files=_asset_list_response(session, [item.asset for item in extracted], include_shares=True))
session.commit()
except ConnectorPolicyDenied as exc:
session.rollback()
@@ -479,9 +561,4 @@ def upload_zip(
finally:
if zip_path:
_cleanup_temp_file(zip_path)
return FileUploadResponse(
files=[
_asset_response(session, item.asset, include_shares=True)
for item in extracted
]
)
return response
+1
View File
@@ -272,6 +272,7 @@ class ArchiveEntryResponse(BaseModel):
class ArchivePreviewResponse(BaseModel):
preview_token: str
staged_upload_id: str | None = None
archive_format: str
entries: list[ArchiveEntryResponse] = Field(default_factory=list)
file_count: int
+208 -69
View File
@@ -6,10 +6,11 @@ import stat
import tarfile
import zipfile
from dataclasses import dataclass
from contextlib import closing
from io import BytesIO
from os import PathLike
from pathlib import Path, PurePosixPath
from typing import Any, BinaryIO, Iterable, Literal
from typing import Any, BinaryIO, Callable, Iterable, Iterator, Literal
import pyzipper
from sqlalchemy.orm import Session
@@ -25,6 +26,7 @@ from govoplan_files.backend.storage.common import (
UploadedStoredFile,
)
from govoplan_files.backend.storage.files import (
archive_storage_backend_scope,
create_file_asset,
current_versions_and_blobs,
)
@@ -33,11 +35,17 @@ from govoplan_files.backend.storage.paths import (
normalize_folder,
normalize_logical_path,
)
from govoplan_files.backend.storage.native_zip import (
native_zip_library,
read_native_zip_members,
)
_ARCHIVE_READ_CHUNK_SIZE = 1024 * 1024
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:")
ARCHIVE_UPLOAD_MAX_ENTRIES = 10_000
ARCHIVE_MAX_PATH_BYTES = 4096
ARCHIVE_MAX_PATH_DEPTH = 128
# Kept for callers and documentation using the former ZIP-specific name.
ZIP_UPLOAD_MAX_FILES = ARCHIVE_UPLOAD_MAX_ENTRIES
SUPPORTED_ARCHIVE_SUFFIXES = (
@@ -51,6 +59,8 @@ SUPPORTED_ARCHIVE_SUFFIXES = (
".zip",
)
ArchiveProgress = Callable[[str, int, int, int, int], None]
class ArchivePasswordError(FileStorageError):
pass
@@ -141,7 +151,13 @@ def inspect_archive(
password=password,
)
else:
entries = _inspect_tar(archive_data)
entries = _inspect_tar(
archive_data,
compressed_size=compressed_size,
max_entries=max_entries,
max_expanded_bytes=max_expanded_bytes,
max_expansion_ratio=max_expansion_ratio,
)
requires_password = False
password_verified = True
_validate_archive_limits(
@@ -151,14 +167,12 @@ def inspect_archive(
max_expanded_bytes=max_expanded_bytes,
max_expansion_ratio=max_expansion_ratio,
)
complete_entries = _with_derived_directories(entries)
complete_entries = _with_derived_directories(entries, max_entries=max_entries)
return ArchiveInspection(
archive_format=archive_format,
entries=tuple(complete_entries),
file_count=sum(entry.kind == "file" for entry in complete_entries),
directory_count=sum(
entry.kind == "directory" for entry in complete_entries
),
directory_count=sum(entry.kind == "directory" for entry in complete_entries),
expanded_size_bytes=sum(
entry.size_bytes for entry in entries if entry.kind == "file"
),
@@ -190,7 +204,10 @@ def extract_archive_upload(
max_file_bytes: int = 50 * 1024 * 1024,
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
max_expansion_ratio: int = 100,
progress: ArchiveProgress | None = None,
) -> list[UploadedStoredFile]:
if progress:
progress("inspecting", 0, 0, 0, 0)
inspection = inspect_archive(
archive_data,
filename=filename,
@@ -204,6 +221,9 @@ def extract_archive_upload(
selected_files = _selected_file_paths(inspection.entries, selected_paths)
if not selected_files:
raise FileStorageError("Select at least one archive file to import")
selected_total_bytes = sum(
entry.size_bytes for entry in inspection.entries if entry.path in selected_files
)
actual_total_limit = min(
max_expanded_bytes,
inspection.compressed_size_bytes * max_expansion_ratio,
@@ -215,6 +235,8 @@ def extract_archive_upload(
password=password,
max_file_bytes=max_file_bytes,
max_total_bytes=actual_total_limit,
progress=progress,
total_bytes=selected_total_bytes,
)
else:
members = _read_selected_tar_members(
@@ -222,22 +244,30 @@ def extract_archive_upload(
selected_files=selected_files,
max_file_bytes=max_file_bytes,
max_total_bytes=actual_total_limit,
progress=progress,
total_bytes=selected_total_bytes,
)
# Release Python/native archive handles immediately if storage or a
# callback fails while the member iterator is suspended at a yield.
with closing(members), archive_storage_backend_scope():
return _store_archive_members(
session,
members=members,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
folder=folder,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
progress=progress,
total_files=len(selected_files),
total_bytes=selected_total_bytes,
)
return _store_archive_members(
session,
members=members,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
folder=folder,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
)
def extract_zip_upload(
@@ -319,11 +349,23 @@ def _inspect_zip(
def _inspect_tar(
archive_data: bytes | str | PathLike[str],
*,
compressed_size: int,
max_entries: int,
max_expanded_bytes: int,
max_expansion_ratio: int,
) -> list[ArchiveEntry]:
try:
with _open_tar(archive_data) as archive:
entries: list[ArchiveEntry] = []
for member in archive.getmembers():
expanded_size = 0
for member in archive:
# Check each header before advancing across its payload. In a
# compressed TAR getmembers() would inflate everything first.
if len(entries) >= max_entries:
raise FileStorageError(
f"Archive contains too many entries (limit {max_entries})"
)
path = _safe_member_path(member.name)
if member.isdir():
entries.append(
@@ -334,6 +376,21 @@ def _inspect_tar(
raise FileStorageError(
f"Archive member {member.name!r} is not a regular file or directory"
)
if member.size < 0:
raise FileStorageError("Archive member has invalid size metadata")
expanded_size += member.size
if expanded_size > max_expanded_bytes:
raise FileStorageError(
"Archive is too large after extraction "
f"(limit {max_expanded_bytes} bytes)"
)
if expanded_size and (
compressed_size <= 0
or expanded_size > compressed_size * max_expansion_ratio
):
raise FileStorageError(
f"Archive expansion ratio exceeds {max_expansion_ratio}:1"
)
entries.append(
ArchiveEntry(
path=path,
@@ -352,21 +409,19 @@ def _zip_entry(info: zipfile.ZipInfo) -> ArchiveEntry:
path = _safe_member_path(info.filename)
unix_mode = (info.external_attr >> 16) & 0xFFFF
file_type = stat.S_IFMT(unix_mode)
if file_type and not (
stat.S_ISREG(unix_mode) or stat.S_ISDIR(unix_mode)
):
if file_type and not (stat.S_ISREG(unix_mode) or stat.S_ISDIR(unix_mode)):
raise FileStorageError(
f"Archive member {info.filename!r} is not a regular file or directory"
)
if info.file_size < 0 or info.compress_size < 0:
raise FileStorageError(f"Archive member {info.filename!r} has invalid size metadata")
raise FileStorageError(
f"Archive member {info.filename!r} has invalid size metadata"
)
return ArchiveEntry(
path=path,
kind="directory" if info.is_dir() else "file",
size_bytes=0 if info.is_dir() else int(info.file_size),
compressed_size_bytes=(
None if info.is_dir() else int(info.compress_size)
),
compressed_size_bytes=(None if info.is_dir() else int(info.compress_size)),
encrypted=bool(info.flag_bits & 0x1),
)
@@ -388,9 +443,7 @@ def _validate_archive_limits(
for entry in entries:
previous_kind = seen.get(entry.path)
if previous_kind is not None:
raise FileStorageError(
f"Archive contains duplicate path {entry.path!r}"
)
raise FileStorageError(f"Archive contains duplicate path {entry.path!r}")
seen[entry.path] = entry.kind
if entry.kind == "file":
expanded_size += entry.size_bytes
@@ -400,17 +453,17 @@ def _validate_archive_limits(
f"(limit {max_expanded_bytes} bytes)"
)
if expanded_size and (
compressed_size <= 0
or expanded_size > compressed_size * max_expansion_ratio
compressed_size <= 0 or expanded_size > compressed_size * max_expansion_ratio
):
raise FileStorageError(
"Archive expansion ratio exceeds "
f"{max_expansion_ratio}:1"
f"Archive expansion ratio exceeds {max_expansion_ratio}:1"
)
def _with_derived_directories(
entries: list[ArchiveEntry],
*,
max_entries: int,
) -> list[ArchiveEntry]:
by_path = {entry.path: entry for entry in entries}
for entry in entries:
@@ -422,10 +475,13 @@ def _with_derived_directories(
raise FileStorageError(
f"Archive path {path!r} is both a file and a directory"
)
by_path.setdefault(
path,
ArchiveEntry(path=path, kind="directory", size_bytes=0),
)
if path not in by_path:
if len(by_path) >= max_entries:
raise FileStorageError(
"Archive contains too many entries including parent "
f"directories (limit {max_entries})"
)
by_path[path] = ArchiveEntry(path=path, kind="directory", size_bytes=0)
parent = parent.parent
return sorted(
by_path.values(),
@@ -458,9 +514,7 @@ def _selected_file_paths(
continue
prefix = f"{path}/"
selected_files.update(
file_path
for file_path in file_paths
if file_path.startswith(prefix)
file_path for file_path in file_paths if file_path.startswith(prefix)
)
return selected_files
@@ -472,12 +526,45 @@ def _read_selected_zip_members(
password: str | None,
max_file_bytes: int,
max_total_bytes: int,
) -> list[tuple[str, bytes]]:
result: list[tuple[str, bytes]] = []
progress: ArchiveProgress | None = None,
total_bytes: int = 0,
) -> Iterator[tuple[str, bytes]]:
total = 0
completed = 0
try:
with pyzipper.AESZipFile(_archive_source(archive_data)) as archive:
for info in archive.infolist():
infos = archive.infolist()
# Accelerate only classic ZIPCrypto using established public C
# decoding APIs; AES and other formats retain their existing path.
native_compatible = (
bool(password)
and "\x00" not in password
and any(info.flag_bits & 1 for info in infos)
and all(
not getattr(info, "wz_aes_version", None)
and info.compress_type in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED}
and (info.filename.isascii() or bool(info.flag_bits & 0x800))
for info in infos
)
)
native = native_zip_library() if native_compatible else None
if native is not None:
yield from read_native_zip_members(
native,
archive_data,
infos_by_path={
_safe_member_path(info.filename): info for info in infos
},
selected_files=selected_files,
normalize_name=_safe_member_path,
password=password,
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
progress=progress,
total_bytes=total_bytes,
)
return
for info in infos:
if info.is_dir():
continue
path = _safe_member_path(info.filename)
@@ -492,6 +579,17 @@ def _read_selected_zip_members(
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
current_total=total,
on_bytes=(
lambda count: progress(
"extracting",
completed,
len(selected_files),
count,
total_bytes,
)
)
if progress
else None,
)
except (RuntimeError, ValueError, zipfile.BadZipFile) as exc:
if info.flag_bits & 0x1:
@@ -499,12 +597,17 @@ def _read_selected_zip_members(
"Archive password is incorrect"
) from exc
raise
result.append((path, data))
completed += 1
if progress:
progress(
"extracting", completed, len(selected_files), total, total_bytes
)
yield path, data
del data
except (FileStorageError, ArchivePasswordError):
raise
except (OSError, ValueError, zipfile.BadZipFile) as exc:
raise FileStorageError("ZIP extraction failed") from exc
return result
def _read_selected_tar_members(
@@ -513,12 +616,16 @@ def _read_selected_tar_members(
selected_files: set[str],
max_file_bytes: int,
max_total_bytes: int,
) -> list[tuple[str, bytes]]:
result: list[tuple[str, bytes]] = []
progress: ArchiveProgress | None = None,
total_bytes: int = 0,
) -> Iterator[tuple[str, bytes]]:
total = 0
completed = 0
try:
with _open_tar(archive_data) as archive:
for member in archive.getmembers():
# Iterate as headers are read instead of inflating the entire TAR
# with getmembers(), then seeking backwards to inflate it again.
for member in archive:
if not member.isfile():
continue
path = _safe_member_path(member.name)
@@ -526,9 +633,7 @@ def _read_selected_tar_members(
continue
source = archive.extractfile(member)
if source is None:
raise FileStorageError(
f"Archive member {path!r} could not be read"
)
raise FileStorageError(f"Archive member {path!r} could not be read")
with source:
data, total = _read_member(
source,
@@ -536,13 +641,29 @@ def _read_selected_tar_members(
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
current_total=total,
on_bytes=(
lambda count: progress(
"extracting",
completed,
len(selected_files),
count,
total_bytes,
)
)
if progress
else None,
)
result.append((path, data))
completed += 1
if progress:
progress(
"extracting", completed, len(selected_files), total, total_bytes
)
yield path, data
del data
except FileStorageError:
raise
except (OSError, tarfile.TarError) as exc:
raise FileStorageError("TAR extraction failed") from exc
return result
def _read_member(
@@ -552,6 +673,7 @@ def _read_member(
max_file_bytes: int,
max_total_bytes: int,
current_total: int,
on_bytes: Callable[[int], None] | None = None,
) -> tuple[bytes, int]:
parts: list[bytes] = []
actual_size = 0
@@ -565,12 +687,12 @@ def _read_member(
break
actual_size += len(chunk)
if actual_size > max_file_bytes:
raise FileStorageError(
f"Archive member {path!r} exceeds per-file limit"
)
raise FileStorageError(f"Archive member {path!r} exceeds per-file limit")
if current_total + actual_size > max_total_bytes:
raise FileStorageError("Archive is too large after extraction")
parts.append(chunk)
if on_bytes:
on_bytes(current_total + actual_size)
return b"".join(parts), current_total + actual_size
@@ -589,13 +711,15 @@ def _store_archive_members(
metadata: dict[str, Any] | None,
is_admin: bool,
encryption_vault_id: str | None,
progress: ArchiveProgress | None = None,
total_files: int = 0,
total_bytes: int = 0,
) -> list[UploadedStoredFile]:
uploaded: list[UploadedStoredFile] = []
base_folder = normalize_folder(folder)
stored_bytes = 0
for inner_path, data in members:
target_path = (
f"{base_folder}/{inner_path}" if base_folder else inner_path
)
target_path = f"{base_folder}/{inner_path}" if base_folder else inner_path
uploaded.append(
create_file_asset(
session,
@@ -616,24 +740,39 @@ def _store_archive_members(
encryption_vault_id=encryption_vault_id,
)
)
stored_bytes += len(data)
if progress:
progress("storing", len(uploaded), total_files, stored_bytes, total_bytes)
del data
return uploaded
def _safe_member_path(value: str) -> str:
raw = str(value or "").replace("\\", "/").strip()
if (
not raw
or "\x00" in raw
or raw.startswith("/")
or _WINDOWS_DRIVE_RE.match(raw)
):
try:
oversized = (
len(raw) > ARCHIVE_MAX_PATH_BYTES
or len(raw.encode("utf-8")) > ARCHIVE_MAX_PATH_BYTES
)
except UnicodeEncodeError as exc:
raise FileStorageError("Archive member path is not valid Unicode") from exc
if oversized:
raise FileStorageError(
f"Archive member path exceeds {ARCHIVE_MAX_PATH_BYTES} UTF-8 bytes"
)
if not raw or "\x00" in raw or raw.startswith("/") or _WINDOWS_DRIVE_RE.match(raw):
raise FileStorageError(f"Unsafe archive member path {value!r}")
if any(part == ".." for part in raw.split("/")):
raise FileStorageError(f"Unsafe archive member path {value!r}")
try:
return normalize_logical_path(raw.rstrip("/"))
normalized = normalize_logical_path(raw.rstrip("/"))
except ValueError as exc:
raise FileStorageError(f"Unsafe archive member path {value!r}") from exc
if normalized.count("/") + 1 > ARCHIVE_MAX_PATH_DEPTH:
raise FileStorageError(
f"Archive member path exceeds {ARCHIVE_MAX_PATH_DEPTH} components"
)
return normalized
def _archive_size(archive_data: bytes | str | PathLike[str]) -> int:
+46 -8
View File
@@ -2,6 +2,8 @@ from __future__ import annotations
import hashlib
import mimetypes
from contextlib import contextmanager
from contextvars import ContextVar
from datetime import datetime
from pathlib import PurePosixPath
from typing import Any, Iterable
@@ -15,6 +17,7 @@ from govoplan_files.backend.db.models import CampaignAttachmentUse, FileAsset, F
from govoplan_files.backend.runtime import get_registry, settings
from govoplan_files.backend.storage.access import ensure_owner_access, ensure_share_target_exists, user_group_ids
from govoplan_files.backend.storage.backends import (
StorageBackend,
StorageBackendError,
StorageObjectMissing,
get_storage_backend,
@@ -30,6 +33,39 @@ from govoplan_files.backend.storage.integrity import (
from govoplan_files.backend.storage.share_state import effective_file_share_clause
_ARCHIVE_STORAGE_BACKEND: ContextVar[dict[str, StorageBackend] | None] = ContextVar(
"govoplan.files.archive_storage_backend", default=None
)
@contextmanager
def archive_storage_backend_scope():
"""Reuse deployment transport only, never principals, policy or file data.
Lazy creation avoids opening storage for rejected or empty work. Recovery
effects retain the backend until post-commit verification is complete.
Nested archive work shares the same deployment transport, and finally
restores the caller context on success, failure or generator cancellation.
"""
if _ARCHIVE_STORAGE_BACKEND.get() is not None:
yield
return
token = _ARCHIVE_STORAGE_BACKEND.set({})
try:
yield
finally:
_ARCHIVE_STORAGE_BACKEND.reset(token)
def _archive_write_backend() -> StorageBackend:
scope = _ARCHIVE_STORAGE_BACKEND.get()
if scope is None:
return get_storage_backend()
if "backend" not in scope:
scope["backend"] = get_storage_backend()
return scope["backend"]
def _campaign_access_provider() -> CampaignAccessProvider:
registry = get_registry()
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_ACCESS):
@@ -88,7 +124,7 @@ def _get_or_create_blob(
.one_or_none()
)
if blob:
backend = get_storage_backend()
backend = _archive_write_backend()
repair_required = (
blob.integrity_status in QUARANTINED_BLOB_STATUSES
or blob.quarantined_at is not None
@@ -162,7 +198,7 @@ def _get_or_create_blob(
blob_id = str(uuid4())
storage_key = _storage_key(tenant_id=tenant_id, checksum=checksum)
backend = get_storage_backend()
backend = _archive_write_backend()
recovery = begin_blob_write_recovery(
session,
backend=backend,
@@ -214,7 +250,6 @@ def _get_or_create_blob(
integrity_checked_at=utcnow(),
)
session.add(blob)
session.flush()
return blob
@@ -265,7 +300,12 @@ def create_file_asset(
logical_path = _next_available_logical_path(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, desired_path=logical_path)
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=encryption_vault_id)
# Assign the immutable identities before the first flush. Publishing an
# incomplete asset and immediately updating its current version doubled
# change events and added a redundant UPDATE for every imported member.
version_id = str(uuid4())
asset = FileAsset(
id=str(uuid4()),
tenant_id=tenant_id,
owner_type=owner_type,
owner_user_id=owner_id if owner_type == "user" else None,
@@ -275,10 +315,10 @@ def create_file_asset(
description=description,
created_by_user_id=user_id,
metadata_=metadata or {},
current_version_id=version_id,
)
session.add(asset)
session.flush()
version = FileVersion(
id=version_id,
tenant_id=tenant_id,
file_asset_id=asset.id,
blob_id=blob.id,
@@ -290,10 +330,8 @@ def create_file_asset(
checksum_sha256=blob.checksum_sha256,
created_by_user_id=user_id,
)
session.add(version)
session.add_all((asset, version))
session.flush()
asset.current_version_id = version.id
session.add(asset)
if campaign_id:
share_file(session, tenant_id=tenant_id, asset=asset, target_type="campaign", target_id=campaign_id, permission="read", user_id=user_id)
return UploadedStoredFile(asset=asset, version=version, blob=blob)
@@ -0,0 +1,214 @@
"""Optional libarchive ZIPCrypto acceleration, without filesystem extraction.
Only public libarchive read APIs are bound. Callers must preflight the complete
ZIP directory with the ordinary archive validator. Missing libraries/symbols
select the Python path before decoding; native failures never retry a decoder.
Public ABI reference: https://github.com/libarchive/libarchive/blob/master/libarchive/archive.h
"""
from __future__ import annotations
from collections.abc import Callable, Iterator, Mapping
import ctypes
from ctypes.util import find_library
from functools import lru_cache
import os
from os import PathLike
import stat
from typing import Any
import zlib
from govoplan_files.backend.storage.common import FileStorageError
_READ_CHUNK_SIZE = 1024 * 1024
_ARCHIVE_OK = 0
_ARCHIVE_EOF = 1
@lru_cache(maxsize=1)
def native_zip_library() -> Any | None:
name = find_library("archive")
if not name:
return None
try:
library = ctypes.CDLL(name)
signatures = {
"archive_read_new": ([], ctypes.c_void_p),
"archive_read_support_filter_none": ([ctypes.c_void_p], ctypes.c_int),
"archive_read_support_format_zip": ([ctypes.c_void_p], ctypes.c_int),
"archive_read_set_format_option": (
[ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p],
ctypes.c_int,
),
"archive_read_add_passphrase": (
[ctypes.c_void_p, ctypes.c_char_p],
ctypes.c_int,
),
"archive_read_open_memory": (
[ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t],
ctypes.c_int,
),
"archive_read_open_filename": (
[ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t],
ctypes.c_int,
),
"archive_read_next_header": (
[ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)],
ctypes.c_int,
),
"archive_read_data": (
[ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t],
ctypes.c_ssize_t,
),
"archive_read_free": ([ctypes.c_void_p], ctypes.c_int),
"archive_entry_pathname_utf8": ([ctypes.c_void_p], ctypes.c_char_p),
"archive_entry_size": ([ctypes.c_void_p], ctypes.c_int64),
"archive_entry_filetype": ([ctypes.c_void_p], ctypes.c_uint),
}
for symbol, (arguments, result) in signatures.items():
function = getattr(library, symbol)
function.argtypes = arguments
function.restype = result
return library
except (OSError, AttributeError):
return None
def _require_ok(status: int) -> None:
if status != _ARCHIVE_OK:
# Do not expose native error strings: they can contain source paths.
raise FileStorageError(
"Native ZIP decoding failed: incorrect password, corrupt or unsupported archive"
)
def read_native_zip_members(
library: Any,
archive_data: bytes | str | PathLike[str],
*,
infos_by_path: Mapping[str, Any],
selected_files: set[str],
normalize_name: Callable[[str], str],
password: str,
max_file_bytes: int,
max_total_bytes: int,
progress: Callable[[str, int, int, int, int], None] | None = None,
total_bytes: int = 0,
) -> Iterator[tuple[str, bytes]]:
# Keep memory input, password and output buffers alive until reader free.
source_buffer = None
password_bytes = password.encode("utf-8")
buffer = ctypes.create_string_buffer(_READ_CHUNK_SIZE)
archive = library.archive_read_new()
if not archive:
raise FileStorageError("Native ZIP reader could not be allocated")
total = 0
completed = 0
seen: set[str] = set()
try:
_require_ok(library.archive_read_support_filter_none(archive))
_require_ok(library.archive_read_support_format_zip(archive))
_require_ok(
library.archive_read_set_format_option(
archive, b"zip", b"hdrcharset", b"UTF-8"
)
)
_require_ok(library.archive_read_add_passphrase(archive, password_bytes))
if isinstance(archive_data, bytes):
source_buffer = ctypes.create_string_buffer(archive_data)
_require_ok(
library.archive_read_open_memory(
archive, source_buffer, len(archive_data)
)
)
else:
_require_ok(
library.archive_read_open_filename(
archive, os.fsencode(archive_data), _READ_CHUNK_SIZE
)
)
entry = ctypes.c_void_p()
while True:
status = library.archive_read_next_header(archive, ctypes.byref(entry))
if status == _ARCHIVE_EOF:
break
_require_ok(status)
raw_path = library.archive_entry_pathname_utf8(entry)
if not raw_path:
raise FileStorageError("Native ZIP entry has no path")
try:
path = normalize_name(raw_path.decode("utf-8"))
except UnicodeDecodeError as exc:
raise FileStorageError(
"Native ZIP path does not match the inspected directory"
) from exc
info = infos_by_path.get(path)
if info is None or path in seen:
raise FileStorageError(
"Native ZIP path does not match the inspected directory"
)
seen.add(path)
directory = info.is_dir()
file_type = library.archive_entry_filetype(entry)
if not (stat.S_ISDIR(file_type) if directory else stat.S_ISREG(file_type)):
raise FileStorageError(
"Native ZIP entry type does not match the inspected directory"
)
if library.archive_entry_size(entry) != info.file_size:
raise FileStorageError(
"Native ZIP entry size does not match the inspected directory"
)
if directory or path not in selected_files:
continue
parts = []
size = 0
checksum = 0
while True:
read_size = min(
_READ_CHUNK_SIZE,
max_file_bytes + 1 - size,
max_total_bytes + 1 - total,
)
if read_size <= 0:
raise FileStorageError("Archive exceeds its extraction limits")
count = library.archive_read_data(archive, buffer, read_size)
if count < 0:
_require_ok(count)
if count == 0:
break
if count > read_size:
raise FileStorageError("Native ZIP reader returned an invalid byte count")
size += count
total += count
if size > max_file_bytes:
raise FileStorageError(
f"Archive member {path!r} exceeds per-file limit"
)
if total > max_total_bytes:
raise FileStorageError("Archive is too large after extraction")
chunk = ctypes.string_at(buffer, count)
checksum = zlib.crc32(chunk, checksum)
parts.append(chunk)
if progress:
progress(
"extracting", completed, len(selected_files), total, total_bytes
)
# Independent checks preserve the Python ZIP reader's CRC and size
# guarantees even when native decoding behavior changes upstream.
if size != info.file_size or checksum != info.CRC:
raise FileStorageError(
"Native ZIP member failed size or CRC verification"
)
completed += 1
if progress:
progress(
"extracting", completed, len(selected_files), total, total_bytes
)
yield path, b"".join(parts)
parts.clear()
if seen != set(infos_by_path) or completed != len(selected_files):
raise FileStorageError(
"Native ZIP directory does not match the inspected archive"
)
finally:
library.archive_read_free(archive)