fix(security): isolate XLSX decoding with hard resource limits

This commit is contained in:
2026-09-08 07:47:18 +02:00
parent 33de5cac55
commit be16f8218e
4 changed files with 153 additions and 8 deletions
@@ -1130,6 +1130,44 @@ manifest = ModuleManifest(
related_modules=("dataflow", "files", "reporting", "risk_compliance"),
order=40,
),
DocumentationTopic(
id="connectors.xlsx-worker-limits",
title="XLSX processing resource limits",
summary="Understand isolated workbook parsing and explicit capacity failures.",
body=(
"Authorized XLSX bytes are parsed in a fresh disposable process with a 15-second wall limit, 10 CPU seconds, "
"512 MiB address-space limit, no file output, and 8 MiB input/64 MiB result transport ceilings. Existing limits remain "
"5,000,000 input bytes, 50,000,000 expanded archive bytes, 5,000 archive entries, 100:1 expansion ratio, "
"500 columns and 10,000 row positions after the header. Typed transport additionally limits nesting to 64 levels "
"and 1,000,000 value nodes. Limit failures reject the whole parse; reduce the workbook before retrying. "
"The shared Core GOVOPLAN_ISOLATED_PROCESS_CONCURRENCY capacity is per API/worker process and does not queue: busy work "
"fails explicitly and may be retried later. POSIX resource controls are required; missing controls, cancellation "
"or worker failure produce sanitized unavailable diagnostics. No in-process fallback occurs. File authorization, "
"credentials, SQL sessions and durable source changes remain in the parent. Operators must budget aggregate "
"memory across process slots and keep the required Core worker API available."
),
layer="static",
documentation_types=("user", "admin"),
audience=("user", "module_admin", "operator"),
translations={"de": {
"title": "Ressourcengrenzen der XLSX-Verarbeitung",
"summary": "Isoliertes Einlesen von Arbeitsmappen und ausdrückliche Kapazitätsfehler verstehen.",
"body": (
"Autorisierte XLSX-Bytes werden in einem neuen kurzlebigen Prozess verarbeitet: höchstens 15 Sekunden Gesamtdauer, "
"10 CPU-Sekunden, 512 MiB Adressraum, keine Dateiausgabe und 8 MiB Eingabe-/64 MiB Ergebnistransport. Weiterhin gelten "
"5.000.000 Eingabebytes, 50.000.000 entpackte Archivbytes, 5.000 Archiveinträge, ein Entpackverhältnis von 100:1, "
"500 Spalten und 10.000 Zeilenpositionen nach der Kopfzeile. Der typisierte Transport begrenzt zusätzlich die "
"Verschachtelung auf 64 Ebenen und 1.000.000 Wertknoten. Grenzverletzungen lehnen den gesamten Lesevorgang ab; "
"verkleinern Sie die Arbeitsmappe vor einem erneuten Versuch. Die gemeinsame Core-Einstellung GOVOPLAN_ISOLATED_PROCESS_CONCURRENCY "
"gilt pro API-/Worker-Prozess und bildet keine Warteschlange: Bei belegter Kapazität erfolgt ein ausdrücklicher Fehler; "
"versuchen Sie es später erneut. POSIX-Ressourcenbegrenzungen sind erforderlich. Fehlende Kontrollen, Abbruch oder "
"Worker-Fehler melden eine bereinigte Nichtverfügbarkeit. Es gibt keinen Rückfall auf Verarbeitung im Elternprozess. "
"Dateiberechtigungen, Zugangsdaten, SQL-Sitzungen und dauerhafte Quelländerungen bleiben im Elternprozess. Betreiber "
"müssen den Gesamtspeicher aller Prozessplätze berücksichtigen und die benötigte Core-Worker-API bereitstellen."
),
}},
order=41,
),
DocumentationTopic(
id="connectors.rss-atom",
title="RSS and Atom feeds",
@@ -59,7 +59,17 @@ from govoplan_core.security.credential_envelopes import (
CredentialEnvelopeError,
resolve_credential_envelope,
)
from govoplan_core.security.bounded_process import (
ProcessBudgetError,
ProcessLimits,
run_bounded_operation,
)
from govoplan_core.security.redaction import is_sensitive_key
from govoplan_core.security.worker_payload import (
WorkerPayloadError,
decode_worker_payload,
encode_worker_payload,
)
from govoplan_connectors.backend.db.models import ConnectorConfiguration
@@ -69,6 +79,13 @@ MAX_FILE_COLUMNS = 500
MAX_XLSX_ENTRIES = 5_000
MAX_XLSX_EXPANDED_BYTES = 50_000_000
MAX_XLSX_COMPRESSION_RATIO = 100
XLSX_PROCESS_LIMITS = ProcessLimits(
wall_seconds=15,
cpu_seconds=10,
memory_bytes=512 * 1024 * 1024,
input_bytes=8 * 1024 * 1024,
output_bytes=64 * 1024 * 1024,
)
POSTGRESQL_SCHEMES = frozenset({"postgresql", "postgresql+psycopg"})
_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_$-]{0,127}$")
@@ -573,6 +590,55 @@ def _parse_xlsx(
payload: bytes,
*,
sheet_name: str | None,
) -> tuple[tuple[Mapping[str, object], ...], str]:
# Only already-authorized workbook bytes and a worksheet selector leave
# the parent; Files access, credentials and SQL sessions never cross.
try:
encoded = encode_worker_payload(
{"payload": payload, "sheet_name": sheet_name},
max_bytes=XLSX_PROCESS_LIMITS.input_bytes,
)
result = decode_worker_payload(
run_bounded_operation(_parse_xlsx_worker, encoded, limits=XLSX_PROCESS_LIMITS),
max_bytes=XLSX_PROCESS_LIMITS.output_bytes,
)
except ProcessBudgetError as exc:
error_type = (
TabularSourceUnavailableError
if exc.code in {"busy", "cancelled", "unavailable", "worker_failed"}
else TabularSourceValidationError
)
raise error_type(f"Managed XLSX processing failed ({exc.code}): {exc}") from exc
except (TypeError, ValueError, RecursionError) as exc:
raise TabularSourceValidationError("Managed XLSX data could not be safely transferred.") from exc
if not isinstance(result, dict):
raise TabularSourceUnavailableError("Managed XLSX worker returned an invalid result.")
if "validation_error" in result:
raise TabularSourceValidationError(str(result["validation_error"]))
rows, selected_name = result.get("rows"), result.get("sheet_name")
if not isinstance(rows, tuple) or not isinstance(selected_name, str):
raise TabularSourceUnavailableError("Managed XLSX worker returned an invalid result.")
return rows, selected_name
def _parse_xlsx_worker(payload: bytes) -> bytes:
data = decode_worker_payload(payload, max_bytes=XLSX_PROCESS_LIMITS.input_bytes)
try:
rows, sheet_name = _parse_xlsx_content(data["payload"], sheet_name=data["sheet_name"])
return encode_worker_payload(
{"rows": rows, "sheet_name": sheet_name},
max_bytes=XLSX_PROCESS_LIMITS.output_bytes,
)
except TabularSourceValidationError as exc:
return encode_worker_payload({"validation_error": str(exc)})
except WorkerPayloadError:
return encode_worker_payload({"validation_error": "Managed XLSX parsed data exceeds the worker transport limit."})
def _parse_xlsx_content(
payload: bytes,
*,
sheet_name: str | None,
) -> tuple[tuple[Mapping[str, object], ...], str]:
_validate_xlsx_archive(payload)
try:
@@ -582,6 +648,8 @@ def _parse_xlsx(
data_only=True,
keep_links=False,
)
except MemoryError:
raise
except Exception as exc:
raise TabularSourceValidationError(
"Managed XLSX content could not be parsed."