fix(connectors): retain exact CSV source evidence
Module Package Release / publish-packages (push) Successful in 17s

Release v0.1.27. Coordinated integrity review: GovOPlaN/govoplan-core#298.
This commit is contained in:
2026-09-08 12:19:38 +02:00
parent be16f8218e
commit 889cafaf20
17 changed files with 322 additions and 145 deletions
@@ -47,6 +47,8 @@ class ConnectorTabularSource(Base, TimestampMixin):
row_count: Mapped[int] = mapped_column(Integer, nullable=False)
byte_count: Mapped[int] = mapped_column(Integer, nullable=False)
metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict, nullable=False)
# Original upload content is deliberately excluded from ordinary catalogue loads/DTOs.
csv_source_: Mapped[dict[str, Any] | None] = mapped_column("csv_source", JSON, nullable=True, deferred=True)
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
updated_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
@@ -1,9 +1,8 @@
from __future__ import annotations
from dataclasses import replace
from typing import Iterable
from govoplan_core.core.modules import DocumentationTopic
from govoplan_core.core.modules import DocumentationTopic, localize_documentation_topics as _localize_topics
_TRANSLATIONS = {
@@ -64,15 +63,4 @@ _TRANSLATIONS = {
def localize_documentation_topics(
topics: Iterable[DocumentationTopic],
) -> tuple[DocumentationTopic, ...]:
localized: list[DocumentationTopic] = []
for topic in topics:
german = _TRANSLATIONS.get(topic.id)
if german is None:
localized.append(topic)
continue
translations = {
locale: dict(value) for locale, value in topic.translations.items()
}
translations["de"] = {**translations.get("de", {}), **german}
localized.append(replace(topic, translations=translations))
return tuple(localized)
return _localize_topics(topics, locale="de", translations=_TRANSLATIONS)
@@ -1,5 +1,7 @@
from __future__ import annotations
from govoplan_connectors.backend.search_principal import principal_acl_tokens as _principal_acl_tokens
from collections.abc import Mapping, Sequence
from urllib.parse import quote
@@ -217,30 +219,6 @@ def search_document(
)
def _principal_acl_tokens(principal: object) -> tuple[str, ...]:
values: list[str] = []
for prefix, attribute in (
("account", "account_id"),
("membership", "membership_id"),
("identity", "identity_id"),
):
value = getattr(principal, attribute, None)
if value:
values.append(f"{prefix}:{value}")
for prefix, attribute in (
("group", "group_ids"),
("role", "role_ids"),
("function", "function_assignment_ids"),
("scope", "scopes"),
):
values.extend(
f"{prefix}:{value}"
for value in getattr(principal, attribute, ())
if value
)
return tuple(dict.fromkeys(values))[:500]
def _has_scope(principal: object, required: str) -> bool:
check = getattr(principal, "has", None)
if callable(check):
+33 -1
View File
@@ -126,7 +126,7 @@ from govoplan_connectors.backend.german_documentation import (
MODULE_ID = "connectors"
MODULE_VERSION = "0.1.26"
MODULE_VERSION = "0.1.27"
TABULAR_SOURCE_INTERFACE_VERSION = "0.1.0"
DATASOURCE_ORIGIN_INTERFACE_VERSION = "0.1.0"
SANCTIONS_SNAPSHOT_INTERFACE_VERSION = "1.0.0"
@@ -967,6 +967,38 @@ manifest = ModuleManifest(
),
),
documentation=localize_documentation_topics((
DocumentationTopic(
id="connectors.csv-source-fidelity",
title="Preserving original CSV input",
summary="Choose explicit text or legacy type inference, and retrieve the verified original of new CSV snapshots.",
body=(
"CSV snapshot and managed-file creation accept csv_value_mode=text to preserve field whitespace, decimal digits, large identifiers, boolean-looking text and explicit empty records as strings. "
"Text mode rejects missing/extra fields and malformed quoting rather than dropping values. Headers remain normalized and blank physical lines are not table rows; the original upload preserves these lexical details. "
"Omitting csv_value_mode keeps legacy_typed API behavior for existing clients. Legacy inference can trim values, infer numbers/booleans and omit empty rows; choose text when exact values matter. "
"New CSV snapshots retain the original supplied text, delimiter, parser profile and UTF-8 SHA-256 separately from catalogue responses; csv_source metadata is reserved verified evidence, not a place to supply content. "
"GET /api/v1/connectors/tabular-sources/{source_ref}/original-csv requires current connectors:source:read or connectors:source:admin authority in the same tenant, verifies the checksum, audits the export without source content, and returns an uncached attachment. "
"The export preserves the text submitted to the API as UTF-8, not an earlier file encoding. It does not sanitize spreadsheet formulas: treat imported originals as untrusted data. "
"Original text and parsed rows each remain limited to 5 MB, with at most 10,000 parsed rows. Each snapshot can retain up to 5 MB of original text, plus serialization and metadata storage overhead. Source text follows the snapshot lifecycle; retired/deleted sources are not downloadable. "
"Managed files retain their original through the exact Files version and its access/lifecycle policy; text mode is pinned in source metadata and preserved on refresh. Existing snapshot fingerprints and historical rows are never rewritten. "
"Original text cannot be reconstructed for older snapshots: the endpoint reports it unavailable. Schema inference now shares one ordered, bounded-state mechanism with Datasources while preserving legacy provider type classifications."
),
layer="always", documentation_types=("user", "admin"), audience=("user", "module_admin", "operator"), order=8,
translations={"de": {
"title": "Ursprüngliche CSV-Eingaben erhalten",
"summary": "Textwerte oder bisherige Typableitung ausdrücklich wählen und das geprüfte Original neuer CSV-Snapshots abrufen.",
"body": (
"CSV-Snapshots und verwaltete Dateiquellen unterstützen csv_value_mode=text. Dieser Modus erhält Leerzeichen in Feldwerten, Dezimalstellen, große Kennungen, boolesch wirkenden Text und ausdrücklich leere Datensätze als Zeichenketten. "
"Fehlende oder zusätzliche Felder und fehlerhafte Anführungszeichen werden abgelehnt. Überschriften werden weiterhin normalisiert; vollständig leere physische Zeilen sind keine Tabellenzeilen. Das Original erhält auch diese Texteigenschaften. "
"Ohne csv_value_mode bleibt für bestehende API-Aufrufe legacy_typed aktiv. Dabei können Werte gekürzt, Zahlen/Wahrheitswerte abgeleitet und leere Zeilen ausgelassen werden. Für genaue Werte wählen Sie Text. "
"Neue CSV-Snapshots speichern den gelieferten Originaltext, Trennzeichen, Parserprofil und UTF-8-SHA-256 getrennt vom Katalog. csv_source-Metadaten sind reservierter geprüfter Nachweis. "
"GET /api/v1/connectors/tabular-sources/{source_ref}/original-csv benötigt aktuelle Rechte connectors:source:read oder connectors:source:admin im selben Mandanten, prüft die Prüfsumme, protokolliert den Export ohne Quellinhalt und liefert einen nicht zwischengespeicherten Download. "
"Exportiert wird der an die API übermittelte Text als UTF-8, nicht eine frühere Dateikodierung. Tabellenformeln werden nicht verändert; behandeln Sie Originaldateien als nicht vertrauenswürdige Daten. "
"Originaltext und verarbeitete Zeilen sind jeweils auf 5 MB begrenzt; höchstens 10.000 Zeilen werden angenommen. Pro Snapshot werden bis zu 5 MB Originaltext zuzüglich Speicher für Serialisierung und Metadaten aufbewahrt. Der Originaltext folgt dem Snapshot-Lebenszyklus; stillgelegte oder gelöschte Quellen sind nicht abrufbar. "
"Verwaltete Dateien behalten ihr Original in der genauen Files-Version mit deren Zugriffs- und Lebenszyklusregeln; der Textmodus bleibt bei Aktualisierungen erhalten. Bestehende Fingerprints und historische Zeilen werden nicht umgeschrieben. "
"Für ältere Snapshots kann das Original nicht rekonstruiert werden; der Abruf meldet es als nicht verfügbar. Die gemeinsame Schemaableitung erhält die bisherigen Typregeln und Spaltenreihenfolge."
),
}},
),
DocumentationTopic(
id="connectors.data-subject-requests",
title="Connector data-subject requests",
@@ -0,0 +1,20 @@
"""Retain original CSV source evidence for new durable snapshots."""
from alembic import op
import sqlalchemy as sa
revision = "d2a4c6e8f0b1"
down_revision = "c0f1a2b3c4d5"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"connector_tabular_sources", sa.Column("csv_source", sa.JSON(), nullable=True)
)
def downgrade() -> None:
with op.batch_alter_table("connector_tabular_sources") as batch:
batch.drop_column("csv_source")
+37 -1
View File
@@ -10,6 +10,7 @@ from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_event
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
from govoplan_core.core.tabular_sources import (
TabularCsvSource,
TabularReadRequest,
TabularSnapshotInput,
TabularSource,
@@ -539,7 +540,7 @@ def api_create_tabular_snapshot(
rows = (
tuple(payload.rows or ())
if payload.format == "json"
else parse_csv_snapshot(payload.csv_text or "", delimiter=payload.delimiter)
else parse_csv_snapshot(payload.csv_text or "", delimiter=payload.delimiter, value_mode=payload.csv_value_mode)
)
source = provider.create_snapshot(
session,
@@ -550,6 +551,11 @@ def api_create_tabular_snapshot(
description=payload.description,
rows=rows,
metadata={"import_format": payload.format},
csv_source=(TabularCsvSource(
text=payload.csv_text or "",
delimiter=payload.delimiter,
value_mode=payload.csv_value_mode,
) if payload.format == "csv" else None),
),
)
except TabularSourceError as exc:
@@ -573,6 +579,35 @@ def api_create_tabular_snapshot(
return _source_response(source)
@router.get("/tabular-sources/{source_ref}/original-csv")
def api_original_tabular_csv(
source_ref: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> Response:
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
try:
text = provider.original_csv(session, principal, source_ref=source_ref)
except TabularSourceError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="connectors.original_csv.exported",
object_type="connector_tabular_source",
object_id=source_ref,
details={"sha256": hashlib.sha256(text.encode("utf-8")).hexdigest()},
)
session.commit()
return Response(
content=text.encode("utf-8"),
media_type="text/csv; charset=utf-8",
headers={"Content-Disposition": 'attachment; filename="original.csv"', "Cache-Control": "no-store"},
)
@router.post(
"/tabular-sources/files",
response_model=TabularSourceResponse,
@@ -595,6 +630,7 @@ def api_create_managed_file_source(
file_version_id=payload.file_version_id,
delimiter=payload.delimiter,
sheet_name=payload.sheet_name,
csv_value_mode=payload.csv_value_mode,
)
except TabularSourceError as exc:
raise _http_error(exc) from exc
@@ -90,6 +90,7 @@ class SnapshotCreateRequest(BaseModel):
format: Literal["json", "csv"] = "json"
rows: list[dict[str, Any]] | None = Field(default=None, max_length=10_000)
csv_text: str | None = Field(default=None, max_length=5_000_000)
csv_value_mode: Literal["legacy_typed", "text"] = "legacy_typed"
delimiter: Literal[",", ";", "\t", "|"] = ","
@model_validator(mode="after")
@@ -117,6 +118,7 @@ class ManagedFileSourceCreateRequest(BaseModel):
file_version_id: str | None = Field(default=None, min_length=1, max_length=36)
delimiter: Literal[",", ";", "\t", "|"] = ","
sheet_name: str | None = Field(default=None, min_length=1, max_length=255)
csv_value_mode: Literal["legacy_typed", "text"] = "legacy_typed"
class SqlSourceCreateRequest(BaseModel):
+26
View File
@@ -0,0 +1,26 @@
"""Shared connector-search ACL token projection; owner authorization stays local."""
def principal_acl_tokens(principal: object) -> tuple[str, ...]:
# Keep legacy first-seen ordering and the exact 500-token authorization cap.
values: dict[str, None] = {}
for prefix, attribute in (
("account", "account_id"),
("membership", "membership_id"),
("identity", "identity_id"),
):
value = getattr(principal, attribute, None)
if value:
values[f"{prefix}:{value}"] = None
for prefix, attribute in (
("group", "group_ids"),
("role", "role_ids"),
("function", "function_assignment_ids"),
("scope", "scopes"),
):
for value in getattr(principal, attribute, ()):
if value:
values[f"{prefix}:{value}"] = None
if len(values) >= 500:
return tuple(values)
return tuple(values)
@@ -1,5 +1,7 @@
from __future__ import annotations
from govoplan_connectors.backend.search_principal import principal_acl_tokens as _principal_acl_tokens
from collections.abc import Mapping, Sequence
from urllib.parse import quote
@@ -288,30 +290,6 @@ def search_document(
)
def _principal_acl_tokens(principal: object) -> tuple[str, ...]:
values: list[str] = []
for prefix, attribute in (
("account", "account_id"),
("membership", "membership_id"),
("identity", "identity_id"),
):
value = getattr(principal, attribute, None)
if value:
values.append(f"{prefix}:{value}")
for prefix, attribute in (
("group", "group_ids"),
("role", "role_ids"),
("function", "function_assignment_ids"),
("scope", "scopes"),
):
values.extend(
f"{prefix}:{value}"
for value in getattr(principal, attribute, ())
if value
)
return tuple(dict.fromkeys(values))[:500]
def _has_scope(principal: object, required: str) -> bool:
check = getattr(principal, "has", None)
if callable(check):
@@ -53,6 +53,9 @@ from govoplan_core.core.tabular_sources import (
TabularSourceUnavailableError,
TabularSourceValidationError,
parse_tabular_csv,
CsvValueMode,
infer_tabular_schema as _infer_schema,
tabular_type_name,
)
from govoplan_core.security.credential_envelopes import (
CredentialAccessContext,
@@ -125,6 +128,7 @@ class ManagedFileTabularAdapter:
file_version_id: str | None,
delimiter: str = ",",
sheet_name: str | None = None,
csv_value_mode: CsvValueMode = "legacy_typed",
) -> TabularOriginInspection:
provider = managed_tabular_file_provider(self._registry)
if provider is None:
@@ -163,6 +167,7 @@ class ManagedFileTabularAdapter:
content_type=content.file.content_type,
delimiter=delimiter,
sheet_name=sheet_name,
csv_value_mode=csv_value_mode,
)
schema = infer_tabular_schema(rows)
fingerprint = origin_fingerprint(
@@ -174,6 +179,7 @@ class ManagedFileTabularAdapter:
content.file.sha256,
resolved_sheet or "",
delimiter,
*(("csv_text_values",) if csv_value_mode == "text" else ()),
),
)
try:
@@ -227,6 +233,7 @@ class ManagedFileTabularAdapter:
"content_type": content.file.content_type,
"format": "xlsx" if _is_xlsx(content.file.filename, content.file.content_type) else "csv",
"delimiter": delimiter,
"csv_value_mode": csv_value_mode,
"sheet_name": resolved_sheet,
},
health=health,
@@ -263,6 +270,7 @@ class ManagedFileTabularAdapter:
file_version_id=_required_metadata(metadata, "file_version_id"),
delimiter=str(metadata.get("delimiter") or ","),
sheet_name=_optional_text(metadata.get("sheet_name")),
csv_value_mode=str(metadata.get("csv_value_mode") or "legacy_typed"),
)
expected_sha256 = _required_metadata(metadata, "file_sha256")
if inspection.metadata.get("file_sha256") != expected_sha256:
@@ -567,6 +575,7 @@ def parse_managed_tabular_content(
content_type: str | None,
delimiter: str,
sheet_name: str | None,
csv_value_mode: CsvValueMode = "legacy_typed",
) -> tuple[tuple[Mapping[str, object], ...], str | None]:
if len(payload) > MAX_FILE_BYTES:
raise TabularSourceValidationError(
@@ -581,7 +590,7 @@ def parse_managed_tabular_content(
"Managed CSV files must use UTF-8 encoding."
) from exc
return (
tuple(parse_tabular_csv(text, delimiter=delimiter, max_rows=MAX_FILE_ROWS)),
tuple(parse_tabular_csv(text, delimiter=delimiter, max_rows=MAX_FILE_ROWS, max_bytes=MAX_FILE_BYTES, value_mode=csv_value_mode)),
None,
)
@@ -809,26 +818,7 @@ def _xlsx_headers(values: Sequence[object]) -> tuple[str, ...]:
def infer_tabular_schema(
rows: Sequence[Mapping[str, object]],
) -> tuple[TabularColumn, ...]:
names: list[str] = []
for row in rows:
for name in row:
if name not in names:
names.append(name)
result: list[TabularColumn] = []
for name in names:
values = [row.get(name) for row in rows]
concrete = [value for value in values if value is not None]
data_type = _type_name(concrete[0]) if concrete else "unknown"
if any(_type_name(value) != data_type for value in concrete[1:]):
data_type = "mixed"
result.append(
TabularColumn(
name=name,
data_type=data_type,
nullable=len(concrete) != len(values),
)
)
return tuple(result)
return _infer_schema(rows, type_name=_type_name)
def origin_fingerprint(
@@ -951,19 +941,7 @@ def _json_value(value: object) -> object:
def _type_name(value: object) -> str:
if isinstance(value, bool):
return "boolean"
if isinstance(value, int):
return "integer"
if isinstance(value, (float, Decimal)):
return "number"
if isinstance(value, str):
return "string"
if isinstance(value, list):
return "array"
if isinstance(value, dict):
return "object"
return type(value).__name__.casefold()
return tabular_type_name(value, casefold_unknown=True)
__all__ = [
@@ -13,6 +13,7 @@ from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, has_scope
from govoplan_core.core.tabular_sources import (
CsvValueMode,
TabularColumn,
TabularPreviewDiagnostic,
TabularPushdown,
@@ -26,6 +27,12 @@ from govoplan_core.core.tabular_sources import (
TabularSourceUnavailableError,
TabularSourceValidationError,
parse_tabular_csv,
csv_source_payload,
csv_source_summary,
csv_projection_matches,
verified_csv_source_text,
infer_tabular_schema,
tabular_type_name as _type_name,
)
from govoplan_core.core.runtime import get_registry
from govoplan_core.db.base import utcnow
@@ -281,6 +288,7 @@ class SqlTabularSourceProvider:
description: str | None = None,
delimiter: str = ",",
sheet_name: str | None = None,
csv_value_mode: CsvValueMode = "legacy_typed",
) -> TabularSource:
db, api_principal = _context(session, principal, WRITE_SCOPE)
inspection = self._file_adapter().inspect(
@@ -290,6 +298,7 @@ class SqlTabularSourceProvider:
file_version_id=file_version_id,
delimiter=delimiter,
sheet_name=sheet_name,
csv_value_mode=csv_value_mode,
)
return self._create_origin(
db,
@@ -353,6 +362,7 @@ class SqlTabularSourceProvider:
file_version_id=None,
delimiter=str(item.metadata_.get("delimiter") or ","),
sheet_name=_clean_optional(item.metadata_.get("sheet_name")),
csv_value_mode=str(item.metadata_.get("csv_value_mode") or "legacy_typed"),
)
elif item.provider == "postgresql":
inspection = self._sql_adapter.inspect(
@@ -456,6 +466,20 @@ class SqlTabularSourceProvider:
f"Snapshots are limited to {MAX_SNAPSHOT_ROWS:,} rows."
)
rows = [_json_row(row) for row in snapshot.rows]
csv_payload = None
if snapshot.csv_source is not None:
if "csv_source" in snapshot.metadata:
raise TabularSourceValidationError("csv_source metadata is reserved for verified CSV import evidence.")
if snapshot.csv_source.parser_profile != "core.csv.v1":
raise TabularSourceValidationError("Unsupported original CSV parser profile.")
csv_payload = csv_source_payload(snapshot.csv_source, max_bytes=MAX_SNAPSHOT_BYTES)
expected = parse_csv_snapshot(
snapshot.csv_source.text,
delimiter=snapshot.csv_source.delimiter,
value_mode=snapshot.csv_source.value_mode,
)
if not csv_projection_matches(expected, rows):
raise TabularSourceValidationError("Snapshot rows do not match their original CSV source and parsing mode.")
encoded = json.dumps(rows, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
if len(encoded) > MAX_SNAPSHOT_BYTES:
raise TabularSourceValidationError(
@@ -487,7 +511,11 @@ class SqlTabularSourceProvider:
fingerprint=fingerprint,
row_count=len(rows),
byte_count=len(encoded),
metadata_=dict(snapshot.metadata),
metadata_={
**dict(snapshot.metadata),
**({"csv_source": csv_source_summary(csv_payload)} if csv_payload is not None else {}),
},
csv_source_=csv_payload,
created_by=actor_id,
updated_by=actor_id,
)
@@ -497,6 +525,15 @@ class SqlTabularSourceProvider:
db.flush()
return _source_dto(item)
def original_csv(self, session: object, principal: object, *, source_ref: str) -> str:
db, api_principal = _context(session, principal, READ_SCOPE)
item = _source_record(db, tenant_id=api_principal.tenant_id, source_ref=source_ref)
if item is None or item.status != "active":
raise TabularSourceNotFoundError("Tabular source not found.")
if not item.csv_source_:
raise TabularSourceNotFoundError("Original CSV was not retained for this source; historical typed snapshots cannot reconstruct it.")
return verified_csv_source_text(item.csv_source_, expected_summary=item.metadata_.get("csv_source") or {})
def delete_snapshot(
self,
session: object,
@@ -519,35 +556,18 @@ class SqlTabularSourceProvider:
return _source_dto(item)
def parse_csv_snapshot(csv_text: str, *, delimiter: str) -> tuple[Mapping[str, object], ...]:
def parse_csv_snapshot(csv_text: str, *, delimiter: str, value_mode: CsvValueMode = "legacy_typed") -> tuple[Mapping[str, object], ...]:
return parse_tabular_csv(
csv_text,
delimiter=delimiter,
max_rows=MAX_SNAPSHOT_ROWS,
max_bytes=MAX_SNAPSHOT_BYTES,
value_mode=value_mode,
)
def infer_schema(rows: Sequence[Mapping[str, object]]) -> tuple[TabularColumn, ...]:
names: list[str] = []
for row in rows:
for name in row:
if name not in names:
names.append(name)
result: list[TabularColumn] = []
for name in names:
values = [row.get(name) for row in rows]
concrete = [value for value in values if value is not None]
data_type = _type_name(concrete[0]) if concrete else "unknown"
if any(_type_name(value) != data_type for value in concrete[1:]):
data_type = "mixed"
result.append(
TabularColumn(
name=name,
data_type=data_type,
nullable=len(concrete) != len(values),
)
)
return tuple(result)
return infer_tabular_schema(rows, type_name=_type_name)
def snapshot_fingerprint(
@@ -717,22 +737,6 @@ def _unsupported_json(value: object) -> object:
raise TypeError(f"{type(value).__name__} is not JSON serializable")
def _type_name(value: object) -> str:
if isinstance(value, bool):
return "boolean"
if isinstance(value, int):
return "integer"
if isinstance(value, (float, Decimal)):
return "number"
if isinstance(value, str):
return "string"
if isinstance(value, list):
return "array"
if isinstance(value, dict):
return "object"
return type(value).__name__.lower()
def _column_payload(column: TabularColumn) -> dict[str, object]:
return {
"name": column.name,