Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f338cae70 | ||
|
|
9d067f1bad | ||
|
|
8b8c6c548e | ||
|
|
930302b535 | ||
|
|
ad8235028f |
@@ -70,3 +70,21 @@ Published or referenced state, immutable materializations, lifecycle evidence,
|
|||||||
governance evidence,
|
governance evidence,
|
||||||
holds, and operator attribution require data-steward review. Dataflow and
|
holds, and operator attribution require data-steward review. Dataflow and
|
||||||
Reporting derivatives must be refreshed after the source correction.
|
Reporting derivatives must be refreshed after the source correction.
|
||||||
|
|
||||||
|
## Git-source WebUI package
|
||||||
|
|
||||||
|
The repository root exposes `@govoplan/datasources-webui` for Git-tagged release
|
||||||
|
dependencies. It mirrors the owning `webui/package.json` version, public
|
||||||
|
TypeScript/CSS exports and peer requirements, with entry paths under
|
||||||
|
`webui/src`. Consumers provide the shared Core/React peers; the facade runs no
|
||||||
|
development or install scripts. The source archive contains `webui/src`, this
|
||||||
|
README and any repository license file. Run module development checks from `webui/`; Python
|
||||||
|
installation remains governed by `pyproject.toml`.
|
||||||
|
|
||||||
|
Das Repository stellt `@govoplan/datasources-webui` am Wurzelpfad für versionierte
|
||||||
|
Git-Abhängigkeiten bereit. Version, öffentliche TypeScript-/CSS-Exporte und
|
||||||
|
Peer-Anforderungen entsprechen `webui/package.json`; die Einstiegspfade liegen
|
||||||
|
unter `webui/src`. Gemeinsame Core-/React-Peers stellt die einbindende Anwendung
|
||||||
|
bereit. Die Fassade führt keine Entwicklungs- oder Installationsskripte aus.
|
||||||
|
Entwicklungsprüfungen bleiben in `webui/`, die Python-Installation weiterhin in
|
||||||
|
`pyproject.toml` definiert.
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"name": "@govoplan/datasources-webui",
|
||||||
|
"version": "0.1.26",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"main": "webui/src/index.ts",
|
||||||
|
"module": "webui/src/index.ts",
|
||||||
|
"types": "webui/src/index.ts",
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"types": "./webui/src/index.ts",
|
||||||
|
"import": "./webui/src/index.ts"
|
||||||
|
},
|
||||||
|
"./styles/datasources.css": "./webui/src/styles/datasources.css"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"@govoplan/core-webui": "^0.1.46",
|
||||||
|
"lucide-react": "^1.23.0",
|
||||||
|
"react": ">=19.2.7 <20",
|
||||||
|
"react-dom": ">=19.2.7 <20",
|
||||||
|
"react-router": ">=8.3.0 <9",
|
||||||
|
"typescript": "^5.7.2"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"@govoplan/core-webui": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"files": [
|
||||||
|
"webui/src",
|
||||||
|
"README.md",
|
||||||
|
"LICENSE"
|
||||||
|
]
|
||||||
|
}
|
||||||
+2
-2
@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "govoplan-datasources"
|
name = "govoplan-datasources"
|
||||||
version = "0.1.23"
|
version = "0.1.26"
|
||||||
description = "Governed datasource catalogue, staging, and materialization lifecycle for GovOPlaN."
|
description = "Governed datasource catalogue, staging, and materialization lifecycle for GovOPlaN."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
license = "AGPL-3.0-or-later"
|
license = "AGPL-3.0-or-later"
|
||||||
authors = [{ name = "GovOPlaN" }]
|
authors = [{ name = "GovOPlaN" }]
|
||||||
dependencies = ["govoplan-core>=0.1.20"]
|
dependencies = ["govoplan-core>=0.1.46"]
|
||||||
|
|
||||||
[tool.setuptools.packages.find]
|
[tool.setuptools.packages.find]
|
||||||
where = ["src"]
|
where = ["src"]
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
"""GovOPlaN Datasources module."""
|
"""GovOPlaN Datasources module."""
|
||||||
|
|
||||||
__version__ = "0.1.23"
|
__version__ = "0.1.26"
|
||||||
|
|||||||
+71
@@ -0,0 +1,71 @@
|
|||||||
|
"""Durable CSV source fidelity; no preview retention or catalogue content."""
|
||||||
|
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
|
||||||
|
from govoplan_core.core.datasources import DatasourceValidationError
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularCsvSource,
|
||||||
|
TabularSourceError,
|
||||||
|
csv_source_payload,
|
||||||
|
parse_tabular_csv,
|
||||||
|
verified_csv_source_text,
|
||||||
|
csv_projection_matches,
|
||||||
|
)
|
||||||
|
from govoplan_datasources.backend.tabular import (
|
||||||
|
MAX_STAGE_BYTES,
|
||||||
|
MAX_STAGE_ROWS,
|
||||||
|
normalize_rows,
|
||||||
|
parse_csv_rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_csv_source(
|
||||||
|
source: TabularCsvSource | None, rows: Sequence[Mapping[str, object]]
|
||||||
|
) -> dict[str, object] | None:
|
||||||
|
if source is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
payload = csv_source_payload(source, max_bytes=MAX_STAGE_BYTES)
|
||||||
|
if source.parser_profile == "core.csv.v1":
|
||||||
|
expected = parse_tabular_csv(
|
||||||
|
source.text,
|
||||||
|
delimiter=source.delimiter,
|
||||||
|
value_mode=source.value_mode,
|
||||||
|
max_rows=MAX_STAGE_ROWS,
|
||||||
|
max_bytes=MAX_STAGE_BYTES,
|
||||||
|
)
|
||||||
|
elif source.parser_profile == "datasources.csv.v1":
|
||||||
|
expected = parse_csv_rows(
|
||||||
|
source.text, delimiter=source.delimiter, value_mode=source.value_mode
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise DatasourceValidationError("Unsupported original CSV parser profile.")
|
||||||
|
if not csv_projection_matches(normalize_rows(expected), rows):
|
||||||
|
raise DatasourceValidationError(
|
||||||
|
"Stage rows do not match their original CSV source and parsing mode."
|
||||||
|
)
|
||||||
|
return payload
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise DatasourceValidationError(str(exc)) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def verify_stage_csv(
|
||||||
|
payload: Mapping[str, object] | None, rows: Sequence[Mapping[str, object]]
|
||||||
|
) -> None:
|
||||||
|
if payload is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
text = verified_csv_source_text(payload)
|
||||||
|
prepare_csv_source(
|
||||||
|
TabularCsvSource(
|
||||||
|
text=text,
|
||||||
|
delimiter=payload["delimiter"],
|
||||||
|
value_mode=payload["value_mode"],
|
||||||
|
parser_profile=payload["parser_profile"],
|
||||||
|
),
|
||||||
|
rows,
|
||||||
|
)
|
||||||
|
except (KeyError, TypeError, TabularSourceError) as exc:
|
||||||
|
raise DatasourceValidationError(
|
||||||
|
"Original CSV source evidence is invalid."
|
||||||
|
) from exc
|
||||||
@@ -245,6 +245,7 @@ class DatasourceMaterializationRecord(Base, TimestampMixin):
|
|||||||
# Kept nullable-in-practice for rolling upgrades. New materializations use
|
# Kept nullable-in-practice for rolling upgrades. New materializations use
|
||||||
# immutable payload rows and leave this legacy field empty.
|
# immutable payload rows and leave this legacy field empty.
|
||||||
rows: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
|
rows: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
|
||||||
|
csv_source_: Mapped[dict[str, Any] | None] = mapped_column("csv_source", JSON, nullable=True, deferred=True)
|
||||||
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
row_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
row_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
byte_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
byte_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
@@ -405,6 +406,7 @@ class DatasourceStageRecord(Base, TimestampMixin):
|
|||||||
nullable=False,
|
nullable=False,
|
||||||
)
|
)
|
||||||
rows: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
|
rows: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
|
||||||
|
csv_source_: Mapped[dict[str, Any] | None] = mapped_column("csv_source", JSON, nullable=True, deferred=True)
|
||||||
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
row_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
row_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
byte_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
byte_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
|||||||
@@ -286,6 +286,10 @@ def stage_subject_digest(
|
|||||||
"fingerprint": stage.fingerprint,
|
"fingerprint": stage.fingerprint,
|
||||||
"validation_policy_hash": stage.validation_.get("policy_hash"),
|
"validation_policy_hash": stage.validation_.get("policy_hash"),
|
||||||
"approval_policy": dict(policy or {}),
|
"approval_policy": dict(policy or {}),
|
||||||
|
**({"csv_source": {
|
||||||
|
key: stage.csv_source_.get(key)
|
||||||
|
for key in ("sha256", "byte_count", "delimiter", "value_mode", "parser_profile")
|
||||||
|
}} if stage.csv_source_ is not None else {}),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -562,6 +566,7 @@ def apply_retention_plan(
|
|||||||
evidence_hashes.append(evidence.event_hash)
|
evidence_hashes.append(evidence.event_hash)
|
||||||
materialization.payload_id = None
|
materialization.payload_id = None
|
||||||
materialization.rows = []
|
materialization.rows = []
|
||||||
|
materialization.csv_source_ = None
|
||||||
materialization.state = "disposed"
|
materialization.state = "disposed"
|
||||||
materialization.disposed_at = plan.as_of
|
materialization.disposed_at = plan.as_of
|
||||||
materialization.disposition_ = {
|
materialization.disposition_ = {
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ from govoplan_datasources.backend.payloads import ExternalArtifactPayloadBackend
|
|||||||
|
|
||||||
MODULE_ID = "datasources"
|
MODULE_ID = "datasources"
|
||||||
MODULE_NAME = "Datasources"
|
MODULE_NAME = "Datasources"
|
||||||
MODULE_VERSION = "0.1.23"
|
MODULE_VERSION = "0.1.26"
|
||||||
DATASOURCE_INTERFACE_VERSION = "0.2.0"
|
DATASOURCE_INTERFACE_VERSION = "0.2.0"
|
||||||
|
|
||||||
ARCHITECTURE = ModuleArchitectureDeclaration(
|
ARCHITECTURE = ModuleArchitectureDeclaration(
|
||||||
@@ -487,6 +487,91 @@ manifest = ModuleManifest(
|
|||||||
),
|
),
|
||||||
architecture=ARCHITECTURE,
|
architecture=ARCHITECTURE,
|
||||||
documentation=(
|
documentation=(
|
||||||
|
DocumentationTopic(
|
||||||
|
id="datasources.csv-source-fidelity",
|
||||||
|
title="CSV input fidelity, original export and retention",
|
||||||
|
summary="Keep original upload text alongside durable stages and immutable materializations without bypassing visibility or retention.",
|
||||||
|
body=(
|
||||||
|
"New CSV upload dialogs default to Preserve text; values remain strings, including whitespace, exact decimal digits, large identifiers and explicit empty records. Infer types (legacy) is an explicit compatibility option. "
|
||||||
|
"Existing API calls without csv_value_mode retain their legacy behavior. The API accepts text or legacy_typed; parsed headers are normalized, and malformed text-mode row shapes are rejected. "
|
||||||
|
"Durable CSV stages retain the original supplied UTF-8 text separately from metadata, bind it to the exact scalar types and values of the parsed rows, and include source evidence in approval. Promotion and freezing preserve the original with the corresponding immutable materialization. This does not persist preview rows. "
|
||||||
|
"Original text and rows each have a 5 MB limit; there are at most 10,000 rows. Each stage/materialization, including a frozen copy, can retain up to 5 MB of original text plus serialization and metadata storage overhead. "
|
||||||
|
"GET /api/v1/datasources/{datasource_id}/materializations/{materialization_id}/original-csv is an audited, uncached attachment export for current datasources:source:admin authority. "
|
||||||
|
"Current and historical visibility and policy-provider decisions still apply. Any row filter or denied field prevents raw export, even for an administrator; use governed row previews instead. No raw input is returned in catalogue, stage or history DTOs. "
|
||||||
|
"Freezing verifies the existing original evidence before copying and retains inherited access restrictions; the original may carry at most 32 distinct governance snapshots, with explicit rejection beyond that limit. "
|
||||||
|
"Original text is checksum-verified and follows the same governed retention as its stage/materialization. Stage deletion removes its original; materialization payload disposal also clears that original while retaining audit evidence. "
|
||||||
|
"No original is invented for historical imports. Missing or disposed originals are reported unavailable. Export preserves the API's supplied text as UTF-8, including unsanitized spreadsheet formulas, and is not a reconstruction of an earlier file encoding."
|
||||||
|
),
|
||||||
|
layer="always", documentation_types=("user", "admin"), audience=("user", "module_admin", "operator"), order=8,
|
||||||
|
translations={"de": {
|
||||||
|
"title": "CSV-Original, genauer Import und Aufbewahrung",
|
||||||
|
"summary": "Originaltext mit dauerhaften Staging-Ständen und Materialisierungen erhalten, ohne Sichtbarkeit oder Aufbewahrung zu umgehen.",
|
||||||
|
"body": (
|
||||||
|
"Neue CSV-Importdialoge wählen standardmäßig Text erhalten: Werte einschließlich Leerzeichen, genauer Dezimalstellen, großer Kennungen und ausdrücklich leerer Datensätze bleiben Zeichenketten. Typen ableiten (bisheriges Verhalten) ist eine ausdrückliche Kompatibilitätsoption. "
|
||||||
|
"Bestehende API-Aufrufe ohne csv_value_mode bleiben unverändert; unterstützt werden text und legacy_typed. Überschriften werden normalisiert, fehlerhafte Zeilenformen im Textmodus abgelehnt. "
|
||||||
|
"Dauerhafte CSV-Staging-Stände speichern den gelieferten UTF-8-Originaltext getrennt von Metadaten, prüfen genaue Typen und Werte der abgeleiteten Zeilen und binden den Quellnachweis in die Freigabe ein. Übernahme und Einfrieren erhalten das Original bei der jeweiligen unveränderlichen Materialisierung. Vorschauzeilen werden dadurch nicht gespeichert. "
|
||||||
|
"Original und Zeilen sind jeweils auf 5 MB und die Tabelle auf 10.000 Zeilen begrenzt. Pro Staging-Stand oder Materialisierung einschließlich eingefrorener Kopien werden bis zu 5 MB Originaltext zuzüglich Speicher für Serialisierung und Metadaten aufbewahrt. "
|
||||||
|
"GET /api/v1/datasources/{datasource_id}/materializations/{materialization_id}/original-csv liefert einen protokollierten, nicht zwischengespeicherten Download mit aktuellem Recht datasources:source:admin. "
|
||||||
|
"Aktuelle und historische Sichtbarkeit sowie externe Richtlinienentscheidungen gelten weiterhin. Jeder Zeilenfilter oder ein nicht freigegebenes Feld verhindert den Originalexport auch für die Administration; verwenden Sie dann die gesteuerte Zeilenvorschau. Katalog, Staging- und Verlaufsantworten enthalten keinen Originaltext. "
|
||||||
|
"Einfrieren prüft den vorhandenen Originalnachweis vor dem Kopieren und erhält geerbte Zugriffsbeschränkungen. Höchstens 32 unterschiedliche Richtlinienstände dürfen ein Original begleiten; darüber hinaus wird ausdrücklich abgewiesen. "
|
||||||
|
"Prüfsummen werden vor dem Export geprüft. Das Original folgt der gesteuerten Aufbewahrung: Löschen eines Staging-Stands entfernt dessen Original; Entsorgung einer Materialisierung entfernt auch deren Original, behält aber Auditnachweise. "
|
||||||
|
"Für historische Importe wird kein Original erfunden. Fehlende oder entsorgte Originale werden als nicht verfügbar gemeldet. Der Download erhält den an die API gelieferten Text als UTF-8 einschließlich unveränderter Tabellenformeln, nicht eine frühere Dateikodierung."
|
||||||
|
),
|
||||||
|
}},
|
||||||
|
),
|
||||||
|
DocumentationTopic(
|
||||||
|
id="datasources.detail-freshness",
|
||||||
|
title="Preview and revision history after a datasource change",
|
||||||
|
summary="Reload the selected datasource details even when its catalogue reference stays the same.",
|
||||||
|
body=(
|
||||||
|
"After a successful catalogue reload, the selected datasource preview and materialization history are read again. "
|
||||||
|
"This includes manual Reload and reloads following refresh, freezing, stage promotion or governance changes, even when the selected reference or current materialization is unchanged. "
|
||||||
|
"While reloading, previous rows and history are not presented as the newly selected or updated datasource. "
|
||||||
|
"An older request cannot replace results for a newer selection, reload or authentication context. "
|
||||||
|
"Callbacks retained from an earlier authorization context cannot start a new reload or change success/error messages and selection. "
|
||||||
|
"After a real identity, credential or permission change, the old catalogue is hidden until the new context has loaded it; open dialog inputs are reset to avoid carrying drafts into another authorization context. "
|
||||||
|
"Cosmetic profile changes and ordinary refreshes with equivalent authority do not reset dialogs or trigger duplicate detail reads. "
|
||||||
|
"Preview and history are independent reads: a preview failure can still leave authorized history available. "
|
||||||
|
"Live datasource previews may read the provider through the existing bounded, authorized preview API; this is not background polling or automatic source refresh. "
|
||||||
|
"Rows are not normalized, reordered or persisted by this display update. Snapshot immutability, retention rules and all server-side permission checks remain unchanged."
|
||||||
|
),
|
||||||
|
layer="always", documentation_types=("user", "admin"), audience=("user", "module_admin", "operator"), order=6,
|
||||||
|
translations={"de": {
|
||||||
|
"title": "Vorschau und Revisionsverlauf nach einer Datenquellenänderung",
|
||||||
|
"summary": "Details der ausgewählten Datenquelle auch bei unverändertem Katalogverweis neu laden.",
|
||||||
|
"body": (
|
||||||
|
"Nach erfolgreichem Neuladen des Katalogs werden Vorschau und Materialisierungsverlauf der ausgewählten Datenquelle erneut abgerufen. "
|
||||||
|
"Dies gilt für Neu laden sowie für das Neuladen nach Aktualisierung, Einfrieren, Staging-Übernahme oder Governance-Änderungen, auch wenn Auswahlverweis oder aktuelle Materialisierung gleich bleiben. "
|
||||||
|
"Während des Neuladens werden bisherige Zeilen und Verläufe nicht als neue Auswahl oder aktualisierte Datenquelle dargestellt. "
|
||||||
|
"Eine ältere Anfrage kann Ergebnisse einer neueren Auswahl, eines neueren Abrufs oder eines geänderten Authentifizierungskontexts nicht ersetzen. "
|
||||||
|
"Rückrufe eines früheren Berechtigungskontexts starten keinen neuen Abruf und ändern weder Erfolgs-/Fehlermeldungen noch die Auswahl. "
|
||||||
|
"Nach einem tatsächlichen Identitäts-, Zugangsdaten- oder Rechtewechsel bleibt der alte Katalog verborgen, bis er im neuen Kontext geladen wurde; offene Dialogeingaben werden zurückgesetzt, damit Entwürfe nicht in einen anderen Berechtigungskontext gelangen. "
|
||||||
|
"Rein optische Profiländerungen und gewöhnliche Aktualisierungen mit gleichwertigen Rechten setzen Dialoge nicht zurück und lösen keine doppelten Detailabrufe aus. "
|
||||||
|
"Vorschau und Verlauf sind getrennte Lesezugriffe: Bei einem Vorschaufehler kann ein berechtigter Verlauf weiterhin verfügbar sein. "
|
||||||
|
"Vorschauen lebender Datenquellen können den Anbieter über die bestehende begrenzte und berechtigungsgeprüfte Vorschau-API lesen; dies ist weder Hintergrund-Polling noch eine automatische Quellenaktualisierung. "
|
||||||
|
"Die Anzeigeaktualisierung normalisiert, sortiert oder speichert keine Zeilen dauerhaft. Unveränderlichkeit von Snapshots, Aufbewahrungsregeln und sämtliche serverseitigen Rechteprüfungen bleiben erhalten."
|
||||||
|
),
|
||||||
|
}},
|
||||||
|
),
|
||||||
|
DocumentationTopic(
|
||||||
|
id="datasources.workspace-layout",
|
||||||
|
title="Datasources workspace actions",
|
||||||
|
summary="Find collection-wide commands in their consistent workspace position.",
|
||||||
|
body="Open the documentation book beside Data for workspace guidance; field documentation stays "
|
||||||
|
"beside the relevant label. "
|
||||||
|
"Reload and Add datasource or stage use the persistent full-width workspace header at the upper right; Reload sits immediately before creation. Selecting a record, changing filters, or opening an editor does not move these collection-wide commands into the left pane. Catalogue, staging, and origin filters remain in the collection pane. Snapshot and freeze actions remain with their selected resource. Existing permissions, disabled-state rules, and unsaved-change guards still apply. Administrators configure authority through the existing permission system; no new permission or automatic operation is introduced.",
|
||||||
|
layer="static",
|
||||||
|
documentation_types=("user", "admin"),
|
||||||
|
audience=("user", "module_admin", "operator"),
|
||||||
|
order=5,
|
||||||
|
translations={"de": {
|
||||||
|
"title": "Datenquellen: Aktionen im Arbeitsbereich",
|
||||||
|
"summary": "Sammlungsweite Aktionen an ihrer einheitlichen Position im Arbeitsbereich finden.",
|
||||||
|
"body": "Öffnen Sie das Dokumentationsbuch neben Daten für Hinweise zum Arbeitsbereich; "
|
||||||
|
"Felddokumentation bleibt neben der jeweiligen Feldbezeichnung. "
|
||||||
|
"Neu laden und Datenquelle oder Staging hinzufügen stehen oben rechts in der dauerhaft sichtbaren, arbeitsbereichsweiten Leiste; Neu laden steht unmittelbar vor dem Anlegen. Auswahl, Filterwechsel und Bearbeitung verschieben diese sammlungsweiten Aktionen nicht in den linken Bereich. Katalog-, Staging- und Herkunftsfilter bleiben im Sammlungsbereich. Snapshot- und Einfrieraktionen bleiben bei der ausgewählten Ressource. Bestehende Berechtigungen, Deaktivierungsregeln und der Schutz ungespeicherter Änderungen gelten weiterhin. Administratoren konfigurieren Rechte im bestehenden Berechtigungssystem; es entstehen weder neue Rechte noch automatische Vorgänge.",
|
||||||
|
}},
|
||||||
|
),
|
||||||
DocumentationTopic(
|
DocumentationTopic(
|
||||||
id="datasources.data-subject-requests",
|
id="datasources.data-subject-requests",
|
||||||
title="Datasources data-subject requests",
|
title="Datasources data-subject requests",
|
||||||
|
|||||||
+24
@@ -0,0 +1,24 @@
|
|||||||
|
"""Preserve original CSV for new durable imports, separate from catalogue DTOs."""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision = "e2b8d4a0f6c3"
|
||||||
|
down_revision = "d1a7c3e9f5b2"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"datasource_stages", sa.Column("csv_source", sa.JSON(), nullable=True)
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"datasource_materializations", sa.Column("csv_source", sa.JSON(), nullable=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
for table in ("datasource_materializations", "datasource_stages"):
|
||||||
|
with op.batch_alter_table(table) as batch:
|
||||||
|
batch.drop_column("csv_source")
|
||||||
@@ -5,7 +5,7 @@ import json
|
|||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from govoplan_core.audit.logging import audit_event
|
from govoplan_core.audit.logging import audit_event
|
||||||
@@ -25,6 +25,7 @@ from govoplan_core.core.datasources import (
|
|||||||
datasource_origins,
|
datasource_origins,
|
||||||
)
|
)
|
||||||
from govoplan_core.db.session import get_session
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_core.core.tabular_sources import TabularCsvSource
|
||||||
from govoplan_datasources.backend.runtime import get_registry
|
from govoplan_datasources.backend.runtime import get_registry
|
||||||
from govoplan_datasources.backend.schemas import (
|
from govoplan_datasources.backend.schemas import (
|
||||||
DatasourceFieldResponse,
|
DatasourceFieldResponse,
|
||||||
@@ -203,7 +204,7 @@ def api_create_stage(
|
|||||||
rows = (
|
rows = (
|
||||||
tuple(payload.rows or ())
|
tuple(payload.rows or ())
|
||||||
if payload.format == "json"
|
if payload.format == "json"
|
||||||
else parse_csv_rows(payload.csv_text or "", delimiter=payload.delimiter)
|
else parse_csv_rows(payload.csv_text or "", delimiter=payload.delimiter, value_mode=payload.csv_value_mode)
|
||||||
)
|
)
|
||||||
stage = _provider().create_stage(
|
stage = _provider().create_stage(
|
||||||
session,
|
session,
|
||||||
@@ -225,6 +226,12 @@ def api_create_stage(
|
|||||||
},
|
},
|
||||||
metadata=payload.metadata,
|
metadata=payload.metadata,
|
||||||
governance=_governance(payload.governance),
|
governance=_governance(payload.governance),
|
||||||
|
csv_source=(TabularCsvSource(
|
||||||
|
text=payload.csv_text or "",
|
||||||
|
delimiter=payload.delimiter,
|
||||||
|
value_mode=payload.csv_value_mode,
|
||||||
|
parser_profile="datasources.csv.v1",
|
||||||
|
) if payload.format == "csv" else None),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
except DatasourceError as exc:
|
except DatasourceError as exc:
|
||||||
@@ -629,6 +636,27 @@ def api_list_materializations(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{datasource_id}/materializations/{materialization_id}/original-csv")
|
||||||
|
def api_original_csv(
|
||||||
|
datasource_id: str,
|
||||||
|
materialization_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> Response:
|
||||||
|
_require_any_scope(principal, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
text = _provider().original_csv(
|
||||||
|
session, principal,
|
||||||
|
datasource_ref=f"datasource:{datasource_id}",
|
||||||
|
materialization_ref=f"materialization:{materialization_id}",
|
||||||
|
)
|
||||||
|
except DatasourceError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
_audit(session, principal, action="datasources.original_csv.exported", object_type="datasource_materialization", object_id=f"materialization:{materialization_id}", details={"datasource_ref": f"datasource:{datasource_id}", "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest()})
|
||||||
|
session.commit()
|
||||||
|
return Response(content=text.encode("utf-8"), media_type="text/csv; charset=utf-8", headers={"Content-Disposition": 'attachment; filename="original.csv"', "Cache-Control": "no-store"})
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/{datasource_id}/refresh",
|
"/{datasource_id}/refresh",
|
||||||
response_model=DatasourceStagePromoteResponse,
|
response_model=DatasourceStagePromoteResponse,
|
||||||
|
|||||||
@@ -209,6 +209,7 @@ class DatasourceStageCreateRequest(BaseModel):
|
|||||||
format: Literal["json", "csv"] = "json"
|
format: Literal["json", "csv"] = "json"
|
||||||
rows: list[dict[str, Any]] | None = Field(default=None, max_length=10_000)
|
rows: list[dict[str, Any]] | None = Field(default=None, max_length=10_000)
|
||||||
csv_text: str | None = Field(default=None, max_length=5_000_000)
|
csv_text: str | None = Field(default=None, max_length=5_000_000)
|
||||||
|
csv_value_mode: Literal["legacy_typed", "text"] = "legacy_typed"
|
||||||
delimiter: Literal[",", ";", "\t", "|"] = ","
|
delimiter: Literal[",", ";", "\t", "|"] = ","
|
||||||
target_datasource_ref: str | None = None
|
target_datasource_ref: str | None = None
|
||||||
provenance: dict[str, Any] = Field(default_factory=dict)
|
provenance: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ from govoplan_core.core.datasources import (
|
|||||||
datasource_visibility_policy_provider,
|
datasource_visibility_policy_provider,
|
||||||
)
|
)
|
||||||
from govoplan_core.db.base import utcnow
|
from govoplan_core.db.base import utcnow
|
||||||
|
from govoplan_core.core.tabular_sources import csv_source_summary, verified_csv_source_text, TabularSourceError
|
||||||
|
from govoplan_datasources.backend.csv_sources import prepare_csv_source, verify_stage_csv
|
||||||
from govoplan_datasources.backend.db.models import (
|
from govoplan_datasources.backend.db.models import (
|
||||||
DatasourceGovernanceReferenceRecord,
|
DatasourceGovernanceReferenceRecord,
|
||||||
DatasourceMaterializationRecord,
|
DatasourceMaterializationRecord,
|
||||||
@@ -471,6 +473,40 @@ class SqlDatasourceProvider:
|
|||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
def original_csv(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
datasource_ref: str,
|
||||||
|
materialization_ref: str,
|
||||||
|
) -> str:
|
||||||
|
"""Export original input only to admins with unrestricted current/history visibility."""
|
||||||
|
db, api_principal = _context(session, principal, ADMIN_SCOPE)
|
||||||
|
item = _required_datasource(db, tenant_id=api_principal.tenant_id, datasource_ref=datasource_ref)
|
||||||
|
request = DatasourceReadRequest(datasource_ref=datasource_ref, materialization_ref=materialization_ref)
|
||||||
|
materialization = _selected_materialization(db, item=item, request=request)
|
||||||
|
if materialization is None or materialization.disposed_at is not None:
|
||||||
|
raise DatasourceUnavailableError("Original CSV source is unavailable or was disposed under retention policy.")
|
||||||
|
if materialization.csv_source_ is None:
|
||||||
|
raise DatasourceNotFoundError("Original CSV was not retained for this materialization; historical typed rows cannot reconstruct it.")
|
||||||
|
try:
|
||||||
|
original = verified_csv_source_text(materialization.csv_source_, expected_summary=materialization.metadata_.get("csv_source") or {})
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise DatasourceUnavailableError(str(exc)) from exc
|
||||||
|
history = materialization.csv_source_.get("governance_history", [])
|
||||||
|
if not isinstance(history, list) or len(history) > 32 or any(not isinstance(value, Mapping) for value in history):
|
||||||
|
raise DatasourceAccessError("Original CSV inherited governance is invalid.")
|
||||||
|
for snapshot in (None, materialization.governance_snapshot_, *history):
|
||||||
|
plan = self._visibility_plan(
|
||||||
|
db, api_principal, item=item, request=request, materialized=True,
|
||||||
|
schema=_fields(materialization.schema_),
|
||||||
|
governance_override=DatasourceGovernance.from_mapping(snapshot) if snapshot is not None else None,
|
||||||
|
)
|
||||||
|
if plan.row_filters or any(not rule.allowed for rule in plan.field_rules):
|
||||||
|
raise DatasourceAccessError("Original CSV export requires unrestricted row and field visibility; use the governed row preview instead.")
|
||||||
|
return original
|
||||||
|
|
||||||
def _read_materialized(
|
def _read_materialized(
|
||||||
self,
|
self,
|
||||||
session: Session,
|
session: Session,
|
||||||
@@ -732,6 +768,9 @@ class SqlDatasourceProvider:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
rows = normalize_rows(stage.rows)
|
rows = normalize_rows(stage.rows)
|
||||||
|
if stage.csv_source is not None and "csv_source" in stage.metadata:
|
||||||
|
raise DatasourceValidationError("csv_source metadata is reserved for verified CSV import evidence.")
|
||||||
|
csv_payload = prepare_csv_source(stage.csv_source, rows)
|
||||||
schema = infer_schema(rows)
|
schema = infer_schema(rows)
|
||||||
governance = replace(
|
governance = replace(
|
||||||
governance,
|
governance,
|
||||||
@@ -768,7 +807,11 @@ class SqlDatasourceProvider:
|
|||||||
byte_count=encoded_size(rows),
|
byte_count=encoded_size(rows),
|
||||||
validation_=validation,
|
validation_=validation,
|
||||||
provenance_=dict(stage.provenance),
|
provenance_=dict(stage.provenance),
|
||||||
metadata_=dict(stage.metadata),
|
metadata_={
|
||||||
|
**dict(stage.metadata),
|
||||||
|
**({"csv_source": csv_source_summary(csv_payload)} if csv_payload is not None else {}),
|
||||||
|
},
|
||||||
|
csv_source_=csv_payload,
|
||||||
governance_=governance.to_dict(),
|
governance_=governance.to_dict(),
|
||||||
created_by=_actor_id(api_principal),
|
created_by=_actor_id(api_principal),
|
||||||
)
|
)
|
||||||
@@ -880,6 +923,7 @@ class SqlDatasourceProvider:
|
|||||||
)
|
)
|
||||||
if stage.state != "ready":
|
if stage.state != "ready":
|
||||||
raise DatasourceValidationError("Only ready stages can be promoted.")
|
raise DatasourceValidationError("Only ready stages can be promoted.")
|
||||||
|
verify_stage_csv(stage.csv_source_, stage.rows)
|
||||||
datasource = (
|
datasource = (
|
||||||
db.get(DatasourceRecord, stage.target_datasource_id)
|
db.get(DatasourceRecord, stage.target_datasource_id)
|
||||||
if stage.target_datasource_id
|
if stage.target_datasource_id
|
||||||
@@ -960,6 +1004,7 @@ class SqlDatasourceProvider:
|
|||||||
"stage_approval": dict(stage.approval_),
|
"stage_approval": dict(stage.approval_),
|
||||||
},
|
},
|
||||||
metadata=dict(stage.metadata_),
|
metadata=dict(stage.metadata_),
|
||||||
|
csv_source=stage.csv_source_,
|
||||||
set_current=True,
|
set_current=True,
|
||||||
)
|
)
|
||||||
stage.state = "promoted"
|
stage.state = "promoted"
|
||||||
@@ -1283,6 +1328,11 @@ class SqlDatasourceProvider:
|
|||||||
raise DatasourceUnavailableError(
|
raise DatasourceUnavailableError(
|
||||||
"The datasource has no current state to freeze."
|
"The datasource has no current state to freeze."
|
||||||
)
|
)
|
||||||
|
if current.csv_source_ is not None:
|
||||||
|
try:
|
||||||
|
verified_csv_source_text(current.csv_source_, expected_summary=current.metadata_.get("csv_source") or {})
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise DatasourceUnavailableError(str(exc)) from exc
|
||||||
current_payload = payload_for_materialization(db, current)
|
current_payload = payload_for_materialization(db, current)
|
||||||
materialization = _append_materialization(
|
materialization = _append_materialization(
|
||||||
db,
|
db,
|
||||||
@@ -1295,6 +1345,8 @@ class SqlDatasourceProvider:
|
|||||||
frozen=True,
|
frozen=True,
|
||||||
frozen_label=label,
|
frozen_label=label,
|
||||||
source_timestamp=current.source_timestamp,
|
source_timestamp=current.source_timestamp,
|
||||||
|
csv_source=current.csv_source_,
|
||||||
|
csv_source_governance=current.governance_snapshot_,
|
||||||
provenance={
|
provenance={
|
||||||
**dict(current.provenance_),
|
**dict(current.provenance_),
|
||||||
"frozen_from": _materialization_ref(current.id),
|
"frozen_from": _materialization_ref(current.id),
|
||||||
@@ -1550,8 +1602,9 @@ class SqlDatasourceProvider:
|
|||||||
schema: Sequence[DatasourceField],
|
schema: Sequence[DatasourceField],
|
||||||
action: str = "read",
|
action: str = "read",
|
||||||
additional_policies: Sequence[Mapping[str, object]] = (),
|
additional_policies: Sequence[Mapping[str, object]] = (),
|
||||||
|
governance_override: DatasourceGovernance | None = None,
|
||||||
) -> VisibilityPlan:
|
) -> VisibilityPlan:
|
||||||
governance = _datasource_governance(item)
|
governance = governance_override or _datasource_governance(item)
|
||||||
policies: list[Mapping[str, object]] = []
|
policies: list[Mapping[str, object]] = []
|
||||||
if governance.visibility_policy:
|
if governance.visibility_policy:
|
||||||
policies.append(governance.visibility_policy)
|
policies.append(governance.visibility_policy)
|
||||||
@@ -1766,6 +1819,8 @@ def _append_materialization(
|
|||||||
set_current: bool,
|
set_current: bool,
|
||||||
reusable_payload: DatasourcePayloadRecord | None = None,
|
reusable_payload: DatasourcePayloadRecord | None = None,
|
||||||
state: str = "published",
|
state: str = "published",
|
||||||
|
csv_source: Mapping[str, object] | None = None,
|
||||||
|
csv_source_governance: Mapping[str, object] | None = None,
|
||||||
) -> DatasourceMaterializationRecord:
|
) -> DatasourceMaterializationRecord:
|
||||||
datasource = _lock_datasource_for_materialization(session, datasource)
|
datasource = _lock_datasource_for_materialization(session, datasource)
|
||||||
revision = _allocate_materialization_revision(session, datasource)
|
revision = _allocate_materialization_revision(session, datasource)
|
||||||
@@ -1788,6 +1843,18 @@ def _append_materialization(
|
|||||||
datasource,
|
datasource,
|
||||||
schema,
|
schema,
|
||||||
)
|
)
|
||||||
|
csv_payload = dict(csv_source) if csv_source is not None else None
|
||||||
|
if csv_payload is not None:
|
||||||
|
history = csv_payload.get("governance_history", [])
|
||||||
|
if not isinstance(history, list) or any(not isinstance(value, Mapping) for value in history):
|
||||||
|
raise DatasourceValidationError("Original CSV inherited governance is invalid.")
|
||||||
|
history = [dict(value) for value in history]
|
||||||
|
for snapshot in (csv_source_governance, _datasource_governance(datasource).to_dict()):
|
||||||
|
if snapshot is not None and dict(snapshot) not in history:
|
||||||
|
history.append(dict(snapshot))
|
||||||
|
if len(history) > 32:
|
||||||
|
raise DatasourceValidationError("Original CSV governance history exceeds the safe copy limit; review the source before creating a new stage.")
|
||||||
|
csv_payload["governance_history"] = history
|
||||||
materialization = DatasourceMaterializationRecord(
|
materialization = DatasourceMaterializationRecord(
|
||||||
tenant_id=datasource.tenant_id,
|
tenant_id=datasource.tenant_id,
|
||||||
datasource_id=datasource.id,
|
datasource_id=datasource.id,
|
||||||
@@ -1798,6 +1865,7 @@ def _append_materialization(
|
|||||||
payload_id=payload.id,
|
payload_id=payload.id,
|
||||||
payload_checksum=payload.checksum,
|
payload_checksum=payload.checksum,
|
||||||
rows=[],
|
rows=[],
|
||||||
|
csv_source_=csv_payload,
|
||||||
fingerprint=fingerprint,
|
fingerprint=fingerprint,
|
||||||
row_count=payload.row_count,
|
row_count=payload.row_count,
|
||||||
byte_count=payload.byte_count,
|
byte_count=payload.byte_count,
|
||||||
@@ -1805,7 +1873,7 @@ def _append_materialization(
|
|||||||
frozen_label=_clean_optional(frozen_label),
|
frozen_label=_clean_optional(frozen_label),
|
||||||
source_timestamp=source_timestamp,
|
source_timestamp=source_timestamp,
|
||||||
provenance_=dict(provenance or {}),
|
provenance_=dict(provenance or {}),
|
||||||
metadata_=dict(metadata or {}),
|
metadata_={**dict(metadata or {}), **({"csv_source": csv_source_summary(csv_payload)} if csv_payload is not None else {})},
|
||||||
governance_snapshot_=_datasource_governance(datasource).to_dict(),
|
governance_snapshot_=_datasource_governance(datasource).to_dict(),
|
||||||
created_by=actor_id,
|
created_by=actor_id,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ from decimal import Decimal
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from govoplan_core.core.datasources import DatasourceField, DatasourceValidationError
|
from govoplan_core.core.datasources import DatasourceField, DatasourceValidationError
|
||||||
|
from govoplan_core.core.tabular_sources import CsvValueMode, TabularSourceError, parse_tabular_csv
|
||||||
|
from govoplan_core.core.tabular_sources import infer_tabular_schema, tabular_type_name as _type_name
|
||||||
|
|
||||||
|
|
||||||
MAX_STAGE_ROWS = 10_000
|
MAX_STAGE_ROWS = 10_000
|
||||||
@@ -36,10 +38,22 @@ def parse_csv_rows(
|
|||||||
csv_text: str,
|
csv_text: str,
|
||||||
*,
|
*,
|
||||||
delimiter: str = ",",
|
delimiter: str = ",",
|
||||||
|
value_mode: CsvValueMode = "legacy_typed",
|
||||||
) -> tuple[dict[str, Any], ...]:
|
) -> tuple[dict[str, Any], ...]:
|
||||||
if delimiter not in {",", ";", "\t", "|"}:
|
if delimiter not in {",", ";", "\t", "|"}:
|
||||||
raise DatasourceValidationError("Unsupported CSV delimiter.")
|
raise DatasourceValidationError("Unsupported CSV delimiter.")
|
||||||
if len(csv_text.encode("utf-8")) > MAX_STAGE_BYTES:
|
if value_mode == "text":
|
||||||
|
try:
|
||||||
|
return normalize_rows(parse_tabular_csv(csv_text, delimiter=delimiter, value_mode="text", max_rows=MAX_STAGE_ROWS, max_bytes=MAX_STAGE_BYTES))
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise DatasourceValidationError(str(exc)) from exc
|
||||||
|
if value_mode != "legacy_typed":
|
||||||
|
raise DatasourceValidationError("Unsupported CSV value mode.")
|
||||||
|
try:
|
||||||
|
encoded_text = csv_text.encode("utf-8")
|
||||||
|
except UnicodeError as exc:
|
||||||
|
raise DatasourceValidationError("CSV input must be valid Unicode encodable as UTF-8.") from exc
|
||||||
|
if len(encoded_text) > MAX_STAGE_BYTES:
|
||||||
raise DatasourceValidationError(
|
raise DatasourceValidationError(
|
||||||
f"Staging is limited to {MAX_STAGE_BYTES // 1_000_000} MB."
|
f"Staging is limited to {MAX_STAGE_BYTES // 1_000_000} MB."
|
||||||
)
|
)
|
||||||
@@ -58,6 +72,8 @@ def parse_csv_rows(
|
|||||||
raise DatasourceValidationError("CSV headers must be unique.")
|
raise DatasourceValidationError("CSV headers must be unique.")
|
||||||
rows: list[dict[str, object]] = []
|
rows: list[dict[str, object]] = []
|
||||||
for line_number, values in enumerate(reader, start=2):
|
for line_number, values in enumerate(reader, start=2):
|
||||||
|
if len(rows) >= MAX_STAGE_ROWS:
|
||||||
|
raise DatasourceValidationError(f"Staging is limited to {MAX_STAGE_ROWS:,} rows.")
|
||||||
if len(values) != len(header):
|
if len(values) != len(header):
|
||||||
raise DatasourceValidationError(
|
raise DatasourceValidationError(
|
||||||
f"CSV row {line_number} has {len(values)} values; expected {len(header)}."
|
f"CSV row {line_number} has {len(values)} values; expected {len(header)}."
|
||||||
@@ -76,26 +92,7 @@ def parse_csv_rows(
|
|||||||
def infer_schema(
|
def infer_schema(
|
||||||
rows: Sequence[Mapping[str, object]],
|
rows: Sequence[Mapping[str, object]],
|
||||||
) -> tuple[DatasourceField, ...]:
|
) -> tuple[DatasourceField, ...]:
|
||||||
names: list[str] = []
|
return tuple(DatasourceField(name=column.name, data_type=column.data_type, nullable=column.nullable) for column in infer_tabular_schema(rows, type_name=_type_name))
|
||||||
for row in rows:
|
|
||||||
for name in row:
|
|
||||||
if name not in names:
|
|
||||||
names.append(name)
|
|
||||||
result: list[DatasourceField] = []
|
|
||||||
for name in names:
|
|
||||||
values = [row.get(name) for row in rows]
|
|
||||||
concrete = [value for value in values if value is not None]
|
|
||||||
data_type = _type_name(concrete[0]) if concrete else "unknown"
|
|
||||||
if any(_type_name(value) != data_type for value in concrete[1:]):
|
|
||||||
data_type = "mixed"
|
|
||||||
result.append(
|
|
||||||
DatasourceField(
|
|
||||||
name=name,
|
|
||||||
data_type=data_type,
|
|
||||||
nullable=len(concrete) != len(values),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return tuple(result)
|
|
||||||
|
|
||||||
|
|
||||||
def fingerprint_rows(
|
def fingerprint_rows(
|
||||||
@@ -168,22 +165,6 @@ def _json_value(value: object) -> object:
|
|||||||
raise TypeError(f"{type(value).__name__} is not JSON serializable")
|
raise TypeError(f"{type(value).__name__} is not JSON serializable")
|
||||||
|
|
||||||
|
|
||||||
def _type_name(value: object) -> str:
|
|
||||||
if isinstance(value, bool):
|
|
||||||
return "boolean"
|
|
||||||
if isinstance(value, int):
|
|
||||||
return "integer"
|
|
||||||
if isinstance(value, (float, Decimal)):
|
|
||||||
return "number"
|
|
||||||
if isinstance(value, str):
|
|
||||||
return "string"
|
|
||||||
if isinstance(value, list):
|
|
||||||
return "array"
|
|
||||||
if isinstance(value, dict):
|
|
||||||
return "object"
|
|
||||||
return type(value).__name__.lower()
|
|
||||||
|
|
||||||
|
|
||||||
def _csv_value(value: str) -> object:
|
def _csv_value(value: str) -> object:
|
||||||
cleaned = value.strip()
|
cleaned = value.strip()
|
||||||
if not cleaned:
|
if not cleaned:
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
from dataclasses import replace
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from fastapi import HTTPException
|
||||||
|
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine
|
||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
@@ -24,6 +28,9 @@ from govoplan_core.core.datasources import (
|
|||||||
DatasourceValidationError,
|
DatasourceValidationError,
|
||||||
)
|
)
|
||||||
from govoplan_core.core.tabular_sources import (
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularCsvSource,
|
||||||
|
parse_tabular_csv,
|
||||||
|
csv_source_payload,
|
||||||
TabularPreviewDiagnostic,
|
TabularPreviewDiagnostic,
|
||||||
TabularPushdown,
|
TabularPushdown,
|
||||||
TabularSourceHealth,
|
TabularSourceHealth,
|
||||||
@@ -40,6 +47,7 @@ from govoplan_datasources.backend.db.models import (
|
|||||||
DatasourceStageRecord,
|
DatasourceStageRecord,
|
||||||
)
|
)
|
||||||
from govoplan_datasources.backend.service import (
|
from govoplan_datasources.backend.service import (
|
||||||
|
ADMIN_SCOPE,
|
||||||
CATALOGUE_READ_SCOPE,
|
CATALOGUE_READ_SCOPE,
|
||||||
SOURCE_WRITE_SCOPE,
|
SOURCE_WRITE_SCOPE,
|
||||||
STAGE_WRITE_SCOPE,
|
STAGE_WRITE_SCOPE,
|
||||||
@@ -263,6 +271,85 @@ class DatasourceLifecycleTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.engine.dispose()
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_original_csv_preserves_source_but_respects_current_and_historical_visibility(self) -> None:
|
||||||
|
text = 'value\r\n" keep me "\r\n0.123456789012345678901234567890\r\n" "\r\n'
|
||||||
|
source = TabularCsvSource(text=text, value_mode="text")
|
||||||
|
request = DatasourceStageInput(name="Text", source_name="csv_text", kind="upload", mode="static", shape="tabular", rows=parse_tabular_csv(text, value_mode="text"), csv_source=source)
|
||||||
|
with self.assertRaises(DatasourceValidationError):
|
||||||
|
self.provider.create_stage(self.session, principal(), stage=replace(request, rows=({"value": "changed"},)))
|
||||||
|
stage = self.provider.create_stage(self.session, principal(), stage=request)
|
||||||
|
descriptor, materialization = self.provider.promote_stage(self.session, principal(), stage_ref=stage.ref)
|
||||||
|
self.session.commit()
|
||||||
|
self.session.expunge_all()
|
||||||
|
admin = principal(scopes=(ADMIN_SCOPE,))
|
||||||
|
self.assertEqual(text, self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref))
|
||||||
|
from govoplan_datasources.backend.router import api_original_csv
|
||||||
|
with patch("govoplan_datasources.backend.router._provider", return_value=self.provider), patch("govoplan_datasources.backend.router._audit") as audit:
|
||||||
|
response = api_original_csv(descriptor.ref.split(":", 1)[1], materialization.ref.split(":", 1)[1], session=self.session, principal=admin)
|
||||||
|
self.assertEqual(text.encode("utf-8"), response.body)
|
||||||
|
self.assertEqual("no-store", response.headers["cache-control"])
|
||||||
|
self.assertEqual("datasources.original_csv.exported", audit.call_args.kwargs["action"])
|
||||||
|
self.assertNotIn(text, repr(audit.call_args.kwargs["details"]))
|
||||||
|
with patch("govoplan_datasources.backend.router._provider") as read, self.assertRaises(HTTPException) as denied:
|
||||||
|
api_original_csv("anything", "anything", session=self.session, principal=principal())
|
||||||
|
self.assertEqual(403, denied.exception.status_code)
|
||||||
|
read.assert_not_called()
|
||||||
|
frozen = self.provider.freeze_datasource(self.session, admin, datasource_ref=descriptor.ref)
|
||||||
|
self.assertEqual(text, self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=frozen.ref))
|
||||||
|
self.assertNotIn("text", descriptor.metadata["csv_source"])
|
||||||
|
self.assertNotIn("text", materialization.metadata["csv_source"])
|
||||||
|
with self.assertRaises(DatasourceAccessError):
|
||||||
|
self.provider.original_csv(self.session, principal(), datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
|
||||||
|
from govoplan_core.core.datasources import DatasourceNotFoundError
|
||||||
|
with self.assertRaises(DatasourceNotFoundError):
|
||||||
|
self.provider.original_csv(self.session, principal("other-tenant", scopes=(ADMIN_SCOPE,)), datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
|
||||||
|
record = self.session.get(DatasourceRecord, descriptor.ref.split(":", 1)[1])
|
||||||
|
record.visibility_policy = {"fields": {"value": {"classification": "restricted", "action": "omit", "allow": {"account_ids": ["other"]}}}}
|
||||||
|
self.session.flush()
|
||||||
|
with self.assertRaises(DatasourceAccessError):
|
||||||
|
self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
|
||||||
|
record.visibility_policy = {}
|
||||||
|
stored = self.session.get(DatasourceMaterializationRecord, materialization.ref.split(":", 1)[1])
|
||||||
|
stored.governance_snapshot_ = {**stored.governance_snapshot_, "visibility_policy": {"row_filters": [{"field": "value", "claim": "account_id"}]}}
|
||||||
|
self.session.flush()
|
||||||
|
with self.assertRaises(DatasourceAccessError):
|
||||||
|
self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
|
||||||
|
stored.governance_snapshot_ = {**stored.governance_snapshot_, "visibility_policy": {}}
|
||||||
|
stored.csv_source_ = {**stored.csv_source_, "text": "changed"}
|
||||||
|
self.session.flush()
|
||||||
|
with self.assertRaises(DatasourceUnavailableError):
|
||||||
|
self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
|
||||||
|
|
||||||
|
def test_csv_stage_projection_binding_rejects_equal_but_different_types(self) -> None:
|
||||||
|
for text, value in (("value\ntrue\n", 1), ("value\n1\n", True), ("value\n1\n", 1.0)):
|
||||||
|
with self.subTest(text=text, value=value), self.assertRaises(DatasourceValidationError):
|
||||||
|
self.provider.create_stage(self.session, principal(), stage=DatasourceStageInput(name="Invalid", source_name="invalid", kind="upload", mode="static", shape="tabular", rows=({"value": value},), csv_source=TabularCsvSource(text=text)))
|
||||||
|
|
||||||
|
def test_csv_invalid_unicode_is_validation_in_both_modes(self) -> None:
|
||||||
|
from govoplan_datasources.backend.tabular import parse_csv_rows
|
||||||
|
for mode in ("text", "legacy_typed"):
|
||||||
|
with self.subTest(mode=mode), self.assertRaisesRegex(DatasourceValidationError, "valid Unicode"):
|
||||||
|
parse_csv_rows("value\nprivate-\ud800\n", value_mode=mode)
|
||||||
|
|
||||||
|
def test_freezing_csv_preserves_historical_restrictions_and_rejects_resealed_drift(self) -> None:
|
||||||
|
source = TabularCsvSource(text='value\nprivate\n', value_mode="text")
|
||||||
|
governance = DatasourceGovernance(visibility_policy={"fields": {"value": {"classification": "restricted", "action": "omit", "allow": {"account_ids": ["other"]}}}})
|
||||||
|
stage = self.provider.create_stage(self.session, principal(), stage=DatasourceStageInput(name="Private CSV", source_name="private_csv", kind="upload", mode="static", shape="tabular", rows=parse_tabular_csv(source.text, value_mode="text"), csv_source=source, governance=governance))
|
||||||
|
descriptor, original = self.provider.promote_stage(self.session, principal(), stage_ref=stage.ref)
|
||||||
|
record = self.session.get(DatasourceRecord, descriptor.ref.split(":", 1)[1])
|
||||||
|
record.visibility_policy = {}
|
||||||
|
self.session.flush()
|
||||||
|
admin = principal(scopes=(ADMIN_SCOPE,))
|
||||||
|
frozen = self.provider.freeze_datasource(self.session, admin, datasource_ref=descriptor.ref)
|
||||||
|
for ref in (original.ref, frozen.ref):
|
||||||
|
with self.subTest(ref=ref), self.assertRaises(DatasourceAccessError):
|
||||||
|
self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=ref)
|
||||||
|
stored = self.session.get(DatasourceMaterializationRecord, original.ref.split(":", 1)[1])
|
||||||
|
stored.csv_source_ = csv_source_payload(TabularCsvSource(text="value\nchanged\n", value_mode="text"))
|
||||||
|
self.session.flush()
|
||||||
|
with self.assertRaises(DatasourceUnavailableError):
|
||||||
|
self.provider.freeze_datasource(self.session, admin, datasource_ref=descriptor.ref)
|
||||||
|
|
||||||
def test_static_stage_promote_update_and_frozen_read(self) -> None:
|
def test_static_stage_promote_update_and_frozen_read(self) -> None:
|
||||||
first_stage = self.provider.create_stage(
|
first_stage = self.provider.create_stage(
|
||||||
self.session,
|
self.session,
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from govoplan_core.core.tabular_sources import TabularCsvSource
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
@@ -345,6 +347,7 @@ class DatasourceLifecycleGovernanceTests(unittest.TestCase):
|
|||||||
mode="static",
|
mode="static",
|
||||||
shape="tabular",
|
shape="tabular",
|
||||||
rows=({"id": 1},),
|
rows=({"id": 1},),
|
||||||
|
csv_source=TabularCsvSource(text="id\n1\n"),
|
||||||
governance=governance,
|
governance=governance,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -389,6 +392,7 @@ class DatasourceLifecycleGovernanceTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
payload_id = first_row.payload_id
|
payload_id = first_row.payload_id
|
||||||
|
self.assertIsNotNone(first_row.csv_source_)
|
||||||
disposed, evidence_hashes = self.provider.apply_retention(
|
disposed, evidence_hashes = self.provider.apply_retention(
|
||||||
self.session,
|
self.session,
|
||||||
principal("admin", scopes=("datasources:source:admin",)),
|
principal("admin", scopes=("datasources:source:admin",)),
|
||||||
@@ -400,6 +404,7 @@ class DatasourceLifecycleGovernanceTests(unittest.TestCase):
|
|||||||
self.assertEqual(1, len(evidence_hashes))
|
self.assertEqual(1, len(evidence_hashes))
|
||||||
self.assertIsNone(self.session.get(DatasourcePayloadRecord, payload_id))
|
self.assertIsNone(self.session.get(DatasourcePayloadRecord, payload_id))
|
||||||
self.assertIsNotNone(first_row.disposed_at)
|
self.assertIsNotNone(first_row.disposed_at)
|
||||||
|
self.assertIsNone(first_row.csv_source_)
|
||||||
self.assertEqual("disposed", first_row.state)
|
self.assertEqual("disposed", first_row.state)
|
||||||
with self.assertRaises(DatasourceUnavailableError):
|
with self.assertRaises(DatasourceUnavailableError):
|
||||||
self.provider.read_datasource(
|
self.provider.read_datasource(
|
||||||
@@ -446,6 +451,7 @@ class DatasourceLifecycleGovernanceTests(unittest.TestCase):
|
|||||||
mode="static",
|
mode="static",
|
||||||
shape="tabular",
|
shape="tabular",
|
||||||
rows=({"id": 1},),
|
rows=({"id": 1},),
|
||||||
|
csv_source=TabularCsvSource(text="id\n1\n"),
|
||||||
governance=DatasourceGovernance(
|
governance=DatasourceGovernance(
|
||||||
retention_policy={
|
retention_policy={
|
||||||
"version": "stage-retention-v1",
|
"version": "stage-retention-v1",
|
||||||
|
|||||||
@@ -14,6 +14,47 @@ from govoplan_datasources.backend.manifest import get_manifest
|
|||||||
|
|
||||||
|
|
||||||
class DatasourceMigrationTests(unittest.TestCase):
|
class DatasourceMigrationTests(unittest.TestCase):
|
||||||
|
def test_csv_evidence_upgrade_preserves_legacy_rows_and_fingerprints(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-datasources-csv-migration-") as directory:
|
||||||
|
url = f"sqlite:///{Path(directory) / 'datasources.db'}"
|
||||||
|
config = self._config(url)
|
||||||
|
command.upgrade(config, "d1a7c3e9f5b2")
|
||||||
|
engine = create_engine(url)
|
||||||
|
try:
|
||||||
|
metadata = MetaData()
|
||||||
|
tables = {name: Table(name, metadata, autoload_with=engine) for name in (
|
||||||
|
"datasource_catalogue", "datasource_stages", "datasource_materializations",
|
||||||
|
)}
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
common = dict(tenant_id="tenant-1", schema=[], fingerprint="a" * 64,
|
||||||
|
metadata={"retained": True}, provenance={}, created_at=now, updated_at=now)
|
||||||
|
with engine.begin() as connection:
|
||||||
|
connection.execute(tables["datasource_catalogue"].insert().values(
|
||||||
|
**common, id="legacy-source", source_name="legacy", name="Legacy", kind="upload",
|
||||||
|
mode="static", shape="tabular", status="active", schema_version=1,
|
||||||
|
))
|
||||||
|
connection.execute(tables["datasource_stages"].insert().values(
|
||||||
|
**common, id="legacy-stage", source_name="staged", name="Staged", kind="upload",
|
||||||
|
mode="static", shape="tabular", state="ready", rows=[{"id": 1}],
|
||||||
|
row_count=1, byte_count=10, validation={}, governance={}, approval={},
|
||||||
|
))
|
||||||
|
connection.execute(tables["datasource_materializations"].insert().values(
|
||||||
|
**common, id="legacy-materialization", datasource_id="legacy-source", revision=1,
|
||||||
|
state="published", schema_version=1, rows=[{"id": 1}], row_count=1,
|
||||||
|
byte_count=10, governance_snapshot={},
|
||||||
|
))
|
||||||
|
before = {name: dict(connection.execute(select(table)).mappings().one()) for name, table in tables.items()}
|
||||||
|
command.upgrade(config, "e2b8d4a0f6c3")
|
||||||
|
with engine.connect() as connection:
|
||||||
|
for name in tables:
|
||||||
|
upgraded = Table(name, MetaData(), autoload_with=connection)
|
||||||
|
after = dict(connection.execute(select(upgraded)).mappings().one())
|
||||||
|
if name != "datasource_catalogue":
|
||||||
|
self.assertIsNone(after.pop("csv_source"))
|
||||||
|
self.assertEqual(before[name], after)
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _config(url: str):
|
def _config(url: str):
|
||||||
return alembic_config(
|
return alembic_config(
|
||||||
@@ -34,7 +75,7 @@ class DatasourceMigrationTests(unittest.TestCase):
|
|||||||
try:
|
try:
|
||||||
with engine.connect() as connection:
|
with engine.connect() as connection:
|
||||||
self.assertIn(
|
self.assertIn(
|
||||||
"d1a7c3e9f5b2",
|
"e2b8d4a0f6c3",
|
||||||
set(MigrationContext.configure(connection).get_current_heads()),
|
set(MigrationContext.configure(connection).get_current_heads()),
|
||||||
)
|
)
|
||||||
catalogue_columns = {
|
catalogue_columns = {
|
||||||
|
|||||||
+4
-3
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@govoplan/datasources-webui",
|
"name": "@govoplan/datasources-webui",
|
||||||
"version": "0.1.23",
|
"version": "0.1.26",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "src/index.ts",
|
"main": "src/index.ts",
|
||||||
@@ -14,10 +14,11 @@
|
|||||||
"./styles/datasources.css": "./src/styles/datasources.css"
|
"./styles/datasources.css": "./src/styles/datasources.css"
|
||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"typecheck": "tsc --noEmit"
|
"typecheck": "tsc --noEmit",
|
||||||
|
"test:detail-refresh": "node --test scripts/test-detail-refresh.mjs"
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"@govoplan/core-webui": "^0.1.18",
|
"@govoplan/core-webui": "^0.1.46",
|
||||||
"lucide-react": "^1.23.0",
|
"lucide-react": "^1.23.0",
|
||||||
"react": ">=19.2.7 <20",
|
"react": ">=19.2.7 <20",
|
||||||
"react-dom": ">=19.2.7 <20",
|
"react-dom": ">=19.2.7 <20",
|
||||||
|
|||||||
Executable
+258
@@ -0,0 +1,258 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { createRequire } from "node:module";
|
||||||
|
import test from "node:test";
|
||||||
|
|
||||||
|
const require = createRequire(new URL("../../../govoplan-core/webui/package.json", import.meta.url));
|
||||||
|
const { transformSync } = require("esbuild");
|
||||||
|
const page = readFileSync(new URL("../src/features/datasources/DatasourcesPage.tsx", import.meta.url), "utf8");
|
||||||
|
function extract(startMarker, endMarker) {
|
||||||
|
const start = page.indexOf(startMarker);
|
||||||
|
const end = page.indexOf(endMarker, start);
|
||||||
|
assert.ok(start >= 0 && end > start, "test the actual page closures");
|
||||||
|
return page.slice(start + startMarker.length, end);
|
||||||
|
}
|
||||||
|
const reloadBody = extract(" const reload = useCallback(async (preferredDatasourceRef?: string) => {", " }, [dialogScope]);");
|
||||||
|
const detailBody = extract(" useEffect(() => {\n if (!selectedDatasourceRef || loading || !catalogueReady)", " }, [selectedDatasourceRef, detailScope, loading, catalogueReady]);");
|
||||||
|
function evaluate(body, bindings) {
|
||||||
|
const code = transformSync(body, { loader: "ts", target: "es2022" }).code;
|
||||||
|
return new Function(...Object.keys(bindings), code)(...Object.values(bindings));
|
||||||
|
}
|
||||||
|
const deferred = () => {
|
||||||
|
let resolve, reject;
|
||||||
|
const promise = new Promise((yes, no) => { resolve = yes; reject = no; });
|
||||||
|
return { promise, resolve, reject };
|
||||||
|
};
|
||||||
|
const settle = async () => { await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); };
|
||||||
|
function harness() {
|
||||||
|
const state = { selectedDatasourceRef: "source-1", loading: false, detailRevision: 0, preview: null, materializations: [], detailLoading: false, catalogueAuthority: 0 };
|
||||||
|
const settings = {};
|
||||||
|
const auth = {};
|
||||||
|
const bindings = {
|
||||||
|
settings, auth, reloadRequestId: { current: 0 }, authority: { current: "authority-A" }, authorityKey: "authority-A", dialogScope: 0, authorityEpoch: { current: { revision: 0 } },
|
||||||
|
currentDetailScope: { current: null }, apiErrorMessage: String,
|
||||||
|
listDatasources: async () => [{ ref: "source-1" }], listDatasourceStages: async () => [],
|
||||||
|
listDatasourceOrigins: async () => ({ origins: [], available: false })
|
||||||
|
};
|
||||||
|
bindings.isCurrentAuthority = () => bindings.authority.current === "authority-A" && bindings.authorityEpoch.current.revision === 0;
|
||||||
|
for (const key of ["loading", "error", "datasources", "stages", "origins", "originsAvailable", "detailRevision", "selectedDatasourceRef", "selectedStageRef", "selectedOriginRef", "preview", "materializations", "detailLoading", "detailResponseScope", "catalogueAuthority", "working", "success", "view", "addOpen", "freezeOpen", "freezeLabel", "retireOpen", "promoteOpen", "governanceOpen", "decisionOpen", "retentionOpen"]) {
|
||||||
|
bindings[`set${key[0].toUpperCase()}${key.slice(1)}`] = (value) => { state[key] = typeof value === "function" ? value(state[key]) : value; };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
state, bindings,
|
||||||
|
reload: (preferredDatasourceRef) => evaluate(`return (async () => {${reloadBody}})();`, { ...bindings, preferredDatasourceRef }),
|
||||||
|
startDetail: () => {
|
||||||
|
const preview = deferred();
|
||||||
|
const history = deferred();
|
||||||
|
const scope = {};
|
||||||
|
bindings.currentDetailScope.current = scope;
|
||||||
|
const cleanup = evaluate(`if (!selectedDatasourceRef || loading || !catalogueReady)${detailBody}`, {
|
||||||
|
...bindings, selectedDatasourceRef: state.selectedDatasourceRef, loading: state.loading, detailScope: scope,
|
||||||
|
catalogueReady: state.catalogueAuthority === bindings.authorityEpoch.current.revision,
|
||||||
|
previewDatasource: () => preview.promise, listDatasourceMaterializations: () => history.promise
|
||||||
|
});
|
||||||
|
return { preview, history, scope, cleanup };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test("same-reference reload invalidates preview/history once, including freeze with an unchanged current materialization", async () => {
|
||||||
|
const h = harness();
|
||||||
|
for (const operation of ["manual reload", "refresh", "freeze", "promotion"]) {
|
||||||
|
const previous = h.state.detailRevision;
|
||||||
|
await h.reload("source-1");
|
||||||
|
assert.equal(h.state.selectedDatasourceRef, "source-1", operation);
|
||||||
|
assert.equal(h.state.detailRevision, previous + 1, operation);
|
||||||
|
const detail = h.startDetail();
|
||||||
|
detail.preview.resolve({ rows: [{ value: operation }] });
|
||||||
|
detail.history.resolve([{ revision: h.state.detailRevision }]);
|
||||||
|
await settle();
|
||||||
|
assert.equal(h.state.preview.rows[0].value, operation);
|
||||||
|
assert.equal(h.state.materializations[0].revision, h.state.detailRevision);
|
||||||
|
assert.equal(h.state.detailResponseScope, detail.scope);
|
||||||
|
detail.cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a changed scope immediately suppresses old preview, history and errors, even before effect cleanup", async () => {
|
||||||
|
const h = harness();
|
||||||
|
const old = h.startDetail();
|
||||||
|
const current = h.startDetail();
|
||||||
|
current.preview.resolve({ rows: [{ value: "current" }] });
|
||||||
|
current.history.resolve([{ revision: 2 }]);
|
||||||
|
await settle();
|
||||||
|
old.preview.reject(new Error("stale error"));
|
||||||
|
old.history.resolve([{ revision: 1 }]);
|
||||||
|
await settle();
|
||||||
|
assert.equal(h.state.preview.rows[0].value, "current");
|
||||||
|
assert.equal(h.state.materializations[0].revision, 2);
|
||||||
|
assert.equal(h.state.error, undefined);
|
||||||
|
current.cleanup();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("unmount cancellation and partial detail failures preserve allSettled behavior", async () => {
|
||||||
|
const h = harness();
|
||||||
|
const cancelled = h.startDetail();
|
||||||
|
cancelled.cleanup();
|
||||||
|
cancelled.preview.resolve({ rows: ["obsolete"] });
|
||||||
|
cancelled.history.resolve([]);
|
||||||
|
await settle();
|
||||||
|
assert.equal(h.state.preview, null);
|
||||||
|
const current = h.startDetail();
|
||||||
|
current.preview.reject(new Error("current preview denied"));
|
||||||
|
current.history.resolve([{ revision: 3 }]);
|
||||||
|
await settle();
|
||||||
|
assert.equal(h.state.preview, null);
|
||||||
|
assert.equal(h.state.materializations[0].revision, 3);
|
||||||
|
assert.match(h.state.error, /current preview denied/);
|
||||||
|
assert.equal(h.state.detailLoading, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("initial/loading catalogue does not issue duplicate detail reads; scoped rendering cannot flash old rows", () => {
|
||||||
|
const h = harness();
|
||||||
|
h.state.loading = true;
|
||||||
|
const detail = h.startDetail();
|
||||||
|
assert.equal(detail.cleanup, undefined);
|
||||||
|
assert.equal(h.state.preview, null);
|
||||||
|
assert.match(page, /useMemo\(\(\) => \(\{\}\), \[selectedDatasourceRef, dialogScope, detailRevision\]\)/);
|
||||||
|
assert.match(page, /preview=\{!loading && detailResponseScope === detailScope \? preview : null\}/);
|
||||||
|
assert.match(page, /materializations=\{!loading && detailResponseScope === detailScope \? materializations : \[\]\}/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("older catalogue reloads and revoked authority cannot invalidate or replace the current catalogue", async () => {
|
||||||
|
const h = harness();
|
||||||
|
const older = deferred();
|
||||||
|
h.bindings.listDatasources = () => older.promise;
|
||||||
|
const pending = h.reload();
|
||||||
|
h.bindings.listDatasources = async () => [{ ref: "source-1", revision: 2 }];
|
||||||
|
await h.reload();
|
||||||
|
older.resolve([{ ref: "source-1", revision: 1 }]);
|
||||||
|
await pending;
|
||||||
|
assert.equal(h.state.datasources[0].revision, 2);
|
||||||
|
assert.equal(h.state.detailRevision, 1);
|
||||||
|
const revoked = deferred();
|
||||||
|
h.bindings.listDatasources = () => revoked.promise;
|
||||||
|
const revokedPending = h.reload();
|
||||||
|
h.bindings.authority.current = "authority-B";
|
||||||
|
revoked.resolve([{ ref: "source-1", revision: 3 }]);
|
||||||
|
await revokedPending;
|
||||||
|
assert.equal(h.state.datasources[0].revision, 2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an old reload invoked only after B is active cannot issue requests or invalidate B state", async () => {
|
||||||
|
const h = harness();
|
||||||
|
const oldReload = h.reload;
|
||||||
|
h.bindings.authority.current = "authority-B";
|
||||||
|
h.bindings.authorityEpoch.current.revision = 1;
|
||||||
|
h.state.catalogueAuthority = 1;
|
||||||
|
h.state.datasources = [{ ref: "B-source" }];
|
||||||
|
h.state.detailRevision = 8;
|
||||||
|
let calls = 0;
|
||||||
|
h.bindings.listDatasources = async () => { calls += 1; return [{ ref: "A-source" }]; };
|
||||||
|
const requestId = h.bindings.reloadRequestId.current;
|
||||||
|
await oldReload();
|
||||||
|
assert.equal(calls, 0);
|
||||||
|
assert.equal(h.bindings.reloadRequestId.current, requestId);
|
||||||
|
assert.equal(h.state.datasources[0].ref, "B-source");
|
||||||
|
assert.equal(h.state.detailRevision, 8);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the first authority-transition render cannot request details from a stale catalogue", () => {
|
||||||
|
const h = harness();
|
||||||
|
h.bindings.authority.current = "authority-B";
|
||||||
|
h.bindings.authorityEpoch.current.revision = 1;
|
||||||
|
h.state.loading = false; // The new reload's effect has not committed loading=true yet.
|
||||||
|
const detail = h.startDetail();
|
||||||
|
assert.equal(detail.cleanup, undefined);
|
||||||
|
assert.equal(h.state.detailLoading, false);
|
||||||
|
assert.match(page, /const datasources = catalogueReady \? loadedDatasources : \[\]/);
|
||||||
|
assert.match(page, /const stages = catalogueReady \? loadedStages : \[\]/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("real-authority setup closes confirmations and clears freeze input; unmount blocks old callbacks", () => {
|
||||||
|
const h = harness();
|
||||||
|
const body = extract(" useEffect(() => {\n authority.current = authorityKey;", " }, [reload]);");
|
||||||
|
const flags = ["addOpen", "freezeOpen", "retireOpen", "promoteOpen", "governanceOpen", "decisionOpen", "retentionOpen"];
|
||||||
|
for (const flag of flags) {
|
||||||
|
h.state[flag] = true;
|
||||||
|
// Hide the old intent already in the first transition render, before the
|
||||||
|
// passive effect clears it; it cannot be clicked against a new selection.
|
||||||
|
assert.ok(page.includes(`open={${flag} && catalogueReady}`), flag);
|
||||||
|
}
|
||||||
|
h.state.freezeLabel = "Evidence from prior authority";
|
||||||
|
let reloads = 0;
|
||||||
|
const cleanup = evaluate(`authority.current = authorityKey;${body}`, {
|
||||||
|
...h.bindings, reload: () => { reloads += 1; }
|
||||||
|
});
|
||||||
|
assert.equal(reloads, 1);
|
||||||
|
for (const flag of flags) assert.equal(h.state[flag], false, flag);
|
||||||
|
assert.equal(h.state.freezeLabel, "");
|
||||||
|
assert.equal(h.bindings.isCurrentAuthority(), true);
|
||||||
|
cleanup();
|
||||||
|
assert.equal(h.bindings.isCurrentAuthority(), false);
|
||||||
|
// The reload callback/effect depend on the stable numeric authority epoch,
|
||||||
|
// not cosmetic settings/auth object identities.
|
||||||
|
assert.match(page, /\}, \[dialogScope\]\);\s*useEffect/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("stale mutation success/failure/finally cannot reload, select or overwrite B messages", async () => {
|
||||||
|
for (const operation of ["promoteStage", "refreshSelected", "freezeSelected", "retireSelected"]) {
|
||||||
|
for (const outcome of ["resolve", "reject"]) {
|
||||||
|
const h = harness();
|
||||||
|
const pending = deferred();
|
||||||
|
let reloads = 0;
|
||||||
|
const signature = operation === "freezeSelected" ? "async (): Promise<boolean>" : "async ()";
|
||||||
|
const body = extract(` const ${operation} = ${signature} => {`, "\n };");
|
||||||
|
const result = evaluate(`return (async () => {${body}})();`, {
|
||||||
|
...h.bindings,
|
||||||
|
selectedDatasource: { ref: "A-source", governance: { approval_policy: { required: false } } },
|
||||||
|
selectedStage: { ref: "A-stage", state: "ready" }, freezeLabel: "Freeze A",
|
||||||
|
reload: async () => { reloads += 1; },
|
||||||
|
promoteDatasourceStage: () => pending.promise, refreshDatasource: () => pending.promise,
|
||||||
|
freezeDatasource: () => pending.promise, retireDatasource: () => pending.promise
|
||||||
|
});
|
||||||
|
h.bindings.authority.current = "authority-B";
|
||||||
|
Object.assign(h.state, { success: "B success", error: "B error", view: "origins", working: true });
|
||||||
|
if (outcome === "reject") pending.reject(new Error("A error"));
|
||||||
|
else pending.resolve({ datasource: { ref: "A-source", name: "A" }, materialization: { revision: 1 }, revision: 1 });
|
||||||
|
await result;
|
||||||
|
assert.equal(reloads, 0, operation);
|
||||||
|
assert.equal(h.state.success, "B success", operation);
|
||||||
|
assert.equal(h.state.error, "B error", operation);
|
||||||
|
assert.equal(h.state.view, "origins", operation);
|
||||||
|
assert.equal(h.state.working, true, operation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("CSV stage dialog sends its explicit mode and exact text; JSON rows remain independent", async () => {
|
||||||
|
const dialog = page.slice(page.indexOf("function AddDatasourceDialog("));
|
||||||
|
const start = dialog.indexOf(" const create = async (): Promise<boolean> => {");
|
||||||
|
const end = dialog.indexOf("\n };", start);
|
||||||
|
assert.ok(start >= 0 && end > start);
|
||||||
|
const body = dialog.slice(start + " const create = async (): Promise<boolean> => {".length, end);
|
||||||
|
const csvText = 'code;value\r\n001;" text "\r\n';
|
||||||
|
for (const format of ["csv", "json"]) {
|
||||||
|
for (const csvValueMode of ["text", "legacy_typed"]) {
|
||||||
|
let payload;
|
||||||
|
const result = await evaluate(`return (async () => {${body}})();`, {
|
||||||
|
kind: "upload", settings: {}, format, csvValueMode, csvText, delimiter: ";",
|
||||||
|
mode: "static", name: "Fixture", sourceName: "fixture", description: "", targetRef: "",
|
||||||
|
rowsText: '[{"code":"001","value":" text "}]', parseRows: JSON.parse,
|
||||||
|
createDatasourceStage: async (_settings, value) => { payload = value; return {}; },
|
||||||
|
onCreated: async () => {}, setBusy: () => {}, setError: () => {}, apiErrorMessage: String
|
||||||
|
});
|
||||||
|
assert.equal(result, true);
|
||||||
|
if (format === "csv") {
|
||||||
|
assert.equal(payload.csv_value_mode, csvValueMode);
|
||||||
|
assert.equal(payload.csv_text, csvText);
|
||||||
|
} else {
|
||||||
|
assert.equal("csv_value_mode" in payload, false);
|
||||||
|
assert.deepEqual(payload.rows, [{ code: "001", value: " text " }]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.match(dialog, /\[csvValueMode, setCsvValueMode\] = useState<"text" \| "legacy_typed">\("text"\)/);
|
||||||
|
assert.match(dialog, /setCsvValueMode\("text"\)/);
|
||||||
|
assert.match(dialog, /csvValueMode: "text"/);
|
||||||
|
});
|
||||||
@@ -324,7 +324,7 @@ export function createDatasourceStage(
|
|||||||
governance?: DatasourceGovernance | null;
|
governance?: DatasourceGovernance | null;
|
||||||
} & (
|
} & (
|
||||||
{ format: "json"; rows: Record<string, unknown>[] }
|
{ format: "json"; rows: Record<string, unknown>[] }
|
||||||
| { format: "csv"; csv_text: string; delimiter: string }
|
| { format: "csv"; csv_text: string; delimiter: string; csv_value_mode?: "text" | "legacy_typed" }
|
||||||
)
|
)
|
||||||
): Promise<DatasourceStage> {
|
): Promise<DatasourceStage> {
|
||||||
return apiFetch(settings, "/api/v1/datasources/stages", {
|
return apiFetch(settings, "/api/v1/datasources/stages", {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import {
|
|||||||
useCallback,
|
useCallback,
|
||||||
useEffect,
|
useEffect,
|
||||||
useMemo,
|
useMemo,
|
||||||
|
useRef,
|
||||||
useState
|
useState
|
||||||
} from "react";
|
} from "react";
|
||||||
import {
|
import {
|
||||||
@@ -43,6 +44,7 @@ import { FormGrid, DialogSection, ActionToolbar,
|
|||||||
WorkspaceFrame,
|
WorkspaceFrame,
|
||||||
WorkspaceLayout,
|
WorkspaceLayout,
|
||||||
hasScope,
|
hasScope,
|
||||||
|
authAuthorityKey,
|
||||||
isApiError,
|
isApiError,
|
||||||
useUnsavedChanges,
|
useUnsavedChanges,
|
||||||
useUnsavedDraftGuard,
|
useUnsavedDraftGuard,
|
||||||
@@ -96,10 +98,10 @@ export default function DatasourcesPage({
|
|||||||
auth: AuthInfo;
|
auth: AuthInfo;
|
||||||
}) {
|
}) {
|
||||||
const [view, setView] = useState<CatalogueView>("catalogue");
|
const [view, setView] = useState<CatalogueView>("catalogue");
|
||||||
const [datasources, setDatasources] = useState<Datasource[]>([]);
|
const [loadedDatasources, setDatasources] = useState<Datasource[]>([]);
|
||||||
const [stages, setStages] = useState<DatasourceStage[]>([]);
|
const [loadedStages, setStages] = useState<DatasourceStage[]>([]);
|
||||||
const [origins, setOrigins] = useState<DatasourceOrigin[]>([]);
|
const [loadedOrigins, setOrigins] = useState<DatasourceOrigin[]>([]);
|
||||||
const [originsAvailable, setOriginsAvailable] = useState(false);
|
const [loadedOriginsAvailable, setOriginsAvailable] = useState(false);
|
||||||
const [selectedDatasourceRef, setSelectedDatasourceRef] = useState(
|
const [selectedDatasourceRef, setSelectedDatasourceRef] = useState(
|
||||||
initialDatasourceRef
|
initialDatasourceRef
|
||||||
);
|
);
|
||||||
@@ -110,6 +112,28 @@ export default function DatasourcesPage({
|
|||||||
const [search, setSearch] = useState("");
|
const [search, setSearch] = useState("");
|
||||||
const [loading, setLoading] = useState(true);
|
const [loading, setLoading] = useState(true);
|
||||||
const [detailLoading, setDetailLoading] = useState(false);
|
const [detailLoading, setDetailLoading] = useState(false);
|
||||||
|
const [detailRevision, setDetailRevision] = useState(0);
|
||||||
|
const authorityKey = authAuthorityKey(auth, settings);
|
||||||
|
const authority = useRef(authorityKey);
|
||||||
|
authority.current = authorityKey;
|
||||||
|
const authorityEpoch = useRef({ key: authorityKey, revision: 0 });
|
||||||
|
if (authorityEpoch.current.key !== authorityKey) {
|
||||||
|
authorityEpoch.current = { key: authorityKey, revision: authorityEpoch.current.revision + 1 };
|
||||||
|
}
|
||||||
|
const dialogScope = authorityEpoch.current.revision;
|
||||||
|
const isCurrentAuthority = () => authority.current === authorityKey
|
||||||
|
&& authorityEpoch.current.revision === dialogScope;
|
||||||
|
const [catalogueAuthority, setCatalogueAuthority] = useState<number | null>(null);
|
||||||
|
const catalogueReady = catalogueAuthority === dialogScope;
|
||||||
|
const datasources = catalogueReady ? loadedDatasources : [];
|
||||||
|
const stages = catalogueReady ? loadedStages : [];
|
||||||
|
const origins = catalogueReady ? loadedOrigins : [];
|
||||||
|
const originsAvailable = catalogueReady && loadedOriginsAvailable;
|
||||||
|
const detailScope = useMemo(() => ({}), [selectedDatasourceRef, dialogScope, detailRevision]);
|
||||||
|
const currentDetailScope = useRef(detailScope);
|
||||||
|
currentDetailScope.current = detailScope;
|
||||||
|
const [detailResponseScope, setDetailResponseScope] = useState<object | null>(null);
|
||||||
|
const reloadRequestId = useRef(0);
|
||||||
const [working, setWorking] = useState(false);
|
const [working, setWorking] = useState(false);
|
||||||
const [error, setError] = useState("");
|
const [error, setError] = useState("");
|
||||||
const [success, setSuccess] = useState("");
|
const [success, setSuccess] = useState("");
|
||||||
@@ -132,6 +156,12 @@ export default function DatasourcesPage({
|
|||||||
const canAdmin = hasScope(auth, "datasources:source:admin");
|
const canAdmin = hasScope(auth, "datasources:source:admin");
|
||||||
|
|
||||||
const reload = useCallback(async (preferredDatasourceRef?: string) => {
|
const reload = useCallback(async (preferredDatasourceRef?: string) => {
|
||||||
|
// An old mutation callback can invoke this closure AFTER authority changed.
|
||||||
|
// Compare the closure's scope before any request, state write or generation bump.
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
|
const requestId = ++reloadRequestId.current;
|
||||||
|
const isCurrent = () => requestId === reloadRequestId.current
|
||||||
|
&& isCurrentAuthority();
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
setError("");
|
setError("");
|
||||||
try {
|
try {
|
||||||
@@ -140,10 +170,15 @@ export default function DatasourcesPage({
|
|||||||
listDatasourceStages(settings),
|
listDatasourceStages(settings),
|
||||||
listDatasourceOrigins(settings)
|
listDatasourceOrigins(settings)
|
||||||
]);
|
]);
|
||||||
|
if (!isCurrent()) return;
|
||||||
|
setCatalogueAuthority(dialogScope);
|
||||||
setDatasources(nextDatasources);
|
setDatasources(nextDatasources);
|
||||||
setStages(nextStages);
|
setStages(nextStages);
|
||||||
setOrigins(originCatalogue.origins);
|
setOrigins(originCatalogue.origins);
|
||||||
setOriginsAvailable(originCatalogue.available);
|
setOriginsAvailable(originCatalogue.available);
|
||||||
|
// Reload also invalidates history/preview when the selected reference did
|
||||||
|
// not change (refresh, freeze, promotion, governance and manual reload).
|
||||||
|
setDetailRevision((current) => current + 1);
|
||||||
setSelectedDatasourceRef((current) => {
|
setSelectedDatasourceRef((current) => {
|
||||||
const preferred = preferredDatasourceRef || current;
|
const preferred = preferredDatasourceRef || current;
|
||||||
return nextDatasources.some((item) => item.ref === preferred)
|
return nextDatasources.some((item) => item.ref === preferred)
|
||||||
@@ -157,20 +192,38 @@ export default function DatasourcesPage({
|
|||||||
? current
|
? current
|
||||||
: originCatalogue.origins[0]?.ref ?? "");
|
: originCatalogue.origins[0]?.ref ?? "");
|
||||||
} catch (loadError) {
|
} catch (loadError) {
|
||||||
setError(apiErrorMessage(loadError));
|
if (isCurrent()) setError(apiErrorMessage(loadError));
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false);
|
if (isCurrent()) setLoading(false);
|
||||||
}
|
}
|
||||||
}, [settings]);
|
}, [dialogScope]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
authority.current = authorityKey;
|
||||||
|
setWorking(false);
|
||||||
|
setSuccess("");
|
||||||
|
// Confirmations and their input must not follow a selection into a new
|
||||||
|
// authority context. Equivalent session refreshes keep this effect stable.
|
||||||
|
setAddOpen(false);
|
||||||
|
setFreezeOpen(false);
|
||||||
|
setFreezeLabel("");
|
||||||
|
setRetireOpen(false);
|
||||||
|
setPromoteOpen(false);
|
||||||
|
setGovernanceOpen(false);
|
||||||
|
setDecisionOpen(false);
|
||||||
|
setRetentionOpen(false);
|
||||||
void reload();
|
void reload();
|
||||||
|
return () => {
|
||||||
|
reloadRequestId.current += 1;
|
||||||
|
if (authority.current === authorityKey) authority.current = "";
|
||||||
|
};
|
||||||
}, [reload]);
|
}, [reload]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!selectedDatasourceRef) {
|
if (!selectedDatasourceRef || loading || !catalogueReady) {
|
||||||
setPreview(null);
|
setPreview(null);
|
||||||
setMaterializations([]);
|
setMaterializations([]);
|
||||||
|
setDetailLoading(false);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let cancelled = false;
|
let cancelled = false;
|
||||||
@@ -179,7 +232,8 @@ export default function DatasourcesPage({
|
|||||||
previewDatasource(settings, selectedDatasourceRef),
|
previewDatasource(settings, selectedDatasourceRef),
|
||||||
listDatasourceMaterializations(settings, selectedDatasourceRef)
|
listDatasourceMaterializations(settings, selectedDatasourceRef)
|
||||||
]).then(([previewResult, materializationResult]) => {
|
]).then(([previewResult, materializationResult]) => {
|
||||||
if (cancelled) return;
|
if (cancelled || currentDetailScope.current !== detailScope) return;
|
||||||
|
setDetailResponseScope(detailScope);
|
||||||
if (previewResult.status === "fulfilled") {
|
if (previewResult.status === "fulfilled") {
|
||||||
setPreview(previewResult.value);
|
setPreview(previewResult.value);
|
||||||
} else {
|
} else {
|
||||||
@@ -196,7 +250,7 @@ export default function DatasourcesPage({
|
|||||||
return () => {
|
return () => {
|
||||||
cancelled = true;
|
cancelled = true;
|
||||||
};
|
};
|
||||||
}, [selectedDatasourceRef, settings]);
|
}, [selectedDatasourceRef, detailScope, loading, catalogueReady]);
|
||||||
|
|
||||||
const selectedDatasource = datasources.find((item) => item.ref === selectedDatasourceRef) ?? null;
|
const selectedDatasource = datasources.find((item) => item.ref === selectedDatasourceRef) ?? null;
|
||||||
const selectedStage = stages.find((item) => item.ref === selectedStageRef) ?? null;
|
const selectedStage = stages.find((item) => item.ref === selectedStageRef) ?? null;
|
||||||
@@ -219,46 +273,50 @@ export default function DatasourcesPage({
|
|||||||
);
|
);
|
||||||
|
|
||||||
const promoteStage = async () => {
|
const promoteStage = async () => {
|
||||||
if (!selectedStage || selectedStage.state !== "ready") return;
|
if (!isCurrentAuthority() || !selectedStage || selectedStage.state !== "ready") return;
|
||||||
setWorking(true);
|
setWorking(true);
|
||||||
setError("");
|
setError("");
|
||||||
try {
|
try {
|
||||||
const result = await promoteDatasourceStage(settings, selectedStage.ref);
|
const result = await promoteDatasourceStage(settings, selectedStage.ref);
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
setSuccess(`Promoted ${result.datasource.name} as revision ${result.materialization.revision}.`);
|
setSuccess(`Promoted ${result.datasource.name} as revision ${result.materialization.revision}.`);
|
||||||
setView("catalogue");
|
setView("catalogue");
|
||||||
await reload(result.datasource.ref);
|
await reload(result.datasource.ref);
|
||||||
} catch (operationError) {
|
} catch (operationError) {
|
||||||
setError(apiErrorMessage(operationError));
|
if (isCurrentAuthority()) setError(apiErrorMessage(operationError));
|
||||||
} finally {
|
} finally {
|
||||||
setWorking(false);
|
if (isCurrentAuthority()) setWorking(false);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const refreshSelected = async () => {
|
const refreshSelected = async () => {
|
||||||
if (!selectedDatasource) return;
|
if (!isCurrentAuthority() || !selectedDatasource) return;
|
||||||
setWorking(true);
|
setWorking(true);
|
||||||
setError("");
|
setError("");
|
||||||
try {
|
try {
|
||||||
if (selectedDatasource.governance.approval_policy.required === true) {
|
if (selectedDatasource.governance.approval_policy.required === true) {
|
||||||
const stage = await prepareDatasourceRefresh(settings, selectedDatasource.ref);
|
const stage = await prepareDatasourceRefresh(settings, selectedDatasource.ref);
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
await reload(selectedDatasource.ref);
|
await reload(selectedDatasource.ref);
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
setSelectedStageRef(stage.ref);
|
setSelectedStageRef(stage.ref);
|
||||||
setView("staging");
|
setView("staging");
|
||||||
setSuccess(`Prepared refresh stage ${stage.name}. It must be approved before promotion.`);
|
setSuccess(`Prepared refresh stage ${stage.name}. It must be approved before promotion.`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const result = await refreshDatasource(settings, selectedDatasource.ref);
|
const result = await refreshDatasource(settings, selectedDatasource.ref);
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
setSuccess(`Refreshed ${result.datasource.name} as revision ${result.materialization.revision}.`);
|
setSuccess(`Refreshed ${result.datasource.name} as revision ${result.materialization.revision}.`);
|
||||||
await reload(result.datasource.ref);
|
await reload(result.datasource.ref);
|
||||||
} catch (operationError) {
|
} catch (operationError) {
|
||||||
setError(apiErrorMessage(operationError));
|
if (isCurrentAuthority()) setError(apiErrorMessage(operationError));
|
||||||
} finally {
|
} finally {
|
||||||
setWorking(false);
|
if (isCurrentAuthority()) setWorking(false);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const freezeSelected = async (): Promise<boolean> => {
|
const freezeSelected = async (): Promise<boolean> => {
|
||||||
if (!selectedDatasource) return false;
|
if (!isCurrentAuthority() || !selectedDatasource) return false;
|
||||||
setWorking(true);
|
setWorking(true);
|
||||||
setError("");
|
setError("");
|
||||||
try {
|
try {
|
||||||
@@ -267,32 +325,34 @@ export default function DatasourcesPage({
|
|||||||
selectedDatasource.ref,
|
selectedDatasource.ref,
|
||||||
freezeLabel
|
freezeLabel
|
||||||
);
|
);
|
||||||
|
if (!isCurrentAuthority()) return false;
|
||||||
setSuccess(`Created frozen revision ${materialization.revision}.`);
|
setSuccess(`Created frozen revision ${materialization.revision}.`);
|
||||||
setFreezeOpen(false);
|
setFreezeOpen(false);
|
||||||
setFreezeLabel("");
|
setFreezeLabel("");
|
||||||
await reload(selectedDatasource.ref);
|
await reload(selectedDatasource.ref);
|
||||||
return true;
|
return isCurrentAuthority();
|
||||||
} catch (operationError) {
|
} catch (operationError) {
|
||||||
setError(apiErrorMessage(operationError));
|
if (isCurrentAuthority()) setError(apiErrorMessage(operationError));
|
||||||
return false;
|
return false;
|
||||||
} finally {
|
} finally {
|
||||||
setWorking(false);
|
if (isCurrentAuthority()) setWorking(false);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const retireSelected = async () => {
|
const retireSelected = async () => {
|
||||||
if (!selectedDatasource) return;
|
if (!isCurrentAuthority() || !selectedDatasource) return;
|
||||||
setWorking(true);
|
setWorking(true);
|
||||||
setError("");
|
setError("");
|
||||||
try {
|
try {
|
||||||
await retireDatasource(settings, selectedDatasource.ref);
|
await retireDatasource(settings, selectedDatasource.ref);
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
setSuccess(`Retired ${selectedDatasource.name}.`);
|
setSuccess(`Retired ${selectedDatasource.name}.`);
|
||||||
setRetireOpen(false);
|
setRetireOpen(false);
|
||||||
await reload();
|
await reload();
|
||||||
} catch (operationError) {
|
} catch (operationError) {
|
||||||
setError(apiErrorMessage(operationError));
|
if (isCurrentAuthority()) setError(apiErrorMessage(operationError));
|
||||||
} finally {
|
} finally {
|
||||||
setWorking(false);
|
if (isCurrentAuthority()) setWorking(false);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -315,6 +375,22 @@ export default function DatasourcesPage({
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<WorkspaceFrame as="main" height="viewport" surface="plain" className="datasources-page" label="Datasource workspace">
|
<WorkspaceFrame as="main" height="viewport" surface="plain" className="datasources-page" label="Datasource workspace">
|
||||||
|
<WorkspaceActionBar
|
||||||
|
title="Data"
|
||||||
|
titleHelp={<DocumentationHelpLink reference={DATASOURCES_DOCUMENTATION} />}
|
||||||
|
scope="workspace"
|
||||||
|
variant="collection"
|
||||||
|
refreshable
|
||||||
|
reloadAction={{ onReload: () => void reload(selectedDatasourceRef), loading: loading || working }}
|
||||||
|
createAction={<IconButton
|
||||||
|
label="Add datasource or stage"
|
||||||
|
icon={<Plus size={17} />}
|
||||||
|
variant="primary"
|
||||||
|
onClick={() => setAddOpen(true)}
|
||||||
|
disabled={!canStage && !canManage}
|
||||||
|
disabledReason={!canStage && !canManage ? DATASOURCES_I18N.manageReason : undefined}
|
||||||
|
/>}
|
||||||
|
/>
|
||||||
<WorkspaceLayout
|
<WorkspaceLayout
|
||||||
variant="split"
|
variant="split"
|
||||||
primarySize="compact"
|
primarySize="compact"
|
||||||
@@ -325,21 +401,6 @@ export default function DatasourcesPage({
|
|||||||
contentLabel="Datasource workspace"
|
contentLabel="Datasource workspace"
|
||||||
contentClassName="datasources-workspace"
|
contentClassName="datasources-workspace"
|
||||||
primary={<>
|
primary={<>
|
||||||
<WorkspaceActionBar
|
|
||||||
scope="collection-pane"
|
|
||||||
variant="collection"
|
|
||||||
refreshable
|
|
||||||
reloadAction={{ onReload: () => void reload(selectedDatasourceRef), loading: loading || working }}
|
|
||||||
contextActions={<strong>Data</strong>}
|
|
||||||
createAction={<IconButton
|
|
||||||
label="Add datasource or stage"
|
|
||||||
icon={<Plus size={17} />}
|
|
||||||
variant="primary"
|
|
||||||
onClick={() => setAddOpen(true)}
|
|
||||||
disabled={!canStage && !canManage}
|
|
||||||
disabledReason={!canStage && !canManage ? DATASOURCES_I18N.manageReason : undefined}
|
|
||||||
/>}
|
|
||||||
/>
|
|
||||||
<div className="datasources-view-switch">
|
<div className="datasources-view-switch">
|
||||||
<SegmentedControl<CatalogueView>
|
<SegmentedControl<CatalogueView>
|
||||||
ariaLabel="Datasource view"
|
ariaLabel="Datasource view"
|
||||||
@@ -440,7 +501,6 @@ export default function DatasourcesPage({
|
|||||||
</small>
|
</small>
|
||||||
</span>
|
</span>
|
||||||
</span>}
|
</span>}
|
||||||
helpAction={<DocumentationHelpLink reference={DATASOURCES_DOCUMENTATION} />}
|
|
||||||
primaryActions={<>
|
primaryActions={<>
|
||||||
{view === "catalogue" && selectedDatasource?.mode === "cached" ? (
|
{view === "catalogue" && selectedDatasource?.mode === "cached" ? (
|
||||||
<Button onClick={() => void refreshSelected()} disabled={!canManage || working} disabledReason={working ? DATASOURCES_I18N.working : !canManage ? DATASOURCES_I18N.manageReason : undefined}>
|
<Button onClick={() => void refreshSelected()} disabled={!canManage || working} disabledReason={working ? DATASOURCES_I18N.working : !canManage ? DATASOURCES_I18N.manageReason : undefined}>
|
||||||
@@ -448,7 +508,7 @@ export default function DatasourcesPage({
|
|||||||
</Button>
|
</Button>
|
||||||
) : null}
|
) : null}
|
||||||
{view === "catalogue" && canAdmin ? (
|
{view === "catalogue" && canAdmin ? (
|
||||||
<Button onClick={() => setRetentionOpen(true)} disabled={working}>
|
<Button helpContextId="datasources.retention" helpModuleId="datasources" onClick={() => setRetentionOpen(true)} disabled={working}>
|
||||||
<Archive size={16} /> Retention
|
<Archive size={16} /> Retention
|
||||||
</Button>
|
</Button>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -539,9 +599,9 @@ export default function DatasourcesPage({
|
|||||||
selectedDatasource ? (
|
selectedDatasource ? (
|
||||||
<DatasourceDetail
|
<DatasourceDetail
|
||||||
datasource={selectedDatasource}
|
datasource={selectedDatasource}
|
||||||
preview={preview}
|
preview={!loading && detailResponseScope === detailScope ? preview : null}
|
||||||
materializations={materializations}
|
materializations={!loading && detailResponseScope === detailScope ? materializations : []}
|
||||||
loading={detailLoading}
|
loading={loading || detailLoading || detailResponseScope !== detailScope}
|
||||||
/>
|
/>
|
||||||
) : <EmptyWorkspace icon={<DatabaseZap size={32} />} label="No datasource selected" />
|
) : <EmptyWorkspace icon={<DatabaseZap size={32} />} label="No datasource selected" />
|
||||||
) : null}
|
) : null}
|
||||||
@@ -562,7 +622,8 @@ export default function DatasourcesPage({
|
|||||||
</WorkspaceLayout>
|
</WorkspaceLayout>
|
||||||
|
|
||||||
<AddDatasourceDialog
|
<AddDatasourceDialog
|
||||||
open={addOpen}
|
key={`add:${dialogScope}`}
|
||||||
|
open={addOpen && catalogueReady}
|
||||||
settings={settings}
|
settings={settings}
|
||||||
initialKind={view === "origins" ? "origin" : "upload"}
|
initialKind={view === "origins" ? "origin" : "upload"}
|
||||||
initialOrigin={view === "origins" ? selectedOrigin : null}
|
initialOrigin={view === "origins" ? selectedOrigin : null}
|
||||||
@@ -573,6 +634,7 @@ export default function DatasourcesPage({
|
|||||||
canManage={canManage}
|
canManage={canManage}
|
||||||
onClose={() => setAddOpen(false)}
|
onClose={() => setAddOpen(false)}
|
||||||
onCreated={async (result) => {
|
onCreated={async (result) => {
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
setAddOpen(false);
|
setAddOpen(false);
|
||||||
if ("state" in result) {
|
if ("state" in result) {
|
||||||
setView("staging");
|
setView("staging");
|
||||||
@@ -587,40 +649,47 @@ export default function DatasourcesPage({
|
|||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
<GovernanceDialog
|
<GovernanceDialog
|
||||||
open={governanceOpen}
|
key={`governance:${dialogScope}`}
|
||||||
|
open={governanceOpen && catalogueReady}
|
||||||
settings={settings}
|
settings={settings}
|
||||||
datasource={selectedDatasource}
|
datasource={selectedDatasource}
|
||||||
onClose={() => setGovernanceOpen(false)}
|
onClose={() => setGovernanceOpen(false)}
|
||||||
onSaved={async (updated) => {
|
onSaved={async (updated) => {
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
setGovernanceOpen(false);
|
setGovernanceOpen(false);
|
||||||
setSuccess(`Updated governance for ${updated.name}.`);
|
setSuccess(`Updated governance for ${updated.name}.`);
|
||||||
await reload(updated.ref);
|
await reload(updated.ref);
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
<StageDecisionDialog
|
<StageDecisionDialog
|
||||||
open={decisionOpen}
|
key={`decision:${dialogScope}`}
|
||||||
|
open={decisionOpen && catalogueReady}
|
||||||
settings={settings}
|
settings={settings}
|
||||||
stage={selectedStage}
|
stage={selectedStage}
|
||||||
onClose={() => setDecisionOpen(false)}
|
onClose={() => setDecisionOpen(false)}
|
||||||
onDecided={async (updated) => {
|
onDecided={async (updated) => {
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
setDecisionOpen(false);
|
setDecisionOpen(false);
|
||||||
await reload();
|
await reload();
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
setSelectedStageRef(updated.ref);
|
setSelectedStageRef(updated.ref);
|
||||||
setSuccess(`Recorded ${updated.approval.state ?? "approval"} decision state for ${updated.name}.`);
|
setSuccess(`Recorded ${updated.approval.state ?? "approval"} decision state for ${updated.name}.`);
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
<RetentionDialog
|
<RetentionDialog
|
||||||
open={retentionOpen}
|
key={`retention:${dialogScope}`}
|
||||||
|
open={retentionOpen && catalogueReady}
|
||||||
settings={settings}
|
settings={settings}
|
||||||
onClose={() => setRetentionOpen(false)}
|
onClose={() => setRetentionOpen(false)}
|
||||||
onApplied={async (count) => {
|
onApplied={async (count) => {
|
||||||
|
if (!isCurrentAuthority()) return;
|
||||||
setRetentionOpen(false);
|
setRetentionOpen(false);
|
||||||
setSuccess(`Applied retention to ${count} eligible target${count === 1 ? "" : "s"}.`);
|
setSuccess(`Applied retention to ${count} eligible target${count === 1 ? "" : "s"}.`);
|
||||||
await reload(selectedDatasourceRef);
|
await reload(selectedDatasourceRef);
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
<Dialog
|
<Dialog
|
||||||
open={freezeOpen}
|
open={freezeOpen && catalogueReady}
|
||||||
title="Freeze datasource state"
|
title="Freeze datasource state"
|
||||||
onClose={closeFreeze}
|
onClose={closeFreeze}
|
||||||
footer={(
|
footer={(
|
||||||
@@ -644,7 +713,7 @@ export default function DatasourcesPage({
|
|||||||
</FormField>
|
</FormField>
|
||||||
</Dialog>
|
</Dialog>
|
||||||
<ConfirmDialog
|
<ConfirmDialog
|
||||||
open={promoteOpen}
|
open={promoteOpen && catalogueReady}
|
||||||
title="i18n:govoplan-datasources.promote_title"
|
title="i18n:govoplan-datasources.promote_title"
|
||||||
message="i18n:govoplan-datasources.promote_message"
|
message="i18n:govoplan-datasources.promote_message"
|
||||||
confirmLabel="Promote"
|
confirmLabel="Promote"
|
||||||
@@ -656,7 +725,7 @@ export default function DatasourcesPage({
|
|||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
<ConfirmDialog
|
<ConfirmDialog
|
||||||
open={retireOpen}
|
open={retireOpen && catalogueReady}
|
||||||
title="Retire datasource"
|
title="Retire datasource"
|
||||||
message={`Retire ${selectedDatasource?.name ?? "this datasource"}? Existing materialization references remain in the database, but the datasource will no longer be available to new definitions.`}
|
message={`Retire ${selectedDatasource?.name ?? "this datasource"}? Existing materialization references remain in the database, but the datasource will no longer be available to new definitions.`}
|
||||||
confirmLabel="Retire"
|
confirmLabel="Retire"
|
||||||
@@ -686,7 +755,7 @@ function DatasourceDetail({
|
|||||||
<MetricCard density="compact" label="Mode" value={datasource.mode} valueTitle={datasource.mode} />
|
<MetricCard density="compact" label="Mode" value={datasource.mode} valueTitle={datasource.mode} />
|
||||||
<MetricCard density="compact" label="Rows" value={formatNumber(datasource.row_count)} />
|
<MetricCard density="compact" label="Rows" value={formatNumber(datasource.row_count)} />
|
||||||
<MetricCard density="compact" label="Fields" value={String(datasource.schema.length)} />
|
<MetricCard density="compact" label="Fields" value={String(datasource.schema.length)} />
|
||||||
<MetricCard density="compact" label="Revisions" value={String(materializations.length)} />
|
<MetricCard density="compact" label="Revisions" value={loading ? "…" : String(materializations.length)} />
|
||||||
<MetricCard density="compact" label="Updated" value={formatDate(datasource.updated_at)} />
|
<MetricCard density="compact" label="Updated" value={formatDate(datasource.updated_at)} />
|
||||||
</MetricGrid>
|
</MetricGrid>
|
||||||
{datasource.description ? (
|
{datasource.description ? (
|
||||||
@@ -764,7 +833,9 @@ function DatasourceDetail({
|
|||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
))}
|
))}
|
||||||
{!materializations.length ? (
|
{loading ? (
|
||||||
|
<tr><td colSpan={5}>Loading materializations...</td></tr>
|
||||||
|
) : !materializations.length ? (
|
||||||
<tr><td colSpan={5}>Live source without materializations</td></tr>
|
<tr><td colSpan={5}>Live source without materializations</td></tr>
|
||||||
) : null}
|
) : null}
|
||||||
</tbody>
|
</tbody>
|
||||||
@@ -1337,7 +1408,7 @@ function GovernanceDialog({
|
|||||||
<GovernanceListField label="Purposes" values={draft.purposes} onChange={(values) => setValue("purposes", values)} />
|
<GovernanceListField label="Purposes" values={draft.purposes} onChange={(values) => setValue("purposes", values)} />
|
||||||
<GovernanceListField label="Legal basis references" values={draft.legal_basis_refs} onChange={(values) => setValue("legal_basis_refs", values)} />
|
<GovernanceListField label="Legal basis references" values={draft.legal_basis_refs} onChange={(values) => setValue("legal_basis_refs", values)} />
|
||||||
<GovernanceListField label="Official keys" values={draft.official_keys} onChange={(values) => setValue("official_keys", values)} />
|
<GovernanceListField label="Official keys" values={draft.official_keys} onChange={(values) => setValue("official_keys", values)} />
|
||||||
<GovernanceListField label="Legal hold references" values={draft.hold_refs} onChange={(values) => setValue("hold_refs", values)} />
|
<GovernanceListField label="Legal hold references" helpContextId="datasources.field.retention-policy-contract" values={draft.hold_refs} onChange={(values) => setValue("hold_refs", values)} />
|
||||||
<GovernanceListField label="Affected services and processes" values={draft.affected_refs} onChange={(values) => setValue("affected_refs", values)} />
|
<GovernanceListField label="Affected services and processes" values={draft.affected_refs} onChange={(values) => setValue("affected_refs", values)} />
|
||||||
<GovernanceListField label="Dependent flows, reports, controls and decisions" values={draft.dependency_refs} onChange={(values) => setValue("dependency_refs", values)} />
|
<GovernanceListField label="Dependent flows, reports, controls and decisions" values={draft.dependency_refs} onChange={(values) => setValue("dependency_refs", values)} />
|
||||||
<GovernanceListField label="Known limits" values={draft.known_limits} onChange={(values) => setValue("known_limits", values)} />
|
<GovernanceListField label="Known limits" values={draft.known_limits} onChange={(values) => setValue("known_limits", values)} />
|
||||||
@@ -1370,15 +1441,17 @@ function GovernanceDialog({
|
|||||||
|
|
||||||
function GovernanceListField({
|
function GovernanceListField({
|
||||||
label,
|
label,
|
||||||
|
helpContextId,
|
||||||
values,
|
values,
|
||||||
onChange
|
onChange
|
||||||
}: {
|
}: {
|
||||||
label: string;
|
label: string;
|
||||||
|
helpContextId?: string;
|
||||||
values: string[];
|
values: string[];
|
||||||
onChange: (values: string[]) => void;
|
onChange: (values: string[]) => void;
|
||||||
}) {
|
}) {
|
||||||
return (
|
return (
|
||||||
<FormField label={label}>
|
<FormField label={label} helpContextId={helpContextId} helpModuleId="datasources">
|
||||||
<textarea
|
<textarea
|
||||||
value={values.join("\n")}
|
value={values.join("\n")}
|
||||||
onChange={(event) => onChange(splitLines(event.target.value))}
|
onChange={(event) => onChange(splitLines(event.target.value))}
|
||||||
@@ -1426,6 +1499,7 @@ function AddDatasourceDialog({
|
|||||||
const [rowsText, setRowsText] = useState('[\n { "id": 1 }\n]');
|
const [rowsText, setRowsText] = useState('[\n { "id": 1 }\n]');
|
||||||
const [csvText, setCsvText] = useState("");
|
const [csvText, setCsvText] = useState("");
|
||||||
const [delimiter, setDelimiter] = useState(";");
|
const [delimiter, setDelimiter] = useState(";");
|
||||||
|
const [csvValueMode, setCsvValueMode] = useState<"text" | "legacy_typed">("text");
|
||||||
const [baselineKey, setBaselineKey] = useState("");
|
const [baselineKey, setBaselineKey] = useState("");
|
||||||
const [busy, setBusy] = useState(false);
|
const [busy, setBusy] = useState(false);
|
||||||
const [error, setError] = useState("");
|
const [error, setError] = useState("");
|
||||||
@@ -1444,6 +1518,7 @@ function AddDatasourceDialog({
|
|||||||
setRowsText('[\n { "id": 1 }\n]');
|
setRowsText('[\n { "id": 1 }\n]');
|
||||||
setCsvText("");
|
setCsvText("");
|
||||||
setDelimiter(";");
|
setDelimiter(";");
|
||||||
|
setCsvValueMode("text");
|
||||||
setBaselineKey(addDatasourceDraftKey({
|
setBaselineKey(addDatasourceDraftKey({
|
||||||
kind: initialKind,
|
kind: initialKind,
|
||||||
format: "csv",
|
format: "csv",
|
||||||
@@ -1455,7 +1530,8 @@ function AddDatasourceDialog({
|
|||||||
description: initialOrigin?.description ?? "",
|
description: initialOrigin?.description ?? "",
|
||||||
rowsText: '[\n { "id": 1 }\n]',
|
rowsText: '[\n { "id": 1 }\n]',
|
||||||
csvText: "",
|
csvText: "",
|
||||||
delimiter: ";"
|
delimiter: ";",
|
||||||
|
csvValueMode: "text"
|
||||||
}));
|
}));
|
||||||
setError("");
|
setError("");
|
||||||
}, [initialKind, initialOrigin, open]);
|
}, [initialKind, initialOrigin, open]);
|
||||||
@@ -1472,7 +1548,8 @@ function AddDatasourceDialog({
|
|||||||
description,
|
description,
|
||||||
rowsText,
|
rowsText,
|
||||||
csvText,
|
csvText,
|
||||||
delimiter
|
delimiter,
|
||||||
|
csvValueMode
|
||||||
}) !== baselineKey);
|
}) !== baselineKey);
|
||||||
|
|
||||||
const chooseOrigin = (ref: string) => {
|
const chooseOrigin = (ref: string) => {
|
||||||
@@ -1524,7 +1601,8 @@ function AddDatasourceDialog({
|
|||||||
...common,
|
...common,
|
||||||
format: "csv",
|
format: "csv",
|
||||||
csv_text: csvText,
|
csv_text: csvText,
|
||||||
delimiter
|
delimiter,
|
||||||
|
csv_value_mode: csvValueMode
|
||||||
})
|
})
|
||||||
: await createDatasourceStage(settings, {
|
: await createDatasourceStage(settings, {
|
||||||
...common,
|
...common,
|
||||||
@@ -1718,6 +1796,16 @@ function AddDatasourceDialog({
|
|||||||
<option value="|">Pipe</option>
|
<option value="|">Pipe</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</FormField>
|
||||||
|
<FormField label="CSV values" documentation={DATASOURCE_FIELDS_DOCUMENTATION}>
|
||||||
|
<select
|
||||||
|
value={csvValueMode}
|
||||||
|
onChange={(event) => setCsvValueMode(event.target.value as "text" | "legacy_typed")}
|
||||||
|
disabled={busy}
|
||||||
|
>
|
||||||
|
<option value="text">Preserve text (no automatic conversion)</option>
|
||||||
|
<option value="legacy_typed">Infer types (legacy)</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
<FormField label="CSV data">
|
<FormField label="CSV data">
|
||||||
<textarea
|
<textarea
|
||||||
value={csvText}
|
value={csvText}
|
||||||
@@ -1836,6 +1924,7 @@ function addDatasourceDraftKey(value: {
|
|||||||
rowsText: string;
|
rowsText: string;
|
||||||
csvText: string;
|
csvText: string;
|
||||||
delimiter: string;
|
delimiter: string;
|
||||||
|
csvValueMode: "text" | "legacy_typed";
|
||||||
}): string {
|
}): string {
|
||||||
return JSON.stringify(value);
|
return JSON.stringify(value);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
import type { PlatformTranslations } from "@govoplan/core-webui";
|
import type { PlatformTranslations } from "@govoplan/core-webui";
|
||||||
|
|
||||||
const en = {
|
const en = {
|
||||||
|
"CSV values": "CSV values",
|
||||||
|
"Preserve text (no automatic conversion)": "Preserve text (no automatic conversion)",
|
||||||
|
"Infer types (legacy)": "Infer types (legacy)",
|
||||||
|
"Loading materializations...": "Loading materializations...",
|
||||||
"i18n:govoplan-datasources.datasources": "Datasources",
|
"i18n:govoplan-datasources.datasources": "Datasources",
|
||||||
"i18n:govoplan-datasources.catalogue": "Datasource catalogue",
|
"i18n:govoplan-datasources.catalogue": "Datasource catalogue",
|
||||||
"i18n:govoplan-datasources.staging": "Datasource staging",
|
"i18n:govoplan-datasources.staging": "Datasource staging",
|
||||||
@@ -78,6 +82,10 @@ const en = {
|
|||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
const de: Record<keyof typeof en, string> = {
|
const de: Record<keyof typeof en, string> = {
|
||||||
|
"CSV values": "CSV-Werte",
|
||||||
|
"Preserve text (no automatic conversion)": "Text erhalten (keine automatische Umwandlung)",
|
||||||
|
"Infer types (legacy)": "Typen ableiten (bisheriges Verhalten)",
|
||||||
|
"Loading materializations...": "Materialisierungen werden geladen...",
|
||||||
"i18n:govoplan-datasources.datasources": "Datenquellen",
|
"i18n:govoplan-datasources.datasources": "Datenquellen",
|
||||||
"i18n:govoplan-datasources.catalogue": "Datenquellenkatalog",
|
"i18n:govoplan-datasources.catalogue": "Datenquellenkatalog",
|
||||||
"i18n:govoplan-datasources.staging": "Datenquellen-Staging",
|
"i18n:govoplan-datasources.staging": "Datenquellen-Staging",
|
||||||
|
|||||||
Reference in New Issue
Block a user