import { useCallback, useEffect, useMemo, useState } from "react"; import { Archive, Database, DatabaseZap, Download, Eye, Layers3, Pencil, Plus, Snowflake, ShieldCheck, Trash2, Upload } from "lucide-react"; import { FormGrid, DialogSection, ActionToolbar, ActionBlockerHint, Button, ConfirmDialog, ContentSection, Dialog, DocumentationHelpLink, DismissibleAlert, FilterBar, FormField, IconButton, LoadingFrame, MetricCard, MetricGrid, SegmentedControl, SelectionList, SelectionListItem, SelectionListItemContent, StatePanel, StatusBadge, WorkspaceActionBar, WorkspaceFrame, WorkspaceLayout, hasScope, isApiError, useUnsavedChanges, useUnsavedDraftGuard, type ApiSettings, type AuthInfo } from "@govoplan/core-webui"; import { createDatasourceStage, freezeDatasource, listDatasourceMaterializations, listDatasourceOrigins, listDatasourceStages, listDatasources, previewDatasource, promoteDatasourceStage, refreshDatasource, registerDatasourceOrigin, retireDatasource, updateDatasourceGovernance, type Datasource, type DatasourceGovernance, type DatasourceMaterialization, type DatasourceOrigin, type DatasourcePreview, type DatasourceSchemaChange, type DatasourceStage, type DatasourceValidationDiagnostic } from "../../api/datasources"; import { DATASOURCE_FIELDS_DOCUMENTATION, DATASOURCE_GOVERNANCE_DOCUMENTATION, DATASOURCE_VISIBILITY_DOCUMENTATION, DATASOURCES_DOCUMENTATION, DATASOURCES_I18N } from "./interfacePatterns"; type CatalogueView = "catalogue" | "staging" | "origins"; type AddKind = "upload" | "origin"; type UploadFormat = "json" | "csv"; export default function DatasourcesPage({ settings, auth }: { settings: ApiSettings; auth: AuthInfo; }) { const [view, setView] = useState("catalogue"); const [datasources, setDatasources] = useState([]); const [stages, setStages] = useState([]); const [origins, setOrigins] = useState([]); const [originsAvailable, setOriginsAvailable] = useState(false); const [selectedDatasourceRef, setSelectedDatasourceRef] = useState( initialDatasourceRef ); const [selectedStageRef, setSelectedStageRef] = useState(""); const [selectedOriginRef, setSelectedOriginRef] = useState(""); const [preview, setPreview] = useState(null); const [materializations, setMaterializations] = useState([]); const [search, setSearch] = useState(""); const [loading, setLoading] = useState(true); const [detailLoading, setDetailLoading] = useState(false); const [working, setWorking] = useState(false); const [error, setError] = useState(""); const [success, setSuccess] = useState(""); const [addOpen, setAddOpen] = useState(false); const [freezeOpen, setFreezeOpen] = useState(false); const [freezeLabel, setFreezeLabel] = useState(""); const [retireOpen, setRetireOpen] = useState(false); const [promoteOpen, setPromoteOpen] = useState(false); const [governanceOpen, setGovernanceOpen] = useState(false); const { requestDiscard } = useUnsavedChanges(); const canManage = hasScope(auth, "datasources:source:write") || hasScope(auth, "datasources:source:admin"); const canStage = hasScope(auth, "datasources:stage:write") || hasScope(auth, "datasources:source:admin"); const reload = useCallback(async (preferredDatasourceRef?: string) => { setLoading(true); setError(""); try { const [nextDatasources, nextStages, originCatalogue] = await Promise.all([ listDatasources(settings), listDatasourceStages(settings), listDatasourceOrigins(settings) ]); setDatasources(nextDatasources); setStages(nextStages); setOrigins(originCatalogue.origins); setOriginsAvailable(originCatalogue.available); setSelectedDatasourceRef((current) => { const preferred = preferredDatasourceRef || current; return nextDatasources.some((item) => item.ref === preferred) ? preferred : nextDatasources[0]?.ref ?? ""; }); setSelectedStageRef((current) => nextStages.some((item) => item.ref === current) ? current : nextStages[0]?.ref ?? ""); setSelectedOriginRef((current) => originCatalogue.origins.some((item) => item.ref === current) ? current : originCatalogue.origins[0]?.ref ?? ""); } catch (loadError) { setError(apiErrorMessage(loadError)); } finally { setLoading(false); } }, [settings]); useEffect(() => { void reload(); }, [reload]); useEffect(() => { if (!selectedDatasourceRef) { setPreview(null); setMaterializations([]); return; } let cancelled = false; setDetailLoading(true); Promise.allSettled([ previewDatasource(settings, selectedDatasourceRef), listDatasourceMaterializations(settings, selectedDatasourceRef) ]).then(([previewResult, materializationResult]) => { if (cancelled) return; if (previewResult.status === "fulfilled") { setPreview(previewResult.value); } else { setPreview(null); setError(apiErrorMessage(previewResult.reason)); } setMaterializations( materializationResult.status === "fulfilled" ? materializationResult.value : [] ); setDetailLoading(false); }); return () => { cancelled = true; }; }, [selectedDatasourceRef, settings]); const selectedDatasource = datasources.find((item) => item.ref === selectedDatasourceRef) ?? null; const selectedStage = stages.find((item) => item.ref === selectedStageRef) ?? null; const selectedOrigin = origins.find((item) => item.ref === selectedOriginRef) ?? null; const visibleDatasources = useMemo( () => filterItems(datasources, search, (item) => `${item.name} ${item.source_name} ${item.description ?? ""} ${item.mode} ${item.kind}`), [datasources, search] ); const visibleStages = useMemo( () => filterItems(stages, search, (item) => `${item.name} ${item.source_name} ${item.state} ${item.mode}`), [stages, search] ); const visibleOrigins = useMemo( () => filterItems(origins, search, (item) => `${item.name} ${item.source_name} ${item.provider} ${item.kind}`), [origins, search] ); const promoteStage = async () => { if (!selectedStage || selectedStage.state !== "ready") return; setWorking(true); setError(""); try { const result = await promoteDatasourceStage(settings, selectedStage.ref); setSuccess(`Promoted ${result.datasource.name} as revision ${result.materialization.revision}.`); setView("catalogue"); await reload(result.datasource.ref); } catch (operationError) { setError(apiErrorMessage(operationError)); } finally { setWorking(false); } }; const refreshSelected = async () => { if (!selectedDatasource) return; setWorking(true); setError(""); try { const result = await refreshDatasource(settings, selectedDatasource.ref); setSuccess(`Refreshed ${result.datasource.name} as revision ${result.materialization.revision}.`); await reload(result.datasource.ref); } catch (operationError) { setError(apiErrorMessage(operationError)); } finally { setWorking(false); } }; const freezeSelected = async (): Promise => { if (!selectedDatasource) return false; setWorking(true); setError(""); try { const materialization = await freezeDatasource( settings, selectedDatasource.ref, freezeLabel ); setSuccess(`Created frozen revision ${materialization.revision}.`); setFreezeOpen(false); setFreezeLabel(""); await reload(selectedDatasource.ref); return true; } catch (operationError) { setError(apiErrorMessage(operationError)); return false; } finally { setWorking(false); } }; const retireSelected = async () => { if (!selectedDatasource) return; setWorking(true); setError(""); try { await retireDatasource(settings, selectedDatasource.ref); setSuccess(`Retired ${selectedDatasource.name}.`); setRetireOpen(false); await reload(); } catch (operationError) { setError(apiErrorMessage(operationError)); } finally { setWorking(false); } }; useUnsavedDraftGuard({ dirty: Boolean(freezeOpen && freezeLabel.trim()), onSave: freezeSelected, onDiscard: () => { setFreezeOpen(false); setFreezeLabel(""); }, title: "i18n:govoplan-datasources.unsaved_freeze_title", message: "i18n:govoplan-datasources.unsaved_freeze_message" }); const closeFreeze = () => { if (working) return; if (freezeLabel.trim()) requestDiscard(() => setFreezeOpen(false)); else setFreezeOpen(false); }; return ( void reload(selectedDatasourceRef), loading: loading || working }} contextActions={Data} createAction={} variant="primary" onClick={() => setAddOpen(true)} disabled={!canStage && !canManage} disabledReason={!canStage && !canManage ? DATASOURCES_I18N.manageReason : undefined} />} />
ariaLabel="Datasource view" size="equal" width="fill" options={[ { id: "catalogue", label: "Catalogue" }, { id: "staging", label: "Staging" }, { id: "origins", label: "Origins" } ]} value={view} onChange={(next) => { setView(next); setSearch(""); setError(""); }} />
setSearch(event.target.value)} placeholder={`Search ${view}`} aria-label={`Search ${view}`} /> {view === "catalogue" ? visibleDatasources.map((item) => ( setSelectedDatasourceRef(item.ref)} > )) : null} {view === "staging" ? visibleStages.map((item) => ( setSelectedStageRef(item.ref)} > )) : null} {view === "origins" ? visibleOrigins.map((item) => ( setSelectedOriginRef(item.ref)} > )) : null} {view === "catalogue" && !visibleDatasources.length ? ( ) : null} {view === "staging" && !visibleStages.length ? ( ) : null} {view === "origins" && !visibleOrigins.length ? ( ) : null} } > {view === "catalogue" ? : view === "staging" ? : } {view === "catalogue" ? selectedDatasource?.name ?? "Catalogue" : view === "staging" ? selectedStage?.name ?? "Staging" : selectedOrigin?.name ?? "Connector origins"} {view === "catalogue" ? selectedDatasource?.source_name ?? "Governed data" : view === "staging" ? selectedStage?.source_name ?? "Inspect before promotion" : selectedOrigin?.provider ?? "External acquisition"} } helpAction={} primaryActions={<> {view === "catalogue" && selectedDatasource?.mode === "cached" ? ( ) : null} {view === "catalogue" && selectedDatasource ? ( <> } onClick={() => setGovernanceOpen(true)} disabled={!canManage || working} disabledReason={working ? DATASOURCES_I18N.working : !canManage ? DATASOURCES_I18N.manageReason : undefined} /> ) : null} {view === "staging" && selectedStage?.state === "ready" ? ( ) : null} {view === "origins" && selectedOrigin ? ( ) : null} } destructiveActions={view === "catalogue" && selectedDatasource ? ( } variant="danger" onClick={() => setRetireOpen(true)} disabled={!canManage || working} disabledReason={working ? DATASOURCES_I18N.working : !canManage ? DATASOURCES_I18N.manageReason : undefined} /> ) : undefined} />
{error ? ( {error} ) : null} {success ? ( {success} ) : null} {!canManage && !canStage ? : null}
{view === "catalogue" ? ( selectedDatasource ? ( ) : } label="No datasource selected" /> ) : null} {view === "staging" ? ( selectedStage ? : } label="No stage selected" /> ) : null} {view === "origins" ? ( selectedOrigin ? : } label={ originsAvailable ? "No origin selected" : "Connectors unavailable" } /> ) : null}
{working ?
Working...
: null}
setAddOpen(false)} onCreated={async (result) => { setAddOpen(false); if ("state" in result) { setView("staging"); setSelectedStageRef(result.ref); setSuccess(`Created stage ${result.name}. Review it before promotion.`); await reload(); } else { setView("catalogue"); setSuccess(`Registered ${result.name}.`); await reload(result.ref); } }} /> setGovernanceOpen(false)} onSaved={async (updated) => { setGovernanceOpen(false); setSuccess(`Updated governance for ${updated.name}.`); await reload(updated.ref); }} /> )} >

Create an immutable, addressable state for reproducible runs and evidence.

setFreezeLabel(event.target.value)} placeholder="Optional evidence label" />
setPromoteOpen(false)} onConfirm={() => { setPromoteOpen(false); void promoteStage(); }} /> setRetireOpen(false)} onConfirm={() => void retireSelected()} />
); } function DatasourceDetail({ datasource, preview, materializations, loading }: { datasource: Datasource; preview: DatasourcePreview | null; materializations: DatasourceMaterialization[]; loading: boolean; }) { return ( <> {datasource.description ? (
{datasource.description}
) : null} Governance
Authority{readableToken(datasource.governance.authority_mode)} Classification{datasource.governance.classification} Owner{datasource.governance.owner_ref || "Not assigned"} Steward{datasource.governance.steward_ref || "Not assigned"} Responsible organization{datasource.governance.responsible_organization_ref || "Not assigned"} Purposes{datasource.governance.purposes.join(", ") || "Not declared"}
{datasource.governance.semantic_definition ? (

{datasource.governance.semantic_definition}

) : null}
Preview {preview ? `${formatNumber(preview.total_rows)} total rows` : ""} {loading ? (
Loading preview...
) : preview ? ( ) : ( )}
Materializations Immutable revisions
{materializations.map((item) => ( ))} {!materializations.length ? ( ) : null}
Revision Created Rows State Fingerprint
{item.revision} {formatDate(item.created_at)} {formatNumber(item.row_count)} {item.frozen_at ? ( ) : } {shortFingerprint(item.fingerprint)}
Live source without materializations
Schema Version {datasource.schema_version} ); } function StageDetail({ stage }: { stage: DatasourceStage }) { const errors = stage.validation.errors ?? []; const warnings = stage.validation.warnings ?? []; const schemaChanges = stage.validation.schema_change?.changes ?? []; const schemaClassification = stage.validation.schema_change?.classification ?? "new"; return ( <> Validation
Fingerprint{shortFingerprint(stage.fingerprint)} Target{stage.target_datasource_ref || "New datasource"} Promoted revision{stage.promoted_materialization_ref || "Not promoted"} Quality policy{stage.validation.policy_version || "Local default"} Policy hash{shortFingerprint(stage.validation.policy_hash || "")} Schema change{readableToken(schemaClassification)}
{errors.length || warnings.length ? (
{errors.length ? ( Promotion blockers ) : null} {warnings.length ? ( Review warnings ) : null}
) : (
All configured quality rules passed and no blocking schema change was detected.
)}
{schemaChanges.length ? ( Schema comparison
    {schemaChanges.map((change, index) => ( ))}
) : null} Detected schema {stage.shape} ); } function ValidationDiagnosticList({ diagnostics }: { diagnostics: DatasourceValidationDiagnostic[]; }) { return (
    {diagnostics.map((diagnostic, index) => { const details = [ diagnostic.rule_id ? `Rule ${diagnostic.rule_id}` : "", diagnostic.affected_rows !== undefined ? `${diagnostic.affected_rows} affected row${diagnostic.affected_rows === 1 ? "" : "s"}` : "", diagnostic.row_numbers?.length ? `Rows ${diagnostic.row_numbers.join(", ")}${diagnostic.row_numbers_truncated ? ", …" : ""}` : "" ].filter(Boolean); return (
  • {diagnostic.message} {details.length ? {details.join(" · ")} : null}
  • ); })}
); } function SchemaChangeItem({ change }: { change: DatasourceSchemaChange }) { return (
  • {change.message} {change.field ? {change.field} : null}
  • ); } function OriginDetail({ origin }: { origin: DatasourceOrigin }) { return ( <> {origin.description ? (
    {origin.description}
    ) : null} Registration options {origin.supported_modes.join(", ")}
    Origin reference{origin.ref} Kind{readableToken(origin.kind)} Shape{readableToken(origin.shape)} Fingerprint{shortFingerprint(origin.fingerprint)} Health{origin.health.summary} Pushdown{pushdownSummary(origin)}
    Discovered schema Version {origin.schema_version} ); } function pushdownSummary(origin: DatasourceOrigin): string { const capabilities = [ origin.pushdown.projections ? "projections" : "", origin.pushdown.pagination ? "pagination" : "", ...origin.pushdown.filters.map((item) => `filter:${item}`), ...origin.pushdown.aggregations.map((item) => `aggregate:${item}`), ...origin.pushdown.sorting.map((item) => `sort:${item}`) ].filter(Boolean); return capabilities.join(", ") || "None declared"; } function GovernanceDialog({ open, settings, datasource, onClose, onSaved }: { open: boolean; settings: ApiSettings; datasource: Datasource | null; onClose: () => void; onSaved: (datasource: Datasource) => void | Promise; }) { const [draft, setDraft] = useState(null); const [freshness, setFreshness] = useState("{}"); const [quality, setQuality] = useState("{}"); const [visibility, setVisibility] = useState("{}"); const [baselineKey, setBaselineKey] = useState(""); const [busy, setBusy] = useState(false); const [error, setError] = useState(""); const { requestDiscard } = useUnsavedChanges(); useEffect(() => { if (!open || !datasource) return; const nextDraft = structuredClone(datasource.governance); const nextFreshness = JSON.stringify(datasource.governance.freshness_policy, null, 2); const nextQuality = JSON.stringify(datasource.governance.quality_policy, null, 2); const nextVisibility = JSON.stringify(datasource.governance.visibility_policy ?? {}, null, 2); setDraft(nextDraft); setFreshness(nextFreshness); setQuality(nextQuality); setVisibility(nextVisibility); setBaselineKey(JSON.stringify({ draft: nextDraft, freshness: nextFreshness, quality: nextQuality, visibility: nextVisibility })); setError(""); }, [datasource, open]); const dirty = Boolean(open && draft && JSON.stringify({ draft, freshness, quality, visibility }) !== baselineKey); const save = async (): Promise => { if (!datasource || !draft) return false; setBusy(true); setError(""); try { const updated = await updateDatasourceGovernance(settings, datasource.ref, { ...draft, freshness_policy: parseObject(freshness, "Freshness policy"), quality_policy: parseObject(quality, "Quality policy"), visibility_policy: parseObject(visibility, "Visibility policy") }); await onSaved(updated); return true; } catch (saveError) { setError(apiErrorMessage(saveError)); return false; } finally { setBusy(false); } }; useUnsavedDraftGuard({ dirty, onSave: save, onDiscard: onClose, title: "i18n:govoplan-datasources.unsaved_governance_title", message: "i18n:govoplan-datasources.unsaved_governance_message" }); const close = () => { if (busy) return; if (dirty) requestDiscard(onClose); else onClose(); }; const setValue = ( key: K, value: DatasourceGovernance[K] ) => setDraft((current) => current ? { ...current, [key]: value } : current); return ( )} > {error ? {error} : null} {draft ? ( setValue("authoritative_source_ref", event.target.value || null)} /> setValue("classification", event.target.value)} /> setValue("publication_state", event.target.value)} /> setValue("owner_ref", event.target.value || null)} /> setValue("steward_ref", event.target.value || null)} /> setValue("responsible_organization_ref", event.target.value || null)} /> setValue("responsible_function_ref", event.target.value || null)} /> setValue("schema_owner_ref", event.target.value || null)} /> setValue("privacy_profile_ref", event.target.value || null)} /> setValue("retention_policy_ref", event.target.value || null)} /> setValue("access_policy_ref", event.target.value || null)} placeholder="Optional Policy module target" /> setValue("transfer_agreement_ref", event.target.value || null)} /> setValue("correction_procedure_ref", event.target.value || null)} />