feat: implement governed datasource catalogue metadata
This commit is contained in:
@@ -5,6 +5,39 @@ import {
|
||||
|
||||
export type DatasourceMode = "live" | "cached" | "static";
|
||||
export type DatasourceShape = "tabular" | "document" | "binary" | "directory" | "stream";
|
||||
export type SourceAuthorityMode =
|
||||
| "native_authoritative"
|
||||
| "external_authoritative"
|
||||
| "external_mirror"
|
||||
| "governed_sync"
|
||||
| "governance_overlay"
|
||||
| "linked_reference";
|
||||
|
||||
export type DatasourceGovernance = {
|
||||
owner_ref?: string | null;
|
||||
steward_ref?: string | null;
|
||||
responsible_organization_ref?: string | null;
|
||||
responsible_function_ref?: string | null;
|
||||
authoritative_source_ref?: string | null;
|
||||
authority_mode: SourceAuthorityMode;
|
||||
legal_basis_refs: string[];
|
||||
purposes: string[];
|
||||
semantic_definition?: string | null;
|
||||
schema_owner_ref?: string | null;
|
||||
official_keys: string[];
|
||||
classification: string;
|
||||
privacy_profile_ref?: string | null;
|
||||
retention_policy_ref?: string | null;
|
||||
hold_refs: string[];
|
||||
publication_state: string;
|
||||
transfer_agreement_ref?: string | null;
|
||||
freshness_policy: Record<string, unknown>;
|
||||
quality_policy: Record<string, unknown>;
|
||||
known_limits: string[];
|
||||
correction_procedure_ref?: string | null;
|
||||
affected_refs: string[];
|
||||
dependency_refs: string[];
|
||||
};
|
||||
|
||||
export type DatasourceField = {
|
||||
name: string;
|
||||
@@ -33,6 +66,7 @@ export type Datasource = {
|
||||
capabilities: string[];
|
||||
provenance: Record<string, unknown>;
|
||||
metadata: Record<string, unknown>;
|
||||
governance: DatasourceGovernance;
|
||||
};
|
||||
|
||||
export type DatasourceMaterialization = {
|
||||
@@ -50,6 +84,7 @@ export type DatasourceMaterialization = {
|
||||
created_at?: string | null;
|
||||
provenance: Record<string, unknown>;
|
||||
metadata: Record<string, unknown>;
|
||||
governance: DatasourceGovernance;
|
||||
};
|
||||
|
||||
export type DatasourceStage = {
|
||||
@@ -71,6 +106,7 @@ export type DatasourceStage = {
|
||||
promoted_materialization_ref?: string | null;
|
||||
provenance: Record<string, unknown>;
|
||||
metadata: Record<string, unknown>;
|
||||
governance: DatasourceGovernance;
|
||||
};
|
||||
|
||||
export type DatasourceOrigin = {
|
||||
@@ -102,9 +138,18 @@ export type DatasourcePreview = {
|
||||
|
||||
export async function listDatasources(
|
||||
settings: ApiSettings,
|
||||
query = ""
|
||||
query = "",
|
||||
filters: Partial<Pick<DatasourceGovernance, "authority_mode" | "classification" | "publication_state" | "owner_ref" | "responsible_organization_ref">> & {
|
||||
affected_ref?: string;
|
||||
dependency_ref?: string;
|
||||
} = {}
|
||||
): Promise<Datasource[]> {
|
||||
const suffix = query.trim() ? `?query=${encodeURIComponent(query.trim())}` : "";
|
||||
const params = new URLSearchParams();
|
||||
if (query.trim()) params.set("query", query.trim());
|
||||
for (const [key, value] of Object.entries(filters)) {
|
||||
if (String(value ?? "").trim()) params.set(key, String(value).trim());
|
||||
}
|
||||
const suffix = params.size ? `?${params.toString()}` : "";
|
||||
const response = await apiFetch<{ datasources: Datasource[] }>(
|
||||
settings,
|
||||
`/api/v1/datasources${suffix}`
|
||||
@@ -158,6 +203,7 @@ export function createDatasourceStage(
|
||||
description?: string | null;
|
||||
mode: "static" | "cached";
|
||||
target_datasource_ref?: string | null;
|
||||
governance?: DatasourceGovernance | null;
|
||||
} & (
|
||||
{ format: "json"; rows: Record<string, unknown>[] }
|
||||
| { format: "csv"; csv_text: string; delimiter: string }
|
||||
@@ -188,6 +234,7 @@ export function registerDatasourceOrigin(
|
||||
source_name: string;
|
||||
mode: "live" | "cached";
|
||||
description?: string | null;
|
||||
governance?: DatasourceGovernance | null;
|
||||
}
|
||||
): Promise<Datasource> {
|
||||
return apiFetch(settings, "/api/v1/datasources/origins/register", {
|
||||
@@ -225,6 +272,17 @@ export function retireDatasource(
|
||||
});
|
||||
}
|
||||
|
||||
export function updateDatasourceGovernance(
|
||||
settings: ApiSettings,
|
||||
datasourceRef: string,
|
||||
governance: DatasourceGovernance
|
||||
): Promise<Datasource> {
|
||||
return apiFetch(settings, `/api/v1/datasources/${refId(datasourceRef)}/governance`, {
|
||||
method: "PATCH",
|
||||
body: JSON.stringify({ governance })
|
||||
});
|
||||
}
|
||||
|
||||
function refId(ref: string): string {
|
||||
const separator = ref.indexOf(":");
|
||||
return encodeURIComponent(separator >= 0 ? ref.slice(separator + 1) : ref);
|
||||
|
||||
@@ -11,9 +11,11 @@ import {
|
||||
Download,
|
||||
Eye,
|
||||
Layers3,
|
||||
Pencil,
|
||||
Plus,
|
||||
RefreshCw,
|
||||
Snowflake,
|
||||
ShieldCheck,
|
||||
Trash2,
|
||||
Upload
|
||||
} from "lucide-react";
|
||||
@@ -44,7 +46,9 @@ import {
|
||||
refreshDatasource,
|
||||
registerDatasourceOrigin,
|
||||
retireDatasource,
|
||||
updateDatasourceGovernance,
|
||||
type Datasource,
|
||||
type DatasourceGovernance,
|
||||
type DatasourceMaterialization,
|
||||
type DatasourceOrigin,
|
||||
type DatasourcePreview,
|
||||
@@ -82,6 +86,7 @@ export default function DatasourcesPage({
|
||||
const [freezeOpen, setFreezeOpen] = useState(false);
|
||||
const [freezeLabel, setFreezeLabel] = useState("");
|
||||
const [retireOpen, setRetireOpen] = useState(false);
|
||||
const [governanceOpen, setGovernanceOpen] = useState(false);
|
||||
|
||||
const canManage = hasScope(auth, "datasources:source:write")
|
||||
|| hasScope(auth, "datasources:source:admin");
|
||||
@@ -386,6 +391,12 @@ export default function DatasourcesPage({
|
||||
) : null}
|
||||
{view === "catalogue" && selectedDatasource ? (
|
||||
<>
|
||||
<IconButton
|
||||
label="Edit datasource governance"
|
||||
icon={<Pencil size={16} />}
|
||||
onClick={() => setGovernanceOpen(true)}
|
||||
disabled={!canManage || working}
|
||||
/>
|
||||
<Button onClick={() => setFreezeOpen(true)} disabled={!canManage || working}>
|
||||
<Snowflake size={16} /> Freeze
|
||||
</Button>
|
||||
@@ -485,6 +496,17 @@ export default function DatasourcesPage({
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<GovernanceDialog
|
||||
open={governanceOpen}
|
||||
settings={settings}
|
||||
datasource={selectedDatasource}
|
||||
onClose={() => setGovernanceOpen(false)}
|
||||
onSaved={async (updated) => {
|
||||
setGovernanceOpen(false);
|
||||
setSuccess(`Updated governance for ${updated.name}.`);
|
||||
await reload(updated.ref);
|
||||
}}
|
||||
/>
|
||||
<Dialog
|
||||
open={freezeOpen}
|
||||
title="Freeze datasource state"
|
||||
@@ -548,6 +570,26 @@ function DatasourceDetail({
|
||||
{datasource.description ? (
|
||||
<div className="datasources-description">{datasource.description}</div>
|
||||
) : null}
|
||||
<section className="datasources-detail-section">
|
||||
<div className="datasources-section-heading">
|
||||
<span><ShieldCheck size={16} /> Governance</span>
|
||||
<StatusBadge
|
||||
status={datasource.governance.publication_state}
|
||||
label={datasource.governance.publication_state}
|
||||
/>
|
||||
</div>
|
||||
<div className="datasources-key-values">
|
||||
<span><small>Authority</small><strong>{readableToken(datasource.governance.authority_mode)}</strong></span>
|
||||
<span><small>Classification</small><strong>{datasource.governance.classification}</strong></span>
|
||||
<span><small>Owner</small><strong>{datasource.governance.owner_ref || "Not assigned"}</strong></span>
|
||||
<span><small>Steward</small><strong>{datasource.governance.steward_ref || "Not assigned"}</strong></span>
|
||||
<span><small>Responsible organization</small><strong>{datasource.governance.responsible_organization_ref || "Not assigned"}</strong></span>
|
||||
<span><small>Purposes</small><strong>{datasource.governance.purposes.join(", ") || "Not declared"}</strong></span>
|
||||
</div>
|
||||
{datasource.governance.semantic_definition ? (
|
||||
<p className="datasources-dialog-copy">{datasource.governance.semantic_definition}</p>
|
||||
) : null}
|
||||
</section>
|
||||
<section className="datasources-detail-section">
|
||||
<div className="datasources-section-heading">
|
||||
<span><Eye size={16} /> Preview</span>
|
||||
@@ -684,6 +726,173 @@ function OriginDetail({ origin }: { origin: DatasourceOrigin }) {
|
||||
);
|
||||
}
|
||||
|
||||
function GovernanceDialog({
|
||||
open,
|
||||
settings,
|
||||
datasource,
|
||||
onClose,
|
||||
onSaved
|
||||
}: {
|
||||
open: boolean;
|
||||
settings: ApiSettings;
|
||||
datasource: Datasource | null;
|
||||
onClose: () => void;
|
||||
onSaved: (datasource: Datasource) => void | Promise<void>;
|
||||
}) {
|
||||
const [draft, setDraft] = useState<DatasourceGovernance | null>(null);
|
||||
const [freshness, setFreshness] = useState("{}");
|
||||
const [quality, setQuality] = useState("{}");
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
useEffect(() => {
|
||||
if (!open || !datasource) return;
|
||||
setDraft(structuredClone(datasource.governance));
|
||||
setFreshness(JSON.stringify(datasource.governance.freshness_policy, null, 2));
|
||||
setQuality(JSON.stringify(datasource.governance.quality_policy, null, 2));
|
||||
setError("");
|
||||
}, [datasource, open]);
|
||||
|
||||
const save = async () => {
|
||||
if (!datasource || !draft) return;
|
||||
setBusy(true);
|
||||
setError("");
|
||||
try {
|
||||
const updated = await updateDatasourceGovernance(settings, datasource.ref, {
|
||||
...draft,
|
||||
freshness_policy: parseObject(freshness, "Freshness policy"),
|
||||
quality_policy: parseObject(quality, "Quality policy")
|
||||
});
|
||||
await onSaved(updated);
|
||||
} catch (saveError) {
|
||||
setError(apiErrorMessage(saveError));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
const setValue = <K extends keyof DatasourceGovernance>(
|
||||
key: K,
|
||||
value: DatasourceGovernance[K]
|
||||
) => setDraft((current) => current ? { ...current, [key]: value } : current);
|
||||
|
||||
return (
|
||||
<Dialog
|
||||
open={open}
|
||||
title="Datasource governance"
|
||||
className="datasources-governance-dialog"
|
||||
onClose={() => { if (!busy) onClose(); }}
|
||||
footer={(
|
||||
<>
|
||||
<Button onClick={onClose} disabled={busy}>Cancel</Button>
|
||||
<Button variant="primary" onClick={() => void save()} disabled={!draft || busy}>
|
||||
Save governance
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
>
|
||||
{error ? <DismissibleAlert tone="danger" resetKey={error}>{error}</DismissibleAlert> : null}
|
||||
{draft ? (
|
||||
<div className="datasources-dialog-fields">
|
||||
<div className="datasources-dialog-grid">
|
||||
<FormField label="Authority mode">
|
||||
<select
|
||||
value={draft.authority_mode}
|
||||
onChange={(event) => setValue("authority_mode", event.target.value as DatasourceGovernance["authority_mode"])}
|
||||
>
|
||||
{[
|
||||
"native_authoritative",
|
||||
"external_authoritative",
|
||||
"external_mirror",
|
||||
"governed_sync",
|
||||
"governance_overlay",
|
||||
"linked_reference"
|
||||
].map((value) => <option key={value} value={value}>{readableToken(value)}</option>)}
|
||||
</select>
|
||||
</FormField>
|
||||
<FormField label="Authoritative source">
|
||||
<input value={draft.authoritative_source_ref ?? ""} onChange={(event) => setValue("authoritative_source_ref", event.target.value || null)} />
|
||||
</FormField>
|
||||
<FormField label="Classification">
|
||||
<input value={draft.classification} onChange={(event) => setValue("classification", event.target.value)} />
|
||||
</FormField>
|
||||
<FormField label="Publication state">
|
||||
<input value={draft.publication_state} onChange={(event) => setValue("publication_state", event.target.value)} />
|
||||
</FormField>
|
||||
<FormField label="Owner reference">
|
||||
<input value={draft.owner_ref ?? ""} onChange={(event) => setValue("owner_ref", event.target.value || null)} />
|
||||
</FormField>
|
||||
<FormField label="Steward reference">
|
||||
<input value={draft.steward_ref ?? ""} onChange={(event) => setValue("steward_ref", event.target.value || null)} />
|
||||
</FormField>
|
||||
<FormField label="Responsible organization">
|
||||
<input value={draft.responsible_organization_ref ?? ""} onChange={(event) => setValue("responsible_organization_ref", event.target.value || null)} />
|
||||
</FormField>
|
||||
<FormField label="Responsible function">
|
||||
<input value={draft.responsible_function_ref ?? ""} onChange={(event) => setValue("responsible_function_ref", event.target.value || null)} />
|
||||
</FormField>
|
||||
<FormField label="Schema owner">
|
||||
<input value={draft.schema_owner_ref ?? ""} onChange={(event) => setValue("schema_owner_ref", event.target.value || null)} />
|
||||
</FormField>
|
||||
<FormField label="Privacy profile">
|
||||
<input value={draft.privacy_profile_ref ?? ""} onChange={(event) => setValue("privacy_profile_ref", event.target.value || null)} />
|
||||
</FormField>
|
||||
<FormField label="Retention policy">
|
||||
<input value={draft.retention_policy_ref ?? ""} onChange={(event) => setValue("retention_policy_ref", event.target.value || null)} />
|
||||
</FormField>
|
||||
<FormField label="Transfer agreement">
|
||||
<input value={draft.transfer_agreement_ref ?? ""} onChange={(event) => setValue("transfer_agreement_ref", event.target.value || null)} />
|
||||
</FormField>
|
||||
<FormField label="Correction procedure">
|
||||
<input value={draft.correction_procedure_ref ?? ""} onChange={(event) => setValue("correction_procedure_ref", event.target.value || null)} />
|
||||
</FormField>
|
||||
</div>
|
||||
<FormField label="Semantic definition">
|
||||
<textarea value={draft.semantic_definition ?? ""} onChange={(event) => setValue("semantic_definition", event.target.value || null)} />
|
||||
</FormField>
|
||||
<div className="datasources-dialog-grid">
|
||||
<GovernanceListField label="Purposes" values={draft.purposes} onChange={(values) => setValue("purposes", values)} />
|
||||
<GovernanceListField label="Legal basis references" values={draft.legal_basis_refs} onChange={(values) => setValue("legal_basis_refs", values)} />
|
||||
<GovernanceListField label="Official keys" values={draft.official_keys} onChange={(values) => setValue("official_keys", values)} />
|
||||
<GovernanceListField label="Legal hold references" values={draft.hold_refs} onChange={(values) => setValue("hold_refs", values)} />
|
||||
<GovernanceListField label="Affected services and processes" values={draft.affected_refs} onChange={(values) => setValue("affected_refs", values)} />
|
||||
<GovernanceListField label="Dependent flows, reports, controls and decisions" values={draft.dependency_refs} onChange={(values) => setValue("dependency_refs", values)} />
|
||||
<GovernanceListField label="Known limits" values={draft.known_limits} onChange={(values) => setValue("known_limits", values)} />
|
||||
</div>
|
||||
<div className="datasources-dialog-grid">
|
||||
<FormField label="Freshness policy (JSON)">
|
||||
<textarea value={freshness} onChange={(event) => setFreshness(event.target.value)} spellCheck={false} />
|
||||
</FormField>
|
||||
<FormField label="Quality policy (JSON)">
|
||||
<textarea value={quality} onChange={(event) => setQuality(event.target.value)} spellCheck={false} />
|
||||
</FormField>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
||||
function GovernanceListField({
|
||||
label,
|
||||
values,
|
||||
onChange
|
||||
}: {
|
||||
label: string;
|
||||
values: string[];
|
||||
onChange: (values: string[]) => void;
|
||||
}) {
|
||||
return (
|
||||
<FormField label={label}>
|
||||
<textarea
|
||||
value={values.join("\n")}
|
||||
onChange={(event) => onChange(splitLines(event.target.value))}
|
||||
placeholder="One reference or value per line"
|
||||
/>
|
||||
</FormField>
|
||||
);
|
||||
}
|
||||
|
||||
function AddDatasourceDialog({
|
||||
open,
|
||||
settings,
|
||||
@@ -1080,6 +1289,20 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function parseObject(value: string, label: string): Record<string, unknown> {
|
||||
const parsed: unknown = JSON.parse(value || "{}");
|
||||
if (!isRecord(parsed)) throw new Error(`${label} must be a JSON object.`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function splitLines(value: string): string[] {
|
||||
return [...new Set(value.split(/\r?\n/).map((item) => item.trim()).filter(Boolean))];
|
||||
}
|
||||
|
||||
function readableToken(value: string): string {
|
||||
return value.replace(/_/g, " ").replace(/^./, (first: string) => first.toUpperCase());
|
||||
}
|
||||
|
||||
function displayValue(value: unknown): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
if (typeof value === "object") return JSON.stringify(value);
|
||||
|
||||
@@ -397,6 +397,15 @@
|
||||
width: min(760px, calc(100vw - 32px));
|
||||
}
|
||||
|
||||
.datasources-governance-dialog {
|
||||
width: min(980px, calc(100vw - 32px));
|
||||
max-height: min(860px, calc(100vh - 32px));
|
||||
}
|
||||
|
||||
.datasources-governance-dialog .datasources-dialog-fields textarea {
|
||||
min-height: 76px;
|
||||
}
|
||||
|
||||
.datasources-dialog-fields {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
|
||||
Reference in New Issue
Block a user