fix(core): preserve data integrity and bound shared UI and response work

Release v0.1.46. Coordinated integrity review: GovOPlaN/govoplan-core#298.

src/govoplan_core/core/concurrency.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import copy
 import hashlib
+import heapq
 import json
 from dataclasses import dataclass, field
 from typing import Any, Callable, Iterable, Mapping, Sequence
@@ -436,10 +437,33 @@ def _merge_list(
             result_by_id[identity] = merged.value
 
     order_source = local_order if local_reordered and not current_reordered else current_order
-    merged_order = [identity for identity in order_source if identity in result_by_id]
-    for identity in identities:
-        if identity in result_by_id and identity not in merged_order:
-            merged_order.append(identity)
+    secondary_order = current_order if order_source is local_order else local_order
+    # Keep the chosen side's order and both sides' insertion anchors. Appending
+    # missing IDs would silently relocate an insertion during a disjoint edit.
+    # An incompatible reorder/insertion cycle is an explicit conflict.
+    edges: dict[str, set[str]] = {identity: set() for identity in result_by_id}
+    incoming = dict.fromkeys(result_by_id, 0)
+    for order, insertions_only in ((order_source, False), (secondary_order, True)):
+        selected = [identity for identity in order if identity in result_by_id]
+        for left, right in zip(selected, selected[1:]):
+            if insertions_only and left in base_by_id and right in base_by_id:
+                continue
+            if right not in edges[left]:
+                edges[left].add(right)
+                incoming[right] += 1
+    priority = {identity: index for index, identity in enumerate(identities)}
+    ready = [(priority[identity], identity) for identity, count in incoming.items() if count == 0]
+    heapq.heapify(ready)
+    merged_order: list[str] = []
+    while ready:
+        _, identity = heapq.heappop(ready)
+        merged_order.append(identity)
+        for following in edges[identity]:
+            incoming[following] -= 1
+            if incoming[following] == 0:
+                heapq.heappush(ready, (priority[following], following))
+    if len(merged_order) != len(result_by_id):
+        return _conflict(path, "collection_reorder", base, local, current)
     return ThreeWayMergeResult(
         value=[result_by_id[identity] for identity in merged_order],
         conflicts=conflicts,

src/govoplan_core/core/datasources.py

@@ -14,6 +14,7 @@ from govoplan_core.core.tabular_sources import (
     DEFAULT_PREVIEW_BYTES,
     DEFAULT_PREVIEW_TIMEOUT_MS,
     TabularPreviewDiagnostic,
+    TabularCsvSource,
     TabularPushdown,
     TabularSourceHealth,
     TabularSourceMode,
@@ -369,6 +370,7 @@ class DatasourceStageInput:
     provenance: Mapping[str, object] = field(default_factory=dict)
     metadata: Mapping[str, object] = field(default_factory=dict)
     governance: DatasourceGovernance | None = None
+    csv_source: TabularCsvSource | None = None
 
 
 @dataclass(frozen=True, slots=True)

src/govoplan_core/core/modules.py

@@ -409,6 +409,25 @@ class DocumentationTopic:
     metadata: Mapping[str, Any] = field(default_factory=dict)
 
 
+def localize_documentation_topics(
+    topics: Iterable[DocumentationTopic],
+    *,
+    locale: str,
+    translations: Mapping[str, Mapping[str, str]],
+) -> tuple[DocumentationTopic, ...]:
+    """Merge owner-supplied text translations without moving feature content."""
+    localized: list[DocumentationTopic] = []
+    for topic in topics:
+        translated = translations.get(topic.id)
+        if translated is None:
+            localized.append(topic)
+            continue
+        values = {name: dict(value) for name, value in topic.translations.items()}
+        values[locale] = {**values.get(locale, {}), **translated}
+        localized.append(replace(topic, translations=values))
+    return tuple(localized)
+
+
 def localizable_documentation_metadata_keys(
     topic: DocumentationTopic,
 ) -> tuple[str, ...]:

src/govoplan_core/core/principal_helpers.py

@@ -0,0 +1,37 @@
+"""Pure principal attribution mechanics, not authorization or tenant resolution.
+
+The two existing contracts intentionally differ in precedence and whitespace.
+Callers retain their own service-account, scope and resource-access decisions.
+"""
+
+
+def principal_actor_ids(principal: object) -> tuple[str, ...]:
+    """Account-first legacy IDs, unique in encounter order; retain nonblank text."""
+    user = getattr(principal, "user", None)
+    return tuple(
+        dict.fromkeys(
+            str(value)
+            for value in (
+                getattr(principal, "account_id", None),
+                getattr(principal, "identity_id", None),
+                getattr(principal, "membership_id", None),
+                getattr(user, "id", None),
+            )
+            if str(value or "").strip()
+        )
+    )
+
+
+def principal_user_first_actor(principal: object) -> str | None:
+    """First nonblank user/account/identity/membership ID, with trimmed text."""
+    user = getattr(principal, "user", None)
+    for value in (
+        getattr(user, "id", None),
+        getattr(principal, "account_id", None),
+        getattr(principal, "identity_id", None),
+        getattr(principal, "membership_id", None),
+    ):
+        candidate = str(value or "").strip()
+        if candidate:
+            return candidate
+    return None

src/govoplan_core/core/tabular_sources.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
 import csv
+import hashlib
 import io
+import json
+import math
 import re
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass, field
 from datetime import datetime
+from decimal import Decimal
 from typing import Literal, Protocol, runtime_checkable
 
 
@@ -17,6 +21,77 @@ DEFAULT_PREVIEW_TIMEOUT_MS = 2_000
 TabularSourceMode = Literal["live", "cached", "file_backed", "static"]
 TabularHealthStatus = Literal["healthy", "warning", "error", "unknown"]
 TabularDiagnosticSeverity = Literal["info", "warning", "error"]
+CsvValueMode = Literal["legacy_typed", "text"]
+
+
+@dataclass(frozen=True, slots=True)
+class TabularCsvSource:
+    """Original upload text, retained only with an explicitly durable import.
+
+    This is never catalogue metadata or transient preview retention. Owners
+    enforce access, size limits, lifecycle and export authorization separately.
+    """
+
+    text: str
+    delimiter: str = ","
+    value_mode: CsvValueMode = "legacy_typed"
+    parser_profile: str = "core.csv.v1"
+
+
+def csv_source_payload(source: TabularCsvSource, *, max_bytes: int = 5_000_000) -> dict[str, object]:
+    encoded = _csv_utf8_bytes(source.text)
+    if len(encoded) > max_bytes:
+        raise TabularSourceValidationError(f"Original CSV input is limited to {max_bytes:,} UTF-8 bytes.")
+    if source.value_mode not in {"text", "legacy_typed"} or len(source.delimiter) != 1:
+        raise TabularSourceValidationError("Invalid CSV source parsing options.")
+    return {
+        "text": source.text,
+        "delimiter": source.delimiter,
+        "value_mode": source.value_mode,
+        "parser_profile": source.parser_profile,
+        "sha256": hashlib.sha256(encoded).hexdigest(),
+        "byte_count": len(encoded),
+    }
+
+
+def csv_source_summary(payload: Mapping[str, object]) -> dict[str, object]:
+    """Allowlist the small, non-content evidence safe for catalogue DTOs."""
+    result = {key: payload[key] for key in ("delimiter", "value_mode", "parser_profile", "sha256", "byte_count")}
+    if "governance_history" in payload:
+        result["governance_sha256"] = hashlib.sha256(json.dumps(payload["governance_history"], sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8")).hexdigest()
+    return result
+
+
+def verified_csv_source_text(payload: Mapping[str, object], *, expected_summary: Mapping[str, object] | None = None) -> str:
+    text = payload.get("text")
+    if not isinstance(text, str):
+        raise TabularSourceUnavailableError("Original CSV source text is unavailable.")
+    try:
+        encoded = text.encode("utf-8")
+    except UnicodeError as exc:
+        raise TabularSourceUnavailableError("Original CSV source encoding is invalid.") from exc
+    if len(encoded) != payload.get("byte_count") or hashlib.sha256(encoded).hexdigest() != payload.get("sha256"):
+        raise TabularSourceUnavailableError("Original CSV source integrity verification failed.")
+    if expected_summary is not None:
+        try:
+            actual = json.dumps(csv_source_summary(payload), sort_keys=True, separators=(",", ":"), allow_nan=False)
+            expected = json.dumps(expected_summary, sort_keys=True, separators=(",", ":"), allow_nan=False)
+        except (KeyError, TypeError, ValueError) as exc:
+            raise TabularSourceUnavailableError("Original CSV source evidence is invalid.") from exc
+        if actual != expected:
+            raise TabularSourceUnavailableError("Original CSV source no longer matches its recorded evidence.")
+    return text
+
+
+def csv_projection_matches(expected: Sequence[Mapping[str, object]], actual: Sequence[Mapping[str, object]]) -> bool:
+    """CSV cells are scalar: booleans, integers and floats are not interchangeable."""
+    return len(expected) == len(actual) and all(
+        left.keys() == right.keys() and all(
+            type(value) is type(right[name]) and value == right[name]
+            for name, value in left.items()
+        )
+        for left, right in zip(expected, actual, strict=True)
+    )
 
 
 class TabularSourceError(ValueError):
@@ -39,29 +114,44 @@ class TabularSourceUnavailableError(TabularSourceError):
     pass
 
 
+def _csv_utf8_bytes(text: str) -> bytes:
+    try:
+        return text.encode("utf-8")
+    except UnicodeError as exc:
+        raise TabularSourceValidationError("CSV input must be valid Unicode encodable as UTF-8.") from exc
+
+
 def parse_tabular_csv(
     csv_text: str,
     *,
     delimiter: str = ",",
     max_rows: int = 10_000,
+    max_bytes: int = 5_000_000,
+    value_mode: CsvValueMode = "legacy_typed",
 ) -> tuple[Mapping[str, object], ...]:
-    """Parse a bounded CSV document into JSON-compatible tabular rows."""
+    """Parse CSV with an explicit lexical-text or backward-compatible typed mode."""
 
     if len(delimiter) != 1:
         raise TabularSourceValidationError("CSV delimiter must be one character.")
+    if value_mode not in {"legacy_typed", "text"}:
+        raise TabularSourceValidationError("Unsupported CSV value mode.")
+    if len(_csv_utf8_bytes(csv_text)) > max_bytes:
+        raise TabularSourceValidationError(f"CSV input is limited to {max_bytes:,} UTF-8 bytes.")
     try:
-        reader = csv.DictReader(io.StringIO(csv_text), delimiter=delimiter)
+        reader = csv.DictReader(io.StringIO(csv_text), delimiter=delimiter, strict=value_mode == "text")
         original_headers, normalized_headers = _csv_headers(reader.fieldnames)
         rows: list[dict[str, object]] = []
         for row in reader:
             _validate_csv_row_shape(row)
-            if _csv_row_is_empty(row, original_headers):
+            if value_mode == "text" and (None in row or any(row.get(header) is None for header in original_headers)):
+                raise TabularSourceValidationError("CSV text rows must have exactly the number of values defined by the header.")
+            if value_mode == "legacy_typed" and _csv_row_is_empty(row, original_headers):
                 continue
             if len(rows) >= max_rows:
                 raise TabularSourceValidationError(
                     f"CSV snapshots are limited to {max_rows:,} rows."
                 )
-            rows.append(_csv_row(row, original_headers, normalized_headers))
+            rows.append(_csv_row(row, original_headers, normalized_headers, value_mode=value_mode))
         return tuple(rows)
     except csv.Error as exc:
         raise TabularSourceValidationError(f"CSV input could not be parsed: {exc}") from exc
@@ -106,9 +196,11 @@ def _csv_row(
     row: Mapping[str | None, str | list[str] | None],
     original_headers: Sequence[str],
     normalized_headers: Sequence[str],
+    *,
+    value_mode: CsvValueMode = "legacy_typed",
 ) -> dict[str, object]:
     return {
-        normalized: _csv_scalar(value if isinstance(value, str) else None)
+        normalized: (value if value_mode == "text" else _csv_scalar(value if isinstance(value, str) else None))
         for original, normalized in zip(
             original_headers,
             normalized_headers,
@@ -128,9 +220,15 @@ def _csv_scalar(value: str | None) -> object:
     if lowered in {"true", "false"}:
         return lowered == "true"
     if re.fullmatch(r"-?(?:0|[1-9][0-9]*)", text):
-        return int(text)
+        try:
+            return int(text)
+        except ValueError as exc:
+            raise TabularSourceValidationError("CSV integer exceeds the conversion limit; use text mode to preserve it.") from exc
     if re.fullmatch(r"-?(?:0|[1-9][0-9]*)\.[0-9]+", text):
-        return float(text)
+        value = float(text)
+        if not math.isfinite(value):
+            raise TabularSourceValidationError("CSV numeric value exceeds the finite number range; use text mode to preserve it.")
+        return value
     return text
 
 
@@ -141,6 +239,48 @@ class TabularColumn:
     nullable: bool = True
 
 
+def tabular_type_name(value: object, *, casefold_unknown: bool = False) -> str:
+    if isinstance(value, bool):
+        return "boolean"
+    if isinstance(value, int):
+        return "integer"
+    if isinstance(value, (float, Decimal)):
+        return "number"
+    if isinstance(value, str):
+        return "string"
+    if isinstance(value, list):
+        return "array"
+    if isinstance(value, dict):
+        return "object"
+    name = type(value).__name__
+    return name.casefold() if casefold_unknown else name.lower()
+
+
+def infer_tabular_schema(
+    rows: Sequence[Mapping[str, object]],
+    *,
+    type_name: Callable[[object], str] = tabular_type_name,
+) -> tuple[TabularColumn, ...]:
+    """Infer first-seen columns in one pass without retaining column values.
+
+    The classifier is explicit so legacy providers can preserve their exact
+    type naming. Missing keys and explicit None both make a column nullable.
+    """
+    states: dict[str, tuple[str | None, int]] = {}
+    for row in rows:
+        for name, value in row.items():
+            kind, concrete = states.get(name, (None, 0))
+            if value is not None:
+                value_kind = type_name(value)
+                kind = value_kind if kind is None else kind if kind == value_kind else "mixed"
+                concrete += 1
+            states[name] = (kind, concrete)
+    return tuple(
+        TabularColumn(name=name, data_type=kind if kind is not None else "unknown", nullable=concrete != len(rows))
+        for name, (kind, concrete) in states.items()
+    )
+
+
 @dataclass(frozen=True, slots=True)
 class TabularPushdown:
     projections: bool = False
@@ -221,6 +361,7 @@ class TabularSnapshotInput:
     rows: tuple[Mapping[str, object], ...]
     description: str | None = None
     metadata: Mapping[str, object] = field(default_factory=dict)
+    csv_source: TabularCsvSource | None = None
 
 
 @runtime_checkable
@@ -296,6 +437,10 @@ __all__ = [
     "CAPABILITY_CONNECTORS_TABULAR_SOURCES",
     "DEFAULT_PREVIEW_BYTES",
     "DEFAULT_PREVIEW_TIMEOUT_MS",
+    "CsvValueMode",
+    "TabularCsvSource",
+    "tabular_type_name",
+    "infer_tabular_schema",
     "TabularColumn",
     "TabularPreviewDiagnostic",
     "TabularPushdown",
@@ -313,6 +458,10 @@ __all__ = [
     "TabularSourceUnavailableError",
     "TabularSourceValidationError",
     "parse_tabular_csv",
+    "csv_source_payload",
+    "csv_source_summary",
+    "verified_csv_source_text",
+    "csv_projection_matches",
     "tabular_snapshot_writer",
     "tabular_source_provider",
 ]