fix(core): preserve data integrity and bound shared UI and response work

Release v0.1.46. Coordinated integrity review: GovOPlaN/govoplan-core#298.

src/govoplan_core/core/concurrency.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import copy
 import hashlib
+import heapq
 import json
 from dataclasses import dataclass, field
 from typing import Any, Callable, Iterable, Mapping, Sequence
@@ -436,10 +437,33 @@ def _merge_list(
             result_by_id[identity] = merged.value
 
     order_source = local_order if local_reordered and not current_reordered else current_order
-    merged_order = [identity for identity in order_source if identity in result_by_id]
-    for identity in identities:
-        if identity in result_by_id and identity not in merged_order:
-            merged_order.append(identity)
+    secondary_order = current_order if order_source is local_order else local_order
+    # Keep the chosen side's order and both sides' insertion anchors. Appending
+    # missing IDs would silently relocate an insertion during a disjoint edit.
+    # An incompatible reorder/insertion cycle is an explicit conflict.
+    edges: dict[str, set[str]] = {identity: set() for identity in result_by_id}
+    incoming = dict.fromkeys(result_by_id, 0)
+    for order, insertions_only in ((order_source, False), (secondary_order, True)):
+        selected = [identity for identity in order if identity in result_by_id]
+        for left, right in zip(selected, selected[1:]):
+            if insertions_only and left in base_by_id and right in base_by_id:
+                continue
+            if right not in edges[left]:
+                edges[left].add(right)
+                incoming[right] += 1
+    priority = {identity: index for index, identity in enumerate(identities)}
+    ready = [(priority[identity], identity) for identity, count in incoming.items() if count == 0]
+    heapq.heapify(ready)
+    merged_order: list[str] = []
+    while ready:
+        _, identity = heapq.heappop(ready)
+        merged_order.append(identity)
+        for following in edges[identity]:
+            incoming[following] -= 1
+            if incoming[following] == 0:
+                heapq.heappush(ready, (priority[following], following))
+    if len(merged_order) != len(result_by_id):
+        return _conflict(path, "collection_reorder", base, local, current)
     return ThreeWayMergeResult(
         value=[result_by_id[identity] for identity in merged_order],
         conflicts=conflicts,

src/govoplan_core/core/datasources.py

@@ -14,6 +14,7 @@ from govoplan_core.core.tabular_sources import (
     DEFAULT_PREVIEW_BYTES,
     DEFAULT_PREVIEW_TIMEOUT_MS,
     TabularPreviewDiagnostic,
+    TabularCsvSource,
     TabularPushdown,
     TabularSourceHealth,
     TabularSourceMode,
@@ -369,6 +370,7 @@ class DatasourceStageInput:
     provenance: Mapping[str, object] = field(default_factory=dict)
     metadata: Mapping[str, object] = field(default_factory=dict)
     governance: DatasourceGovernance | None = None
+    csv_source: TabularCsvSource | None = None
 
 
 @dataclass(frozen=True, slots=True)

src/govoplan_core/core/modules.py

@@ -409,6 +409,25 @@ class DocumentationTopic:
     metadata: Mapping[str, Any] = field(default_factory=dict)
 
 
+def localize_documentation_topics(
+    topics: Iterable[DocumentationTopic],
+    *,
+    locale: str,
+    translations: Mapping[str, Mapping[str, str]],
+) -> tuple[DocumentationTopic, ...]:
+    """Merge owner-supplied text translations without moving feature content."""
+    localized: list[DocumentationTopic] = []
+    for topic in topics:
+        translated = translations.get(topic.id)
+        if translated is None:
+            localized.append(topic)
+            continue
+        values = {name: dict(value) for name, value in topic.translations.items()}
+        values[locale] = {**values.get(locale, {}), **translated}
+        localized.append(replace(topic, translations=values))
+    return tuple(localized)
+
+
 def localizable_documentation_metadata_keys(
     topic: DocumentationTopic,
 ) -> tuple[str, ...]:

src/govoplan_core/core/principal_helpers.py

@@ -0,0 +1,37 @@
+"""Pure principal attribution mechanics, not authorization or tenant resolution.
+
+The two existing contracts intentionally differ in precedence and whitespace.
+Callers retain their own service-account, scope and resource-access decisions.
+"""
+
+
+def principal_actor_ids(principal: object) -> tuple[str, ...]:
+    """Account-first legacy IDs, unique in encounter order; retain nonblank text."""
+    user = getattr(principal, "user", None)
+    return tuple(
+        dict.fromkeys(
+            str(value)
+            for value in (
+                getattr(principal, "account_id", None),
+                getattr(principal, "identity_id", None),
+                getattr(principal, "membership_id", None),
+                getattr(user, "id", None),
+            )
+            if str(value or "").strip()
+        )
+    )
+
+
+def principal_user_first_actor(principal: object) -> str | None:
+    """First nonblank user/account/identity/membership ID, with trimmed text."""
+    user = getattr(principal, "user", None)
+    for value in (
+        getattr(user, "id", None),
+        getattr(principal, "account_id", None),
+        getattr(principal, "identity_id", None),
+        getattr(principal, "membership_id", None),
+    ):
+        candidate = str(value or "").strip()
+        if candidate:
+            return candidate
+    return None

src/govoplan_core/core/tabular_sources.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
 import csv
+import hashlib
 import io
+import json
+import math
 import re
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass, field
 from datetime import datetime
+from decimal import Decimal
 from typing import Literal, Protocol, runtime_checkable
 
 
@@ -17,6 +21,77 @@ DEFAULT_PREVIEW_TIMEOUT_MS = 2_000
 TabularSourceMode = Literal["live", "cached", "file_backed", "static"]
 TabularHealthStatus = Literal["healthy", "warning", "error", "unknown"]
 TabularDiagnosticSeverity = Literal["info", "warning", "error"]
+CsvValueMode = Literal["legacy_typed", "text"]
+
+
+@dataclass(frozen=True, slots=True)
+class TabularCsvSource:
+    """Original upload text, retained only with an explicitly durable import.
+
+    This is never catalogue metadata or transient preview retention. Owners
+    enforce access, size limits, lifecycle and export authorization separately.
+    """
+
+    text: str
+    delimiter: str = ","
+    value_mode: CsvValueMode = "legacy_typed"
+    parser_profile: str = "core.csv.v1"
+
+
+def csv_source_payload(source: TabularCsvSource, *, max_bytes: int = 5_000_000) -> dict[str, object]:
+    encoded = _csv_utf8_bytes(source.text)
+    if len(encoded) > max_bytes:
+        raise TabularSourceValidationError(f"Original CSV input is limited to {max_bytes:,} UTF-8 bytes.")
+    if source.value_mode not in {"text", "legacy_typed"} or len(source.delimiter) != 1:
+        raise TabularSourceValidationError("Invalid CSV source parsing options.")
+    return {
+        "text": source.text,
+        "delimiter": source.delimiter,
+        "value_mode": source.value_mode,
+        "parser_profile": source.parser_profile,
+        "sha256": hashlib.sha256(encoded).hexdigest(),
+        "byte_count": len(encoded),
+    }
+
+
+def csv_source_summary(payload: Mapping[str, object]) -> dict[str, object]:
+    """Allowlist the small, non-content evidence safe for catalogue DTOs."""
+    result = {key: payload[key] for key in ("delimiter", "value_mode", "parser_profile", "sha256", "byte_count")}
+    if "governance_history" in payload:
+        result["governance_sha256"] = hashlib.sha256(json.dumps(payload["governance_history"], sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8")).hexdigest()
+    return result
+
+
+def verified_csv_source_text(payload: Mapping[str, object], *, expected_summary: Mapping[str, object] | None = None) -> str:
+    text = payload.get("text")
+    if not isinstance(text, str):
+        raise TabularSourceUnavailableError("Original CSV source text is unavailable.")
+    try:
+        encoded = text.encode("utf-8")
+    except UnicodeError as exc:
+        raise TabularSourceUnavailableError("Original CSV source encoding is invalid.") from exc
+    if len(encoded) != payload.get("byte_count") or hashlib.sha256(encoded).hexdigest() != payload.get("sha256"):
+        raise TabularSourceUnavailableError("Original CSV source integrity verification failed.")
+    if expected_summary is not None:
+        try:
+            actual = json.dumps(csv_source_summary(payload), sort_keys=True, separators=(",", ":"), allow_nan=False)
+            expected = json.dumps(expected_summary, sort_keys=True, separators=(",", ":"), allow_nan=False)
+        except (KeyError, TypeError, ValueError) as exc:
+            raise TabularSourceUnavailableError("Original CSV source evidence is invalid.") from exc
+        if actual != expected:
+            raise TabularSourceUnavailableError("Original CSV source no longer matches its recorded evidence.")
+    return text
+
+
+def csv_projection_matches(expected: Sequence[Mapping[str, object]], actual: Sequence[Mapping[str, object]]) -> bool:
+    """CSV cells are scalar: booleans, integers and floats are not interchangeable."""
+    return len(expected) == len(actual) and all(
+        left.keys() == right.keys() and all(
+            type(value) is type(right[name]) and value == right[name]
+            for name, value in left.items()
+        )
+        for left, right in zip(expected, actual, strict=True)
+    )
 
 
 class TabularSourceError(ValueError):
@@ -39,29 +114,44 @@ class TabularSourceUnavailableError(TabularSourceError):
     pass
 
 
+def _csv_utf8_bytes(text: str) -> bytes:
+    try:
+        return text.encode("utf-8")
+    except UnicodeError as exc:
+        raise TabularSourceValidationError("CSV input must be valid Unicode encodable as UTF-8.") from exc
+
+
 def parse_tabular_csv(
     csv_text: str,
     *,
     delimiter: str = ",",
     max_rows: int = 10_000,
+    max_bytes: int = 5_000_000,
+    value_mode: CsvValueMode = "legacy_typed",
 ) -> tuple[Mapping[str, object], ...]:
-    """Parse a bounded CSV document into JSON-compatible tabular rows."""
+    """Parse CSV with an explicit lexical-text or backward-compatible typed mode."""
 
     if len(delimiter) != 1:
         raise TabularSourceValidationError("CSV delimiter must be one character.")
+    if value_mode not in {"legacy_typed", "text"}:
+        raise TabularSourceValidationError("Unsupported CSV value mode.")
+    if len(_csv_utf8_bytes(csv_text)) > max_bytes:
+        raise TabularSourceValidationError(f"CSV input is limited to {max_bytes:,} UTF-8 bytes.")
     try:
-        reader = csv.DictReader(io.StringIO(csv_text), delimiter=delimiter)
+        reader = csv.DictReader(io.StringIO(csv_text), delimiter=delimiter, strict=value_mode == "text")
         original_headers, normalized_headers = _csv_headers(reader.fieldnames)
         rows: list[dict[str, object]] = []
         for row in reader:
             _validate_csv_row_shape(row)
-            if _csv_row_is_empty(row, original_headers):
+            if value_mode == "text" and (None in row or any(row.get(header) is None for header in original_headers)):
+                raise TabularSourceValidationError("CSV text rows must have exactly the number of values defined by the header.")
+            if value_mode == "legacy_typed" and _csv_row_is_empty(row, original_headers):
                 continue
             if len(rows) >= max_rows:
                 raise TabularSourceValidationError(
                     f"CSV snapshots are limited to {max_rows:,} rows."
                 )
-            rows.append(_csv_row(row, original_headers, normalized_headers))
+            rows.append(_csv_row(row, original_headers, normalized_headers, value_mode=value_mode))
         return tuple(rows)
     except csv.Error as exc:
         raise TabularSourceValidationError(f"CSV input could not be parsed: {exc}") from exc
@@ -106,9 +196,11 @@ def _csv_row(
     row: Mapping[str | None, str | list[str] | None],
     original_headers: Sequence[str],
     normalized_headers: Sequence[str],
+    *,
+    value_mode: CsvValueMode = "legacy_typed",
 ) -> dict[str, object]:
     return {
-        normalized: _csv_scalar(value if isinstance(value, str) else None)
+        normalized: (value if value_mode == "text" else _csv_scalar(value if isinstance(value, str) else None))
         for original, normalized in zip(
             original_headers,
             normalized_headers,
@@ -128,9 +220,15 @@ def _csv_scalar(value: str | None) -> object:
     if lowered in {"true", "false"}:
         return lowered == "true"
     if re.fullmatch(r"-?(?:0|[1-9][0-9]*)", text):
-        return int(text)
+        try:
+            return int(text)
+        except ValueError as exc:
+            raise TabularSourceValidationError("CSV integer exceeds the conversion limit; use text mode to preserve it.") from exc
     if re.fullmatch(r"-?(?:0|[1-9][0-9]*)\.[0-9]+", text):
-        return float(text)
+        value = float(text)
+        if not math.isfinite(value):
+            raise TabularSourceValidationError("CSV numeric value exceeds the finite number range; use text mode to preserve it.")
+        return value
     return text
 
 
@@ -141,6 +239,48 @@ class TabularColumn:
     nullable: bool = True
 
 
+def tabular_type_name(value: object, *, casefold_unknown: bool = False) -> str:
+    if isinstance(value, bool):
+        return "boolean"
+    if isinstance(value, int):
+        return "integer"
+    if isinstance(value, (float, Decimal)):
+        return "number"
+    if isinstance(value, str):
+        return "string"
+    if isinstance(value, list):
+        return "array"
+    if isinstance(value, dict):
+        return "object"
+    name = type(value).__name__
+    return name.casefold() if casefold_unknown else name.lower()
+
+
+def infer_tabular_schema(
+    rows: Sequence[Mapping[str, object]],
+    *,
+    type_name: Callable[[object], str] = tabular_type_name,
+) -> tuple[TabularColumn, ...]:
+    """Infer first-seen columns in one pass without retaining column values.
+
+    The classifier is explicit so legacy providers can preserve their exact
+    type naming. Missing keys and explicit None both make a column nullable.
+    """
+    states: dict[str, tuple[str | None, int]] = {}
+    for row in rows:
+        for name, value in row.items():
+            kind, concrete = states.get(name, (None, 0))
+            if value is not None:
+                value_kind = type_name(value)
+                kind = value_kind if kind is None else kind if kind == value_kind else "mixed"
+                concrete += 1
+            states[name] = (kind, concrete)
+    return tuple(
+        TabularColumn(name=name, data_type=kind if kind is not None else "unknown", nullable=concrete != len(rows))
+        for name, (kind, concrete) in states.items()
+    )
+
+
 @dataclass(frozen=True, slots=True)
 class TabularPushdown:
     projections: bool = False
@@ -221,6 +361,7 @@ class TabularSnapshotInput:
     rows: tuple[Mapping[str, object], ...]
     description: str | None = None
     metadata: Mapping[str, object] = field(default_factory=dict)
+    csv_source: TabularCsvSource | None = None
 
 
 @runtime_checkable
@@ -296,6 +437,10 @@ __all__ = [
     "CAPABILITY_CONNECTORS_TABULAR_SOURCES",
     "DEFAULT_PREVIEW_BYTES",
     "DEFAULT_PREVIEW_TIMEOUT_MS",
+    "CsvValueMode",
+    "TabularCsvSource",
+    "tabular_type_name",
+    "infer_tabular_schema",
     "TabularColumn",
     "TabularPreviewDiagnostic",
     "TabularPushdown",
@@ -313,6 +458,10 @@ __all__ = [
     "TabularSourceUnavailableError",
     "TabularSourceValidationError",
     "parse_tabular_csv",
+    "csv_source_payload",
+    "csv_source_summary",
+    "verified_csv_source_text",
+    "csv_projection_matches",
     "tabular_snapshot_writer",
     "tabular_source_provider",
 ]

src/govoplan_core/db/json_predicates.py

@@ -0,0 +1,147 @@
+"""Exact, bound-parameter JSON permission predicates for SQLite/PostgreSQL.
+
+These primitives only match strings (never coerced numbers/booleans) and
+objects inside actual arrays. Owners still define tenant, subject, permission,
+purpose and current-state policy. Unsupported dialects fail at compilation.
+"""
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+
+from sqlalchemy import Boolean, literal
+from sqlalchemy.exc import CompileError
+from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.sql.functions import FunctionElement
+
+
+class _ArrayString(FunctionElement):
+    type = Boolean()
+    inherit_cache = True
+
+
+class _ObjectStrings(FunctionElement):
+    type = Boolean()
+    inherit_cache = True
+
+
+class _ArrayObjectStrings(FunctionElement):
+    type = Boolean()
+    inherit_cache = True
+
+
+def json_array_contains_string(column, value: str):
+    if type(value) is not str:
+        raise TypeError("JSON string membership requires a string value.")
+    return _ArrayString(column, literal(value))
+
+
+def _field_arguments(fields: Mapping[str, str]):
+    if not isinstance(fields, Mapping) or not 1 <= len(fields) <= 16:
+        raise ValueError("JSON object matching requires between 1 and 16 string fields.")
+    arguments = []
+    for key, value in sorted(fields.items()):
+        if type(key) is not str or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,127}", key) is None:
+            raise ValueError("JSON object field names must be simple identifiers.")
+        if type(value) is not str:
+            raise TypeError("JSON object matching requires string values.")
+        arguments.extend((literal(key), literal(value)))
+    return arguments
+
+
+def json_object_matches_strings(column, fields: Mapping[str, str]):
+    return _ObjectStrings(column, *_field_arguments(fields))
+
+
+def json_array_contains_object_strings(column, fields: Mapping[str, str]):
+    return _ArrayObjectStrings(column, *_field_arguments(fields))
+
+
+@compiles(_ArrayString)
+@compiles(_ObjectStrings)
+@compiles(_ArrayObjectStrings)
+def _unsupported(element, compiler, **kwargs):
+    raise CompileError("Exact JSON permission predicates support only SQLite and PostgreSQL.")
+
+
+def _parts(element, compiler, kwargs):
+    return [compiler.process(item, **kwargs) for item in element.clauses]
+
+
+def _sqlite_array(value):
+    return f"CASE WHEN json_type({value}) = 'array' THEN {value} ELSE '[]' END"
+
+
+def _postgres_array(value):
+    value = f"CAST({value} AS JSON)"
+    return f"CASE WHEN json_typeof({value}) = 'array' THEN {value} ELSE '[]'::json END"
+
+
+def _sqlite_fields(value, fields):
+    terms = []
+    for index in range(0, len(fields), 2):
+        key, expected = fields[index:index + 2]
+        path = f"('$.' || {key})"
+        terms.extend((f"json_type({value}, {path}) = 'text'", f"json_extract({value}, {path}) = {expected}"))
+    return " AND ".join(terms)
+
+
+def _postgres_fields(value, fields):
+    terms = []
+    for index in range(0, len(fields), 2):
+        key, expected = fields[index:index + 2]
+        terms.extend((f"json_typeof(({value}) -> {key}) = 'string'", f"(({value}) ->> {key}) = {expected}"))
+    return " AND ".join(terms)
+
+
+@compiles(_ArrayString, "sqlite")
+def _array_string_sqlite(element, compiler, **kwargs):
+    value, expected = _parts(element, compiler, kwargs)
+    return (
+        f"EXISTS (SELECT 1 FROM json_each({_sqlite_array(value)}) AS gp_json_string "
+        f"WHERE gp_json_string.type = 'text' AND gp_json_string.value = {expected})"
+    )
+
+
+@compiles(_ArrayString, "postgresql")
+def _array_string_postgres(element, compiler, **kwargs):
+    value, expected = _parts(element, compiler, kwargs)
+    return (
+        f"EXISTS (SELECT 1 FROM json_array_elements({_postgres_array(value)}) AS gp_json_string(value) "
+        f"WHERE json_typeof(gp_json_string.value) = 'string' "
+        f"AND (gp_json_string.value #>> '{{}}') = {expected})"
+    )
+
+
+@compiles(_ObjectStrings, "sqlite")
+def _object_sqlite(element, compiler, **kwargs):
+    value, *fields = _parts(element, compiler, kwargs)
+    value = f"CASE WHEN json_type({value}) = 'object' THEN {value} ELSE '{{}}' END"
+    return f"({_sqlite_fields(value, fields)})"
+
+
+@compiles(_ObjectStrings, "postgresql")
+def _object_postgres(element, compiler, **kwargs):
+    value, *fields = _parts(element, compiler, kwargs)
+    value = f"CAST({value} AS JSON)"
+    return f"(json_typeof({value}) = 'object' AND {_postgres_fields(value, fields)})"
+
+
+@compiles(_ArrayObjectStrings, "sqlite")
+def _array_object_sqlite(element, compiler, **kwargs):
+    value, *fields = _parts(element, compiler, kwargs)
+    item = "CASE WHEN gp_json_object.type = 'object' THEN gp_json_object.value ELSE '{}' END"
+    return (
+        f"EXISTS (SELECT 1 FROM json_each({_sqlite_array(value)}) AS gp_json_object "
+        f"WHERE {_sqlite_fields(item, fields)})"
+    )
+
+
+@compiles(_ArrayObjectStrings, "postgresql")
+def _array_object_postgres(element, compiler, **kwargs):
+    value, *fields = _parts(element, compiler, kwargs)
+    return (
+        f"EXISTS (SELECT 1 FROM json_array_elements({_postgres_array(value)}) AS gp_json_object(value) "
+        f"WHERE json_typeof(gp_json_object.value) = 'object' "
+        f"AND {_postgres_fields('gp_json_object.value', fields)})"
+    )

src/govoplan_core/server/conditional_requests.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
 import hashlib
-from collections.abc import Awaitable, Callable
+from collections.abc import AsyncIterator, Awaitable, Callable
 
 from fastapi import Request
 from starlette.responses import Response
 
 JSON_CACHE_CONTROL = "private, no-cache"
+# This bounds middleware-owned buffering, not the size of a route response.
+# Larger responses retain their streaming iterator and are never truncated.
+MAX_CONDITIONAL_JSON_BYTES = 1_048_576
 JSON_ETAG_VARY_HEADERS = (
     "Authorization",
     "Cookie",
@@ -26,15 +29,34 @@ async def conditional_json_get_middleware(
 
     The middleware deliberately works after route handling. That keeps the
     contract platform-wide without requiring every module router to learn about
-    conditional requests, while still limiting buffering to successful JSON GET
-    responses.
+    conditional requests. Only small successful JSON responses are buffered;
+    larger responses stream unchanged. Authorization still runs on every GET.
     """
 
     response = await call_next(request)
     if not _eligible_for_conditional_json_get(request, response):
         return response
 
-    body = b"".join([chunk async for chunk in response.body_iterator])
+    response.headers["cache-control"] = _conditional_cache_control(response.headers.get("cache-control"))
+    response.headers["vary"] = _merge_vary(response.headers.get("vary"), JSON_ETAG_VARY_HEADERS)
+    content_length = response.headers.get("content-length", "")
+    if content_length.isascii() and content_length.isdecimal() and int(content_length) > MAX_CONDITIONAL_JSON_BYTES:
+        return response
+
+    chunks: list[bytes] = []
+    size = 0
+    iterator = response.body_iterator
+    async for chunk in iterator:
+        if not chunk:
+            continue
+        chunks.append(chunk)
+        size += len(chunk)
+        if size > MAX_CONDITIONAL_JSON_BYTES:
+            # Include the crossing chunk exactly once, without draining the
+            # rest of the producer or copying a potentially large chunk.
+            response.body_iterator = _replay_prefix(chunks, iterator)
+            return response
+    body = b"".join(chunks)
     etag = response.headers.get("etag") or json_response_etag(body)
     headers = dict(response.headers)
     headers["etag"] = etag
@@ -48,6 +70,14 @@ async def conditional_json_get_middleware(
     return Response(content=body, status_code=response.status_code, headers=headers, background=response.background)
 
 
+async def _replay_prefix(chunks: list[bytes], iterator: AsyncIterator[bytes]) -> AsyncIterator[bytes]:
+    for chunk in chunks:
+        yield chunk
+    chunks.clear()
+    async for chunk in iterator:
+        yield chunk
+
+
 def json_response_etag(body: bytes) -> str:
     digest = hashlib.sha256(body).hexdigest()
     return f'W/"sha256-{digest}"'