feat(datasources): enforce policy-aware data visibility

src/govoplan_datasources/backend/service.py

@@ -33,7 +33,9 @@ from govoplan_core.core.datasources import (
     DatasourceStageInput,
     DatasourceUnavailableError,
     DatasourceValidationError,
+    DatasourceVisibilityPolicyRequest,
     datasource_origins,
+    datasource_visibility_policy_provider,
 )
 from govoplan_core.db.base import utcnow
 from govoplan_datasources.backend.db.models import (
@@ -62,6 +64,14 @@ from govoplan_datasources.backend.tabular import (
     infer_schema,
     normalize_rows,
 )
+from govoplan_datasources.backend.visibility import (
+    VisibilityPlan,
+    apply_visibility,
+    normalize_visibility_policy,
+    visibility_diagnostics,
+    visibility_plan,
+    visible_schema,
+)
 
 
 CATALOGUE_READ_SCOPE = "datasources:catalogue:read"
@@ -335,7 +345,15 @@ class SqlDatasourceProvider:
                         DatasourceGovernanceReferenceRecord.reference == cleaned,
                     )
                 )
-        return tuple(_datasource_dto(item) for item in db.scalars(statement))
+        result: list[DatasourceDescriptor] = []
+        for item in db.scalars(statement):
+            if has_scope(api_principal, ADMIN_SCOPE) or self._can_discover(
+                db,
+                api_principal,
+                item=item,
+            ):
+                result.append(_datasource_dto(item))
+        return tuple(result)
 
     def get_datasource(
         self,
@@ -350,7 +368,15 @@ class SqlDatasourceProvider:
             tenant_id=api_principal.tenant_id,
             datasource_ref=datasource_ref,
         )
-        return _datasource_dto(item) if item is not None else None
+        if item is None:
+            return None
+        if not has_scope(api_principal, ADMIN_SCOPE) and not self._can_discover(
+            db,
+            api_principal,
+            item=item,
+        ):
+            return None
+        return _datasource_dto(item)
 
     def read_datasource(
         self,
@@ -374,41 +400,67 @@ class SqlDatasourceProvider:
             and not request.materialization_ref
             and request.consistency == "current"
         )
-        if read_live:
-            if not item.provider_ref:
-                raise DatasourceUnavailableError(
-                    "This datasource has no live origin."
+        try:
+            if read_live:
+                if not item.provider_ref:
+                    raise DatasourceUnavailableError(
+                        "This datasource has no live origin."
+                    )
+                result = self._read_live(
+                    db,
+                    api_principal,
+                    item=item,
+                    request=request,
+                    limit=limit,
+                    offset=offset,
+                    columns=columns,
                 )
-            result = self._read_live(
+            else:
+                result = self._read_materialized(
+                    db,
+                    api_principal,
+                    item=item,
+                    request=request,
+                    limit=limit,
+                    offset=offset,
+                    columns=columns,
+                )
+        except DatasourceAccessError:
+            _audit_visibility_read(
                 db,
                 api_principal,
                 item=item,
-                request=request,
-                limit=limit,
-                offset=offset,
-                columns=columns,
+                outcome="denied",
+                consistency=request.consistency,
             )
-            if (
-                request.expected_fingerprint
-                and request.expected_fingerprint != result.datasource.fingerprint
-            ):
-                raise DatasourceValidationError(
-                    "The live datasource fingerprint changed; refresh the consuming definition."
-                )
-            return result
-
-        return self._read_materialized(
-            db,
-            item=item,
-            request=request,
-            limit=limit,
-            offset=offset,
-            columns=columns,
+            raise
+        if (
+            request.expected_fingerprint
+            and request.expected_fingerprint != result.datasource.fingerprint
+        ):
+            raise DatasourceValidationError(
+                "The datasource permitted-view fingerprint changed; refresh the consuming definition."
+            )
+        visibility_evidence = tuple(
+            item
+            for item in result.diagnostics
+            if item.code.startswith("datasource.visibility_")
         )
+        if visibility_evidence:
+            _audit_visibility_read(
+                db,
+                api_principal,
+                item=item,
+                outcome="filtered",
+                consistency=request.consistency,
+                diagnostics=visibility_evidence,
+            )
+        return result
 
     def _read_materialized(
         self,
         session: Session,
+        principal: ApiPrincipal,
         *,
         item: DatasourceRecord,
         request: DatasourceReadRequest,
@@ -428,41 +480,114 @@ class SqlDatasourceProvider:
                 else "This datasource has no materialized state."
             )
             raise DatasourceUnavailableError(message)
-        if (
-            request.expected_fingerprint
-            and request.expected_fingerprint != materialization.fingerprint
-        ):
-            raise DatasourceValidationError(
-                "The datasource fingerprint changed; refresh the consuming definition."
-            )
-        _validate_columns(materialization.schema_, columns)
+        source_schema = _fields(materialization.schema_)
+        snapshot_governance = DatasourceGovernance.from_mapping(
+            materialization.governance_snapshot_
+        )
+        snapshot_policies = (
+            (snapshot_governance.visibility_policy,)
+            if snapshot_governance.visibility_policy
+            and snapshot_governance.visibility_policy
+            != _datasource_governance(item).visibility_policy
+            else ()
+        )
+        plan = self._visibility_plan(
+            session,
+            principal,
+            item=item,
+            request=request,
+            materialized=True,
+            schema=source_schema,
+            additional_policies=snapshot_policies,
+        )
+        permitted_schema = visible_schema(source_schema, plan)
+        _validate_field_names(permitted_schema, columns)
         payload = payload_for_materialization(session, materialization)
-        if payload is None:
-            window = materialization.rows[offset : offset + limit]
-        else:
-            backend = self._payload_backends.require(payload.backend)
-            backend.verify(session, payload)
-            window = backend.read_rows(
-                session,
-                payload,
-                offset=offset,
-                limit=limit,
+        if plan.applied:
+            if payload is None:
+                scanned_rows = tuple(materialization.rows[:MAX_STAGE_ROWS])
+            else:
+                backend = self._payload_backends.require(payload.backend)
+                backend.verify(session, payload)
+                scanned_rows = tuple(
+                    backend.read_rows(
+                        session,
+                        payload,
+                        offset=0,
+                        limit=MAX_STAGE_ROWS,
+                    )
+                )
+            permitted_rows, filtered_rows = apply_visibility(
+                scanned_rows,
+                principal=principal.principal,
+                plan=plan,
+                columns=columns,
             )
-        rows = tuple(_select_columns(row, columns) for row in window)
+            candidate_rows = permitted_rows[offset : offset + limit]
+            rows = _bounded_result_rows(
+                candidate_rows,
+                max_bytes=request.max_bytes,
+            )
+            byte_limited = len(rows) < len(candidate_rows)
+            total_rows = len(permitted_rows)
+            scan_limited = len(scanned_rows) < materialization.row_count
+        else:
+            if payload is None:
+                window = materialization.rows[offset : offset + limit]
+            else:
+                backend = self._payload_backends.require(payload.backend)
+                backend.verify(session, payload)
+                window = backend.read_rows(
+                    session,
+                    payload,
+                    offset=offset,
+                    limit=limit,
+                )
+            rows = tuple(_select_columns(row, columns) for row in window)
+            scanned_rows = rows
+            total_rows = materialization.row_count
+            filtered_rows = 0
+            scan_limited = False
+            byte_limited = False
+        view_fingerprint = plan.view_fingerprint(materialization.fingerprint)
         descriptor = _datasource_dto(item)
+        result_descriptor = replace(
+            descriptor,
+            schema=permitted_schema,
+            schema_version=str(materialization.schema_version),
+            fingerprint=view_fingerprint,
+            row_count=total_rows,
+            byte_count=encoded_size(rows),
+        )
+        materialization_dto = replace(
+            _materialization_dto(materialization),
+            schema=permitted_schema,
+            fingerprint=view_fingerprint,
+            row_count=total_rows,
+            byte_count=encoded_size(rows),
+        )
+        if plan.applied:
+            result_descriptor = _opaque_descriptor(result_descriptor)
+            materialization_dto = _opaque_materialization(materialization_dto)
         return DatasourceReadResult(
-            datasource=replace(
-                descriptor,
-                schema=_fields(materialization.schema_),
-                schema_version=str(materialization.schema_version),
-                fingerprint=materialization.fingerprint,
-                row_count=materialization.row_count,
-                byte_count=materialization.byte_count,
-            ),
+            datasource=result_descriptor,
             rows=rows,
-            total_rows=materialization.row_count,
-            truncated=offset + len(rows) < materialization.row_count,
-            materialization=_materialization_dto(materialization),
+            total_rows=total_rows,
+            truncated=(
+                scan_limited or byte_limited or offset + len(rows) < total_rows
+            ),
+            materialization=materialization_dto,
+            returned_bytes=encoded_size(rows),
+            effective_row_limit=limit,
+            effective_byte_limit=request.max_bytes,
+            effective_timeout_ms=request.timeout_ms,
+            diagnostics=visibility_diagnostics(
+                plan,
+                scanned_rows=len(scanned_rows),
+                permitted_rows=total_rows,
+                filtered_rows=filtered_rows,
+                scan_limited=scan_limited,
+            ),
         )
 
     def list_materializations(
@@ -486,7 +611,46 @@ class SqlDatasourceProvider:
             )
             .order_by(DatasourceMaterializationRecord.revision.desc())
         )
-        return tuple(_materialization_dto(row) for row in db.scalars(statement))
+        result: list[DatasourceMaterialization] = []
+        for row in db.scalars(statement):
+            schema = _fields(row.schema_)
+            snapshot = DatasourceGovernance.from_mapping(row.governance_snapshot_)
+            additional_policies = (
+                (snapshot.visibility_policy,)
+                if snapshot.visibility_policy
+                and snapshot.visibility_policy
+                != _datasource_governance(item).visibility_policy
+                else ()
+            )
+            try:
+                plan = self._visibility_plan(
+                    db,
+                    api_principal,
+                    item=item,
+                    request=DatasourceReadRequest(
+                        datasource_ref=datasource_ref,
+                        materialization_ref=_materialization_ref(row.id),
+                        consistency="frozen",
+                    ),
+                    materialized=True,
+                    schema=schema,
+                    additional_policies=additional_policies,
+                )
+            except DatasourceAccessError:
+                continue
+            dto = _materialization_dto(row)
+            if plan.applied:
+                dto = _opaque_materialization(
+                    replace(
+                        dto,
+                        schema=visible_schema(schema, plan),
+                        fingerprint=plan.view_fingerprint(row.fingerprint),
+                        row_count=None,
+                        byte_count=None,
+                    )
+                )
+            result.append(dto)
+        return tuple(result)
 
     def list_stages(
         self,
@@ -551,6 +715,13 @@ class SqlDatasourceProvider:
         )
         rows = normalize_rows(stage.rows)
         schema = infer_schema(rows)
+        governance = replace(
+            governance,
+            visibility_policy=normalize_visibility_policy(
+                governance.visibility_policy,
+                schema=schema,
+            ),
+        )
         fingerprint = fingerprint_rows(rows, schema)
         validation = validate_stage(
             rows=rows,
@@ -919,6 +1090,26 @@ class SqlDatasourceProvider:
             raise DatasourceUnavailableError(
                 "No connector origin provider is currently available."
             )
+        plan = self._visibility_plan(
+            session,
+            principal,
+            item=item,
+            request=request,
+            materialized=False,
+            schema=_fields(item.schema_),
+        )
+        if plan.applied:
+            return self._read_live_visible(
+                session,
+                principal,
+                provider=provider,
+                item=item,
+                request=request,
+                plan=plan,
+                limit=limit,
+                offset=offset,
+                columns=columns,
+            )
         try:
             result = provider.read_origin(
                 session,
@@ -962,6 +1153,203 @@ class SqlDatasourceProvider:
             diagnostics=result.diagnostics,
         )
 
+    def _read_live_visible(
+        self,
+        session: Session,
+        principal: ApiPrincipal,
+        *,
+        provider: object,
+        item: DatasourceRecord,
+        request: DatasourceReadRequest,
+        plan: VisibilityPlan,
+        limit: int,
+        offset: int,
+        columns: tuple[str, ...],
+    ) -> DatasourceReadResult:
+        rows: list[Mapping[str, object]] = []
+        first_result = None
+        last_result = None
+        base_fingerprint: str | None = None
+        elapsed_ms = 0
+        while len(rows) < MAX_STAGE_ROWS:
+            page_limit = min(MAX_READ_ROWS, MAX_STAGE_ROWS - len(rows))
+            try:
+                page = provider.read_origin(
+                    session,
+                    principal,
+                    request=DatasourceOriginReadRequest(
+                        origin_ref=str(item.provider_ref),
+                        limit=page_limit,
+                        offset=len(rows),
+                        expected_fingerprint=base_fingerprint,
+                        max_bytes=request.max_bytes,
+                        timeout_ms=request.timeout_ms,
+                    ),
+                )
+            except DatasourceError:
+                raise
+            except Exception as exc:
+                raise DatasourceUnavailableError(
+                    f"The datasource origin could not be read: {exc}"
+                ) from exc
+            if first_result is None:
+                first_result = page
+                base_fingerprint = page.origin.fingerprint
+            last_result = page
+            elapsed_ms += page.elapsed_ms
+            rows.extend(page.rows)
+            if not page.truncated or not page.rows:
+                break
+        if first_result is None or last_result is None:
+            raise DatasourceUnavailableError(
+                "The datasource origin returned no result."
+            )
+        source_schema = first_result.origin.schema
+        plan = self._visibility_plan(
+            session,
+            principal,
+            item=item,
+            request=request,
+            materialized=False,
+            schema=source_schema,
+        )
+        permitted_schema = visible_schema(source_schema, plan)
+        _validate_field_names(permitted_schema, columns)
+        permitted, filtered_rows = apply_visibility(
+            rows,
+            principal=principal.principal,
+            plan=plan,
+            columns=columns,
+        )
+        window = permitted[offset : offset + limit]
+        window = _bounded_result_rows(window, max_bytes=request.max_bytes)
+        scan_limited = bool(
+            last_result.truncated
+            and (len(rows) >= MAX_STAGE_ROWS or not last_result.rows)
+        )
+        total_rows = len(permitted)
+        view_fingerprint = plan.view_fingerprint(first_result.origin.fingerprint)
+        descriptor = replace(
+            _datasource_dto(item),
+            schema=permitted_schema,
+            schema_version=first_result.origin.schema_version,
+            fingerprint=view_fingerprint,
+            row_count=total_rows,
+            byte_count=encoded_size(window),
+            updated_at=first_result.origin.updated_at,
+        )
+        return DatasourceReadResult(
+            datasource=_opaque_descriptor(descriptor),
+            rows=window,
+            total_rows=total_rows,
+            truncated=(
+                scan_limited
+                or offset + len(window) < total_rows
+                or len(window) < min(limit, max(0, total_rows - offset))
+            ),
+            returned_bytes=encoded_size(window),
+            elapsed_ms=elapsed_ms,
+            effective_row_limit=limit,
+            effective_byte_limit=request.max_bytes,
+            effective_timeout_ms=request.timeout_ms,
+            diagnostics=(
+                *first_result.diagnostics,
+                *visibility_diagnostics(
+                    plan,
+                    scanned_rows=len(rows),
+                    permitted_rows=total_rows,
+                    filtered_rows=filtered_rows,
+                    scan_limited=scan_limited,
+                ),
+            ),
+        )
+
+    def _can_discover(
+        self,
+        session: Session,
+        principal: ApiPrincipal,
+        *,
+        item: DatasourceRecord,
+    ) -> bool:
+        try:
+            self._visibility_plan(
+                session,
+                principal,
+                item=item,
+                request=DatasourceReadRequest(datasource_ref=_datasource_ref(item.id)),
+                materialized=False,
+                schema=_fields(item.schema_),
+                action="discover",
+            )
+        except (DatasourceAccessError, DatasourceValidationError):
+            return False
+        return True
+
+    def _visibility_plan(
+        self,
+        session: Session,
+        principal: ApiPrincipal,
+        *,
+        item: DatasourceRecord,
+        request: DatasourceReadRequest,
+        materialized: bool,
+        schema: Sequence[DatasourceField],
+        action: str = "read",
+        additional_policies: Sequence[Mapping[str, object]] = (),
+    ) -> VisibilityPlan:
+        governance = _datasource_governance(item)
+        policies: list[Mapping[str, object]] = []
+        if governance.visibility_policy:
+            policies.append(governance.visibility_policy)
+        policies.extend(additional_policies)
+        decision_refs: list[str] = []
+        policy_provider = datasource_visibility_policy_provider(self._registry)
+        if policy_provider is None:
+            if governance.access_policy_ref:
+                raise DatasourceAccessError(
+                    "The configured datasource access policy provider is unavailable."
+                )
+        else:
+            try:
+                decision = policy_provider.decide_datasource_visibility(
+                    session,
+                    request=DatasourceVisibilityPolicyRequest(
+                        tenant_id=principal.tenant_id,
+                        datasource_ref=_datasource_ref(item.id),
+                        principal=principal.principal,
+                        action=cast(Any, action),
+                        classification=governance.classification,
+                        policy_ref=governance.access_policy_ref,
+                        consistency=request.consistency,
+                        materialization_ref=request.materialization_ref,
+                    ),
+                )
+            except DatasourceError:
+                raise
+            except Exception as exc:
+                raise DatasourceAccessError(
+                    "Datasource policy evaluation is unavailable."
+                ) from exc
+            if not decision.allowed:
+                raise DatasourceAccessError(
+                    decision.reason or "Datasource policy denied access."
+                )
+            policies.extend(decision.policies)
+            if decision.decision_ref:
+                decision_refs.append(decision.decision_ref)
+        try:
+            return visibility_plan(
+                principal=principal.principal,
+                policies=policies,
+                materialized=materialized,
+                schema=schema,
+                decision_refs=decision_refs,
+            )
+        except DatasourceValidationError as exc:
+            raise DatasourceAccessError(
+                "Datasource visibility policy is invalid and access was denied."
+            ) from exc
+
     def _required_origin(
         self,
         session: Session,
@@ -1422,6 +1810,8 @@ def _datasource_governance(item: DatasourceRecord) -> DatasourceGovernance:
             "classification": item.classification,
             "privacy_profile_ref": item.privacy_profile_ref,
             "retention_policy_ref": item.retention_policy_ref,
+            "access_policy_ref": item.access_policy_ref,
+            "visibility_policy": item.visibility_policy,
             "hold_refs": item.hold_refs,
             "publication_state": item.publication_state,
             "transfer_agreement_ref": item.transfer_agreement_ref,
@@ -1453,6 +1843,13 @@ def _apply_datasource_governance(
     item.classification = governance.classification
     item.privacy_profile_ref = governance.privacy_profile_ref
     item.retention_policy_ref = governance.retention_policy_ref
+    item.access_policy_ref = governance.access_policy_ref
+    item.visibility_policy = dict(
+        normalize_visibility_policy(
+            governance.visibility_policy,
+            schema=_fields(item.schema_),
+        )
+    )
     item.hold_refs = list(governance.hold_refs)
     item.publication_state = governance.publication_state
     item.transfer_agreement_ref = governance.transfer_agreement_ref
@@ -1569,6 +1966,7 @@ def _fields(payload: Sequence[Mapping[str, object]]) -> tuple[DatasourceField, .
             name=str(item.get("name", "")),
             data_type=str(item.get("data_type", "unknown")),
             nullable=bool(item.get("nullable", True)),
+            classification=str(item.get("classification") or "internal"),
         )
         for item in payload
     )
@@ -1617,6 +2015,109 @@ def _validate_columns(
         )
 
 
+def _validate_field_names(
+    schema: Sequence[DatasourceField],
+    columns: tuple[str, ...],
+) -> None:
+    _validate_columns(
+        tuple({"name": field.name} for field in schema),
+        columns,
+    )
+
+
+def _bounded_result_rows(
+    rows: Sequence[Mapping[str, object]],
+    *,
+    max_bytes: int,
+) -> tuple[Mapping[str, object], ...]:
+    result: list[Mapping[str, object]] = []
+    byte_limit = max(1, int(max_bytes))
+    for row in rows:
+        candidate = (*result, row)
+        if encoded_size(candidate) > byte_limit:
+            break
+        result.append(row)
+    return tuple(result)
+
+
+def _opaque_descriptor(item: DatasourceDescriptor) -> DatasourceDescriptor:
+    return replace(
+        item,
+        provider=None,
+        provider_ref=None,
+        provenance={"visibility": "policy_filtered"},
+        metadata={},
+        governance=DatasourceGovernance(
+            classification=item.governance.classification,
+            publication_state=item.governance.publication_state,
+        ),
+    )
+
+
+def _opaque_materialization(
+    item: DatasourceMaterialization,
+) -> DatasourceMaterialization:
+    return replace(
+        item,
+        provenance={"visibility": "policy_filtered"},
+        metadata={},
+        governance=DatasourceGovernance(
+            classification=item.governance.classification,
+            publication_state=item.governance.publication_state,
+        ),
+    )
+
+
+def _audit_visibility_read(
+    session: Session,
+    principal: ApiPrincipal,
+    *,
+    item: DatasourceRecord,
+    outcome: str,
+    consistency: str,
+    diagnostics: Sequence[object] = (),
+) -> None:
+    visibility_summary: dict[str, object] = {}
+    for diagnostic in diagnostics:
+        if getattr(diagnostic, "code", None) != "datasource.visibility_applied":
+            continue
+        details = getattr(diagnostic, "details", {})
+        if isinstance(details, Mapping):
+            visibility_summary = {
+                key: details[key]
+                for key in (
+                    "policy_count",
+                    "field_rule_count",
+                    "row_filter_count",
+                    "scanned_rows",
+                    "permitted_rows",
+                    "filtered_rows",
+                )
+                if key in details
+            }
+    audit_event(
+        session,
+        tenant_id=principal.tenant_id,
+        user_id=getattr(principal.user, "id", None) or principal.account_id,
+        api_key_id=principal.api_key_id,
+        action=f"datasource.read.{outcome}",
+        object_type="datasource",
+        object_id=item.id,
+        details={
+            "outcome": outcome,
+            "consistency": consistency,
+            "policy_ref_configured": bool(item.access_policy_ref),
+            "local_policy_configured": bool(item.visibility_policy),
+            "diagnostic_codes": [
+                str(getattr(diagnostic, "code", ""))
+                for diagnostic in diagnostics
+                if getattr(diagnostic, "code", None)
+            ],
+            "visibility_summary": visibility_summary,
+        },
+    )
+
+
 def _select_columns(
     row: Mapping[str, object],
     columns: tuple[str, ...],