from __future__ import annotations from dataclasses import asdict import hashlib from typing import Annotated from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status from sqlalchemy.orm import Session from govoplan_core.audit.logging import audit_event from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope from govoplan_core.core.tabular_sources import ( TabularReadRequest, TabularSnapshotInput, TabularSource, TabularSourceAccessError, TabularSourceError, TabularSourceNotFoundError, TabularSourceUnavailableError, ) from govoplan_core.core.feeds import ( FeedCapabilityError, FeedEntry, FeedRenderRequest, ) from govoplan_core.core.sanctions import SanctionsSnapshotReference from govoplan_core.db.session import get_session from govoplan_connectors.backend.schemas import ( FeedAcquireRequest, FeedDocumentResponse, FeedImportRequest, FeedRenderPayload, SanctionsAcquisitionRunListResponse, SanctionsAcquisitionRunResponse, SanctionsRefreshResponse, SanctionsSnapshotListResponse, SanctionsSnapshotResponse, SanctionsSourceListResponse, SanctionsSourceResponse, SnapshotCreateRequest, TabularColumnResponse, TabularHealthResponse, TabularPreviewDiagnosticResponse, TabularPushdownResponse, TabularSourceDeleteResponse, TabularSourceListResponse, TabularSourcePreviewResponse, TabularSourceResponse, ) from govoplan_connectors.backend.feeds import ConnectorFeedProvider, feed_rows from govoplan_connectors.backend.governed_runtime import ( GovernedConnectorError, create_configuration, execute_run, list_configurations, list_definitions, list_runs, review_run, update_configuration, upsert_definition, ) from govoplan_connectors.backend.governed_schemas import ( ConnectorConfigurationCreateRequest, ConnectorConfigurationItem, ConnectorConfigurationListResponse, ConnectorConfigurationUpdateRequest, ConnectorDefinitionItem, ConnectorDefinitionListResponse, ConnectorDefinitionUpsertRequest, ConnectorReviewRequest, ConnectorRunItem, ConnectorRunListResponse, ConnectorRunRequest, ) from govoplan_connectors.backend.recovery import ( ConnectorRecoveryError, begin_connector_read_snapshot, ) from govoplan_connectors.backend.sanctions_sources import ( SANCTIONS_READ_SCOPE, SANCTIONS_REFRESH_SCOPE, SanctionsSourceAccessError, SanctionsSourceError, SanctionsSourceNotFoundError, SqlSanctionsSnapshotProvider, ) from govoplan_connectors.backend.tabular_sources import ( ADMIN_SCOPE, READ_SCOPE, WRITE_SCOPE, SqlTabularSourceProvider, parse_csv_snapshot, ) router = APIRouter(prefix="/connectors", tags=["connectors"]) provider = SqlTabularSourceProvider() sanctions_provider = SqlSanctionsSnapshotProvider() feed_transport = ConnectorFeedProvider() def _require_any_scope(principal: ApiPrincipal, *scopes: str) -> None: if any(has_scope(principal, scope) for scope in scopes): return raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"Missing one of the required scopes: {', '.join(scopes)}", ) def _http_error(exc: TabularSourceError) -> HTTPException: if isinstance(exc, TabularSourceNotFoundError): return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) if isinstance(exc, TabularSourceAccessError): return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)) if isinstance(exc, TabularSourceUnavailableError): return HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc), ) return HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) def _sanctions_http_error( exc: SanctionsSourceError, ) -> HTTPException: if isinstance(exc, SanctionsSourceNotFoundError): return HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=str(exc), ) if isinstance(exc, SanctionsSourceAccessError): return HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=str(exc), ) return HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc), ) def _feed_http_error(exc: FeedCapabilityError) -> HTTPException: return HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc), ) def _recovery_http_error(exc: ConnectorRecoveryError) -> HTTPException: detail = str(exc) return HTTPException( status_code=( status.HTTP_409_CONFLICT if "already" in detail.casefold() or "active" in detail.casefold() else status.HTTP_503_SERVICE_UNAVAILABLE ), detail=detail, ) def _governed_http_error(exc: GovernedConnectorError) -> HTTPException: if exc.code.endswith("_not_found"): status_code = status.HTTP_404_NOT_FOUND elif exc.code in { "configuration_conflict", "configuration_disabled", "idempotency_conflict", "local_definition_protected", "package_definition_requires_overrides", "run_not_reviewable", }: status_code = status.HTTP_409_CONFLICT else: status_code = status.HTTP_422_UNPROCESSABLE_CONTENT return HTTPException( status_code=status_code, detail={"code": exc.code, "message": str(exc)}, ) @router.post("/feeds/preview", response_model=FeedDocumentResponse) def api_preview_feed( payload: FeedAcquireRequest, principal: ApiPrincipal = Depends(get_api_principal), ) -> FeedDocumentResponse: _require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE) try: document = feed_transport.fetch( payload.url, max_entries=payload.max_entries, ) except FeedCapabilityError as exc: raise _feed_http_error(exc) from exc return FeedDocumentResponse.model_validate(asdict(document)) @router.post( "/feeds/import", response_model=TabularSourceResponse, status_code=status.HTTP_201_CREATED, ) def api_import_feed_snapshot( payload: FeedImportRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), idempotency_key: Annotated[ str | None, Header(alias="Idempotency-Key", max_length=500), ] = None, ) -> TabularSourceResponse: _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE) try: recovery = begin_connector_read_snapshot( session, tenant_id=principal.tenant_id, provider_id="connectors.feed_snapshot", idempotency_key=idempotency_key, source_revision=None, cursor=None, dry_run_evidence={ "performed": False, "reason": "read-only acquisition into an immutable snapshot", }, request_metadata={ "source_url_sha256": hashlib.sha256( payload.url.encode("utf-8") ).hexdigest(), "source_name": payload.source_name, "max_entries": payload.max_entries, }, resource_type="connector_tabular_source", ) except ConnectorRecoveryError as exc: raise _recovery_http_error(exc) from exc if recovery.replayed: try: source = provider.get_source( session, principal, source_ref=f"snapshot:{recovery.resource_id}", ) except TabularSourceError as exc: raise _http_error(exc) from exc return _source_response(source) try: document = feed_transport.fetch( payload.url, max_entries=payload.max_entries, ) source = provider.create_snapshot( session, principal, snapshot=TabularSnapshotInput( name=payload.name, source_name=payload.source_name, description=payload.description or document.description, rows=feed_rows(document), metadata={ "import_format": document.format, "feed": { "source_url": document.source_url, "home_url": document.home_url, "acquired_at": ( document.acquired_at.isoformat() if document.acquired_at else None ), "fresh_until": ( document.fresh_until.isoformat() if document.fresh_until else None ), "etag": document.etag, "last_modified": document.last_modified, "content_type": document.content_type, "sha256": document.sha256, }, }, ), source_id=recovery.resource_id, ) except (FeedCapabilityError, TabularSourceError) as exc: session.rollback() recovery.fail_without_projection( summary="The read-only feed import failed before a snapshot committed", code=exc.__class__.__name__, ) if isinstance(exc, FeedCapabilityError): raise _feed_http_error(exc) from exc raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="connectors.feed_snapshot.created", object_type="connector_tabular_source", object_id=source.ref, details={ "source_url": document.source_url, "format": document.format, "sha256": document.sha256, "row_count": source.row_count, }, ) try: recovery.commit_success( session, evidence={ "verified": True, "checks": { "snapshot_ref": source.ref, "snapshot_fingerprint": source.fingerprint, "feed_sha256": document.sha256, "row_count": source.row_count, "provider_mutation": False, }, }, ) except ConnectorRecoveryError as exc: raise _recovery_http_error(exc) from exc return _source_response(source) @router.post("/feeds/render") def api_render_feed( payload: FeedRenderPayload, principal: ApiPrincipal = Depends(get_api_principal), ) -> Response: _require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE) try: rendered = feed_transport.render( FeedRenderRequest( format=payload.format, title=payload.title, feed_url=payload.feed_url, home_url=payload.home_url, description=payload.description, language=payload.language, entries=tuple( FeedEntry(**item.model_dump()) for item in payload.entries ), allowed_visibilities=frozenset(payload.allowed_visibilities), ) ) except FeedCapabilityError as exc: raise _feed_http_error(exc) from exc return Response( content=rendered.body, media_type=rendered.content_type, headers={ "X-GovOPlaN-Feed-Included": str(rendered.included_entries), "X-GovOPlaN-Feed-Excluded": str(rendered.excluded_entries), }, ) @router.get("/tabular-sources", response_model=TabularSourceListResponse) def api_list_tabular_sources( query: str = Query(default="", max_length=200), limit: int = Query(default=100, ge=1, le=100), session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> TabularSourceListResponse: _require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE) sources = provider.list_sources( session, principal, query=query, limit=limit, ) return TabularSourceListResponse(sources=[_source_response(source) for source in sources]) @router.post( "/tabular-sources/snapshots", response_model=TabularSourceResponse, status_code=status.HTTP_201_CREATED, ) def api_create_tabular_snapshot( payload: SnapshotCreateRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> TabularSourceResponse: _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE) try: rows = ( tuple(payload.rows or ()) if payload.format == "json" else parse_csv_snapshot(payload.csv_text or "", delimiter=payload.delimiter) ) source = provider.create_snapshot( session, principal, snapshot=TabularSnapshotInput( name=payload.name, source_name=payload.source_name, description=payload.description, rows=rows, metadata={"import_format": payload.format}, ), ) except TabularSourceError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="connectors.tabular_snapshot.created", object_type="connector_tabular_source", object_id=source.ref, details={ "provider": source.provider, "source_name": source.source_name, "fingerprint": source.fingerprint, "row_count": source.row_count, }, ) session.commit() return _source_response(source) @router.get( "/tabular-sources/{source_id}/preview", response_model=TabularSourcePreviewResponse, ) def api_preview_tabular_source( source_id: str, limit: int = Query(default=100, ge=1, le=500), offset: int = Query(default=0, ge=0), max_bytes: int = Query(default=1_000_000, ge=2, le=5_000_000), timeout_ms: int = Query(default=2_000, ge=1, le=10_000), session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> TabularSourcePreviewResponse: _require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE) try: result = provider.read_source( session, principal, request=TabularReadRequest( source_ref=f"snapshot:{source_id}", limit=limit, offset=offset, max_bytes=max_bytes, timeout_ms=timeout_ms, ), ) except TabularSourceError as exc: raise _http_error(exc) from exc return TabularSourcePreviewResponse( source=_source_response(result.source), rows=[dict(row) for row in result.rows], total_rows=result.total_rows, truncated=result.truncated, returned_bytes=result.returned_bytes, elapsed_ms=result.elapsed_ms, effective_row_limit=result.effective_row_limit, effective_byte_limit=result.effective_byte_limit, effective_timeout_ms=result.effective_timeout_ms, diagnostics=[ TabularPreviewDiagnosticResponse( severity=item.severity, code=item.code, message=item.message, details=dict(item.details), ) for item in result.diagnostics ], ) @router.delete( "/tabular-sources/{source_id}", response_model=TabularSourceDeleteResponse, ) def api_delete_tabular_source( source_id: str, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> TabularSourceDeleteResponse: _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE) source_ref = f"snapshot:{source_id}" try: source = provider.delete_snapshot( session, principal, source_ref=source_ref, ) except TabularSourceError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="connectors.tabular_snapshot.deleted", object_type="connector_tabular_source", object_id=source_ref, details={"source_name": source.source_name, "fingerprint": source.fingerprint}, ) session.commit() return TabularSourceDeleteResponse(deleted=True, source_ref=source_ref) @router.get( "/sanctions/sources", response_model=SanctionsSourceListResponse, ) def api_list_sanctions_sources( principal: ApiPrincipal = Depends(get_api_principal), ) -> SanctionsSourceListResponse: _require_any_scope( principal, SANCTIONS_READ_SCOPE, SANCTIONS_REFRESH_SCOPE, ADMIN_SCOPE, ) return SanctionsSourceListResponse( sources=[ SanctionsSourceResponse.model_validate( source, from_attributes=True, ) for source in sanctions_provider.available_sources() ] ) @router.post( "/sanctions/sources/{provider_id}/refresh", response_model=SanctionsRefreshResponse, ) def api_refresh_sanctions_source( provider_id: str, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), idempotency_key: Annotated[ str | None, Header(alias="Idempotency-Key", max_length=500), ] = None, ) -> SanctionsRefreshResponse: _require_any_scope( principal, SANCTIONS_REFRESH_SCOPE, ADMIN_SCOPE, ) try: result = sanctions_provider.refresh_source( session, principal, provider_id=provider_id, idempotency_key=idempotency_key, ) except ConnectorRecoveryError as exc: raise _recovery_http_error(exc) from exc except SanctionsSourceError as exc: raise _sanctions_http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="connectors.sanctions_source.refreshed", object_type="connector_sanctions_acquisition_run", object_id=result.run_id, details={ "provider_id": provider_id, "status": result.status, "snapshot_ref": ( result.snapshot.ref if result.snapshot is not None else None ), }, ) session.commit() return SanctionsRefreshResponse( run_id=result.run_id, provider_id=result.provider_id, status=result.status, snapshot=( _sanctions_snapshot_response(result.snapshot) if result.snapshot is not None else None ), error=result.error, ) @router.get( "/sanctions/snapshots", response_model=SanctionsSnapshotListResponse, ) def api_list_sanctions_snapshots( limit: int = Query(default=100, ge=1, le=500), session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> SanctionsSnapshotListResponse: _require_any_scope( principal, SANCTIONS_READ_SCOPE, ADMIN_SCOPE, ) try: snapshots = sanctions_provider.list_snapshots( session, principal, limit=limit, ) except SanctionsSourceError as exc: raise _sanctions_http_error(exc) from exc return SanctionsSnapshotListResponse( snapshots=[ _sanctions_snapshot_response(item) for item in snapshots ] ) @router.get( "/sanctions/runs", response_model=SanctionsAcquisitionRunListResponse, ) def api_list_sanctions_runs( limit: int = Query(default=100, ge=1, le=500), session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> SanctionsAcquisitionRunListResponse: _require_any_scope( principal, SANCTIONS_READ_SCOPE, ADMIN_SCOPE, ) try: runs = sanctions_provider.list_runs( session, principal, limit=limit, ) except SanctionsSourceError as exc: raise _sanctions_http_error(exc) from exc return SanctionsAcquisitionRunListResponse( runs=[ SanctionsAcquisitionRunResponse.model_validate( item, from_attributes=True, ) for item in runs ] ) @router.get( "/governed/definitions", response_model=ConnectorDefinitionListResponse, ) def api_list_governed_definitions( session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> ConnectorDefinitionListResponse: _require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE) try: items = list_definitions(session, tenant_id=principal.tenant_id) except GovernedConnectorError as exc: raise _governed_http_error(exc) from exc return ConnectorDefinitionListResponse(items=items) @router.post( "/governed/definitions", response_model=ConnectorDefinitionItem, ) def api_upsert_governed_definition( payload: ConnectorDefinitionUpsertRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> ConnectorDefinitionItem: _require_any_scope(principal, ADMIN_SCOPE) try: return upsert_definition(session, principal, payload) except GovernedConnectorError as exc: session.rollback() raise _governed_http_error(exc) from exc @router.get( "/governed/configurations", response_model=ConnectorConfigurationListResponse, ) def api_list_governed_configurations( session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> ConnectorConfigurationListResponse: _require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE) try: items = list_configurations(session, tenant_id=principal.tenant_id) except GovernedConnectorError as exc: raise _governed_http_error(exc) from exc return ConnectorConfigurationListResponse(items=items) @router.post( "/governed/configurations", response_model=ConnectorConfigurationItem, status_code=status.HTTP_201_CREATED, ) def api_create_governed_configuration( payload: ConnectorConfigurationCreateRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> ConnectorConfigurationItem: _require_any_scope(principal, ADMIN_SCOPE) try: return create_configuration(session, principal, payload) except GovernedConnectorError as exc: session.rollback() raise _governed_http_error(exc) from exc @router.put( "/governed/configurations/{configuration_id}", response_model=ConnectorConfigurationItem, ) def api_update_governed_configuration( configuration_id: str, payload: ConnectorConfigurationUpdateRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> ConnectorConfigurationItem: _require_any_scope(principal, ADMIN_SCOPE) try: return update_configuration( session, principal, configuration_id=configuration_id, payload=payload, ) except GovernedConnectorError as exc: session.rollback() raise _governed_http_error(exc) from exc def _api_execute_governed_run( *, configuration_id: str, mode: str, payload: ConnectorRunRequest, session: Session, principal: ApiPrincipal, ) -> ConnectorRunItem: _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE) try: return execute_run( session, principal, configuration_id=configuration_id, mode=mode, payload=payload, ) except GovernedConnectorError as exc: session.rollback() raise _governed_http_error(exc) from exc @router.post( "/governed/configurations/{configuration_id}/dry-runs", response_model=ConnectorRunItem, ) def api_dry_run_governed_configuration( configuration_id: str, payload: ConnectorRunRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> ConnectorRunItem: return _api_execute_governed_run( configuration_id=configuration_id, mode="dry_run", payload=payload, session=session, principal=principal, ) @router.post( "/governed/configurations/{configuration_id}/simulations", response_model=ConnectorRunItem, ) def api_simulate_governed_configuration( configuration_id: str, payload: ConnectorRunRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> ConnectorRunItem: return _api_execute_governed_run( configuration_id=configuration_id, mode="simulation", payload=payload, session=session, principal=principal, ) @router.get( "/governed/runs", response_model=ConnectorRunListResponse, ) def api_list_governed_runs( configuration_id: str | None = Query(default=None), review_state: str | None = Query(default=None), limit: int = Query(default=100, ge=1, le=500), session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> ConnectorRunListResponse: _require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE) try: items = list_runs( session, tenant_id=principal.tenant_id, configuration_id=configuration_id, review_state=review_state, limit=limit, ) except GovernedConnectorError as exc: raise _governed_http_error(exc) from exc return ConnectorRunListResponse(items=items) @router.post( "/governed/runs/{run_id}/review", response_model=ConnectorRunItem, ) def api_review_governed_run( run_id: str, payload: ConnectorReviewRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> ConnectorRunItem: _require_any_scope(principal, ADMIN_SCOPE) try: return review_run(session, principal, run_id=run_id, payload=payload) except GovernedConnectorError as exc: session.rollback() raise _governed_http_error(exc) from exc def _source_response(source: TabularSource) -> TabularSourceResponse: return TabularSourceResponse( ref=source.ref, provider=source.provider, source_name=source.source_name, name=source.name, description=source.description, columns=[ TabularColumnResponse( name=column.name, data_type=column.data_type, nullable=column.nullable, ) for column in source.schema ], schema_version=source.schema_version, fingerprint=source.fingerprint, row_count=source.row_count, byte_count=source.byte_count, updated_at=source.updated_at.isoformat() if source.updated_at else None, capabilities=list(source.capabilities), metadata=dict(source.metadata), source_mode=source.source_mode, pushdown=TabularPushdownResponse( projections=source.pushdown.projections, pagination=source.pushdown.pagination, filters=list(source.pushdown.filters), aggregations=list(source.pushdown.aggregations), sorting=list(source.pushdown.sorting), ), health=TabularHealthResponse( status=source.health.status, code=source.health.code, summary=source.health.summary, checked_at=( source.health.checked_at.isoformat() if source.health.checked_at else None ), details=dict(source.health.details), ), ) def _sanctions_snapshot_response( snapshot: SanctionsSnapshotReference, ) -> SanctionsSnapshotResponse: return SanctionsSnapshotResponse.model_validate( { "ref": snapshot.ref, "provider_id": snapshot.provider_id, "publisher": snapshot.publisher, "jurisdiction": snapshot.jurisdiction, "list_type": snapshot.list_type, "source_id": snapshot.source_id, "source_version": snapshot.source_version, "publication_at": snapshot.publication_at, "effective_at": snapshot.effective_at, "acquired_at": snapshot.acquired_at, "content_type": snapshot.content_type, "byte_count": snapshot.byte_count, "sha256": snapshot.sha256, "parser_version": snapshot.parser_version, "raw_evidence_ref": snapshot.raw_evidence_ref, "connector_run_id": snapshot.connector_run_id, "signature_evidence": dict( snapshot.signature_evidence ), "licence_notes": snapshot.licence_notes, "trust_notes": snapshot.trust_notes, "transport_evidence": dict( snapshot.transport_evidence ), } ) __all__ = ["router"]