Files
govoplan-connectors/src/govoplan_connectors/backend/router.py
T

690 lines
22 KiB
Python

from __future__ import annotations
from dataclasses import asdict
import hashlib
from typing import Annotated
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status
from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_event
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
from govoplan_core.core.tabular_sources import (
TabularReadRequest,
TabularSnapshotInput,
TabularSource,
TabularSourceAccessError,
TabularSourceError,
TabularSourceNotFoundError,
TabularSourceUnavailableError,
)
from govoplan_core.core.feeds import (
FeedCapabilityError,
FeedEntry,
FeedRenderRequest,
)
from govoplan_core.core.sanctions import SanctionsSnapshotReference
from govoplan_core.db.session import get_session
from govoplan_connectors.backend.schemas import (
FeedAcquireRequest,
FeedDocumentResponse,
FeedImportRequest,
FeedRenderPayload,
SanctionsAcquisitionRunListResponse,
SanctionsAcquisitionRunResponse,
SanctionsRefreshResponse,
SanctionsSnapshotListResponse,
SanctionsSnapshotResponse,
SanctionsSourceListResponse,
SanctionsSourceResponse,
SnapshotCreateRequest,
TabularColumnResponse,
TabularHealthResponse,
TabularPreviewDiagnosticResponse,
TabularPushdownResponse,
TabularSourceDeleteResponse,
TabularSourceListResponse,
TabularSourcePreviewResponse,
TabularSourceResponse,
)
from govoplan_connectors.backend.feeds import ConnectorFeedProvider, feed_rows
from govoplan_connectors.backend.recovery import (
ConnectorRecoveryError,
begin_connector_read_snapshot,
)
from govoplan_connectors.backend.sanctions_sources import (
SANCTIONS_READ_SCOPE,
SANCTIONS_REFRESH_SCOPE,
SanctionsSourceAccessError,
SanctionsSourceError,
SanctionsSourceNotFoundError,
SqlSanctionsSnapshotProvider,
)
from govoplan_connectors.backend.tabular_sources import (
ADMIN_SCOPE,
READ_SCOPE,
WRITE_SCOPE,
SqlTabularSourceProvider,
parse_csv_snapshot,
)
router = APIRouter(prefix="/connectors", tags=["connectors"])
provider = SqlTabularSourceProvider()
sanctions_provider = SqlSanctionsSnapshotProvider()
feed_transport = ConnectorFeedProvider()
def _require_any_scope(principal: ApiPrincipal, *scopes: str) -> None:
if any(has_scope(principal, scope) for scope in scopes):
return
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"Missing one of the required scopes: {', '.join(scopes)}",
)
def _http_error(exc: TabularSourceError) -> HTTPException:
if isinstance(exc, TabularSourceNotFoundError):
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
if isinstance(exc, TabularSourceAccessError):
return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc))
if isinstance(exc, TabularSourceUnavailableError):
return HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=str(exc),
)
return HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc))
def _sanctions_http_error(
exc: SanctionsSourceError,
) -> HTTPException:
if isinstance(exc, SanctionsSourceNotFoundError):
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=str(exc),
)
if isinstance(exc, SanctionsSourceAccessError):
return HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=str(exc),
)
return HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=str(exc),
)
def _feed_http_error(exc: FeedCapabilityError) -> HTTPException:
return HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=str(exc),
)
def _recovery_http_error(exc: ConnectorRecoveryError) -> HTTPException:
detail = str(exc)
return HTTPException(
status_code=(
status.HTTP_409_CONFLICT
if "already" in detail.casefold() or "active" in detail.casefold()
else status.HTTP_503_SERVICE_UNAVAILABLE
),
detail=detail,
)
@router.post("/feeds/preview", response_model=FeedDocumentResponse)
def api_preview_feed(
payload: FeedAcquireRequest,
principal: ApiPrincipal = Depends(get_api_principal),
) -> FeedDocumentResponse:
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
try:
document = feed_transport.fetch(
payload.url,
max_entries=payload.max_entries,
)
except FeedCapabilityError as exc:
raise _feed_http_error(exc) from exc
return FeedDocumentResponse.model_validate(asdict(document))
@router.post(
"/feeds/import",
response_model=TabularSourceResponse,
status_code=status.HTTP_201_CREATED,
)
def api_import_feed_snapshot(
payload: FeedImportRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
idempotency_key: Annotated[
str | None,
Header(alias="Idempotency-Key", max_length=500),
] = None,
) -> TabularSourceResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
try:
recovery = begin_connector_read_snapshot(
session,
tenant_id=principal.tenant_id,
provider_id="connectors.feed_snapshot",
idempotency_key=idempotency_key,
source_revision=None,
cursor=None,
dry_run_evidence={
"performed": False,
"reason": "read-only acquisition into an immutable snapshot",
},
request_metadata={
"source_url_sha256": hashlib.sha256(
payload.url.encode("utf-8")
).hexdigest(),
"source_name": payload.source_name,
"max_entries": payload.max_entries,
},
resource_type="connector_tabular_source",
)
except ConnectorRecoveryError as exc:
raise _recovery_http_error(exc) from exc
if recovery.replayed:
try:
source = provider.get_source(
session,
principal,
source_ref=f"snapshot:{recovery.resource_id}",
)
except TabularSourceError as exc:
raise _http_error(exc) from exc
return _source_response(source)
try:
document = feed_transport.fetch(
payload.url,
max_entries=payload.max_entries,
)
source = provider.create_snapshot(
session,
principal,
snapshot=TabularSnapshotInput(
name=payload.name,
source_name=payload.source_name,
description=payload.description or document.description,
rows=feed_rows(document),
metadata={
"import_format": document.format,
"feed": {
"source_url": document.source_url,
"home_url": document.home_url,
"acquired_at": (
document.acquired_at.isoformat()
if document.acquired_at
else None
),
"fresh_until": (
document.fresh_until.isoformat()
if document.fresh_until
else None
),
"etag": document.etag,
"last_modified": document.last_modified,
"content_type": document.content_type,
"sha256": document.sha256,
},
},
),
source_id=recovery.resource_id,
)
except (FeedCapabilityError, TabularSourceError) as exc:
session.rollback()
recovery.fail_without_projection(
summary="The read-only feed import failed before a snapshot committed",
code=exc.__class__.__name__,
)
if isinstance(exc, FeedCapabilityError):
raise _feed_http_error(exc) from exc
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="connectors.feed_snapshot.created",
object_type="connector_tabular_source",
object_id=source.ref,
details={
"source_url": document.source_url,
"format": document.format,
"sha256": document.sha256,
"row_count": source.row_count,
},
)
try:
recovery.commit_success(
session,
evidence={
"verified": True,
"checks": {
"snapshot_ref": source.ref,
"snapshot_fingerprint": source.fingerprint,
"feed_sha256": document.sha256,
"row_count": source.row_count,
"provider_mutation": False,
},
},
)
except ConnectorRecoveryError as exc:
raise _recovery_http_error(exc) from exc
return _source_response(source)
@router.post("/feeds/render")
def api_render_feed(
payload: FeedRenderPayload,
principal: ApiPrincipal = Depends(get_api_principal),
) -> Response:
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
try:
rendered = feed_transport.render(
FeedRenderRequest(
format=payload.format,
title=payload.title,
feed_url=payload.feed_url,
home_url=payload.home_url,
description=payload.description,
language=payload.language,
entries=tuple(
FeedEntry(**item.model_dump()) for item in payload.entries
),
allowed_visibilities=frozenset(payload.allowed_visibilities),
)
)
except FeedCapabilityError as exc:
raise _feed_http_error(exc) from exc
return Response(
content=rendered.body,
media_type=rendered.content_type,
headers={
"X-GovOPlaN-Feed-Included": str(rendered.included_entries),
"X-GovOPlaN-Feed-Excluded": str(rendered.excluded_entries),
},
)
@router.get("/tabular-sources", response_model=TabularSourceListResponse)
def api_list_tabular_sources(
query: str = Query(default="", max_length=200),
limit: int = Query(default=100, ge=1, le=100),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> TabularSourceListResponse:
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
sources = provider.list_sources(
session,
principal,
query=query,
limit=limit,
)
return TabularSourceListResponse(sources=[_source_response(source) for source in sources])
@router.post(
"/tabular-sources/snapshots",
response_model=TabularSourceResponse,
status_code=status.HTTP_201_CREATED,
)
def api_create_tabular_snapshot(
payload: SnapshotCreateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> TabularSourceResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
try:
rows = (
tuple(payload.rows or ())
if payload.format == "json"
else parse_csv_snapshot(payload.csv_text or "", delimiter=payload.delimiter)
)
source = provider.create_snapshot(
session,
principal,
snapshot=TabularSnapshotInput(
name=payload.name,
source_name=payload.source_name,
description=payload.description,
rows=rows,
metadata={"import_format": payload.format},
),
)
except TabularSourceError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="connectors.tabular_snapshot.created",
object_type="connector_tabular_source",
object_id=source.ref,
details={
"provider": source.provider,
"source_name": source.source_name,
"fingerprint": source.fingerprint,
"row_count": source.row_count,
},
)
session.commit()
return _source_response(source)
@router.get(
"/tabular-sources/{source_id}/preview",
response_model=TabularSourcePreviewResponse,
)
def api_preview_tabular_source(
source_id: str,
limit: int = Query(default=100, ge=1, le=500),
offset: int = Query(default=0, ge=0),
max_bytes: int = Query(default=1_000_000, ge=2, le=5_000_000),
timeout_ms: int = Query(default=2_000, ge=1, le=10_000),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> TabularSourcePreviewResponse:
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
try:
result = provider.read_source(
session,
principal,
request=TabularReadRequest(
source_ref=f"snapshot:{source_id}",
limit=limit,
offset=offset,
max_bytes=max_bytes,
timeout_ms=timeout_ms,
),
)
except TabularSourceError as exc:
raise _http_error(exc) from exc
return TabularSourcePreviewResponse(
source=_source_response(result.source),
rows=[dict(row) for row in result.rows],
total_rows=result.total_rows,
truncated=result.truncated,
returned_bytes=result.returned_bytes,
elapsed_ms=result.elapsed_ms,
effective_row_limit=result.effective_row_limit,
effective_byte_limit=result.effective_byte_limit,
effective_timeout_ms=result.effective_timeout_ms,
diagnostics=[
TabularPreviewDiagnosticResponse(
severity=item.severity,
code=item.code,
message=item.message,
details=dict(item.details),
)
for item in result.diagnostics
],
)
@router.delete(
"/tabular-sources/{source_id}",
response_model=TabularSourceDeleteResponse,
)
def api_delete_tabular_source(
source_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> TabularSourceDeleteResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
source_ref = f"snapshot:{source_id}"
try:
source = provider.delete_snapshot(
session,
principal,
source_ref=source_ref,
)
except TabularSourceError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="connectors.tabular_snapshot.deleted",
object_type="connector_tabular_source",
object_id=source_ref,
details={"source_name": source.source_name, "fingerprint": source.fingerprint},
)
session.commit()
return TabularSourceDeleteResponse(deleted=True, source_ref=source_ref)
@router.get(
"/sanctions/sources",
response_model=SanctionsSourceListResponse,
)
def api_list_sanctions_sources(
principal: ApiPrincipal = Depends(get_api_principal),
) -> SanctionsSourceListResponse:
_require_any_scope(
principal,
SANCTIONS_READ_SCOPE,
SANCTIONS_REFRESH_SCOPE,
ADMIN_SCOPE,
)
return SanctionsSourceListResponse(
sources=[
SanctionsSourceResponse.model_validate(
source,
from_attributes=True,
)
for source in sanctions_provider.available_sources()
]
)
@router.post(
"/sanctions/sources/{provider_id}/refresh",
response_model=SanctionsRefreshResponse,
)
def api_refresh_sanctions_source(
provider_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
idempotency_key: Annotated[
str | None,
Header(alias="Idempotency-Key", max_length=500),
] = None,
) -> SanctionsRefreshResponse:
_require_any_scope(
principal,
SANCTIONS_REFRESH_SCOPE,
ADMIN_SCOPE,
)
try:
result = sanctions_provider.refresh_source(
session,
principal,
provider_id=provider_id,
idempotency_key=idempotency_key,
)
except ConnectorRecoveryError as exc:
raise _recovery_http_error(exc) from exc
except SanctionsSourceError as exc:
raise _sanctions_http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="connectors.sanctions_source.refreshed",
object_type="connector_sanctions_acquisition_run",
object_id=result.run_id,
details={
"provider_id": provider_id,
"status": result.status,
"snapshot_ref": (
result.snapshot.ref
if result.snapshot is not None
else None
),
},
)
session.commit()
return SanctionsRefreshResponse(
run_id=result.run_id,
provider_id=result.provider_id,
status=result.status,
snapshot=(
_sanctions_snapshot_response(result.snapshot)
if result.snapshot is not None
else None
),
error=result.error,
)
@router.get(
"/sanctions/snapshots",
response_model=SanctionsSnapshotListResponse,
)
def api_list_sanctions_snapshots(
limit: int = Query(default=100, ge=1, le=500),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> SanctionsSnapshotListResponse:
_require_any_scope(
principal,
SANCTIONS_READ_SCOPE,
ADMIN_SCOPE,
)
try:
snapshots = sanctions_provider.list_snapshots(
session,
principal,
limit=limit,
)
except SanctionsSourceError as exc:
raise _sanctions_http_error(exc) from exc
return SanctionsSnapshotListResponse(
snapshots=[
_sanctions_snapshot_response(item)
for item in snapshots
]
)
@router.get(
"/sanctions/runs",
response_model=SanctionsAcquisitionRunListResponse,
)
def api_list_sanctions_runs(
limit: int = Query(default=100, ge=1, le=500),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> SanctionsAcquisitionRunListResponse:
_require_any_scope(
principal,
SANCTIONS_READ_SCOPE,
ADMIN_SCOPE,
)
try:
runs = sanctions_provider.list_runs(
session,
principal,
limit=limit,
)
except SanctionsSourceError as exc:
raise _sanctions_http_error(exc) from exc
return SanctionsAcquisitionRunListResponse(
runs=[
SanctionsAcquisitionRunResponse.model_validate(
item,
from_attributes=True,
)
for item in runs
]
)
def _source_response(source: TabularSource) -> TabularSourceResponse:
return TabularSourceResponse(
ref=source.ref,
provider=source.provider,
source_name=source.source_name,
name=source.name,
description=source.description,
columns=[
TabularColumnResponse(
name=column.name,
data_type=column.data_type,
nullable=column.nullable,
)
for column in source.schema
],
schema_version=source.schema_version,
fingerprint=source.fingerprint,
row_count=source.row_count,
byte_count=source.byte_count,
updated_at=source.updated_at.isoformat() if source.updated_at else None,
capabilities=list(source.capabilities),
metadata=dict(source.metadata),
source_mode=source.source_mode,
pushdown=TabularPushdownResponse(
projections=source.pushdown.projections,
pagination=source.pushdown.pagination,
filters=list(source.pushdown.filters),
aggregations=list(source.pushdown.aggregations),
sorting=list(source.pushdown.sorting),
),
health=TabularHealthResponse(
status=source.health.status,
code=source.health.code,
summary=source.health.summary,
checked_at=(
source.health.checked_at.isoformat()
if source.health.checked_at
else None
),
details=dict(source.health.details),
),
)
def _sanctions_snapshot_response(
snapshot: SanctionsSnapshotReference,
) -> SanctionsSnapshotResponse:
return SanctionsSnapshotResponse.model_validate(
{
"ref": snapshot.ref,
"provider_id": snapshot.provider_id,
"publisher": snapshot.publisher,
"jurisdiction": snapshot.jurisdiction,
"list_type": snapshot.list_type,
"source_id": snapshot.source_id,
"source_version": snapshot.source_version,
"publication_at": snapshot.publication_at,
"effective_at": snapshot.effective_at,
"acquired_at": snapshot.acquired_at,
"content_type": snapshot.content_type,
"byte_count": snapshot.byte_count,
"sha256": snapshot.sha256,
"parser_version": snapshot.parser_version,
"raw_evidence_ref": snapshot.raw_evidence_ref,
"connector_run_id": snapshot.connector_run_id,
"signature_evidence": dict(
snapshot.signature_evidence
),
"licence_notes": snapshot.licence_notes,
"trust_notes": snapshot.trust_notes,
"transport_evidence": dict(
snapshot.transport_evidence
),
}
)
__all__ = ["router"]