feat: govern connector configurations and simulations
This commit is contained in:
@@ -5,6 +5,7 @@ from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
@@ -255,9 +256,213 @@ class ConnectorSanctionsSnapshot(Base, TimestampMixin):
|
||||
)
|
||||
|
||||
|
||||
class ConnectorDefinition(Base, TimestampMixin):
|
||||
__tablename__ = "connector_definitions"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"definition_key",
|
||||
name="uq_connector_definition_tenant_key",
|
||||
),
|
||||
Index(
|
||||
"ix_connector_definitions_tenant_status",
|
||||
"tenant_id",
|
||||
"status",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
definition_key: Mapped[str] = mapped_column(String(160), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(30),
|
||||
default="active",
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
current_revision: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
source_package: Mapped[str | None] = mapped_column(String(300))
|
||||
local_definition: Mapped[bool] = mapped_column(
|
||||
Boolean,
|
||||
default=False,
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
|
||||
class ConnectorDefinitionRevision(Base, TimestampMixin):
|
||||
__tablename__ = "connector_definition_revisions"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"definition_id",
|
||||
"revision",
|
||||
name="uq_connector_definition_revision",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
definition_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("connector_definitions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
specification: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON,
|
||||
default=dict,
|
||||
nullable=False,
|
||||
)
|
||||
definition_hash: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
origin: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
package_ref: Mapped[str | None] = mapped_column(String(300))
|
||||
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||
|
||||
|
||||
class ConnectorConfiguration(Base, TimestampMixin):
|
||||
__tablename__ = "connector_configurations"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"name",
|
||||
name="uq_connector_configuration_tenant_name",
|
||||
),
|
||||
Index(
|
||||
"ix_connector_configurations_tenant_status",
|
||||
"tenant_id",
|
||||
"status",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
definition_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("connector_definitions.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(30),
|
||||
default="draft",
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
endpoint_url: Mapped[str | None] = mapped_column(String(1500))
|
||||
credential_ref: Mapped[str | None] = mapped_column(String(500))
|
||||
base_definition_revision: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
)
|
||||
local_overrides: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON,
|
||||
default=dict,
|
||||
nullable=False,
|
||||
)
|
||||
protected_paths: Mapped[list[str]] = mapped_column(
|
||||
JSON,
|
||||
default=list,
|
||||
nullable=False,
|
||||
)
|
||||
effective_configuration: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON,
|
||||
default=dict,
|
||||
nullable=False,
|
||||
)
|
||||
effective_hash: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
resource_revision: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
default=1,
|
||||
nullable=False,
|
||||
)
|
||||
ambiguity_policy: Mapped[str] = mapped_column(
|
||||
String(30),
|
||||
default="manual_review",
|
||||
nullable=False,
|
||||
)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||
|
||||
|
||||
class ConnectorSimulationRun(Base, TimestampMixin):
|
||||
__tablename__ = "connector_simulation_runs"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"configuration_id",
|
||||
"mode",
|
||||
"idempotency_key",
|
||||
name="uq_connector_simulation_run_idempotency",
|
||||
),
|
||||
Index(
|
||||
"ix_connector_simulation_runs_review",
|
||||
"tenant_id",
|
||||
"review_state",
|
||||
"created_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
configuration_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("connector_configurations.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
mode: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||
review_state: Mapped[str] = mapped_column(
|
||||
String(30),
|
||||
default="not_required",
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
definition_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
configuration_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
configuration_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
input_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
summary: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON,
|
||||
default=dict,
|
||||
nullable=False,
|
||||
)
|
||||
effects: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||
JSON,
|
||||
default=list,
|
||||
nullable=False,
|
||||
)
|
||||
diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||
JSON,
|
||||
default=list,
|
||||
nullable=False,
|
||||
)
|
||||
provenance: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON,
|
||||
default=dict,
|
||||
nullable=False,
|
||||
)
|
||||
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||
reviewed_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||
reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
review_reason: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ConnectorConfiguration",
|
||||
"ConnectorDefinition",
|
||||
"ConnectorDefinitionRevision",
|
||||
"ConnectorSanctionsAcquisitionRun",
|
||||
"ConnectorSanctionsSnapshot",
|
||||
"ConnectorSimulationRun",
|
||||
"ConnectorTabularSource",
|
||||
"new_uuid",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,921 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.audit.logging import audit_from_principal
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.core.connector_runtime import ConnectorContractError, ConnectorEndpoint
|
||||
from govoplan_connectors.backend.db.models import (
|
||||
ConnectorConfiguration,
|
||||
ConnectorDefinition,
|
||||
ConnectorDefinitionRevision,
|
||||
ConnectorSimulationRun,
|
||||
)
|
||||
from govoplan_connectors.backend.governed_schemas import (
|
||||
ConnectorConfigurationCreateRequest,
|
||||
ConnectorConfigurationItem,
|
||||
ConnectorConfigurationUpdateRequest,
|
||||
ConnectorDefinitionItem,
|
||||
ConnectorDefinitionUpsertRequest,
|
||||
ConnectorReviewRequest,
|
||||
ConnectorRunItem,
|
||||
ConnectorRunRequest,
|
||||
GovernedConnectorSpecification,
|
||||
)
|
||||
|
||||
|
||||
class GovernedConnectorError(ValueError):
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
def list_definitions(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
) -> list[ConnectorDefinitionItem]:
|
||||
rows = (
|
||||
session.query(ConnectorDefinition)
|
||||
.filter(
|
||||
ConnectorDefinition.tenant_id == tenant_id,
|
||||
ConnectorDefinition.status == "active",
|
||||
)
|
||||
.order_by(ConnectorDefinition.name.asc())
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
definition_item(
|
||||
session,
|
||||
row,
|
||||
_definition_revision(session, row, row.current_revision),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def upsert_definition(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
payload: ConnectorDefinitionUpsertRequest,
|
||||
) -> ConnectorDefinitionItem:
|
||||
specification = payload.specification.model_dump(mode="json")
|
||||
definition_hash = _hash(specification)
|
||||
definition = (
|
||||
session.query(ConnectorDefinition)
|
||||
.filter(
|
||||
ConnectorDefinition.tenant_id == principal.tenant_id,
|
||||
ConnectorDefinition.definition_key == payload.definition_key,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
expected_local = payload.origin == "local"
|
||||
if definition is None:
|
||||
definition = ConnectorDefinition(
|
||||
tenant_id=principal.tenant_id,
|
||||
definition_key=payload.definition_key,
|
||||
name=payload.name.strip(),
|
||||
description=_optional_text(payload.description),
|
||||
status="active",
|
||||
current_revision=0,
|
||||
source_package=payload.package_ref,
|
||||
local_definition=expected_local,
|
||||
)
|
||||
session.add(definition)
|
||||
session.flush()
|
||||
elif definition.local_definition != expected_local:
|
||||
if definition.local_definition:
|
||||
raise GovernedConnectorError(
|
||||
"local_definition_protected",
|
||||
"A package update cannot replace a locally owned connector definition.",
|
||||
)
|
||||
raise GovernedConnectorError(
|
||||
"package_definition_requires_overrides",
|
||||
"Use configuration overrides instead of converting a package definition into a local definition.",
|
||||
)
|
||||
else:
|
||||
current = _definition_revision(
|
||||
session,
|
||||
definition,
|
||||
definition.current_revision,
|
||||
)
|
||||
if current.definition_hash == definition_hash:
|
||||
return definition_item(session, definition, current)
|
||||
|
||||
definition.name = payload.name.strip()
|
||||
definition.description = _optional_text(payload.description)
|
||||
definition.source_package = payload.package_ref
|
||||
definition.current_revision += 1
|
||||
revision = ConnectorDefinitionRevision(
|
||||
definition_id=definition.id,
|
||||
revision=definition.current_revision,
|
||||
specification=specification,
|
||||
definition_hash=definition_hash,
|
||||
origin=payload.origin,
|
||||
package_ref=payload.package_ref,
|
||||
created_by=principal.user.id,
|
||||
)
|
||||
session.add(revision)
|
||||
session.flush()
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="connectors.definition.revision_created",
|
||||
object_type="connector_definition",
|
||||
object_id=definition.id,
|
||||
details={
|
||||
"definition_key": definition.definition_key,
|
||||
"revision": revision.revision,
|
||||
"definition_hash": definition_hash,
|
||||
"origin": payload.origin,
|
||||
"package_ref": payload.package_ref,
|
||||
"provider": payload.specification.provider,
|
||||
"protocol": payload.specification.protocol,
|
||||
"capabilities": sorted(payload.specification.capabilities),
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return definition_item(session, definition, revision)
|
||||
|
||||
|
||||
def list_configurations(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
) -> list[ConnectorConfigurationItem]:
|
||||
rows = (
|
||||
session.query(ConnectorConfiguration)
|
||||
.filter(ConnectorConfiguration.tenant_id == tenant_id)
|
||||
.order_by(ConnectorConfiguration.name.asc())
|
||||
.all()
|
||||
)
|
||||
return [configuration_item(session, row) for row in rows]
|
||||
|
||||
|
||||
def create_configuration(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
payload: ConnectorConfigurationCreateRequest,
|
||||
) -> ConnectorConfigurationItem:
|
||||
definition = _tenant_definition(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
definition_id=payload.definition_id,
|
||||
)
|
||||
_validate_endpoint(payload.endpoint_url, payload.credential_ref)
|
||||
revision = _definition_revision(session, definition, definition.current_revision)
|
||||
overrides = copy.deepcopy(payload.local_overrides)
|
||||
effective = _effective_specification(revision.specification, overrides)
|
||||
item = ConnectorConfiguration(
|
||||
tenant_id=principal.tenant_id,
|
||||
definition_id=definition.id,
|
||||
name=payload.name.strip(),
|
||||
status=payload.status,
|
||||
endpoint_url=_optional_text(payload.endpoint_url),
|
||||
credential_ref=_optional_text(payload.credential_ref),
|
||||
base_definition_revision=definition.current_revision,
|
||||
local_overrides=overrides,
|
||||
protected_paths=_protected_paths(overrides),
|
||||
effective_configuration=effective,
|
||||
effective_hash=_hash(effective),
|
||||
resource_revision=1,
|
||||
ambiguity_policy=payload.ambiguity_policy,
|
||||
updated_by=principal.user.id,
|
||||
)
|
||||
session.add(item)
|
||||
session.flush()
|
||||
_audit_configuration(
|
||||
session,
|
||||
principal,
|
||||
item,
|
||||
action="connectors.configuration.created",
|
||||
)
|
||||
session.commit()
|
||||
return configuration_item(session, item)
|
||||
|
||||
|
||||
def update_configuration(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
configuration_id: str,
|
||||
payload: ConnectorConfigurationUpdateRequest,
|
||||
) -> ConnectorConfigurationItem:
|
||||
item = _tenant_configuration(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
configuration_id=configuration_id,
|
||||
)
|
||||
if item.resource_revision != payload.expected_revision:
|
||||
raise GovernedConnectorError(
|
||||
"configuration_conflict",
|
||||
"The connector configuration changed; reload it before saving.",
|
||||
)
|
||||
supplied = payload.model_fields_set
|
||||
definition = _tenant_definition(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
definition_id=item.definition_id,
|
||||
)
|
||||
if "name" in supplied and payload.name is not None:
|
||||
item.name = payload.name.strip()
|
||||
if "endpoint_url" in supplied:
|
||||
item.endpoint_url = _optional_text(payload.endpoint_url)
|
||||
if "credential_ref" in supplied:
|
||||
item.credential_ref = _optional_text(payload.credential_ref)
|
||||
if "local_overrides" in supplied and payload.local_overrides is not None:
|
||||
item.local_overrides = copy.deepcopy(payload.local_overrides)
|
||||
if payload.ambiguity_policy is not None:
|
||||
item.ambiguity_policy = payload.ambiguity_policy
|
||||
if payload.status is not None:
|
||||
item.status = payload.status
|
||||
if payload.adopt_latest_definition:
|
||||
item.base_definition_revision = definition.current_revision
|
||||
_validate_endpoint(item.endpoint_url, item.credential_ref)
|
||||
base = _definition_revision(
|
||||
session,
|
||||
definition,
|
||||
item.base_definition_revision,
|
||||
)
|
||||
effective = _effective_specification(base.specification, item.local_overrides)
|
||||
item.protected_paths = _protected_paths(item.local_overrides)
|
||||
item.effective_configuration = effective
|
||||
item.effective_hash = _hash(effective)
|
||||
item.resource_revision += 1
|
||||
item.updated_by = principal.user.id
|
||||
_audit_configuration(
|
||||
session,
|
||||
principal,
|
||||
item,
|
||||
action="connectors.configuration.updated",
|
||||
extra={"adopted_latest_definition": payload.adopt_latest_definition},
|
||||
)
|
||||
session.commit()
|
||||
return configuration_item(session, item)
|
||||
|
||||
|
||||
def execute_run(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
configuration_id: str,
|
||||
mode: str,
|
||||
payload: ConnectorRunRequest,
|
||||
) -> ConnectorRunItem:
|
||||
if mode not in {"dry_run", "simulation"}:
|
||||
raise GovernedConnectorError("invalid_mode", "Unsupported connector run mode.")
|
||||
configuration = _tenant_configuration(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
configuration_id=configuration_id,
|
||||
)
|
||||
if configuration.status == "disabled":
|
||||
raise GovernedConnectorError(
|
||||
"configuration_disabled",
|
||||
"Disabled connector configurations cannot be executed.",
|
||||
)
|
||||
specification = GovernedConnectorSpecification.model_validate(
|
||||
configuration.effective_configuration
|
||||
)
|
||||
if mode == "dry_run" and not specification.dry_run.supported:
|
||||
raise GovernedConnectorError(
|
||||
"dry_run_unsupported",
|
||||
"This connector definition does not support dry runs.",
|
||||
)
|
||||
if mode == "simulation" and not specification.dry_run.simulation_supported:
|
||||
raise GovernedConnectorError(
|
||||
"simulation_unsupported",
|
||||
"This connector definition does not support simulation.",
|
||||
)
|
||||
rows = (
|
||||
list(payload.input_rows)
|
||||
if payload.input_rows is not None
|
||||
else list(specification.dry_run.sample_rows)
|
||||
)
|
||||
request_payload = {
|
||||
"mode": mode,
|
||||
"configuration_id": configuration.id,
|
||||
"configuration_revision": configuration.resource_revision,
|
||||
"configuration_hash": configuration.effective_hash,
|
||||
"external_revision": payload.external_revision,
|
||||
"input_rows": rows,
|
||||
}
|
||||
request_hash = _hash(request_payload)
|
||||
existing = (
|
||||
session.query(ConnectorSimulationRun)
|
||||
.filter(
|
||||
ConnectorSimulationRun.tenant_id == principal.tenant_id,
|
||||
ConnectorSimulationRun.configuration_id == configuration.id,
|
||||
ConnectorSimulationRun.mode == mode,
|
||||
ConnectorSimulationRun.idempotency_key == payload.idempotency_key,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if existing is not None:
|
||||
if existing.request_hash != request_hash:
|
||||
raise GovernedConnectorError(
|
||||
"idempotency_conflict",
|
||||
"The idempotency key was already used with different run inputs.",
|
||||
)
|
||||
return run_item(existing)
|
||||
|
||||
limit = specification.dry_run.max_items
|
||||
truncated = len(rows) > limit
|
||||
bounded_rows = rows[:limit]
|
||||
effects, diagnostics, ambiguous_count = _simulate(
|
||||
bounded_rows,
|
||||
specification,
|
||||
)
|
||||
if truncated:
|
||||
diagnostics.append(
|
||||
{
|
||||
"severity": "warning",
|
||||
"code": "connectors.run.truncated",
|
||||
"message": f"The run was limited to {limit} input rows.",
|
||||
"stage": "planning",
|
||||
"retryable": False,
|
||||
}
|
||||
)
|
||||
errors = sum(1 for item in diagnostics if item["severity"] == "error")
|
||||
if ambiguous_count:
|
||||
status_value, review_state = {
|
||||
"manual_review": ("manual_review", "pending"),
|
||||
"quarantine": ("quarantined", "quarantined"),
|
||||
"reject": ("rejected", "not_required"),
|
||||
}[configuration.ambiguity_policy]
|
||||
elif errors:
|
||||
status_value, review_state = "invalid", "not_required"
|
||||
else:
|
||||
status_value, review_state = "ready", "not_required"
|
||||
input_hash = _hash(bounded_rows)
|
||||
summary = {
|
||||
"total": len(effects),
|
||||
"creates": sum(1 for item in effects if item["effect"] == "create"),
|
||||
"updates": 0,
|
||||
"deletes": 0,
|
||||
"conflicts": sum(1 for item in effects if item["effect"] == "conflict"),
|
||||
"unchanged": 0,
|
||||
"ignored": sum(1 for item in effects if item["effect"] == "ignored"),
|
||||
"ambiguous": ambiguous_count,
|
||||
"errors": errors,
|
||||
"truncated": truncated,
|
||||
}
|
||||
now = datetime.now(UTC)
|
||||
provenance = {
|
||||
"contract_version": "1.0",
|
||||
"definition_id": configuration.definition_id,
|
||||
"definition_revision": configuration.base_definition_revision,
|
||||
"configuration_id": configuration.id,
|
||||
"configuration_revision": configuration.resource_revision,
|
||||
"configuration_hash": configuration.effective_hash,
|
||||
"mapping_version": specification.mapping.version,
|
||||
"input_hash": input_hash,
|
||||
"external_revision": payload.external_revision,
|
||||
"generated_at": now.isoformat(),
|
||||
"actor_id": principal.user.id,
|
||||
"mode": mode,
|
||||
"provider": specification.provider,
|
||||
"protocol": specification.protocol,
|
||||
"privacy_classification": specification.privacy_classification,
|
||||
"retention_class": specification.retention_class,
|
||||
}
|
||||
run = ConnectorSimulationRun(
|
||||
tenant_id=principal.tenant_id,
|
||||
configuration_id=configuration.id,
|
||||
mode=mode,
|
||||
idempotency_key=payload.idempotency_key,
|
||||
request_hash=request_hash,
|
||||
status=status_value,
|
||||
review_state=review_state,
|
||||
definition_revision=configuration.base_definition_revision,
|
||||
configuration_revision=configuration.resource_revision,
|
||||
configuration_hash=configuration.effective_hash,
|
||||
input_hash=input_hash,
|
||||
summary=summary,
|
||||
effects=effects,
|
||||
diagnostics=diagnostics,
|
||||
provenance=provenance,
|
||||
created_by=principal.user.id,
|
||||
)
|
||||
session.add(run)
|
||||
session.flush()
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action=f"connectors.configuration.{mode}_completed",
|
||||
object_type="connector_simulation_run",
|
||||
object_id=run.id,
|
||||
details={
|
||||
"configuration_id": configuration.id,
|
||||
"configuration_revision": configuration.resource_revision,
|
||||
"configuration_hash": configuration.effective_hash,
|
||||
"definition_revision": configuration.base_definition_revision,
|
||||
"input_hash": input_hash,
|
||||
"status": status_value,
|
||||
"review_state": review_state,
|
||||
"summary": summary,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return run_item(run)
|
||||
|
||||
|
||||
def list_runs(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
configuration_id: str | None = None,
|
||||
review_state: str | None = None,
|
||||
limit: int = 100,
|
||||
) -> list[ConnectorRunItem]:
|
||||
query = session.query(ConnectorSimulationRun).filter(
|
||||
ConnectorSimulationRun.tenant_id == tenant_id
|
||||
)
|
||||
if configuration_id:
|
||||
query = query.filter(
|
||||
ConnectorSimulationRun.configuration_id == configuration_id
|
||||
)
|
||||
if review_state:
|
||||
query = query.filter(ConnectorSimulationRun.review_state == review_state)
|
||||
rows = (
|
||||
query.order_by(ConnectorSimulationRun.created_at.desc())
|
||||
.limit(max(1, min(int(limit), 500)))
|
||||
.all()
|
||||
)
|
||||
return [run_item(row) for row in rows]
|
||||
|
||||
|
||||
def review_run(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
run_id: str,
|
||||
payload: ConnectorReviewRequest,
|
||||
) -> ConnectorRunItem:
|
||||
run = (
|
||||
session.query(ConnectorSimulationRun)
|
||||
.filter(
|
||||
ConnectorSimulationRun.id == run_id,
|
||||
ConnectorSimulationRun.tenant_id == principal.tenant_id,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if run is None:
|
||||
raise GovernedConnectorError("run_not_found", "Connector run not found.")
|
||||
if run.review_state not in {"pending", "quarantined"}:
|
||||
raise GovernedConnectorError(
|
||||
"run_not_reviewable",
|
||||
"Only pending or quarantined connector results can be reviewed.",
|
||||
)
|
||||
run.review_state = payload.decision
|
||||
run.status = f"review_{payload.decision}"
|
||||
run.reviewed_by = principal.user.id
|
||||
run.reviewed_at = datetime.now(UTC)
|
||||
run.review_reason = payload.reason.strip()
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action=f"connectors.simulation.{payload.decision}",
|
||||
object_type="connector_simulation_run",
|
||||
object_id=run.id,
|
||||
details={
|
||||
"configuration_id": run.configuration_id,
|
||||
"input_hash": run.input_hash,
|
||||
"configuration_hash": run.configuration_hash,
|
||||
"decision": payload.decision,
|
||||
"reason": run.review_reason,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return run_item(run)
|
||||
|
||||
|
||||
def definition_item(
|
||||
session: Session,
|
||||
definition: ConnectorDefinition,
|
||||
revision: ConnectorDefinitionRevision,
|
||||
) -> ConnectorDefinitionItem:
|
||||
del session
|
||||
return ConnectorDefinitionItem(
|
||||
id=definition.id,
|
||||
tenant_id=definition.tenant_id,
|
||||
definition_key=definition.definition_key,
|
||||
name=definition.name,
|
||||
description=definition.description,
|
||||
status=definition.status,
|
||||
current_revision=definition.current_revision,
|
||||
source_package=definition.source_package,
|
||||
local_definition=definition.local_definition,
|
||||
revision_id=revision.id,
|
||||
definition_hash=revision.definition_hash,
|
||||
origin=revision.origin,
|
||||
package_ref=revision.package_ref,
|
||||
specification=GovernedConnectorSpecification.model_validate(
|
||||
revision.specification
|
||||
),
|
||||
created_at=definition.created_at,
|
||||
updated_at=definition.updated_at,
|
||||
)
|
||||
|
||||
|
||||
def configuration_item(
|
||||
session: Session,
|
||||
item: ConnectorConfiguration,
|
||||
) -> ConnectorConfigurationItem:
|
||||
definition = _tenant_definition(
|
||||
session,
|
||||
tenant_id=item.tenant_id,
|
||||
definition_id=item.definition_id,
|
||||
)
|
||||
return ConnectorConfigurationItem(
|
||||
id=item.id,
|
||||
tenant_id=item.tenant_id,
|
||||
definition_id=item.definition_id,
|
||||
definition_key=definition.definition_key,
|
||||
definition_name=definition.name,
|
||||
name=item.name,
|
||||
status=item.status,
|
||||
endpoint_url=item.endpoint_url,
|
||||
credential_ref=item.credential_ref,
|
||||
base_definition_revision=item.base_definition_revision,
|
||||
latest_definition_revision=definition.current_revision,
|
||||
update_available=definition.current_revision > item.base_definition_revision,
|
||||
local_overrides=dict(item.local_overrides or {}),
|
||||
protected_paths=list(item.protected_paths or []),
|
||||
effective_configuration=dict(item.effective_configuration or {}),
|
||||
effective_hash=item.effective_hash,
|
||||
resource_revision=item.resource_revision,
|
||||
ambiguity_policy=item.ambiguity_policy,
|
||||
updated_at=item.updated_at,
|
||||
)
|
||||
|
||||
|
||||
def run_item(run: ConnectorSimulationRun) -> ConnectorRunItem:
|
||||
return ConnectorRunItem(
|
||||
id=run.id,
|
||||
tenant_id=run.tenant_id,
|
||||
configuration_id=run.configuration_id,
|
||||
mode=run.mode, # type: ignore[arg-type]
|
||||
idempotency_key=run.idempotency_key,
|
||||
status=run.status,
|
||||
review_state=run.review_state,
|
||||
definition_revision=run.definition_revision,
|
||||
configuration_revision=run.configuration_revision,
|
||||
configuration_hash=run.configuration_hash,
|
||||
input_hash=run.input_hash,
|
||||
summary=dict(run.summary or {}),
|
||||
effects=list(run.effects or []),
|
||||
diagnostics=list(run.diagnostics or []),
|
||||
provenance=dict(run.provenance or {}),
|
||||
reviewed_by=run.reviewed_by,
|
||||
reviewed_at=run.reviewed_at,
|
||||
review_reason=run.review_reason,
|
||||
created_at=run.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _simulate(
|
||||
rows: Sequence[Mapping[str, Any]],
|
||||
specification: GovernedConnectorSpecification,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int]:
|
||||
mapped_rows: list[dict[str, Any]] = []
|
||||
row_errors: dict[int, list[dict[str, Any]]] = {}
|
||||
diagnostics: list[dict[str, Any]] = []
|
||||
for index, row in enumerate(rows):
|
||||
mapped: dict[str, Any] = {}
|
||||
for rule in specification.mapping.rules:
|
||||
value, found = _path_value(row, rule.source)
|
||||
if not found:
|
||||
value = rule.default
|
||||
if rule.required and value in (None, ""):
|
||||
diagnostic = _diagnostic(
|
||||
"error",
|
||||
"connectors.mapping.required_source_missing",
|
||||
f"Required source field {rule.source!r} is missing.",
|
||||
"mapping",
|
||||
index=index,
|
||||
field=rule.source,
|
||||
)
|
||||
row_errors.setdefault(index, []).append(diagnostic)
|
||||
diagnostics.append(diagnostic)
|
||||
_set_path(mapped, rule.target, value)
|
||||
mapped_rows.append(mapped)
|
||||
|
||||
ambiguous_indexes: set[int] = set()
|
||||
for rule in specification.validation_rules:
|
||||
if rule.kind == "unique":
|
||||
seen: dict[str, list[int]] = {}
|
||||
for index, row in enumerate(mapped_rows):
|
||||
value, found = _path_value(row, rule.field)
|
||||
if found and value not in (None, ""):
|
||||
seen.setdefault(_stable_value(value), []).append(index)
|
||||
for indexes in seen.values():
|
||||
if len(indexes) > 1:
|
||||
ambiguous_indexes.update(indexes)
|
||||
for index in indexes:
|
||||
diagnostic = _diagnostic(
|
||||
rule.severity,
|
||||
rule.code,
|
||||
rule.message,
|
||||
"validation",
|
||||
index=index,
|
||||
field=rule.field,
|
||||
)
|
||||
row_errors.setdefault(index, []).append(diagnostic)
|
||||
diagnostics.append(diagnostic)
|
||||
continue
|
||||
for index, row in enumerate(mapped_rows):
|
||||
value, found = _path_value(row, rule.field)
|
||||
invalid = (
|
||||
rule.kind == "required" and (not found or value in (None, ""))
|
||||
) or (
|
||||
rule.kind == "one_of" and found and value not in rule.values
|
||||
)
|
||||
if invalid:
|
||||
diagnostic = _diagnostic(
|
||||
rule.severity,
|
||||
rule.code,
|
||||
rule.message,
|
||||
"validation",
|
||||
index=index,
|
||||
field=rule.field,
|
||||
)
|
||||
row_errors.setdefault(index, []).append(diagnostic)
|
||||
diagnostics.append(diagnostic)
|
||||
|
||||
redacted = set(specification.dry_run.redacted_fields)
|
||||
effects: list[dict[str, Any]] = []
|
||||
for index, mapped in enumerate(mapped_rows):
|
||||
errors = row_errors.get(index, [])
|
||||
has_error = any(item["severity"] == "error" for item in errors)
|
||||
effect = "conflict" if has_error or index in ambiguous_indexes else "create"
|
||||
effects.append(
|
||||
{
|
||||
"effect": effect,
|
||||
"source_object_ref": _source_ref(rows[index], index),
|
||||
"target_object_ref": None,
|
||||
"changed_fields": sorted(_leaf_paths(mapped)),
|
||||
"sample": _redact_fields(mapped, redacted),
|
||||
"reason_code": (
|
||||
"ambiguous_external_result"
|
||||
if index in ambiguous_indexes
|
||||
else errors[0]["code"] if errors else None
|
||||
),
|
||||
"outcome": "preview",
|
||||
"revision": specification.mapping.version,
|
||||
}
|
||||
)
|
||||
return effects, diagnostics, len(ambiguous_indexes)
|
||||
|
||||
|
||||
def _audit_configuration(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
item: ConnectorConfiguration,
|
||||
*,
|
||||
action: str,
|
||||
extra: Mapping[str, Any] | None = None,
|
||||
) -> None:
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action=action,
|
||||
object_type="connector_configuration",
|
||||
object_id=item.id,
|
||||
details={
|
||||
"definition_id": item.definition_id,
|
||||
"base_definition_revision": item.base_definition_revision,
|
||||
"resource_revision": item.resource_revision,
|
||||
"effective_hash": item.effective_hash,
|
||||
"protected_paths": list(item.protected_paths or []),
|
||||
"ambiguity_policy": item.ambiguity_policy,
|
||||
"status": item.status,
|
||||
"credential_reference_present": bool(item.credential_ref),
|
||||
**dict(extra or {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _tenant_definition(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
definition_id: str,
|
||||
) -> ConnectorDefinition:
|
||||
item = (
|
||||
session.query(ConnectorDefinition)
|
||||
.filter(
|
||||
ConnectorDefinition.id == definition_id,
|
||||
ConnectorDefinition.tenant_id == tenant_id,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if item is None:
|
||||
raise GovernedConnectorError(
|
||||
"definition_not_found",
|
||||
"Connector definition not found.",
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
def _tenant_configuration(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
configuration_id: str,
|
||||
) -> ConnectorConfiguration:
|
||||
item = (
|
||||
session.query(ConnectorConfiguration)
|
||||
.filter(
|
||||
ConnectorConfiguration.id == configuration_id,
|
||||
ConnectorConfiguration.tenant_id == tenant_id,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if item is None:
|
||||
raise GovernedConnectorError(
|
||||
"configuration_not_found",
|
||||
"Connector configuration not found.",
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
def _definition_revision(
|
||||
session: Session,
|
||||
definition: ConnectorDefinition,
|
||||
revision: int,
|
||||
) -> ConnectorDefinitionRevision:
|
||||
item = (
|
||||
session.query(ConnectorDefinitionRevision)
|
||||
.filter(
|
||||
ConnectorDefinitionRevision.definition_id == definition.id,
|
||||
ConnectorDefinitionRevision.revision == revision,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if item is None:
|
||||
raise GovernedConnectorError(
|
||||
"definition_revision_not_found",
|
||||
"Connector definition revision not found.",
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
def _effective_specification(
|
||||
base: Mapping[str, Any],
|
||||
overrides: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
merged = _deep_merge(base, overrides)
|
||||
return GovernedConnectorSpecification.model_validate(merged).model_dump(mode="json")
|
||||
|
||||
|
||||
def _deep_merge(
|
||||
base: Mapping[str, Any],
|
||||
overrides: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
result = copy.deepcopy(dict(base))
|
||||
for key, value in overrides.items():
|
||||
if isinstance(value, Mapping) and isinstance(result.get(key), Mapping):
|
||||
result[key] = _deep_merge(result[key], value) # type: ignore[arg-type]
|
||||
else:
|
||||
result[key] = copy.deepcopy(value)
|
||||
return result
|
||||
|
||||
|
||||
def _protected_paths(value: Mapping[str, Any], prefix: str = "") -> list[str]:
|
||||
paths: list[str] = []
|
||||
for key in sorted(value):
|
||||
path = f"{prefix}.{key}" if prefix else str(key)
|
||||
item = value[key]
|
||||
if isinstance(item, Mapping) and item:
|
||||
paths.extend(_protected_paths(item, path))
|
||||
else:
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
|
||||
def _leaf_paths(value: Mapping[str, Any], prefix: str = "") -> set[str]:
|
||||
paths: set[str] = set()
|
||||
for key, item in value.items():
|
||||
path = f"{prefix}.{key}" if prefix else str(key)
|
||||
if isinstance(item, Mapping) and item:
|
||||
paths.update(_leaf_paths(item, path))
|
||||
else:
|
||||
paths.add(path)
|
||||
return paths
|
||||
|
||||
|
||||
def _path_value(value: Mapping[str, Any], path: str) -> tuple[Any, bool]:
|
||||
current: Any = value
|
||||
for part in path.split("."):
|
||||
if not isinstance(current, Mapping) or part not in current:
|
||||
return None, False
|
||||
current = current[part]
|
||||
return current, True
|
||||
|
||||
|
||||
def _set_path(target: dict[str, Any], path: str, value: Any) -> None:
|
||||
parts = path.split(".")
|
||||
current = target
|
||||
for part in parts[:-1]:
|
||||
nested = current.get(part)
|
||||
if not isinstance(nested, dict):
|
||||
nested = {}
|
||||
current[part] = nested
|
||||
current = nested
|
||||
current[parts[-1]] = value
|
||||
|
||||
|
||||
def _source_ref(row: Mapping[str, Any], index: int) -> str:
|
||||
for key in ("id", "external_id", "source_id"):
|
||||
value = row.get(key)
|
||||
if value not in (None, ""):
|
||||
return f"sample:{value}"
|
||||
return f"sample:row:{index + 1}"
|
||||
|
||||
|
||||
def _redact_fields(value: Mapping[str, Any], fields: set[str]) -> dict[str, Any]:
|
||||
result = copy.deepcopy(dict(value))
|
||||
for path in fields:
|
||||
parts = path.split(".")
|
||||
current: Any = result
|
||||
for part in parts[:-1]:
|
||||
if not isinstance(current, dict):
|
||||
break
|
||||
current = current.get(part)
|
||||
else:
|
||||
if isinstance(current, dict) and parts[-1] in current:
|
||||
current[parts[-1]] = "<redacted>"
|
||||
return result
|
||||
|
||||
|
||||
def _diagnostic(
|
||||
severity: str,
|
||||
code: str,
|
||||
message: str,
|
||||
stage: str,
|
||||
**details: Any,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"severity": severity,
|
||||
"code": code,
|
||||
"message": message,
|
||||
"stage": stage,
|
||||
"retryable": False,
|
||||
"details": details,
|
||||
}
|
||||
|
||||
|
||||
def _validate_endpoint(
|
||||
endpoint_url: str | None,
|
||||
credential_ref: str | None,
|
||||
) -> None:
|
||||
if endpoint_url:
|
||||
try:
|
||||
ConnectorEndpoint(
|
||||
url=endpoint_url,
|
||||
credential_ref=_optional_text(credential_ref),
|
||||
)
|
||||
except ConnectorContractError as exc:
|
||||
raise GovernedConnectorError("invalid_endpoint", str(exc)) from exc
|
||||
elif credential_ref:
|
||||
raise GovernedConnectorError(
|
||||
"credential_without_endpoint",
|
||||
"A credential reference requires a configured endpoint.",
|
||||
)
|
||||
|
||||
|
||||
def _stable_value(value: Any) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
||||
|
||||
|
||||
def _hash(value: Any) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _optional_text(value: object | None) -> str | None:
|
||||
normalized = str(value or "").strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GovernedConnectorError",
|
||||
"configuration_item",
|
||||
"create_configuration",
|
||||
"execute_run",
|
||||
"list_configurations",
|
||||
"list_definitions",
|
||||
"list_runs",
|
||||
"review_run",
|
||||
"run_item",
|
||||
"update_configuration",
|
||||
"upsert_definition",
|
||||
]
|
||||
@@ -0,0 +1,221 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
class ConnectorMappingRule(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
source: str = Field(min_length=1, max_length=255)
|
||||
target: str = Field(min_length=1, max_length=255)
|
||||
required: bool = False
|
||||
default: Any = None
|
||||
|
||||
|
||||
class ConnectorMappingDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
version: str = Field(min_length=1, max_length=100)
|
||||
rules: list[ConnectorMappingRule] = Field(default_factory=list, max_length=500)
|
||||
|
||||
|
||||
class ConnectorValidationRule(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
kind: Literal["required", "one_of", "unique"]
|
||||
field: str = Field(min_length=1, max_length=255)
|
||||
values: list[Any] = Field(default_factory=list, max_length=500)
|
||||
severity: Literal["warning", "error"] = "error"
|
||||
code: str = Field(min_length=1, max_length=120)
|
||||
message: str = Field(min_length=1, max_length=500)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_values(self) -> "ConnectorValidationRule":
|
||||
if self.kind == "one_of" and not self.values:
|
||||
raise ValueError("one_of validation requires allowed values")
|
||||
if self.kind != "one_of" and self.values:
|
||||
raise ValueError("Only one_of validation accepts values")
|
||||
return self
|
||||
|
||||
|
||||
class ConnectorDryRunMetadata(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
supported: bool = True
|
||||
simulation_supported: bool = True
|
||||
sample_rows: list[dict[str, Any]] = Field(default_factory=list, max_length=500)
|
||||
max_items: int = Field(default=500, ge=1, le=10_000)
|
||||
redacted_fields: list[str] = Field(default_factory=list, max_length=100)
|
||||
|
||||
|
||||
class ConnectorAuditMetadata(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
event_prefix: str = Field(min_length=1, max_length=120)
|
||||
expected_events: list[str] = Field(default_factory=list, max_length=100)
|
||||
evidence_fields: list[str] = Field(default_factory=list, max_length=100)
|
||||
|
||||
|
||||
class GovernedConnectorSpecification(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
provider: str = Field(min_length=1, max_length=120)
|
||||
protocol: str = Field(min_length=1, max_length=80)
|
||||
capabilities: list[str] = Field(min_length=1, max_length=100)
|
||||
input_schema: dict[str, Any]
|
||||
output_schema: dict[str, Any]
|
||||
mapping: ConnectorMappingDefinition
|
||||
validation_rules: list[ConnectorValidationRule] = Field(
|
||||
default_factory=list,
|
||||
max_length=500,
|
||||
)
|
||||
dry_run: ConnectorDryRunMetadata
|
||||
audit: ConnectorAuditMetadata
|
||||
privacy_classification: Literal[
|
||||
"public",
|
||||
"internal",
|
||||
"confidential",
|
||||
"restricted",
|
||||
] = "internal"
|
||||
retention_class: str = Field(min_length=1, max_length=120)
|
||||
operational_limits: dict[str, Any] = Field(default_factory=dict)
|
||||
retry_policy: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ConnectorDefinitionUpsertRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
definition_key: str = Field(pattern=r"^[a-z0-9][a-z0-9._-]{0,159}$")
|
||||
name: str = Field(min_length=1, max_length=300)
|
||||
description: str | None = Field(default=None, max_length=4000)
|
||||
origin: Literal["package", "local"] = "local"
|
||||
package_ref: str | None = Field(default=None, max_length=300)
|
||||
specification: GovernedConnectorSpecification
|
||||
|
||||
@model_validator(mode="after")
|
||||
def package_provenance(self) -> "ConnectorDefinitionUpsertRequest":
|
||||
if self.origin == "package" and not self.package_ref:
|
||||
raise ValueError("Package definitions require package_ref")
|
||||
if self.origin == "local" and self.package_ref:
|
||||
raise ValueError("Local definitions cannot claim package_ref")
|
||||
return self
|
||||
|
||||
|
||||
class ConnectorDefinitionItem(BaseModel):
|
||||
id: str
|
||||
tenant_id: str
|
||||
definition_key: str
|
||||
name: str
|
||||
description: str | None = None
|
||||
status: str
|
||||
current_revision: int
|
||||
source_package: str | None = None
|
||||
local_definition: bool
|
||||
revision_id: str
|
||||
definition_hash: str
|
||||
origin: str
|
||||
package_ref: str | None = None
|
||||
specification: GovernedConnectorSpecification
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class ConnectorDefinitionListResponse(BaseModel):
|
||||
items: list[ConnectorDefinitionItem]
|
||||
|
||||
|
||||
class ConnectorConfigurationCreateRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
definition_id: str = Field(min_length=1, max_length=36)
|
||||
name: str = Field(min_length=1, max_length=300)
|
||||
endpoint_url: str | None = Field(default=None, max_length=1500)
|
||||
credential_ref: str | None = Field(default=None, max_length=500)
|
||||
local_overrides: dict[str, Any] = Field(default_factory=dict)
|
||||
ambiguity_policy: Literal["manual_review", "quarantine", "reject"] = (
|
||||
"manual_review"
|
||||
)
|
||||
status: Literal["draft", "active", "disabled"] = "draft"
|
||||
|
||||
|
||||
class ConnectorConfigurationUpdateRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
expected_revision: int = Field(ge=1)
|
||||
name: str | None = Field(default=None, min_length=1, max_length=300)
|
||||
endpoint_url: str | None = Field(default=None, max_length=1500)
|
||||
credential_ref: str | None = Field(default=None, max_length=500)
|
||||
local_overrides: dict[str, Any] | None = None
|
||||
ambiguity_policy: Literal["manual_review", "quarantine", "reject"] | None = None
|
||||
status: Literal["draft", "active", "disabled"] | None = None
|
||||
adopt_latest_definition: bool = False
|
||||
|
||||
|
||||
class ConnectorConfigurationItem(BaseModel):
|
||||
id: str
|
||||
tenant_id: str
|
||||
definition_id: str
|
||||
definition_key: str
|
||||
definition_name: str
|
||||
name: str
|
||||
status: str
|
||||
endpoint_url: str | None = None
|
||||
credential_ref: str | None = None
|
||||
base_definition_revision: int
|
||||
latest_definition_revision: int
|
||||
update_available: bool
|
||||
local_overrides: dict[str, Any]
|
||||
protected_paths: list[str]
|
||||
effective_configuration: dict[str, Any]
|
||||
effective_hash: str
|
||||
resource_revision: int
|
||||
ambiguity_policy: str
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class ConnectorConfigurationListResponse(BaseModel):
|
||||
items: list[ConnectorConfigurationItem]
|
||||
|
||||
|
||||
class ConnectorRunRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||
input_rows: list[dict[str, Any]] | None = Field(default=None, max_length=500)
|
||||
external_revision: str | None = Field(default=None, max_length=255)
|
||||
|
||||
|
||||
class ConnectorRunItem(BaseModel):
|
||||
id: str
|
||||
tenant_id: str
|
||||
configuration_id: str
|
||||
mode: Literal["dry_run", "simulation"]
|
||||
idempotency_key: str
|
||||
status: str
|
||||
review_state: str
|
||||
definition_revision: int
|
||||
configuration_revision: int
|
||||
configuration_hash: str
|
||||
input_hash: str
|
||||
summary: dict[str, Any]
|
||||
effects: list[dict[str, Any]]
|
||||
diagnostics: list[dict[str, Any]]
|
||||
provenance: dict[str, Any]
|
||||
reviewed_by: str | None = None
|
||||
reviewed_at: datetime | None = None
|
||||
review_reason: str | None = None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ConnectorRunListResponse(BaseModel):
|
||||
items: list[ConnectorRunItem]
|
||||
|
||||
|
||||
class ConnectorReviewRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
decision: Literal["approved", "rejected"]
|
||||
reason: str = Field(min_length=5, max_length=1000)
|
||||
@@ -14,11 +14,13 @@ from govoplan_core.core.datasources import CAPABILITY_DATASOURCE_ORIGINS
|
||||
from govoplan_core.core.feeds import CAPABILITY_CONNECTORS_FEEDS
|
||||
from govoplan_core.core.modules import (
|
||||
DocumentationTopic,
|
||||
FrontendModule,
|
||||
MigrationSpec,
|
||||
ModuleInterfaceProvider,
|
||||
ModuleManifest,
|
||||
PermissionDefinition,
|
||||
RoleTemplate,
|
||||
ViewSurface,
|
||||
)
|
||||
from govoplan_core.core.provider_governance import (
|
||||
ExternalProviderDeclaration,
|
||||
@@ -38,8 +40,12 @@ from govoplan_core.core.sanctions import (
|
||||
)
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_connectors.backend.db.models import (
|
||||
ConnectorConfiguration,
|
||||
ConnectorDefinition,
|
||||
ConnectorDefinitionRevision,
|
||||
ConnectorSanctionsAcquisitionRun,
|
||||
ConnectorSanctionsSnapshot,
|
||||
ConnectorSimulationRun,
|
||||
ConnectorTabularSource,
|
||||
)
|
||||
from govoplan_connectors.backend.sanctions_sources import (
|
||||
@@ -93,6 +99,11 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
|
||||
reference="tests/test_recovery.py",
|
||||
summary="Proves atomic snapshot commits, idempotent replay, distributed fences, tamper rejection, and unknown external-effect handling.",
|
||||
),
|
||||
ModuleMaturityEvidence(
|
||||
kind="test",
|
||||
reference="tests/test_governed_runtime.py",
|
||||
summary="Exercises immutable definition revisions, protected local overrides, idempotent simulations, and explicit ambiguity review.",
|
||||
),
|
||||
ModuleMaturityEvidence(
|
||||
kind="documentation",
|
||||
reference="docs/CONNECTOR_SOURCE_LIFECYCLE.md",
|
||||
@@ -102,6 +113,7 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
|
||||
known_limits=(
|
||||
"The executable generic datasource origin is an immutable tabular snapshot; database and arbitrary REST profiles remain future providers.",
|
||||
"Feed publication renders a governed document but does not yet push it to an external publishing endpoint.",
|
||||
"The generic governed runtime simulates deterministic mapping and validation; provider-specific live writes remain owned by explicit connector adapters.",
|
||||
),
|
||||
supported_authority_modes=(
|
||||
"external_authoritative",
|
||||
@@ -254,7 +266,7 @@ PERMISSIONS = (
|
||||
_permission(
|
||||
ADMIN_SCOPE,
|
||||
"Administer connector sources",
|
||||
"Manage every tenant connector source and future source policies.",
|
||||
"Manage tenant connector sources, versioned definitions, protected overrides, and review policies.",
|
||||
),
|
||||
_permission(
|
||||
SANCTIONS_READ_SCOPE,
|
||||
@@ -269,6 +281,18 @@ PERMISSIONS = (
|
||||
)
|
||||
|
||||
ROLE_TEMPLATES = (
|
||||
RoleTemplate(
|
||||
slug="connector_administrator",
|
||||
name="Connector administrator",
|
||||
description="Govern connector definitions, local configurations, simulations, and manual review.",
|
||||
permissions=(
|
||||
READ_SCOPE,
|
||||
WRITE_SCOPE,
|
||||
ADMIN_SCOPE,
|
||||
SANCTIONS_READ_SCOPE,
|
||||
SANCTIONS_REFRESH_SCOPE,
|
||||
),
|
||||
),
|
||||
RoleTemplate(
|
||||
slug="connector_source_manager",
|
||||
name="Connector source manager",
|
||||
@@ -315,6 +339,21 @@ def _feed_provider(_context) -> ConnectorFeedProvider:
|
||||
|
||||
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
|
||||
return {
|
||||
"connector_definitions": (
|
||||
session.query(ConnectorDefinition)
|
||||
.filter(ConnectorDefinition.tenant_id == tenant_id)
|
||||
.count()
|
||||
),
|
||||
"connector_configurations": (
|
||||
session.query(ConnectorConfiguration)
|
||||
.filter(ConnectorConfiguration.tenant_id == tenant_id)
|
||||
.count()
|
||||
),
|
||||
"connector_simulation_runs": (
|
||||
session.query(ConnectorSimulationRun)
|
||||
.filter(ConnectorSimulationRun.tenant_id == tenant_id)
|
||||
.count()
|
||||
),
|
||||
"connector_tabular_sources": (
|
||||
session.query(ConnectorTabularSource)
|
||||
.filter(
|
||||
@@ -383,6 +422,27 @@ manifest = ModuleManifest(
|
||||
permissions=PERMISSIONS,
|
||||
role_templates=ROLE_TEMPLATES,
|
||||
route_factory=_router,
|
||||
frontend=FrontendModule(
|
||||
module_id=MODULE_ID,
|
||||
package_name="@govoplan/connectors-webui",
|
||||
view_surfaces=(
|
||||
ViewSurface(
|
||||
id="connectors.admin.governed-configurations",
|
||||
module_id=MODULE_ID,
|
||||
kind="section",
|
||||
label="Connector governance",
|
||||
order=45,
|
||||
),
|
||||
ViewSurface(
|
||||
id="connectors.admin.simulation-review",
|
||||
module_id=MODULE_ID,
|
||||
kind="section",
|
||||
label="Connector simulation review",
|
||||
parent_id="connectors.admin.governed-configurations",
|
||||
order=20,
|
||||
),
|
||||
),
|
||||
),
|
||||
capability_factories={
|
||||
CAPABILITY_CONNECTORS_TABULAR_SOURCES: _provider,
|
||||
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER: _provider,
|
||||
@@ -411,6 +471,10 @@ manifest = ModuleManifest(
|
||||
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
||||
retirement_supported=True,
|
||||
retirement_provider=drop_table_retirement_provider(
|
||||
ConnectorSimulationRun,
|
||||
ConnectorConfiguration,
|
||||
ConnectorDefinitionRevision,
|
||||
ConnectorDefinition,
|
||||
ConnectorSanctionsSnapshot,
|
||||
ConnectorSanctionsAcquisitionRun,
|
||||
ConnectorTabularSource,
|
||||
@@ -423,6 +487,10 @@ manifest = ModuleManifest(
|
||||
),
|
||||
uninstall_guard_providers=(
|
||||
persistent_table_uninstall_guard(
|
||||
ConnectorSimulationRun,
|
||||
ConnectorConfiguration,
|
||||
ConnectorDefinitionRevision,
|
||||
ConnectorDefinition,
|
||||
ConnectorSanctionsSnapshot,
|
||||
ConnectorSanctionsAcquisitionRun,
|
||||
ConnectorTabularSource,
|
||||
@@ -430,6 +498,29 @@ manifest = ModuleManifest(
|
||||
),
|
||||
),
|
||||
documentation=(
|
||||
DocumentationTopic(
|
||||
id="connectors.governed-configuration",
|
||||
title="Govern connector definitions and simulations",
|
||||
summary="Version connector schemas and mappings while preserving tenant-local overrides and review evidence.",
|
||||
body=(
|
||||
"Connector administrators create package-managed or local definitions that explicitly declare provider, protocol, capabilities, schemas, mapping rules, validation, preview support, audit expectations, privacy, retention, limits, and retry metadata. Every definition change creates an immutable revision. A tenant configuration pins one revision and stores only a credential reference; package updates remain available but do not change the effective configuration until an administrator adopts them. Local override leaf paths are displayed as protected and are reapplied when an update is adopted. Dry-runs and simulations are bounded, redact configured fields, are idempotent by caller key, and retain configuration, mapping, input, and external revision provenance. Ambiguous results follow the configuration policy: manual review, quarantine, or rejection. Pending and quarantined evidence requires an explicit approve or reject decision with a reason. Provider-specific live writes are not implied by a successful generic simulation."
|
||||
),
|
||||
layer="configured",
|
||||
documentation_types=("admin", "user"),
|
||||
audience=("operator", "module_admin", "integration_admin"),
|
||||
related_modules=("policy", "audit", "dataflow", "ops"),
|
||||
order=38,
|
||||
metadata={
|
||||
"kind": "guide",
|
||||
"help_contexts": ["connectors.admin.governed-configurations"],
|
||||
"prerequisites": [
|
||||
"A connector definition has been installed or authored.",
|
||||
"Credential material is stored outside the connector URL and referenced by an approved secret identifier.",
|
||||
],
|
||||
"outcome": "The active connector behavior is inspectable, version-pinned, testable, and reviewable before any provider-specific write.",
|
||||
"verification": "Reload the configuration, inspect protected paths and effective hash, run a simulation with a new idempotency key, and resolve any pending review result.",
|
||||
},
|
||||
),
|
||||
DocumentationTopic(
|
||||
id="connectors.authority-and-effects",
|
||||
title="Connector authority and effect behavior",
|
||||
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
"""governed connector definitions and simulation evidence
|
||||
|
||||
Revision ID: a8d9e0f1b2c3
|
||||
Revises: f7c8d9e0a1b2
|
||||
Create Date: 2026-08-20 12:30:00.000000
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "a8d9e0f1b2c3"
|
||||
down_revision = "f7c8d9e0a1b2"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"connector_definitions",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("definition_key", sa.String(length=160), nullable=False),
|
||||
sa.Column("name", sa.String(length=300), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
sa.Column("status", sa.String(length=30), nullable=False),
|
||||
sa.Column("current_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("source_package", sa.String(length=300), nullable=True),
|
||||
sa.Column("local_definition", sa.Boolean(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_definitions")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"definition_key",
|
||||
name="uq_connector_definition_tenant_key",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_connector_definitions_tenant_id"),
|
||||
"connector_definitions",
|
||||
["tenant_id"],
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_connector_definitions_status"),
|
||||
"connector_definitions",
|
||||
["status"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_connector_definitions_tenant_status",
|
||||
"connector_definitions",
|
||||
["tenant_id", "status"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"connector_definition_revisions",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("definition_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("specification", sa.JSON(), nullable=False),
|
||||
sa.Column("definition_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("origin", sa.String(length=30), nullable=False),
|
||||
sa.Column("package_ref", sa.String(length=300), nullable=True),
|
||||
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["definition_id"],
|
||||
["connector_definitions.id"],
|
||||
name=op.f(
|
||||
"fk_connector_definition_revisions_definition_id_connector_definitions"
|
||||
),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"id",
|
||||
name=op.f("pk_connector_definition_revisions"),
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"definition_id",
|
||||
"revision",
|
||||
name="uq_connector_definition_revision",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_connector_definition_revisions_definition_id"),
|
||||
"connector_definition_revisions",
|
||||
["definition_id"],
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_connector_definition_revisions_definition_hash"),
|
||||
"connector_definition_revisions",
|
||||
["definition_hash"],
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_connector_definition_revisions_created_by"),
|
||||
"connector_definition_revisions",
|
||||
["created_by"],
|
||||
)
|
||||
op.create_table(
|
||||
"connector_configurations",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("definition_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("name", sa.String(length=300), nullable=False),
|
||||
sa.Column("status", sa.String(length=30), nullable=False),
|
||||
sa.Column("endpoint_url", sa.String(length=1500), nullable=True),
|
||||
sa.Column("credential_ref", sa.String(length=500), nullable=True),
|
||||
sa.Column("base_definition_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("local_overrides", sa.JSON(), nullable=False),
|
||||
sa.Column("protected_paths", sa.JSON(), nullable=False),
|
||||
sa.Column("effective_configuration", sa.JSON(), nullable=False),
|
||||
sa.Column("effective_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("resource_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("ambiguity_policy", sa.String(length=30), nullable=False),
|
||||
sa.Column("updated_by", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["definition_id"],
|
||||
["connector_definitions.id"],
|
||||
name=op.f(
|
||||
"fk_connector_configurations_definition_id_connector_definitions"
|
||||
),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_configurations")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"name",
|
||||
name="uq_connector_configuration_tenant_name",
|
||||
),
|
||||
)
|
||||
for name, columns in (
|
||||
("ix_connector_configurations_tenant_id", ["tenant_id"]),
|
||||
("ix_connector_configurations_definition_id", ["definition_id"]),
|
||||
("ix_connector_configurations_status", ["status"]),
|
||||
("ix_connector_configurations_effective_hash", ["effective_hash"]),
|
||||
("ix_connector_configurations_updated_by", ["updated_by"]),
|
||||
):
|
||||
op.create_index(op.f(name), "connector_configurations", columns)
|
||||
op.create_index(
|
||||
"ix_connector_configurations_tenant_status",
|
||||
"connector_configurations",
|
||||
["tenant_id", "status"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"connector_simulation_runs",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("configuration_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("mode", sa.String(length=30), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("request_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("status", sa.String(length=30), nullable=False),
|
||||
sa.Column("review_state", sa.String(length=30), nullable=False),
|
||||
sa.Column("definition_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("configuration_revision", sa.Integer(), nullable=False),
|
||||
sa.Column("configuration_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("input_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("summary", sa.JSON(), nullable=False),
|
||||
sa.Column("effects", sa.JSON(), nullable=False),
|
||||
sa.Column("diagnostics", sa.JSON(), nullable=False),
|
||||
sa.Column("provenance", sa.JSON(), nullable=False),
|
||||
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||
sa.Column("reviewed_by", sa.String(length=255), nullable=True),
|
||||
sa.Column("reviewed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("review_reason", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["configuration_id"],
|
||||
["connector_configurations.id"],
|
||||
name=op.f(
|
||||
"fk_connector_simulation_runs_configuration_id_connector_configurations"
|
||||
),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"id",
|
||||
name=op.f("pk_connector_simulation_runs"),
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"configuration_id",
|
||||
"mode",
|
||||
"idempotency_key",
|
||||
name="uq_connector_simulation_run_idempotency",
|
||||
),
|
||||
)
|
||||
for name, columns in (
|
||||
("ix_connector_simulation_runs_tenant_id", ["tenant_id"]),
|
||||
("ix_connector_simulation_runs_configuration_id", ["configuration_id"]),
|
||||
("ix_connector_simulation_runs_mode", ["mode"]),
|
||||
("ix_connector_simulation_runs_status", ["status"]),
|
||||
("ix_connector_simulation_runs_review_state", ["review_state"]),
|
||||
("ix_connector_simulation_runs_created_by", ["created_by"]),
|
||||
("ix_connector_simulation_runs_reviewed_by", ["reviewed_by"]),
|
||||
):
|
||||
op.create_index(op.f(name), "connector_simulation_runs", columns)
|
||||
op.create_index(
|
||||
"ix_connector_simulation_runs_review",
|
||||
"connector_simulation_runs",
|
||||
["tenant_id", "review_state", "created_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("connector_simulation_runs")
|
||||
op.drop_table("connector_configurations")
|
||||
op.drop_table("connector_definition_revisions")
|
||||
op.drop_table("connector_definitions")
|
||||
@@ -48,6 +48,30 @@ from govoplan_connectors.backend.schemas import (
|
||||
TabularSourceResponse,
|
||||
)
|
||||
from govoplan_connectors.backend.feeds import ConnectorFeedProvider, feed_rows
|
||||
from govoplan_connectors.backend.governed_runtime import (
|
||||
GovernedConnectorError,
|
||||
create_configuration,
|
||||
execute_run,
|
||||
list_configurations,
|
||||
list_definitions,
|
||||
list_runs,
|
||||
review_run,
|
||||
update_configuration,
|
||||
upsert_definition,
|
||||
)
|
||||
from govoplan_connectors.backend.governed_schemas import (
|
||||
ConnectorConfigurationCreateRequest,
|
||||
ConnectorConfigurationItem,
|
||||
ConnectorConfigurationListResponse,
|
||||
ConnectorConfigurationUpdateRequest,
|
||||
ConnectorDefinitionItem,
|
||||
ConnectorDefinitionListResponse,
|
||||
ConnectorDefinitionUpsertRequest,
|
||||
ConnectorReviewRequest,
|
||||
ConnectorRunItem,
|
||||
ConnectorRunListResponse,
|
||||
ConnectorRunRequest,
|
||||
)
|
||||
from govoplan_connectors.backend.recovery import (
|
||||
ConnectorRecoveryError,
|
||||
begin_connector_read_snapshot,
|
||||
@@ -135,6 +159,26 @@ def _recovery_http_error(exc: ConnectorRecoveryError) -> HTTPException:
|
||||
)
|
||||
|
||||
|
||||
def _governed_http_error(exc: GovernedConnectorError) -> HTTPException:
|
||||
if exc.code.endswith("_not_found"):
|
||||
status_code = status.HTTP_404_NOT_FOUND
|
||||
elif exc.code in {
|
||||
"configuration_conflict",
|
||||
"configuration_disabled",
|
||||
"idempotency_conflict",
|
||||
"local_definition_protected",
|
||||
"package_definition_requires_overrides",
|
||||
"run_not_reviewable",
|
||||
}:
|
||||
status_code = status.HTTP_409_CONFLICT
|
||||
else:
|
||||
status_code = status.HTTP_422_UNPROCESSABLE_CONTENT
|
||||
return HTTPException(
|
||||
status_code=status_code,
|
||||
detail={"code": exc.code, "message": str(exc)},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/feeds/preview", response_model=FeedDocumentResponse)
|
||||
def api_preview_feed(
|
||||
payload: FeedAcquireRequest,
|
||||
@@ -609,6 +653,199 @@ def api_list_sanctions_runs(
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/governed/definitions",
|
||||
response_model=ConnectorDefinitionListResponse,
|
||||
)
|
||||
def api_list_governed_definitions(
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> ConnectorDefinitionListResponse:
|
||||
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
||||
try:
|
||||
items = list_definitions(session, tenant_id=principal.tenant_id)
|
||||
except GovernedConnectorError as exc:
|
||||
raise _governed_http_error(exc) from exc
|
||||
return ConnectorDefinitionListResponse(items=items)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/governed/definitions",
|
||||
response_model=ConnectorDefinitionItem,
|
||||
)
|
||||
def api_upsert_governed_definition(
|
||||
payload: ConnectorDefinitionUpsertRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> ConnectorDefinitionItem:
|
||||
_require_any_scope(principal, ADMIN_SCOPE)
|
||||
try:
|
||||
return upsert_definition(session, principal, payload)
|
||||
except GovernedConnectorError as exc:
|
||||
session.rollback()
|
||||
raise _governed_http_error(exc) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/governed/configurations",
|
||||
response_model=ConnectorConfigurationListResponse,
|
||||
)
|
||||
def api_list_governed_configurations(
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> ConnectorConfigurationListResponse:
|
||||
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
||||
try:
|
||||
items = list_configurations(session, tenant_id=principal.tenant_id)
|
||||
except GovernedConnectorError as exc:
|
||||
raise _governed_http_error(exc) from exc
|
||||
return ConnectorConfigurationListResponse(items=items)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/governed/configurations",
|
||||
response_model=ConnectorConfigurationItem,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def api_create_governed_configuration(
|
||||
payload: ConnectorConfigurationCreateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> ConnectorConfigurationItem:
|
||||
_require_any_scope(principal, ADMIN_SCOPE)
|
||||
try:
|
||||
return create_configuration(session, principal, payload)
|
||||
except GovernedConnectorError as exc:
|
||||
session.rollback()
|
||||
raise _governed_http_error(exc) from exc
|
||||
|
||||
|
||||
@router.put(
|
||||
"/governed/configurations/{configuration_id}",
|
||||
response_model=ConnectorConfigurationItem,
|
||||
)
|
||||
def api_update_governed_configuration(
|
||||
configuration_id: str,
|
||||
payload: ConnectorConfigurationUpdateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> ConnectorConfigurationItem:
|
||||
_require_any_scope(principal, ADMIN_SCOPE)
|
||||
try:
|
||||
return update_configuration(
|
||||
session,
|
||||
principal,
|
||||
configuration_id=configuration_id,
|
||||
payload=payload,
|
||||
)
|
||||
except GovernedConnectorError as exc:
|
||||
session.rollback()
|
||||
raise _governed_http_error(exc) from exc
|
||||
|
||||
|
||||
def _api_execute_governed_run(
|
||||
*,
|
||||
configuration_id: str,
|
||||
mode: str,
|
||||
payload: ConnectorRunRequest,
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
) -> ConnectorRunItem:
|
||||
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
||||
try:
|
||||
return execute_run(
|
||||
session,
|
||||
principal,
|
||||
configuration_id=configuration_id,
|
||||
mode=mode,
|
||||
payload=payload,
|
||||
)
|
||||
except GovernedConnectorError as exc:
|
||||
session.rollback()
|
||||
raise _governed_http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/governed/configurations/{configuration_id}/dry-runs",
|
||||
response_model=ConnectorRunItem,
|
||||
)
|
||||
def api_dry_run_governed_configuration(
|
||||
configuration_id: str,
|
||||
payload: ConnectorRunRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> ConnectorRunItem:
|
||||
return _api_execute_governed_run(
|
||||
configuration_id=configuration_id,
|
||||
mode="dry_run",
|
||||
payload=payload,
|
||||
session=session,
|
||||
principal=principal,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/governed/configurations/{configuration_id}/simulations",
|
||||
response_model=ConnectorRunItem,
|
||||
)
|
||||
def api_simulate_governed_configuration(
|
||||
configuration_id: str,
|
||||
payload: ConnectorRunRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> ConnectorRunItem:
|
||||
return _api_execute_governed_run(
|
||||
configuration_id=configuration_id,
|
||||
mode="simulation",
|
||||
payload=payload,
|
||||
session=session,
|
||||
principal=principal,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/governed/runs",
|
||||
response_model=ConnectorRunListResponse,
|
||||
)
|
||||
def api_list_governed_runs(
|
||||
configuration_id: str | None = Query(default=None),
|
||||
review_state: str | None = Query(default=None),
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> ConnectorRunListResponse:
|
||||
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
||||
try:
|
||||
items = list_runs(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
configuration_id=configuration_id,
|
||||
review_state=review_state,
|
||||
limit=limit,
|
||||
)
|
||||
except GovernedConnectorError as exc:
|
||||
raise _governed_http_error(exc) from exc
|
||||
return ConnectorRunListResponse(items=items)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/governed/runs/{run_id}/review",
|
||||
response_model=ConnectorRunItem,
|
||||
)
|
||||
def api_review_governed_run(
|
||||
run_id: str,
|
||||
payload: ConnectorReviewRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> ConnectorRunItem:
|
||||
_require_any_scope(principal, ADMIN_SCOPE)
|
||||
try:
|
||||
return review_run(session, principal, run_id=run_id, payload=payload)
|
||||
except GovernedConnectorError as exc:
|
||||
session.rollback()
|
||||
raise _governed_http_error(exc) from exc
|
||||
|
||||
|
||||
def _source_response(source: TabularSource) -> TabularSourceResponse:
|
||||
return TabularSourceResponse(
|
||||
ref=source.ref,
|
||||
|
||||
Reference in New Issue
Block a user