Release Connectors v0.1.21 with external knowledge integration
Module Package Release / publish-packages (push) Successful in 12s

This commit is contained in:
2026-08-22 14:34:36 +02:00
parent 2c2b11f860
commit 79e2315e89
28 changed files with 6351 additions and 33 deletions
@@ -456,10 +456,189 @@ class ConnectorSimulationRun(Base, TimestampMixin):
review_reason: Mapped[str | None] = mapped_column(Text)
class ConnectorKnowledgeProfile(Base, TimestampMixin):
__tablename__ = "connector_knowledge_profiles"
__table_args__ = (
UniqueConstraint(
"tenant_id",
"configuration_id",
name="uq_connector_knowledge_profile_configuration",
),
Index(
"ix_connector_knowledge_profiles_tenant_status",
"tenant_id",
"status",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
configuration_id: Mapped[str] = mapped_column(
ForeignKey("connector_configurations.id", ondelete="RESTRICT"),
nullable=False,
index=True,
)
status: Mapped[str] = mapped_column(
String(30), default="active", nullable=False, index=True
)
product: Mapped[str] = mapped_column(
String(50), default="unknown", nullable=False, index=True
)
product_version: Mapped[str | None] = mapped_column(String(100))
desired_maturity: Mapped[str] = mapped_column(
String(30), default="read", nullable=False
)
discovered_maturity: Mapped[str] = mapped_column(
String(30), default="discover", nullable=False
)
source_authority_mode: Mapped[str] = mapped_column(
String(40), default="external_mirror", nullable=False
)
default_visibility: Mapped[str] = mapped_column(
String(30), default="restricted", nullable=False
)
default_acl_tokens: Mapped[list[str]] = mapped_column(
JSON, default=list, nullable=False
)
namespace_mappings: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
capabilities: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
discovery_revision: Mapped[str | None] = mapped_column(String(255), index=True)
discovery_evidence: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
health_status: Mapped[str] = mapped_column(
String(30), default="unknown", nullable=False, index=True
)
health_details: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
discovered_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_sync_cursor: Mapped[str | None] = mapped_column(String(500))
last_high_watermark: Mapped[str | None] = mapped_column(String(500))
resource_revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
updated_by: Mapped[str | None] = mapped_column(String(255), index=True)
class ConnectorKnowledgeObject(Base, TimestampMixin):
__tablename__ = "connector_knowledge_objects"
__table_args__ = (
UniqueConstraint(
"profile_id",
"object_type",
"external_id",
name="uq_connector_knowledge_object_identity",
),
Index(
"ix_connector_knowledge_objects_tenant_profile_status",
"tenant_id",
"profile_id",
"status",
),
Index(
"ix_connector_knowledge_objects_tenant_updated",
"tenant_id",
"source_updated_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
profile_id: Mapped[str] = mapped_column(
ForeignKey("connector_knowledge_profiles.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
object_type: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
external_id: Mapped[str] = mapped_column(String(255), nullable=False)
external_page_id: Mapped[str | None] = mapped_column(String(255), index=True)
external_revision_id: Mapped[str | None] = mapped_column(String(255), index=True)
namespace_id: Mapped[int | None] = mapped_column(Integer, index=True)
title: Mapped[str] = mapped_column(String(500), nullable=False)
canonical_url: Mapped[str | None] = mapped_column(String(1500))
status: Mapped[str] = mapped_column(
String(30), default="active", nullable=False, index=True
)
redirect_target_external_id: Mapped[str | None] = mapped_column(String(255))
source_revision: Mapped[str] = mapped_column(String(255), nullable=False)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
visibility: Mapped[str] = mapped_column(
String(30), default="restricted", nullable=False
)
acl_tokens: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
mapped_data: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
provenance: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
change_cursor: Mapped[str | None] = mapped_column(String(500), index=True)
source_updated_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), index=True
)
observed_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True
)
resource_revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
class ConnectorKnowledgeSyncRun(Base, TimestampMixin):
__tablename__ = "connector_knowledge_sync_runs"
__table_args__ = (
UniqueConstraint(
"tenant_id",
"profile_id",
"mode",
"idempotency_key",
name="uq_connector_knowledge_sync_run_idempotency",
),
Index(
"ix_connector_knowledge_sync_runs_profile_started",
"tenant_id",
"profile_id",
"started_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
profile_id: Mapped[str] = mapped_column(
ForeignKey("connector_knowledge_profiles.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
mode: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
cursor_before: Mapped[str | None] = mapped_column(String(500))
cursor_after: Mapped[str | None] = mapped_column(String(500))
high_watermark: Mapped[str | None] = mapped_column(String(500))
counts: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
effects: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(
JSON, default=list, nullable=False
)
provenance: Mapped[dict[str, Any]] = mapped_column(
JSON, default=dict, nullable=False
)
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
started_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True
)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
__all__ = [
"ConnectorConfiguration",
"ConnectorDefinition",
"ConnectorDefinitionRevision",
"ConnectorKnowledgeObject",
"ConnectorKnowledgeProfile",
"ConnectorKnowledgeSyncRun",
"ConnectorSanctionsAcquisitionRun",
"ConnectorSanctionsSnapshot",
"ConnectorSimulationRun",
@@ -18,6 +18,8 @@ from govoplan_connectors.backend.db.models import (
ConnectorConfiguration,
ConnectorDefinition,
ConnectorDefinitionRevision,
ConnectorKnowledgeProfile,
ConnectorKnowledgeSyncRun,
ConnectorSanctionsAcquisitionRun,
ConnectorSimulationRun,
ConnectorTabularSource,
@@ -37,6 +39,8 @@ class _SubjectSelectors:
definition_id: str | None
configuration_id: str | None
simulation_id: str | None
knowledge_profile_id: str | None
knowledge_run_id: str | None
@property
def narrowed(self) -> bool:
@@ -47,6 +51,8 @@ class _SubjectSelectors:
self.definition_id,
self.configuration_id,
self.simulation_id,
self.knowledge_profile_id,
self.knowledge_run_id,
)
)
@@ -172,6 +178,44 @@ class ConnectorsDsarProvider:
)
)
if not selectors.narrowed or selectors.knowledge_profile_id:
query = db.query(ConnectorKnowledgeProfile).filter(
ConnectorKnowledgeProfile.tenant_id == tenant_id,
ConnectorKnowledgeProfile.updated_by == selectors.account_id,
)
if selectors.knowledge_profile_id:
query = query.filter(
ConnectorKnowledgeProfile.id == selectors.knowledge_profile_id
)
records.extend(
_knowledge_profile_attribution(row)
for row in _limited(
query,
ConnectorKnowledgeProfile.created_at,
ConnectorKnowledgeProfile.id,
label="knowledge profile attribution",
)
)
if not selectors.narrowed or selectors.knowledge_run_id:
query = db.query(ConnectorKnowledgeSyncRun).filter(
ConnectorKnowledgeSyncRun.tenant_id == tenant_id,
ConnectorKnowledgeSyncRun.created_by == selectors.account_id,
)
if selectors.knowledge_run_id:
query = query.filter(
ConnectorKnowledgeSyncRun.id == selectors.knowledge_run_id
)
records.extend(
_knowledge_run_attribution(row)
for row in _limited(
query,
ConnectorKnowledgeSyncRun.started_at,
ConnectorKnowledgeSyncRun.id,
label="knowledge run attribution",
)
)
if len(records) > _MAX_RECORDS:
raise ValueError("Connectors DSAR result limit exceeded; narrow selectors.")
return tuple(
@@ -270,6 +314,14 @@ def _subject_selectors(subject: DsarSubjectRef) -> _SubjectSelectors | None:
references.get("connectors.simulation"),
references.get("connectors.simulation_id"),
),
"knowledge_profile_id": _coalesce(
references.get("connectors.knowledge_profile"),
references.get("connectors.knowledge_profile_id"),
),
"knowledge_run_id": _coalesce(
references.get("connectors.knowledge_run"),
references.get("connectors.knowledge_run_id"),
),
}
if account is _CONFLICT or any(value is _CONFLICT for value in values.values()):
return None
@@ -283,6 +335,8 @@ def _subject_selectors(subject: DsarSubjectRef) -> _SubjectSelectors | None:
definition_id=_optional_string(values["definition_id"]),
configuration_id=_optional_string(values["configuration_id"]),
simulation_id=_optional_string(values["simulation_id"]),
knowledge_profile_id=_optional_string(values["knowledge_profile_id"]),
knowledge_run_id=_optional_string(values["knowledge_run_id"]),
)
@@ -396,6 +450,48 @@ def _simulation_attribution(
)
def _knowledge_profile_attribution(
row: ConnectorKnowledgeProfile,
) -> DsarRecordRef:
return _record(
resource_type="knowledge_profile_actor_attribution",
resource_id=row.id,
title="External knowledge profile actor attribution",
data={
"knowledge_profile_id": row.id,
"configuration_id": row.configuration_id,
"status": row.status,
"desired_maturity": row.desired_maturity,
"source_authority_mode": row.source_authority_mode,
"resource_revision": row.resource_revision,
"activity": "updated_external_knowledge_profile",
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
},
observed_at=row.updated_at,
)
def _knowledge_run_attribution(
row: ConnectorKnowledgeSyncRun,
) -> DsarRecordRef:
return _record(
resource_type="knowledge_run_actor_attribution",
resource_id=row.id,
title="External knowledge operation actor attribution",
data={
"knowledge_run_id": row.id,
"knowledge_profile_id": row.profile_id,
"mode": row.mode,
"status": row.status,
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"activity": "started_external_knowledge_operation",
},
observed_at=row.finished_at or row.started_at,
)
def _record(
*,
resource_type: str,
@@ -462,6 +558,8 @@ _RESOURCE_TYPES = {
"definition_actor_attribution",
"configuration_actor_attribution",
"simulation_actor_attribution",
"knowledge_profile_actor_attribution",
"knowledge_run_actor_attribution",
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,303 @@
from __future__ import annotations
from datetime import datetime
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
KnowledgeMaturity = Literal[
"discover",
"link",
"search",
"read",
"publish",
"synchronize",
"migrate",
]
KnowledgeVisibility = Literal["tenant", "restricted"]
class KnowledgeNamespaceMapping(BaseModel):
model_config = ConfigDict(extra="forbid")
source_namespace_id: int
source_name: str = Field(default="", max_length=200)
target_space_ref: str = Field(min_length=1, max_length=255)
target_path_prefix: str = Field(default="", max_length=500)
include: bool = True
visibility: KnowledgeVisibility | None = None
acl_tokens: list[str] = Field(default_factory=list, max_length=500)
@field_validator("acl_tokens")
@classmethod
def normalize_acl_tokens(cls, values: list[str]) -> list[str]:
return _normalized_tokens(values)
@model_validator(mode="after")
def restricted_mapping_requires_acl(self) -> "KnowledgeNamespaceMapping":
if self.visibility == "restricted" and not self.acl_tokens:
raise ValueError("Restricted namespace mappings require ACL tokens")
return self
class KnowledgeProfileCreateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
configuration_id: str = Field(min_length=1, max_length=36)
desired_maturity: KnowledgeMaturity = "read"
source_authority_mode: Literal[
"external_authoritative",
"external_mirror",
"governed_sync",
"linked_reference",
] = "external_mirror"
default_visibility: KnowledgeVisibility = "restricted"
default_acl_tokens: list[str] = Field(default_factory=list, max_length=500)
namespace_mappings: list[KnowledgeNamespaceMapping] = Field(
min_length=1, max_length=500
)
@field_validator("default_acl_tokens")
@classmethod
def normalize_acl_tokens(cls, values: list[str]) -> list[str]:
return _normalized_tokens(values)
@model_validator(mode="after")
def restricted_profile_requires_acl(self) -> "KnowledgeProfileCreateRequest":
if self.default_visibility == "restricted" and not self.default_acl_tokens:
raise ValueError("Restricted knowledge profiles require default ACL tokens")
return self
class KnowledgeProfileUpdateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
expected_resource_revision: int = Field(ge=1)
status: Literal["active", "paused"] | None = None
desired_maturity: KnowledgeMaturity | None = None
source_authority_mode: Literal[
"external_authoritative",
"external_mirror",
"governed_sync",
"linked_reference",
] | None = None
default_visibility: KnowledgeVisibility | None = None
default_acl_tokens: list[str] | None = Field(default=None, max_length=500)
namespace_mappings: list[KnowledgeNamespaceMapping] | None = Field(
default=None, min_length=1, max_length=500
)
@field_validator("default_acl_tokens")
@classmethod
def normalize_acl_tokens(cls, values: list[str] | None) -> list[str] | None:
return _normalized_tokens(values) if values is not None else None
class KnowledgeProfileItem(BaseModel):
id: str
tenant_id: str
configuration_id: str
status: str
product: str
product_version: str | None = None
desired_maturity: str
discovered_maturity: str
source_authority_mode: str
default_visibility: str
default_acl_tokens: list[str]
namespace_mappings: list[dict[str, Any]]
capabilities: list[str]
discovery_revision: str | None = None
health_status: str
health_details: dict[str, Any]
discovered_at: datetime | None = None
last_sync_cursor: str | None = None
last_high_watermark: str | None = None
resource_revision: int
credential_reference_present: bool = False
endpoint_configured: bool = False
created_at: datetime
updated_at: datetime
class KnowledgeProfileListResponse(BaseModel):
items: list[KnowledgeProfileItem]
class KnowledgeDiagnostic(BaseModel):
severity: Literal["info", "warning", "error"]
code: str
message: str
object_ref: str | None = None
field: str | None = None
retryable: bool = False
details: dict[str, Any] = Field(default_factory=dict)
class KnowledgeDiscoveryResponse(BaseModel):
profile: KnowledgeProfileItem
product: str
product_version: str | None = None
api_version: str | None = None
capabilities: list[str]
namespaces: list[dict[str, Any]]
extensions: list[dict[str, Any]]
maturity: str
health_status: str
diagnostics: list[KnowledgeDiagnostic]
revision: str
class KnowledgeSyncRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
idempotency_key: str = Field(min_length=1, max_length=255)
cursor: str | None = Field(default=None, max_length=500)
force_full: bool = False
limit: int = Field(default=100, ge=1, le=500)
class KnowledgeSyncRunItem(BaseModel):
id: str
tenant_id: str
profile_id: str
mode: str
idempotency_key: str
status: str
cursor_before: str | None = None
cursor_after: str | None = None
high_watermark: str | None = None
counts: dict[str, Any]
effects: list[dict[str, Any]]
diagnostics: list[KnowledgeDiagnostic]
provenance: dict[str, Any]
started_at: datetime
finished_at: datetime | None = None
created_at: datetime
class KnowledgeSyncRunListResponse(BaseModel):
items: list[KnowledgeSyncRunItem]
class KnowledgeExternalReferenceResponse(BaseModel):
system: str
object_type: str
object_id: str
maturity: str
authority_mode: str
connector_id: str | None = None
canonical_url: str | None = None
version: str | None = None
etag: str | None = None
observed_at: str | None = None
metadata: dict[str, Any]
class KnowledgeObjectItem(BaseModel):
id: str
profile_id: str
object_type: str
external_id: str
external_page_id: str | None = None
external_revision_id: str | None = None
namespace_id: int | None = None
title: str
canonical_url: str | None = None
status: str
redirect_target_external_id: str | None = None
source_revision: str
visibility: str
acl_tokens: list[str]
mapped_data: dict[str, Any]
external_reference: KnowledgeExternalReferenceResponse
source_updated_at: datetime | None = None
observed_at: datetime
resource_revision: int
class KnowledgeObjectListResponse(BaseModel):
items: list[KnowledgeObjectItem]
next_cursor: str | None = None
class KnowledgeMigrationTargetState(BaseModel):
model_config = ConfigDict(extra="forbid")
path: str = Field(min_length=1, max_length=500)
source_external_id: str | None = Field(default=None, max_length=255)
attachment_names: list[str] = Field(default_factory=list, max_length=500)
class KnowledgeMigrationDryRunRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
idempotency_key: str = Field(min_length=1, max_length=255)
target_space_ref: str = Field(min_length=1, max_length=255)
max_items: int = Field(default=100, ge=1, le=500)
supported_macros: list[str] = Field(default_factory=list, max_length=200)
existing_targets: list[KnowledgeMigrationTargetState] = Field(
default_factory=list, max_length=5_000
)
class KnowledgeMigrationDryRunResponse(BaseModel):
run: KnowledgeSyncRunItem
target_space_ref: str
source_revision: str
source_fingerprint: str
summary: dict[str, int]
effects: list[dict[str, Any]]
diagnostics: list[KnowledgeDiagnostic]
truncated: bool
can_apply: bool
class KnowledgePublishRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
idempotency_key: str = Field(min_length=1, max_length=255)
title: str = Field(min_length=1, max_length=500)
body: str = Field(max_length=200_000)
summary: str = Field(default="", max_length=500)
expected_external_revision: str | None = Field(default=None, max_length=255)
minor: bool = False
class KnowledgePublishResponse(BaseModel):
run: KnowledgeSyncRunItem
external_reference: KnowledgeExternalReferenceResponse
accepted: bool
outcome_unknown: bool = False
def _normalized_tokens(values: list[str]) -> list[str]:
normalized = [str(item).strip() for item in values]
if any(not item or len(item) > 500 for item in normalized):
raise ValueError("ACL tokens must contain 1 to 500 characters")
if len(normalized) != len(set(normalized)):
raise ValueError("ACL tokens must be unique")
return normalized
__all__ = [
"KnowledgeDiagnostic",
"KnowledgeDiscoveryResponse",
"KnowledgeExternalReferenceResponse",
"KnowledgeMigrationDryRunRequest",
"KnowledgeMigrationDryRunResponse",
"KnowledgeMigrationTargetState",
"KnowledgeNamespaceMapping",
"KnowledgeObjectItem",
"KnowledgeObjectListResponse",
"KnowledgeProfileCreateRequest",
"KnowledgeProfileItem",
"KnowledgeProfileListResponse",
"KnowledgeProfileUpdateRequest",
"KnowledgePublishRequest",
"KnowledgePublishResponse",
"KnowledgeSyncRequest",
"KnowledgeSyncRunItem",
"KnowledgeSyncRunListResponse",
]
@@ -0,0 +1,269 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from urllib.parse import quote
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from govoplan_core.core.external_references import ExternalObjectReference
from govoplan_core.core.modules import ModuleContext
from govoplan_core.core.search import (
SearchAuthorizationRequest,
SearchBackfillPage,
SearchBackfillRequest,
SearchDocument,
SearchResourceType,
)
from govoplan_connectors.backend.db.models import (
ConnectorKnowledgeObject,
ConnectorKnowledgeProfile,
)
from govoplan_connectors.backend.knowledge_connector import (
KNOWLEDGE_PROVIDER_ID,
KNOWLEDGE_READ_SCOPE,
KNOWLEDGE_RESOURCE_TYPE,
)
class ExternalKnowledgeSearchSource:
def resource_types(self) -> Sequence[SearchResourceType]:
return (
SearchResourceType(
provider_id=KNOWLEDGE_PROVIDER_ID,
module_id="connectors",
resource_type=KNOWLEDGE_RESOURCE_TYPE,
label="External knowledge pages",
requires_authorization_recheck=True,
),
)
def backfill(
self, session: object, *, request: SearchBackfillRequest
) -> SearchBackfillPage:
_assert_source(request.provider_id, request.resource_type)
db = _session(session)
query = (
select(ConnectorKnowledgeObject, ConnectorKnowledgeProfile)
.join(
ConnectorKnowledgeProfile,
ConnectorKnowledgeProfile.id == ConnectorKnowledgeObject.profile_id,
)
.where(
ConnectorKnowledgeObject.tenant_id == request.tenant_id,
ConnectorKnowledgeObject.object_type == "page",
ConnectorKnowledgeObject.status != "deleted",
ConnectorKnowledgeProfile.tenant_id == request.tenant_id,
ConnectorKnowledgeProfile.status == "active",
)
)
if request.cursor:
query = query.where(ConnectorKnowledgeObject.id > request.cursor)
rows = tuple(
db.execute(
query.order_by(ConnectorKnowledgeObject.id.asc()).limit(
request.limit + 1
)
)
)
has_more = len(rows) > request.limit
selected = rows[: request.limit]
watermark = db.scalar(
select(func.max(ConnectorKnowledgeObject.updated_at))
.join(
ConnectorKnowledgeProfile,
ConnectorKnowledgeProfile.id == ConnectorKnowledgeObject.profile_id,
)
.where(
ConnectorKnowledgeObject.tenant_id == request.tenant_id,
ConnectorKnowledgeObject.object_type == "page",
ConnectorKnowledgeObject.status != "deleted",
ConnectorKnowledgeProfile.tenant_id == request.tenant_id,
ConnectorKnowledgeProfile.status == "active",
)
)
return SearchBackfillPage(
documents=tuple(search_document(db, row, profile) for row, profile in selected),
next_cursor=selected[-1][0].id if has_more and selected else None,
complete=not has_more,
high_watermark=watermark.isoformat() if watermark else None,
)
def authorize(
self,
session: object,
principal: object,
*,
requests: Sequence[SearchAuthorizationRequest],
) -> Mapping[str, bool]:
db = _session(session)
tenant_id = str(getattr(principal, "tenant_id", "") or "")
can_read = _has_scope(principal, KNOWLEDGE_READ_SCOPE)
tokens = set(_principal_acl_tokens(principal))
decisions = {item.reference.key: False for item in requests}
if not tenant_id or not can_read:
return decisions
for request in requests:
reference = request.reference
if (
reference.tenant_id != tenant_id
or reference.module_id != "connectors"
or reference.resource_type != KNOWLEDGE_RESOURCE_TYPE
):
continue
row = db.execute(
select(ConnectorKnowledgeObject, ConnectorKnowledgeProfile)
.join(
ConnectorKnowledgeProfile,
ConnectorKnowledgeProfile.id
== ConnectorKnowledgeObject.profile_id,
)
.where(
ConnectorKnowledgeObject.tenant_id == tenant_id,
ConnectorKnowledgeObject.id == reference.resource_id,
ConnectorKnowledgeObject.object_type == "page",
ConnectorKnowledgeObject.status != "deleted",
ConnectorKnowledgeProfile.tenant_id == tenant_id,
ConnectorKnowledgeProfile.status == "active",
)
).first()
if row is None:
continue
page, _profile = row
decisions[reference.key] = page.visibility == "tenant" or bool(
tokens.intersection(str(value) for value in page.acl_tokens or ())
)
return decisions
def create_external_knowledge_search_source(
_context: ModuleContext,
) -> ExternalKnowledgeSearchSource:
return ExternalKnowledgeSearchSource()
def search_document(
session: Session,
row: ConnectorKnowledgeObject,
profile: ConnectorKnowledgeProfile | None = None,
) -> SearchDocument:
if profile is None:
profile = session.scalar(
select(ConnectorKnowledgeProfile).where(
ConnectorKnowledgeProfile.tenant_id == row.tenant_id,
ConnectorKnowledgeProfile.id == row.profile_id,
)
)
if profile is None:
raise ValueError("External knowledge profile is unavailable.")
data = dict(row.mapped_data or {})
categories = tuple(str(value)[:200] for value in data.get("categories") or ())
links = tuple(
str(value.get("title") or value.get("external_id") or "")[:200]
for value in data.get("links") or ()
if isinstance(value, Mapping)
)
external_reference = ExternalObjectReference(
system=profile.product if profile.product != "unknown" else "mediawiki",
object_type=row.object_type,
object_id=row.external_id,
maturity=profile.discovered_maturity,
authority_mode=profile.source_authority_mode,
connector_id=profile.id,
canonical_url=row.canonical_url,
version=row.source_revision,
etag=row.content_hash,
observed_at=row.observed_at,
metadata={
"title": row.title,
"namespace_id": row.namespace_id,
"external_revision_id": row.external_revision_id,
},
)
return SearchDocument(
tenant_id=row.tenant_id,
module_id="connectors",
provider_id=KNOWLEDGE_PROVIDER_ID,
resource_type=KNOWLEDGE_RESOURCE_TYPE,
resource_id=row.id,
title=row.title,
url=(
"/connectors/knowledge?profileId="
f"{quote(row.profile_id, safe='')}&objectId={quote(row.id, safe='')}"
),
summary=str(data.get("summary") or data.get("body") or "")[:4_000] or None,
body=str(data.get("body") or "")[:200_000] or None,
keywords=tuple(dict.fromkeys((*categories, *links)))[:100],
visibility=row.visibility,
acl_tokens=(
tuple(str(value) for value in row.acl_tokens or ())
if row.visibility == "restricted"
else ()
),
external_reference=external_reference,
metadata={
"profile_id": row.profile_id,
"external_page_id": row.external_page_id,
"namespace_id": row.namespace_id,
"target_space_ref": data.get("target_space_ref"),
"target_path": data.get("target_path"),
"status": row.status,
"redirect_target_external_id": row.redirect_target_external_id,
},
source_revision=row.source_revision,
change_cursor=row.change_cursor,
source_updated_at=row.source_updated_at or row.observed_at,
requires_authorization_recheck=True,
)
def _principal_acl_tokens(principal: object) -> tuple[str, ...]:
values: list[str] = []
for prefix, attribute in (
("account", "account_id"),
("membership", "membership_id"),
("identity", "identity_id"),
):
value = getattr(principal, attribute, None)
if value:
values.append(f"{prefix}:{value}")
for prefix, attribute in (
("group", "group_ids"),
("role", "role_ids"),
("function", "function_assignment_ids"),
("scope", "scopes"),
):
values.extend(
f"{prefix}:{value}"
for value in getattr(principal, attribute, ())
if value
)
return tuple(dict.fromkeys(values))[:500]
def _has_scope(principal: object, required: str) -> bool:
check = getattr(principal, "has", None)
if callable(check):
return bool(check(required))
return required in getattr(principal, "scopes", ())
def _assert_source(provider_id: str, resource_type: str) -> None:
if (
provider_id != KNOWLEDGE_PROVIDER_ID
or resource_type != KNOWLEDGE_RESOURCE_TYPE
):
raise ValueError("Unsupported external knowledge search source.")
def _session(value: object) -> Session:
if not isinstance(value, Session):
raise TypeError("External knowledge Search requires a SQLAlchemy session.")
return value
__all__ = [
"ExternalKnowledgeSearchSource",
"create_external_knowledge_search_source",
"search_document",
]
+260 -9
View File
@@ -41,16 +41,34 @@ from govoplan_core.core.files import CAPABILITY_FILES_TABULAR_CONTENT
from govoplan_core.core.sanctions import (
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
)
from govoplan_core.core.search import SearchSourceProviderRegistration
from govoplan_core.db.base import Base
from govoplan_connectors.backend.db.models import (
ConnectorConfiguration,
ConnectorDefinition,
ConnectorDefinitionRevision,
ConnectorKnowledgeObject,
ConnectorKnowledgeProfile,
ConnectorKnowledgeSyncRun,
ConnectorSanctionsAcquisitionRun,
ConnectorSanctionsSnapshot,
ConnectorSimulationRun,
ConnectorTabularSource,
)
from govoplan_connectors.backend.knowledge_connector import (
KNOWLEDGE_ADMIN_SCOPE,
KNOWLEDGE_CAPABILITY,
KNOWLEDGE_INTERFACE_VERSION,
KNOWLEDGE_MIGRATE_SCOPE,
KNOWLEDGE_PROVIDER_ID,
KNOWLEDGE_PUBLISH_SCOPE,
KNOWLEDGE_READ_SCOPE,
KNOWLEDGE_SYNC_SCOPE,
ExternalKnowledgeCapability,
)
from govoplan_connectors.backend.knowledge_search import (
create_external_knowledge_search_source,
)
from govoplan_connectors.backend.dsar_provider import (
CONNECTORS_DSAR_CAPABILITY,
ConnectorsDsarProvider,
@@ -77,13 +95,14 @@ from govoplan_connectors.backend.feeds import (
from govoplan_connectors.backend.provider_state import (
SANCTIONS_PROVIDER_ID,
TABULAR_PROVIDER_ID,
knowledge_provider_states,
sanctions_provider_states,
tabular_provider_states,
)
MODULE_ID = "connectors"
MODULE_VERSION = "0.1.20"
MODULE_VERSION = "0.1.21"
TABULAR_SOURCE_INTERFACE_VERSION = "0.1.0"
DATASOURCE_ORIGIN_INTERFACE_VERSION = "0.1.0"
SANCTIONS_SNAPSHOT_INTERFACE_VERSION = "1.0.0"
@@ -125,6 +144,11 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
reference="tests/test_governed_runtime.py",
summary="Exercises immutable definition revisions, protected local overrides, idempotent simulations, and explicit ambiguity review.",
),
ModuleMaturityEvidence(
kind="test",
reference="tests/test_mediawiki_connector.py",
summary="Exercises deterministic MediaWiki/BlueSpice discovery, stable mapping, bounded deltas, ACL-safe Search, migration loss diagnostics, and publication recovery states.",
),
ModuleMaturityEvidence(
kind="documentation",
reference="docs/CONNECTOR_SOURCE_LIFECYCLE.md",
@@ -134,11 +158,13 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
known_limits=(
"Tabular origins support immutable snapshots, exact managed CSV/XLSX versions, and read-only PostgreSQL tables; arbitrary REST and other database adapters remain future providers.",
"Feed publication renders a governed document but does not yet push it to an external publishing endpoint.",
"The generic governed runtime simulates deterministic mapping and validation; provider-specific live writes remain owned by explicit connector adapters.",
"The MediaWiki/BlueSpice adapter publishes revision-checked page edits; generic simulations and all other providers do not imply a live write capability.",
"Migration into native Wiki is preview-only; a target-side write worker and Wiki-owned mutation contract remain future work.",
),
supported_authority_modes=(
"external_authoritative",
"external_mirror",
"governed_sync",
"linked_reference",
),
owned_concepts=(
@@ -146,23 +172,38 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
"protocol interaction",
"immutable connector snapshots",
"connector acquisition health",
"external knowledge synchronization evidence",
),
non_owned_concepts=(
"datasource catalogue identity and lifecycle",
"domain records and business semantics",
"data transformations",
"screening dispositions",
"native Wiki spaces, pages, and revision semantics",
),
target_tested_providers=(
TABULAR_PROVIDER_ID,
SANCTIONS_PROVIDER_ID,
KNOWLEDGE_PROVIDER_ID,
),
documentation=ModuleArchitectureDocumentation(
migration=("src/govoplan_connectors/backend/migrations/versions",),
upgrade=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
recovery=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
security=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
operations=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
upgrade=(
"docs/CONNECTOR_SOURCE_LIFECYCLE.md",
"docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md",
),
recovery=(
"docs/CONNECTOR_SOURCE_LIFECYCLE.md",
"docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md",
),
security=(
"docs/CONNECTOR_SOURCE_LIFECYCLE.md",
"docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md",
),
operations=(
"docs/CONNECTOR_SOURCE_LIFECYCLE.md",
"docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md",
),
),
)
@@ -261,6 +302,88 @@ EXTERNAL_PROVIDERS = (
"connectors.sanctions-snapshots",
),
),
ExternalProviderDeclaration(
id=KNOWLEDGE_PROVIDER_ID,
module_id=MODULE_ID,
label="MediaWiki and BlueSpice knowledge provider",
maturity="migrate",
operations=(
"discover",
"link",
"search",
"read",
"publish",
"synchronize",
"migrate",
"preview",
"dry_run",
),
objects=(
ProviderObjectDeclaration(
object_type="external_knowledge_page",
field_groups=(
"stable_identity",
"revisions",
"namespaces",
"categories",
"links",
"files",
"discussions",
"permissions",
"source_provenance",
),
authority_modes=(
"external_authoritative",
"external_mirror",
"governed_sync",
"linked_reference",
),
default_authority_mode="external_mirror",
),
),
behavior=ProviderBehaviorDeclaration(
revision_tokens="Stable MediaWiki page ids, revision ids, discovery revisions, recent-change cursors, content digests, and provider timestamps are retained.",
concurrency="Profile changes use optimistic revisions; publication supplies an expected provider revision and a durable idempotency fence.",
freshness="The latest discovery time, sync high-watermark, recent-change cursor, source update time, and health state are exposed.",
health="Product discovery, authentication, transport, mapping loss, ACL degradation, Search deferral, and outcome-unknown publication are explicit without exposing credentials.",
max_read_items=500,
idempotency="Backfill, delta synchronization, migration previews, and publication require caller keys; exact replays return the committed evidence and mismatched reuse is rejected.",
retry="Read-only discovery and synchronization may be retried with a new key; a publication with an unknown outcome must be reconciled before retry.",
timeout_seconds=30,
conflicts="Namespace and path mappings are explicit; migration previews block target-path and attachment conflicts, while publication rejects stale expected revisions.",
outcome_unknown="A timed-out publication remains outcome-unknown behind a durable recovery fence until the institutional operator verifies the provider revision.",
outcome_unknown_supported=True,
evidence="Mapped snapshots retain stable external references, revision identity, source hashes, transport provenance, ACLs, and bounded loss diagnostics.",
audit_event_types=(
"connectors.knowledge.profile.created",
"connectors.knowledge.profile.updated",
"connectors.knowledge.profile.discovered",
"connectors.knowledge.profile.synchronized",
"connectors.knowledge.migration.previewed",
"connectors.knowledge.page.published",
),
correction="A later provider revision or explicit publication creates a new connector snapshot revision while synchronization-run evidence remains retained.",
rollback="Local snapshot and terminal recovery evidence commit atomically; remote publication cannot be rolled back by a local transaction.",
reconciliation="Rediscover capabilities, compare stable page and revision ids, run a bounded delta or full backfill, and inspect unresolved publication evidence.",
outage="Existing snapshots remain visible only through current tenant and ACL authorization and are marked stale or unavailable; no provider freshness claim is made.",
classifications=("public", "internal", "confidential", "restricted"),
purposes=(
"external knowledge discovery",
"authorized federated search",
"knowledge synchronization",
"migration planning",
"governed publication",
),
retention="The tenant's connector, Records, and target Wiki policies determine snapshot and operation-evidence retention.",
secret_handling="Credentials are resolved from a scoped Core credential envelope, never placed in endpoint URLs, persisted snapshots, diagnostics, or API responses.",
),
capability_names=(KNOWLEDGE_CAPABILITY,),
interface_names=(KNOWLEDGE_CAPABILITY,),
documentation_topic_ids=(
"connectors.authority-and-effects",
"connectors.mediawiki-bluespice",
),
),
)
@@ -314,6 +437,31 @@ PERMISSIONS = (
"Refresh sanctions sources",
"Acquire a new immutable sanctions source snapshot.",
),
_permission(
KNOWLEDGE_READ_SCOPE,
"View external knowledge",
"View authorized MediaWiki and BlueSpice profiles, pages, and synchronization evidence.",
),
_permission(
KNOWLEDGE_ADMIN_SCOPE,
"Administer external knowledge",
"Configure knowledge profiles, namespace mappings, authority, visibility, and discovery.",
),
_permission(
KNOWLEDGE_SYNC_SCOPE,
"Synchronize external knowledge",
"Run bounded MediaWiki and BlueSpice backfills and change synchronization.",
),
_permission(
KNOWLEDGE_PUBLISH_SCOPE,
"Publish external knowledge",
"Publish a governed page revision with concurrency and recovery evidence.",
),
_permission(
KNOWLEDGE_MIGRATE_SCOPE,
"Preview knowledge migration",
"Dry-run a bounded migration into Wiki and inspect loss or conflict diagnostics.",
),
)
ROLE_TEMPLATES = (
@@ -329,6 +477,11 @@ ROLE_TEMPLATES = (
FEED_PRIVATE_PUBLISH_SCOPE,
SANCTIONS_READ_SCOPE,
SANCTIONS_REFRESH_SCOPE,
KNOWLEDGE_READ_SCOPE,
KNOWLEDGE_ADMIN_SCOPE,
KNOWLEDGE_SYNC_SCOPE,
KNOWLEDGE_PUBLISH_SCOPE,
KNOWLEDGE_MIGRATE_SCOPE,
),
),
RoleTemplate(
@@ -341,13 +494,16 @@ ROLE_TEMPLATES = (
FEED_PUBLISH_SCOPE,
SANCTIONS_READ_SCOPE,
SANCTIONS_REFRESH_SCOPE,
KNOWLEDGE_READ_SCOPE,
KNOWLEDGE_SYNC_SCOPE,
KNOWLEDGE_MIGRATE_SCOPE,
),
),
RoleTemplate(
slug="connector_source_reader",
name="Connector source reader",
description="Discover and preview tabular connector sources.",
permissions=(READ_SCOPE, SANCTIONS_READ_SCOPE),
permissions=(READ_SCOPE, SANCTIONS_READ_SCOPE, KNOWLEDGE_READ_SCOPE),
),
)
@@ -382,6 +538,10 @@ def _dsar_provider(_context) -> ConnectorsDsarProvider:
return ConnectorsDsarProvider()
def _knowledge_provider(_context) -> ExternalKnowledgeCapability:
return ExternalKnowledgeCapability()
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
return {
"connector_definitions": (
@@ -417,6 +577,24 @@ def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
.filter(ConnectorSanctionsAcquisitionRun.tenant_id == tenant_id)
.count()
),
"connector_knowledge_profiles": (
session.query(ConnectorKnowledgeProfile)
.filter(ConnectorKnowledgeProfile.tenant_id == tenant_id)
.count()
),
"connector_knowledge_objects": (
session.query(ConnectorKnowledgeObject)
.filter(
ConnectorKnowledgeObject.tenant_id == tenant_id,
ConnectorKnowledgeObject.status != "deleted",
)
.count()
),
"connector_knowledge_runs": (
session.query(ConnectorKnowledgeSyncRun)
.filter(ConnectorKnowledgeSyncRun.tenant_id == tenant_id)
.count()
),
}
@@ -433,6 +611,8 @@ manifest = ModuleManifest(
"portal",
"reporting",
"risk_compliance",
"search",
"wiki",
),
required_capabilities=(
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
@@ -463,6 +643,10 @@ manifest = ModuleManifest(
name="connectors.runtime_contract",
version=CONNECTOR_RUNTIME_INTERFACE_VERSION,
),
ModuleInterfaceProvider(
name=KNOWLEDGE_CAPABILITY,
version=KNOWLEDGE_INTERFACE_VERSION,
),
ModuleInterfaceProvider(name=CONNECTORS_DSAR_CAPABILITY, version="0.1.0"),
),
requires_interfaces=(
@@ -495,6 +679,14 @@ manifest = ModuleManifest(
parent_id="connectors.admin.governed-configurations",
order=20,
),
ViewSurface(
id="connectors.admin.external-knowledge",
module_id=MODULE_ID,
kind="section",
label="External knowledge",
parent_id="connectors.admin.governed-configurations",
order=30,
),
),
),
capability_factories={
@@ -503,6 +695,7 @@ manifest = ModuleManifest(
CAPABILITY_DATASOURCE_ORIGINS: _datasource_origin_provider,
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS: (_sanctions_snapshot_provider),
CAPABILITY_CONNECTORS_FEEDS: _feed_provider,
KNOWLEDGE_CAPABILITY: _knowledge_provider,
CONNECTORS_DSAR_CAPABILITY: _dsar_provider,
},
capability_documentation={
@@ -516,6 +709,13 @@ manifest = ModuleManifest(
),
},
tenant_summary_providers=(_tenant_summary,),
search_sources=(
SearchSourceProviderRegistration(
id=KNOWLEDGE_PROVIDER_ID,
factory=create_external_knowledge_search_source,
order=65,
),
),
architecture=ARCHITECTURE,
external_providers=EXTERNAL_PROVIDERS,
external_provider_state_providers=(
@@ -529,6 +729,11 @@ manifest = ModuleManifest(
provider_id=SANCTIONS_PROVIDER_ID,
provider=sanctions_provider_states,
),
ExternalProviderStateProviderRegistration(
module_id=MODULE_ID,
provider_id=KNOWLEDGE_PROVIDER_ID,
provider=knowledge_provider_states,
),
),
migration_spec=MigrationSpec(
module_id=MODULE_ID,
@@ -536,6 +741,9 @@ manifest = ModuleManifest(
script_location=str(Path(__file__).with_name("migrations") / "versions"),
retirement_supported=True,
retirement_provider=drop_table_retirement_provider(
ConnectorKnowledgeSyncRun,
ConnectorKnowledgeObject,
ConnectorKnowledgeProfile,
ConnectorSimulationRun,
ConnectorConfiguration,
ConnectorDefinitionRevision,
@@ -552,6 +760,9 @@ manifest = ModuleManifest(
),
uninstall_guard_providers=(
persistent_table_uninstall_guard(
ConnectorKnowledgeSyncRun,
ConnectorKnowledgeObject,
ConnectorKnowledgeProfile,
ConnectorSimulationRun,
ConnectorConfiguration,
ConnectorDefinitionRevision,
@@ -573,8 +784,9 @@ manifest = ModuleManifest(
body=(
"Connectors correlates only an exact tenant account identifier and can "
"narrow an already verified search to one source, acquisition, "
"definition, configuration, or simulation. The export identifies the "
"subject's configuration, acquisition, simulation, and review activity "
"definition, configuration, simulation, external-knowledge profile, or "
"knowledge operation. The export identifies the subject's configuration, "
"acquisition, simulation, knowledge-operation, and review activity "
"using bounded lifecycle metadata. It never includes credential or "
"endpoint references, source rows, external responses, request payloads, "
"mapping and configuration documents, diagnostics, provenance, hashes, "
@@ -709,6 +921,45 @@ manifest = ModuleManifest(
related_modules=("datasources", "dataflow", "portal", "reporting"),
order=42,
),
DocumentationTopic(
id="connectors.mediawiki-bluespice",
title="Connect MediaWiki and BlueSpice knowledge",
summary="Discover, synchronize, search, publish, and preview migration of external knowledge without weakening source permissions.",
body=(
"A connector administrator first creates a governed MediaWiki Action API configuration whose endpoint passes Core egress and peer validation; credentials remain in a scoped credential envelope. A knowledge profile then maps each included source namespace to a target Wiki space and path prefix, selects source authority, and sets a tenant or restricted fallback ACL. Discovery records product, version, extensions, namespaces, authentication state, capabilities, and loss diagnostics. Run a bounded full backfill once, then cursor-based deltas for revisions, moves, deletions, and permission changes. Stable page and revision identifiers, canonical links, redirects, files, categories, links, discussions, provenance, and current ACLs remain attached to the connector snapshot. Search indexes only active, non-deleted pages and rechecks the current profile, tenant, permission, and ACL before returning every result; disabling Connectors removes its Search projection. Publication requires an expected external revision, an idempotency key, and durable outcome evidence. An unknown provider outcome must be reconciled before retry. Migration into native Wiki is deliberately preview-only in this slice: the dry-run reports target-path collisions, attachment-name conflicts, unsupported macros, truncation, and source fingerprints; a successful preview does not write native Wiki pages. Existing authorized snapshots can remain visible during a provider outage, but health and freshness stay explicit and no current-source claim is made."
),
layer="configured",
documentation_types=("admin", "user"),
audience=(
"operator",
"module_admin",
"integration_admin",
"knowledge_manager",
"auditor",
),
related_modules=("core", "search", "wiki", "files", "audit", "policy"),
order=43,
translations={
"de": {
"title": "Wissen aus MediaWiki und BlueSpice anbinden",
"summary": "Externes Wissen erkennen, synchronisieren, durchsuchen, veröffentlichen und eine Migration prüfen, ohne Quellberechtigungen abzuschwächen.",
"body": (
"Eine Connector-Administration erstellt zuerst eine gesteuerte Konfiguration für die MediaWiki Action API. Der Endpunkt unterliegt der zentralen Ausgangs- und Gegenstellenprüfung; Zugangsdaten bleiben in einem zweckgebundenen Zugangsdaten-Umschlag. Das Wissensprofil ordnet jeden eingeschlossenen Quellnamensraum einem Zielbereich und Pfadpräfix in Wiki zu und legt Quellhoheit sowie eine mandantenweite oder eingeschränkte Ersatz-ACL fest. Die Erkennung dokumentiert Produkt, Version, Erweiterungen, Namensräume, Authentifizierungsstatus, Fähigkeiten und Verlustdiagnosen. Nach einem begrenzten Vollabgleich folgen cursorbasierte Änderungen für Revisionen, Verschiebungen, Löschungen und Berechtigungen. Stabile Seiten- und Revisionskennungen, kanonische Links, Weiterleitungen, Dateien, Kategorien, Links, Diskussionen, Herkunft und aktuelle ACLs bleiben am Snapshot. Search indiziert nur aktive, nicht gelöschte Seiten und prüft bei jedem Treffer Profilstatus, Mandant, Berechtigung und ACL erneut; bei Deaktivierung von Connectors wird dessen Suchprojektion entfernt. Veröffentlichungen erfordern die erwartete externe Revision, einen Idempotenzschlüssel und dauerhafte Ergebnisevidenz. Ein unbekanntes Ergebnis muss vor einem erneuten Versuch abgeglichen werden. Die Migration in das native Wiki ist in diesem Ausbauschritt ausschließlich eine Vorschau: Sie meldet Pfad- und Anhangskonflikte, nicht unterstützte Makros, Begrenzungen und Quellfingerabdrücke, schreibt aber keine Wiki-Seiten. Bei einem Ausfall dürfen bestehende Snapshots nur für weiterhin Berechtigte sichtbar bleiben; Zustand und Aktualität bleiben ausdrücklich erkennbar."
),
}
},
metadata={
"kind": "guide",
"help_contexts": ["connectors.admin.external-knowledge"],
"prerequisites": [
"An active governed MediaWiki Action API configuration exists.",
"Namespace targets and fallback ACLs have been reviewed.",
"The Search and Wiki modules are optional and remain capability-separated.",
],
"outcome": "External knowledge remains identity-stable, loss-visible, ACL-safe, and migration-ready.",
"verification": "Rediscover the profile, run a keyed delta, inspect health and diagnostics, verify an allowed and denied Search principal, and run a migration dry-run before any target-side work.",
},
),
DocumentationTopic(
id="connectors.sanctions-snapshots",
title="Sanctions source snapshots",
@@ -0,0 +1,503 @@
from __future__ import annotations
import base64
import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any, Protocol
from urllib.parse import quote, urlencode, urljoin, urlsplit, urlunsplit
from govoplan_core.security.http_fetch import fetch_http
MAX_MEDIAWIKI_RESPONSE_BYTES = 10_000_000
class MediaWikiTransportError(RuntimeError):
def __init__(
self,
code: str,
message: str,
*,
retryable: bool = False,
outcome_unknown: bool = False,
) -> None:
super().__init__(message)
self.code = code
self.retryable = retryable
self.outcome_unknown = outcome_unknown
@dataclass(frozen=True, slots=True)
class MediaWikiChangeBatch:
changes: tuple[Mapping[str, Any], ...]
next_cursor: str | None
complete: bool
high_watermark: str | None
evidence: Mapping[str, Any]
@dataclass(frozen=True, slots=True)
class MediaWikiPublishResult:
page_id: str
revision_id: str
title: str
canonical_url: str | None
evidence: Mapping[str, Any]
class MediaWikiTransport(Protocol):
def discover(
self,
*,
endpoint_url: str,
credential: Mapping[str, Any] | None,
) -> Mapping[str, Any]: ...
def changes(
self,
*,
endpoint_url: str,
credential: Mapping[str, Any] | None,
cursor: str | None,
limit: int,
force_full: bool,
) -> MediaWikiChangeBatch: ...
def publish(
self,
*,
endpoint_url: str,
credential: Mapping[str, Any] | None,
title: str,
body: str,
summary: str,
expected_revision: str | None,
minor: bool,
) -> MediaWikiPublishResult: ...
class HttpMediaWikiTransport:
"""Bounded MediaWiki Action API transport with Core outbound policy enforcement."""
def discover(
self,
*,
endpoint_url: str,
credential: Mapping[str, Any] | None,
) -> Mapping[str, Any]:
return self._request(
endpoint_url,
credential=credential,
params={
"action": "query",
"meta": "siteinfo|userinfo",
"siprop": "general|extensions|namespaces|namespacealiases|rightsinfo",
"uiprop": "rights|groups",
"format": "json",
"formatversion": "2",
"curtimestamp": "1",
},
)
def changes(
self,
*,
endpoint_url: str,
credential: Mapping[str, Any] | None,
cursor: str | None,
limit: int,
force_full: bool,
) -> MediaWikiChangeBatch:
if force_full:
listing = self._request(
endpoint_url,
credential=credential,
params={
"action": "query",
"list": "allpages",
"aplimit": str(limit),
"apcontinue": cursor or "",
"apfilterredir": "all",
"format": "json",
"formatversion": "2",
"curtimestamp": "1",
},
)
rows = _list(listing, "query", "allpages")
page_ids = tuple(
str(item.get("pageid"))
for item in rows
if isinstance(item, Mapping) and item.get("pageid") is not None
)
changes = self._page_details(
endpoint_url,
credential=credential,
page_ids=page_ids,
)
next_cursor = _continue_token(listing, "apcontinue")
return MediaWikiChangeBatch(
changes=changes,
next_cursor=next_cursor,
complete=next_cursor is None,
high_watermark=_optional_text(listing.get("curtimestamp")),
evidence={
"mode": "backfill",
"listed": len(rows),
"resolved": len(changes),
},
)
listing = self._request(
endpoint_url,
credential=credential,
params={
"action": "query",
"list": "recentchanges",
"rclimit": str(limit),
"rccontinue": cursor or "",
"rcdir": "newer",
"rcprop": "title|ids|sizes|flags|user|timestamp|loginfo|tags",
"rctype": "edit|new|log",
"format": "json",
"formatversion": "2",
"curtimestamp": "1",
},
)
recent = tuple(
item
for item in _list(listing, "query", "recentchanges")
if isinstance(item, Mapping)
)
page_ids = tuple(
dict.fromkeys(
str(item.get("pageid"))
for item in recent
if item.get("pageid") is not None
and str(item.get("logtype") or "") != "delete"
)
)
resolved = {
str(item.get("pageid")): item
for item in self._page_details(
endpoint_url,
credential=credential,
page_ids=page_ids,
)
}
changes: list[Mapping[str, Any]] = []
for item in recent:
page_id = _optional_text(item.get("pageid"))
if str(item.get("logtype") or "") == "delete":
changes.append(
{
"change_kind": "delete",
"pageid": page_id or f"log:{item.get('logid')}",
"title": _optional_text(item.get("title")) or "Deleted page",
"ns": item.get("ns"),
"timestamp": item.get("timestamp"),
"logid": item.get("logid"),
"change_cursor": _change_cursor(item),
}
)
continue
page = dict(resolved.get(page_id or "") or item)
page["change_kind"] = "upsert"
page["change_cursor"] = _change_cursor(item)
page["recent_change"] = _safe_recent_change(item)
changes.append(page)
next_cursor = _continue_token(listing, "rccontinue")
return MediaWikiChangeBatch(
changes=tuple(changes),
next_cursor=next_cursor,
complete=next_cursor is None,
high_watermark=_optional_text(listing.get("curtimestamp")),
evidence={
"mode": "delta",
"listed": len(recent),
"resolved": len(resolved),
},
)
def publish(
self,
*,
endpoint_url: str,
credential: Mapping[str, Any] | None,
title: str,
body: str,
summary: str,
expected_revision: str | None,
minor: bool,
) -> MediaWikiPublishResult:
if not credential:
raise MediaWikiTransportError(
"credential_required",
"Publishing requires a governed credential envelope.",
)
csrf_token = _optional_text(credential.get("csrf_token"))
if not csrf_token:
token_payload = self._request(
endpoint_url,
credential=credential,
params={
"action": "query",
"meta": "tokens",
"type": "csrf",
"format": "json",
"formatversion": "2",
},
)
query = token_payload.get("query")
tokens = query.get("tokens") if isinstance(query, Mapping) else None
csrf_token = (
_optional_text(tokens.get("csrftoken"))
if isinstance(tokens, Mapping)
else None
)
if not csrf_token:
raise MediaWikiTransportError(
"csrf_token_unavailable",
"The provider did not issue a CSRF token for the configured credential.",
)
parameters: dict[str, str] = {
"action": "edit",
"title": title,
"text": body,
"summary": summary,
"token": csrf_token,
"format": "json",
"formatversion": "2",
}
if minor:
parameters["minor"] = "1"
if expected_revision:
parameters["baserevid"] = expected_revision
payload = self._request(
endpoint_url,
credential=credential,
params=parameters,
method="POST",
)
edit = payload.get("edit")
if not isinstance(edit, Mapping) or str(edit.get("result")) != "Success":
raise MediaWikiTransportError(
"publish_rejected",
"MediaWiki rejected the page publication.",
)
page_id = _required_text(edit.get("pageid"), "MediaWiki omitted the page id")
revision_id = _required_text(
edit.get("newrevid"), "MediaWiki omitted the accepted revision id"
)
canonical_url = _canonical_page_url(endpoint_url, title)
return MediaWikiPublishResult(
page_id=page_id,
revision_id=revision_id,
title=_optional_text(edit.get("title")) or title,
canonical_url=canonical_url,
evidence={
"result": "Success",
"page_id": page_id,
"revision_id": revision_id,
"old_revision_id": _optional_text(edit.get("oldrevid")),
"new_page": bool(edit.get("new")),
},
)
def _page_details(
self,
endpoint_url: str,
*,
credential: Mapping[str, Any] | None,
page_ids: Sequence[str],
) -> tuple[Mapping[str, Any], ...]:
if not page_ids:
return ()
payload = self._request(
endpoint_url,
credential=credential,
params={
"action": "query",
"pageids": "|".join(page_ids),
"prop": "info|revisions|categories|links|images|pageprops",
"inprop": "url|displaytitle",
"rvlimit": "1",
"rvprop": "ids|timestamp|user|comment|content|contentmodel|sha1|flags",
"rvslots": "main",
"cllimit": "max",
"pllimit": "max",
"imlimit": "max",
"format": "json",
"formatversion": "2",
},
)
return tuple(
item
for item in _list(payload, "query", "pages")
if isinstance(item, Mapping)
)
def _request(
self,
endpoint_url: str,
*,
credential: Mapping[str, Any] | None,
params: Mapping[str, str],
method: str = "GET",
) -> Mapping[str, Any]:
api_url = _api_url(endpoint_url)
headers = {
"Accept": "application/json",
"User-Agent": "GovOPlaN-Connectors/MediaWiki",
**_auth_headers(credential),
}
body = None
target_url = api_url
if method == "POST":
body = urlencode(params).encode("utf-8")
headers["Content-Type"] = "application/x-www-form-urlencoded"
else:
target_url = f"{api_url}?{urlencode({key: value for key, value in params.items() if value != ''})}"
try:
response = fetch_http(
target_url,
method=method,
headers=headers,
body=body,
max_bytes=MAX_MEDIAWIKI_RESPONSE_BYTES,
timeout=30,
label="MediaWiki Action API URL",
)
except Exception as exc:
raise MediaWikiTransportError(
"transport_unavailable",
"MediaWiki transport failed before a valid response was received.",
retryable=True,
outcome_unknown=method == "POST",
) from exc
if response.status < 200 or response.status >= 300:
raise MediaWikiTransportError(
"http_error",
f"MediaWiki returned HTTP {response.status}.",
retryable=response.status >= 500,
outcome_unknown=method == "POST" and response.status >= 500,
)
try:
payload = json.loads(response.body)
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise MediaWikiTransportError(
"invalid_json",
"MediaWiki returned an invalid JSON response.",
outcome_unknown=method == "POST",
) from exc
if not isinstance(payload, Mapping):
raise MediaWikiTransportError(
"invalid_response",
"MediaWiki returned an unsupported response shape.",
outcome_unknown=method == "POST",
)
error = payload.get("error")
if isinstance(error, Mapping):
code = _optional_text(error.get("code")) or "provider_error"
raise MediaWikiTransportError(
code,
_optional_text(error.get("info")) or "MediaWiki rejected the request.",
retryable=code in {"maxlag", "readonly", "ratelimited"},
)
return payload
def _api_url(endpoint_url: str) -> str:
normalized = endpoint_url.strip().rstrip("/")
parsed = urlsplit(normalized)
if parsed.path.endswith("/api.php"):
return normalized
path = f"{parsed.path.rstrip('/')}/api.php"
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
def _canonical_page_url(endpoint_url: str, title: str) -> str:
normalized = endpoint_url.strip().rstrip("/")
parsed = urlsplit(normalized)
root_path = parsed.path
if root_path.endswith("/api.php"):
root_path = root_path[: -len("/api.php")]
base = urlunsplit((parsed.scheme, parsed.netloc, f"{root_path.rstrip('/')}/", "", ""))
return urljoin(base, f"wiki/{quote(title.replace(' ', '_'), safe=':_-./~')}")
def _auth_headers(credential: Mapping[str, Any] | None) -> dict[str, str]:
if not credential:
return {}
token = _optional_text(
credential.get("bearer_token")
or credential.get("access_token")
or credential.get("token")
)
if token:
return {"Authorization": f"Bearer {token}"}
username = _optional_text(credential.get("username") or credential.get("user"))
password = _optional_text(credential.get("password"))
if username and password:
encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
return {"Authorization": f"Basic {encoded}"}
return {}
def _list(payload: Mapping[str, Any], *path: str) -> list[Any]:
value: Any = payload
for part in path:
if not isinstance(value, Mapping):
return []
value = value.get(part)
return list(value) if isinstance(value, list) else []
def _continue_token(payload: Mapping[str, Any], name: str) -> str | None:
value = payload.get("continue")
return _optional_text(value.get(name)) if isinstance(value, Mapping) else None
def _change_cursor(change: Mapping[str, Any]) -> str:
for name in ("rcid", "logid", "revid", "old_revid"):
if change.get(name) is not None:
return f"{name}:{change[name]}"
return _optional_text(change.get("timestamp")) or "unknown"
def _safe_recent_change(change: Mapping[str, Any]) -> dict[str, Any]:
allowed = (
"type",
"ns",
"title",
"pageid",
"revid",
"old_revid",
"timestamp",
"logtype",
"logaction",
"tags",
)
return {key: change[key] for key in allowed if key in change}
def _required_text(value: object, message: str) -> str:
normalized = _optional_text(value)
if not normalized:
raise MediaWikiTransportError("invalid_response", message)
return normalized
def _optional_text(value: object) -> str | None:
normalized = str(value or "").strip()
return normalized or None
__all__ = [
"HttpMediaWikiTransport",
"MAX_MEDIAWIKI_RESPONSE_BYTES",
"MediaWikiChangeBatch",
"MediaWikiPublishResult",
"MediaWikiTransport",
"MediaWikiTransportError",
]
@@ -0,0 +1,207 @@
"""MediaWiki and BlueSpice knowledge connector state
Revision ID: b9e0f1a2c3d4
Revises: a8d9e0f1b2c3
Create Date: 2026-08-22 14:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "b9e0f1a2c3d4"
down_revision = "a8d9e0f1b2c3"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"connector_knowledge_profiles",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("configuration_id", sa.String(length=36), nullable=False),
sa.Column("status", sa.String(length=30), nullable=False),
sa.Column("product", sa.String(length=50), nullable=False),
sa.Column("product_version", sa.String(length=100), nullable=True),
sa.Column("desired_maturity", sa.String(length=30), nullable=False),
sa.Column("discovered_maturity", sa.String(length=30), nullable=False),
sa.Column("source_authority_mode", sa.String(length=40), nullable=False),
sa.Column("default_visibility", sa.String(length=30), nullable=False),
sa.Column("default_acl_tokens", sa.JSON(), nullable=False),
sa.Column("namespace_mappings", sa.JSON(), nullable=False),
sa.Column("capabilities", sa.JSON(), nullable=False),
sa.Column("discovery_revision", sa.String(length=255), nullable=True),
sa.Column("discovery_evidence", sa.JSON(), nullable=False),
sa.Column("health_status", sa.String(length=30), nullable=False),
sa.Column("health_details", sa.JSON(), nullable=False),
sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_sync_cursor", sa.String(length=500), nullable=True),
sa.Column("last_high_watermark", sa.String(length=500), nullable=True),
sa.Column("resource_revision", sa.Integer(), nullable=False),
sa.Column("updated_by", sa.String(length=255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["configuration_id"],
["connector_configurations.id"],
name=op.f(
"fk_connector_knowledge_profiles_configuration_id_connector_configurations"
),
ondelete="RESTRICT",
),
sa.PrimaryKeyConstraint(
"id", name=op.f("pk_connector_knowledge_profiles")
),
sa.UniqueConstraint(
"tenant_id",
"configuration_id",
name="uq_connector_knowledge_profile_configuration",
),
)
for name, columns in (
("ix_connector_knowledge_profiles_tenant_id", ["tenant_id"]),
("ix_connector_knowledge_profiles_configuration_id", ["configuration_id"]),
("ix_connector_knowledge_profiles_status", ["status"]),
("ix_connector_knowledge_profiles_product", ["product"]),
("ix_connector_knowledge_profiles_discovery_revision", ["discovery_revision"]),
("ix_connector_knowledge_profiles_health_status", ["health_status"]),
("ix_connector_knowledge_profiles_updated_by", ["updated_by"]),
):
op.create_index(op.f(name), "connector_knowledge_profiles", columns)
op.create_index(
"ix_connector_knowledge_profiles_tenant_status",
"connector_knowledge_profiles",
["tenant_id", "status"],
)
op.create_table(
"connector_knowledge_objects",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("object_type", sa.String(length=40), nullable=False),
sa.Column("external_id", sa.String(length=255), nullable=False),
sa.Column("external_page_id", sa.String(length=255), nullable=True),
sa.Column("external_revision_id", sa.String(length=255), nullable=True),
sa.Column("namespace_id", sa.Integer(), nullable=True),
sa.Column("title", sa.String(length=500), nullable=False),
sa.Column("canonical_url", sa.String(length=1500), nullable=True),
sa.Column("status", sa.String(length=30), nullable=False),
sa.Column("redirect_target_external_id", sa.String(length=255), nullable=True),
sa.Column("source_revision", sa.String(length=255), nullable=False),
sa.Column("content_hash", sa.String(length=64), nullable=False),
sa.Column("visibility", sa.String(length=30), nullable=False),
sa.Column("acl_tokens", sa.JSON(), nullable=False),
sa.Column("mapped_data", sa.JSON(), nullable=False),
sa.Column("provenance", sa.JSON(), nullable=False),
sa.Column("change_cursor", sa.String(length=500), nullable=True),
sa.Column("source_updated_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("resource_revision", sa.Integer(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["profile_id"],
["connector_knowledge_profiles.id"],
name=op.f(
"fk_connector_knowledge_objects_profile_id_connector_knowledge_profiles"
),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint(
"id", name=op.f("pk_connector_knowledge_objects")
),
sa.UniqueConstraint(
"profile_id",
"object_type",
"external_id",
name="uq_connector_knowledge_object_identity",
),
)
for name, columns in (
("ix_connector_knowledge_objects_tenant_id", ["tenant_id"]),
("ix_connector_knowledge_objects_profile_id", ["profile_id"]),
("ix_connector_knowledge_objects_object_type", ["object_type"]),
("ix_connector_knowledge_objects_external_page_id", ["external_page_id"]),
("ix_connector_knowledge_objects_external_revision_id", ["external_revision_id"]),
("ix_connector_knowledge_objects_namespace_id", ["namespace_id"]),
("ix_connector_knowledge_objects_status", ["status"]),
("ix_connector_knowledge_objects_content_hash", ["content_hash"]),
("ix_connector_knowledge_objects_change_cursor", ["change_cursor"]),
("ix_connector_knowledge_objects_source_updated_at", ["source_updated_at"]),
("ix_connector_knowledge_objects_observed_at", ["observed_at"]),
):
op.create_index(op.f(name), "connector_knowledge_objects", columns)
op.create_index(
"ix_connector_knowledge_objects_tenant_profile_status",
"connector_knowledge_objects",
["tenant_id", "profile_id", "status"],
)
op.create_index(
"ix_connector_knowledge_objects_tenant_updated",
"connector_knowledge_objects",
["tenant_id", "source_updated_at"],
)
op.create_table(
"connector_knowledge_sync_runs",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("mode", sa.String(length=40), nullable=False),
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
sa.Column("request_hash", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=30), nullable=False),
sa.Column("cursor_before", sa.String(length=500), nullable=True),
sa.Column("cursor_after", sa.String(length=500), nullable=True),
sa.Column("high_watermark", sa.String(length=500), nullable=True),
sa.Column("counts", sa.JSON(), nullable=False),
sa.Column("effects", sa.JSON(), nullable=False),
sa.Column("diagnostics", sa.JSON(), nullable=False),
sa.Column("provenance", sa.JSON(), nullable=False),
sa.Column("created_by", sa.String(length=255), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["profile_id"],
["connector_knowledge_profiles.id"],
name=op.f(
"fk_connector_knowledge_sync_runs_profile_id_connector_knowledge_profiles"
),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint(
"id", name=op.f("pk_connector_knowledge_sync_runs")
),
sa.UniqueConstraint(
"tenant_id",
"profile_id",
"mode",
"idempotency_key",
name="uq_connector_knowledge_sync_run_idempotency",
),
)
for name, columns in (
("ix_connector_knowledge_sync_runs_tenant_id", ["tenant_id"]),
("ix_connector_knowledge_sync_runs_profile_id", ["profile_id"]),
("ix_connector_knowledge_sync_runs_mode", ["mode"]),
("ix_connector_knowledge_sync_runs_status", ["status"]),
("ix_connector_knowledge_sync_runs_created_by", ["created_by"]),
("ix_connector_knowledge_sync_runs_started_at", ["started_at"]),
):
op.create_index(op.f(name), "connector_knowledge_sync_runs", columns)
op.create_index(
"ix_connector_knowledge_sync_runs_profile_started",
"connector_knowledge_sync_runs",
["tenant_id", "profile_id", "started_at"],
)
def downgrade() -> None:
op.drop_table("connector_knowledge_sync_runs")
op.drop_table("connector_knowledge_objects")
op.drop_table("connector_knowledge_profiles")
@@ -7,6 +7,9 @@ from sqlalchemy import func, select
from sqlalchemy.orm import Session
from govoplan_connectors.backend.db.models import (
ConnectorKnowledgeObject,
ConnectorKnowledgeProfile,
ConnectorKnowledgeSyncRun,
ConnectorSanctionsAcquisitionRun,
ConnectorSanctionsSnapshot,
ConnectorTabularSource,
@@ -19,6 +22,7 @@ from govoplan_core.core.provider_governance import (
TABULAR_PROVIDER_ID = "connectors.tabular_snapshot"
SANCTIONS_PROVIDER_ID = "connectors.sanctions_snapshot"
KNOWLEDGE_PROVIDER_ID = "connectors.mediawiki.pages"
def tabular_provider_states(
@@ -85,6 +89,37 @@ def sanctions_provider_states(
)
def knowledge_provider_states(
context: ExternalProviderStateContext,
) -> tuple[ExternalProviderRuntimeState, ...]:
session = _session(context)
statement = select(ConnectorKnowledgeProfile)
if context.tenant_id is not None:
statement = statement.where(
ConnectorKnowledgeProfile.tenant_id == context.tenant_id
)
profiles = tuple(
session.scalars(
statement.order_by(
ConnectorKnowledgeProfile.tenant_id,
ConnectorKnowledgeProfile.id,
).limit(context.max_items + 1)
)
)[: context.max_items]
counts = _knowledge_counts(session, profiles)
latest_runs = _latest_knowledge_runs(session, profiles)
observed_at = datetime.now(UTC)
return tuple(
_knowledge_state(
profile,
observed_at=observed_at,
object_count=counts.get(profile.id, 0),
latest_run=latest_runs.get(profile.id),
)
for profile in profiles
)
def _session(context: ExternalProviderStateContext) -> Session:
if not isinstance(context.session, Session):
raise RuntimeError("Connectors provider state requires a database session.")
@@ -161,6 +196,119 @@ def _snapshot_counts(
}
def _knowledge_counts(
session: Session,
profiles: tuple[ConnectorKnowledgeProfile, ...],
) -> dict[str, int]:
profile_ids = tuple(item.id for item in profiles)
if not profile_ids:
return {}
return {
str(profile_id): int(count)
for profile_id, count in session.execute(
select(
ConnectorKnowledgeObject.profile_id,
func.count(ConnectorKnowledgeObject.id),
)
.where(
ConnectorKnowledgeObject.profile_id.in_(profile_ids),
ConnectorKnowledgeObject.status != "deleted",
)
.group_by(ConnectorKnowledgeObject.profile_id)
)
}
def _latest_knowledge_runs(
session: Session,
profiles: tuple[ConnectorKnowledgeProfile, ...],
) -> dict[str, ConnectorKnowledgeSyncRun]:
profile_ids = tuple(item.id for item in profiles)
if not profile_ids:
return {}
rows = tuple(
session.scalars(
select(ConnectorKnowledgeSyncRun)
.where(
ConnectorKnowledgeSyncRun.profile_id.in_(profile_ids),
ConnectorKnowledgeSyncRun.mode.in_(("backfill", "delta")),
)
.order_by(
ConnectorKnowledgeSyncRun.profile_id,
ConnectorKnowledgeSyncRun.started_at.desc(),
ConnectorKnowledgeSyncRun.id.desc(),
)
)
)
latest: dict[str, ConnectorKnowledgeSyncRun] = {}
for row in rows:
latest.setdefault(row.profile_id, row)
return latest
def _knowledge_state(
profile: ConnectorKnowledgeProfile,
*,
observed_at: datetime,
object_count: int,
latest_run: ConnectorKnowledgeSyncRun | None,
) -> ExternalProviderRuntimeState:
active = profile.status == "active"
health = (
"inactive"
if not active
else "healthy"
if profile.health_status == "healthy"
else "warning"
if profile.health_status in {"unknown", "degraded"}
else "error"
)
last_success = (
latest_run.finished_at
if latest_run is not None and latest_run.status == "completed"
else profile.discovered_at
)
return ExternalProviderRuntimeState(
provider_id=KNOWLEDGE_PROVIDER_ID,
binding_ref=f"connectors:knowledge-profile:{profile.id}",
authority_mode=profile.source_authority_mode,
observed_at=observed_at,
configured=True,
active=active,
health=health,
freshness="unknown" if active else "not_applicable",
conflict=(
"pending"
if latest_run is not None and latest_run.status == "outcome_unknown"
else "not_applicable"
),
recovery=(
"attention"
if latest_run is not None
and latest_run.status in {"failed", "outcome_unknown"}
else "ready"
if active
else "not_applicable"
),
last_success_at=_aware(last_success),
detail=(
f"{profile.product} knowledge profile is synchronized and ACL-rechecked."
if active and profile.health_status == "healthy"
else "Knowledge profile requires discovery, synchronization, or health review."
if active
else "Knowledge profile is paused."
),
metrics={
"product": profile.product,
"product_version": profile.product_version,
"desired_maturity": profile.desired_maturity,
"discovered_maturity": profile.discovered_maturity,
"active_objects": int(object_count),
"last_run_status": latest_run.status if latest_run is not None else None,
},
)
def _sanctions_state(
run: ConnectorSanctionsAcquisitionRun,
*,
@@ -209,8 +357,10 @@ def _aware(value: datetime | None) -> datetime | None:
__all__ = [
"KNOWLEDGE_PROVIDER_ID",
"SANCTIONS_PROVIDER_ID",
"TABULAR_PROVIDER_ID",
"sanctions_provider_states",
"knowledge_provider_states",
"tabular_provider_states",
]
+15 -1
View File
@@ -70,7 +70,7 @@ CONNECTOR_RECOVERY_OPERATIONS = (
"unknown outcomes remain unresolved until provider-backed reconciliation",
"never retry the same remote effect solely to reconstruct local state",
),
implemented=False,
implemented=True,
),
)
@@ -262,6 +262,20 @@ class ConnectorExternalMutationRecovery:
if self.operation is not None:
self.operation.succeed(evidence=provider_evidence)
def commit_success(
self,
session: Session,
*,
provider_evidence: dict[str, Any],
) -> None:
if self.operation is not None:
self.operation.commit_verified_success(
session,
evidence=provider_evidence,
)
else:
session.commit()
def reject(self, *, summary: str, provider_code: str) -> None:
if self.operation is not None:
self.operation.reject(
+246
View File
@@ -81,6 +81,33 @@ from govoplan_connectors.backend.governed_schemas import (
ConnectorRunListResponse,
ConnectorRunRequest,
)
from govoplan_connectors.backend.knowledge_connector import (
KnowledgeConnectorError,
create_profile,
discover_profile,
list_objects as list_knowledge_objects,
list_profiles as list_knowledge_profiles,
list_runs as list_knowledge_runs,
migration_dry_run,
publish_page,
synchronize_profile,
update_profile,
)
from govoplan_connectors.backend.knowledge_schemas import (
KnowledgeDiscoveryResponse,
KnowledgeMigrationDryRunRequest,
KnowledgeMigrationDryRunResponse,
KnowledgeObjectListResponse,
KnowledgeProfileCreateRequest,
KnowledgeProfileItem,
KnowledgeProfileListResponse,
KnowledgeProfileUpdateRequest,
KnowledgePublishRequest,
KnowledgePublishResponse,
KnowledgeSyncRequest,
KnowledgeSyncRunItem,
KnowledgeSyncRunListResponse,
)
from govoplan_connectors.backend.recovery import (
ConnectorRecoveryError,
begin_connector_read_snapshot,
@@ -188,6 +215,34 @@ def _governed_http_error(exc: GovernedConnectorError) -> HTTPException:
)
def _knowledge_http_error(exc: KnowledgeConnectorError) -> HTTPException:
if exc.code.endswith("_not_found"):
status_code = status.HTTP_404_NOT_FOUND
elif exc.code == "access_denied":
status_code = status.HTTP_403_FORBIDDEN
elif exc.code in {
"idempotency_conflict",
"operation_unresolved",
"profile_conflict",
"profile_paused",
"publication_conflict",
"publication_reconciliation_required",
}:
status_code = status.HTTP_409_CONFLICT
elif exc.retryable or exc.code.endswith("_unavailable"):
status_code = status.HTTP_503_SERVICE_UNAVAILABLE
else:
status_code = status.HTTP_422_UNPROCESSABLE_CONTENT
return HTTPException(
status_code=status_code,
detail={
"code": exc.code,
"message": str(exc),
"retryable": exc.retryable,
},
)
@router.post("/feeds/preview", response_model=FeedDocumentResponse)
def api_preview_feed(
payload: FeedAcquireRequest,
@@ -986,6 +1041,197 @@ def api_review_governed_run(
raise _governed_http_error(exc) from exc
@router.get(
"/knowledge/profiles",
response_model=KnowledgeProfileListResponse,
)
def api_list_knowledge_profiles(
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> KnowledgeProfileListResponse:
try:
return KnowledgeProfileListResponse(
items=list(list_knowledge_profiles(session, principal))
)
except KnowledgeConnectorError as exc:
raise _knowledge_http_error(exc) from exc
@router.post(
"/knowledge/profiles",
response_model=KnowledgeProfileItem,
status_code=status.HTTP_201_CREATED,
)
def api_create_knowledge_profile(
payload: KnowledgeProfileCreateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> KnowledgeProfileItem:
try:
return create_profile(session, principal, payload)
except KnowledgeConnectorError as exc:
session.rollback()
raise _knowledge_http_error(exc) from exc
@router.put(
"/knowledge/profiles/{profile_id}",
response_model=KnowledgeProfileItem,
)
def api_update_knowledge_profile(
profile_id: str,
payload: KnowledgeProfileUpdateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> KnowledgeProfileItem:
try:
return update_profile(
session,
principal,
profile_id=profile_id,
payload=payload,
)
except KnowledgeConnectorError as exc:
session.rollback()
raise _knowledge_http_error(exc) from exc
@router.post(
"/knowledge/profiles/{profile_id}/discover",
response_model=KnowledgeDiscoveryResponse,
)
def api_discover_knowledge_profile(
profile_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> KnowledgeDiscoveryResponse:
try:
return discover_profile(
session,
principal,
profile_id=profile_id,
)
except KnowledgeConnectorError as exc:
raise _knowledge_http_error(exc) from exc
@router.post(
"/knowledge/profiles/{profile_id}/sync",
response_model=KnowledgeSyncRunItem,
)
def api_synchronize_knowledge_profile(
profile_id: str,
payload: KnowledgeSyncRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> KnowledgeSyncRunItem:
try:
return synchronize_profile(
session,
principal,
profile_id=profile_id,
payload=payload,
)
except KnowledgeConnectorError as exc:
raise _knowledge_http_error(exc) from exc
@router.get(
"/knowledge/profiles/{profile_id}/objects",
response_model=KnowledgeObjectListResponse,
)
def api_list_knowledge_objects(
profile_id: str,
cursor: str | None = Query(default=None, max_length=500),
limit: int = Query(default=100, ge=1, le=500),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> KnowledgeObjectListResponse:
try:
items, next_cursor = list_knowledge_objects(
session,
principal,
profile_id=profile_id,
cursor=cursor,
limit=limit,
)
return KnowledgeObjectListResponse(
items=list(items),
next_cursor=next_cursor,
)
except KnowledgeConnectorError as exc:
raise _knowledge_http_error(exc) from exc
@router.get(
"/knowledge/runs",
response_model=KnowledgeSyncRunListResponse,
)
def api_list_knowledge_runs(
profile_id: str | None = Query(default=None),
limit: int = Query(default=100, ge=1, le=500),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> KnowledgeSyncRunListResponse:
try:
return KnowledgeSyncRunListResponse(
items=list(
list_knowledge_runs(
session,
principal,
profile_id=profile_id,
limit=limit,
)
)
)
except KnowledgeConnectorError as exc:
raise _knowledge_http_error(exc) from exc
@router.post(
"/knowledge/profiles/{profile_id}/migration-dry-runs",
response_model=KnowledgeMigrationDryRunResponse,
)
def api_preview_knowledge_migration(
profile_id: str,
payload: KnowledgeMigrationDryRunRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> KnowledgeMigrationDryRunResponse:
try:
return migration_dry_run(
session,
principal,
profile_id=profile_id,
payload=payload,
)
except KnowledgeConnectorError as exc:
raise _knowledge_http_error(exc) from exc
@router.post(
"/knowledge/profiles/{profile_id}/pages/{external_page_id}/publish",
response_model=KnowledgePublishResponse,
)
def api_publish_knowledge_page(
profile_id: str,
external_page_id: str,
payload: KnowledgePublishRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> KnowledgePublishResponse:
try:
return publish_page(
session,
principal,
profile_id=profile_id,
external_page_id=external_page_id,
payload=payload,
)
except KnowledgeConnectorError as exc:
raise _knowledge_http_error(exc) from exc
def _source_response(source: TabularSource) -> TabularSourceResponse:
return TabularSourceResponse(
ref=source.ref,