From 79e2315e8956e3b308176018fde39f42c987f16c Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Sat, 22 Aug 2026 14:34:36 +0200 Subject: [PATCH] Release Connectors v0.1.21 with external knowledge integration --- README.md | 19 +- docs/CONNECTOR_SOURCE_LIFECYCLE.md | 7 +- docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md | 134 + pyproject.toml | 4 +- src/govoplan_connectors/backend/db/models.py | 179 ++ .../backend/dsar_provider.py | 98 + .../backend/knowledge_connector.py | 2175 +++++++++++++++++ .../backend/knowledge_schemas.py | 303 +++ .../backend/knowledge_search.py | 269 ++ src/govoplan_connectors/backend/manifest.py | 269 +- .../backend/mediawiki_transport.py | 503 ++++ ...0f1a2c3d4_mediawiki_knowledge_connector.py | 207 ++ .../backend/provider_state.py | 150 ++ src/govoplan_connectors/backend/recovery.py | 16 +- src/govoplan_connectors/backend/router.py | 246 ++ tests/test_dsar_provider.py | 52 +- tests/test_manifest.py | 25 + tests/test_mediawiki_connector.py | 659 +++++ tests/test_mediawiki_transport.py | 158 ++ tests/test_migrations.py | 5 +- tests/test_provider_state.py | 71 +- tests/test_recovery.py | 2 +- webui/package.json | 2 +- webui/src/api/externalKnowledge.ts | 212 ++ webui/src/features/ExternalKnowledgePage.tsx | 561 +++++ webui/src/module.ts | 31 +- webui/src/styles/connectors.css | 12 +- ...connector-governance-ui-structure.test.mjs | 15 + 28 files changed, 6351 insertions(+), 33 deletions(-) create mode 100644 docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md create mode 100644 src/govoplan_connectors/backend/knowledge_connector.py create mode 100644 src/govoplan_connectors/backend/knowledge_schemas.py create mode 100644 src/govoplan_connectors/backend/knowledge_search.py create mode 100644 src/govoplan_connectors/backend/mediawiki_transport.py create mode 100644 src/govoplan_connectors/backend/migrations/versions/b9e0f1a2c3d4_mediawiki_knowledge_connector.py create mode 100644 tests/test_mediawiki_connector.py create mode 100644 tests/test_mediawiki_transport.py create mode 100644 webui/src/api/externalKnowledge.ts create mode 100644 webui/src/features/ExternalKnowledgePage.tsx diff --git a/README.md b/README.md index 7edb60f..3bd7b02 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,20 @@ The source revision/cursor, redacted dry-run decision, canonical request digest, and distributed lease are durable before network I/O. Immutable snapshot rows and the terminal recovery checkpoint commit atomically, and an `Idempotency-Key` replays the committed result without contacting the provider. -Current acquisition transports are read-only. The exported external-mutation -recovery contract requires stable idempotency, provider verification, and -operator reconciliation, but no connector currently claims a production write -or delete path. +Most acquisition transports are read-only. The MediaWiki/BlueSpice knowledge +adapter is the first governed publication path: it requires an expected remote +revision, a stable idempotency key, a scoped credential envelope, and durable +forward-recovery evidence. A timeout after dispatch becomes outcome-unknown and +blocks replay until the provider revision has been reconciled. + +The knowledge adapter discovers MediaWiki or BlueSpice product/version and +capabilities, maps namespaces, pages, revisions, users, categories, links, +files, discussions, redirects, and permissions into identity-stable connector +snapshots, and consumes bounded full or recent-change deltas. Optional Search +integration indexes active pages and rechecks current profile state and ACLs on +every result. Migration into native Wiki is preview-only: conflicts, attachment +collisions, unsupported macros, truncation, and source fingerprints are +reported before a target-side write is considered. RSS and Atom emission is a bounded renderer, not an authority shortcut. Every selected entry declares whether it came from a GovOPlaN event, publication, @@ -95,3 +105,4 @@ See: - [OpenProject connector concept](docs/OPENPROJECT_CONNECTOR.md) - [OpenDesk integration map](docs/OPENDESK_INTEGRATION_MAP.md) - [Governed connector configuration](docs/GOVERNED_CONNECTOR_CONFIGURATION.md) +- [MediaWiki and BlueSpice connector](docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md) diff --git a/docs/CONNECTOR_SOURCE_LIFECYCLE.md b/docs/CONNECTOR_SOURCE_LIFECYCLE.md index f7f70d9..2171b8c 100644 --- a/docs/CONNECTOR_SOURCE_LIFECYCLE.md +++ b/docs/CONNECTOR_SOURCE_LIFECYCLE.md @@ -178,9 +178,10 @@ start through the connector mutation recovery contract with a stable idempotency key, SHA-256 request digest, source revision/cursor, and dry-run evidence. Definitive rejection is terminal. A timeout or lost acknowledgement after dispatch is `outcome_unknown` and blocks replay until the owning connector -verifies provider state. The contract and conformance tests exist; no current -connector advertises a production external mutation, so write/delete adoption -remains explicitly planned rather than implied. +verifies provider state. The MediaWiki/BlueSpice page publisher implements this +path for revision-checked edits. Other connector types do not thereby acquire a +write capability; each adapter must declare and prove its own recovery and +reconciliation behavior. ## Provider Declaration diff --git a/docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md b/docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md new file mode 100644 index 0000000..79053fe --- /dev/null +++ b/docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md @@ -0,0 +1,134 @@ +# MediaWiki and BlueSpice Knowledge Connector + +The knowledge connector integrates an external MediaWiki or BlueSpice instance +without making Connectors the owner of native knowledge semantics. Connectors +owns endpoint and credential governance, Action API transport, discovery, +synchronization, identity-stable snapshots, diagnostics, and publication +recovery. Wiki continues to own GovOPlaN-native spaces, pages, drafts, +revisions, comments, and publishing policy. + +## Configure and discover + +Create an active governed connector definition and configuration with: + +- provider `mediawiki` or `bluespice`; +- protocol `mediawiki`, `action_api`, or `mediawiki_action_api`; +- an absolute HTTP(S) endpoint without embedded credentials; and +- an optional Core credential-envelope reference. Publication requires the + credential; public read-only sources may omit it. + +Core validates the endpoint and every redirect against the deployment egress, +DNS/IP-pinning, peer, downgrade, and response-size policies. Credentials can be +bearer/access tokens or username/password values resolved only for the current +tenant, actor, module, target scope, and server. They never enter connector +profiles, snapshots, diagnostics, URLs, or API responses. + +A knowledge profile selects desired maturity and authority and maps each +included source namespace to a target Wiki space reference and path prefix. +Every profile also supplies a tenant or restricted fallback visibility. A +restricted fallback needs at least one normalized Search ACL token such as +`group:`, `role:`, `membership:`, `account:`, +`identity:`, `function:`, or `scope:`. + +Discovery queries site and current-user metadata. It records the product, +version, API/PHP version, extensions, namespaces, advertised edit right, +capabilities, a stable discovery digest, and sanitized diagnostics. BlueSpice +permission and discussion extensions are detected where advertised. Standard +MediaWiki does not expose complete page ACL semantics through the base Action +API, so discovery reports that the configured namespace fallback will be used. + +## Synchronize and preserve identity + +Run one bounded full backfill and then bounded recent-change deltas. Every run +requires an idempotency key; exact replay returns the original evidence without +contacting the source, while reusing the key with different inputs is rejected. +The profile stores only its latest cursor and high-watermark. Runs retain +counts, redacted effects, diagnostics, configuration/discovery provenance, and +transport evidence. + +Snapshots use the MediaWiki page id as stable identity and preserve revision +id, canonical URL, content digest, namespace, title, source timestamp, and +observed timestamp. Page content mapping covers categories, links, file +references, discussion references, revision author references, redirect +targets, permission metadata, target space/path, and detected wikitext macros. +Referenced files remain external references; binary transfer and attachment +ownership remain with Files or DMS. + +Moves update the existing stable page. Deletions use the stable page id when the +provider supplies it and otherwise match a previously synchronized namespace +and title. An unmatched deletion becomes a separate tombstone with an explicit +diagnostic instead of silently deleting an unrelated page. Permission changes +are part of the content digest and therefore update the current snapshot even +when page text is unchanged. + +Desired maturity is an operator ceiling. A read-only profile cannot be used for +synchronization, migration preview, or publication merely because the provider +advertises those capabilities. + +## Search and access safety + +When Search is installed, Connectors registers `connectors.mediawiki.pages` for +the `external_knowledge_page` resource type. Backfill includes only active, +non-deleted pages from active profiles. Restricted documents carry the current +snapshot ACL; tenant documents carry no narrower ACL. + +Search performs a fail-closed authorization recheck for every result. It +requires the external-knowledge read permission, an exact tenant match, an +active profile, a current non-deleted page, and—when restricted—intersection +with the current account, membership, identity, group, role, function, or scope +tokens. Consequently, an ACL change takes effect even before a deferred index +update completes. Immediate index updates remove deleted pages and refresh ACLs; +the next Search rebuild reconciles any transient writer failure. Search also +removes source projections when Connectors is disabled. + +## Publish and reconcile + +Publication requires the publish permission, an active profile and +configuration, discovered provider publication capability, desired maturity at +least `publish`, an idempotency key, page title/body, and an optional expected +external revision. The Action API edit carries `baserevid` when supplied and +uses a credential-provided or freshly acquired CSRF token. + +Before remote I/O, Connectors prepares a durable forward-recovery operation +with request digest, expected revision, cursor, profile/configuration revision, +and resource identity. A confirmed provider response stores the stable page and +new revision and completes the recovery evidence. A definitive provider +rejection is terminal. A timeout, invalid response, or server failure after +dispatch becomes outcome-unknown: do not retry with another key until an +operator compares the provider page/revision and reconciles the recovery +operation. Local rollback cannot undo a confirmed external edit. + +## Migration preview into Wiki + +Migration is intentionally a dry-run in this slice. The request names a target +space, supported macro set, existing target paths/source identities, and +existing attachment names. The preview is bounded to 500 source pages and +records a source fingerprint and high-watermark. It reports create, update, or +conflict effects plus: + +- target path already owned by a different external page; +- attachment filename collisions; +- unsupported macros; +- pages outside the requested target space; and +- truncation at the configured limit. + +`can_apply=true` means the bounded preview contains no error or conflict and is +not truncated. It does not write Wiki pages. A future target-side migration +worker must use the optional connector capability and Wiki-owned mutation +contract, revalidate the preview fingerprint, and keep native Wiki permissions +and revision history authoritative. + +## Operations and privacy + +Provider state exposes profile product/maturity, active-object count, latest run +status, health, recovery attention, and last success time without endpoint, +credential, title, ACL, or content data. Existing snapshots may remain +available during an outage only to principals still authorized by current local +profile and ACL state; their health and freshness remain explicit. + +Data-subject exports include only minimized operator attribution for profile +updates and synchronization/migration/publication runs. They exclude endpoints, +credentials, page content, titles, ACLs, namespace mappings, idempotency keys, +request hashes, effects, diagnostics, provenance, and transport evidence. +Attribution and external-operation evidence is retained for governance rather +than automatically erased. diff --git a/pyproject.toml b/pyproject.toml index e4eacef..0d3931e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "govoplan-connectors" -version = "0.1.20" +version = "0.1.21" description = "Governed connector catalogue and tabular source capabilities for GovOPlaN." readme = "README.md" requires-python = ">=3.12" @@ -12,7 +12,7 @@ license = "AGPL-3.0-or-later" authors = [{ name = "GovOPlaN" }] dependencies = [ "defusedxml>=0.7,<1", - "govoplan-core>=0.1.19", + "govoplan-core>=0.1.32", "openpyxl>=3.1.5,<4", ] diff --git a/src/govoplan_connectors/backend/db/models.py b/src/govoplan_connectors/backend/db/models.py index 6d00658..61395f6 100644 --- a/src/govoplan_connectors/backend/db/models.py +++ b/src/govoplan_connectors/backend/db/models.py @@ -456,10 +456,189 @@ class ConnectorSimulationRun(Base, TimestampMixin): review_reason: Mapped[str | None] = mapped_column(Text) +class ConnectorKnowledgeProfile(Base, TimestampMixin): + __tablename__ = "connector_knowledge_profiles" + __table_args__ = ( + UniqueConstraint( + "tenant_id", + "configuration_id", + name="uq_connector_knowledge_profile_configuration", + ), + Index( + "ix_connector_knowledge_profiles_tenant_status", + "tenant_id", + "status", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + configuration_id: Mapped[str] = mapped_column( + ForeignKey("connector_configurations.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + status: Mapped[str] = mapped_column( + String(30), default="active", nullable=False, index=True + ) + product: Mapped[str] = mapped_column( + String(50), default="unknown", nullable=False, index=True + ) + product_version: Mapped[str | None] = mapped_column(String(100)) + desired_maturity: Mapped[str] = mapped_column( + String(30), default="read", nullable=False + ) + discovered_maturity: Mapped[str] = mapped_column( + String(30), default="discover", nullable=False + ) + source_authority_mode: Mapped[str] = mapped_column( + String(40), default="external_mirror", nullable=False + ) + default_visibility: Mapped[str] = mapped_column( + String(30), default="restricted", nullable=False + ) + default_acl_tokens: Mapped[list[str]] = mapped_column( + JSON, default=list, nullable=False + ) + namespace_mappings: Mapped[list[dict[str, Any]]] = mapped_column( + JSON, default=list, nullable=False + ) + capabilities: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) + discovery_revision: Mapped[str | None] = mapped_column(String(255), index=True) + discovery_evidence: Mapped[dict[str, Any]] = mapped_column( + JSON, default=dict, nullable=False + ) + health_status: Mapped[str] = mapped_column( + String(30), default="unknown", nullable=False, index=True + ) + health_details: Mapped[dict[str, Any]] = mapped_column( + JSON, default=dict, nullable=False + ) + discovered_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_sync_cursor: Mapped[str | None] = mapped_column(String(500)) + last_high_watermark: Mapped[str | None] = mapped_column(String(500)) + resource_revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + updated_by: Mapped[str | None] = mapped_column(String(255), index=True) + + +class ConnectorKnowledgeObject(Base, TimestampMixin): + __tablename__ = "connector_knowledge_objects" + __table_args__ = ( + UniqueConstraint( + "profile_id", + "object_type", + "external_id", + name="uq_connector_knowledge_object_identity", + ), + Index( + "ix_connector_knowledge_objects_tenant_profile_status", + "tenant_id", + "profile_id", + "status", + ), + Index( + "ix_connector_knowledge_objects_tenant_updated", + "tenant_id", + "source_updated_at", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + profile_id: Mapped[str] = mapped_column( + ForeignKey("connector_knowledge_profiles.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + object_type: Mapped[str] = mapped_column(String(40), nullable=False, index=True) + external_id: Mapped[str] = mapped_column(String(255), nullable=False) + external_page_id: Mapped[str | None] = mapped_column(String(255), index=True) + external_revision_id: Mapped[str | None] = mapped_column(String(255), index=True) + namespace_id: Mapped[int | None] = mapped_column(Integer, index=True) + title: Mapped[str] = mapped_column(String(500), nullable=False) + canonical_url: Mapped[str | None] = mapped_column(String(1500)) + status: Mapped[str] = mapped_column( + String(30), default="active", nullable=False, index=True + ) + redirect_target_external_id: Mapped[str | None] = mapped_column(String(255)) + source_revision: Mapped[str] = mapped_column(String(255), nullable=False) + content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + visibility: Mapped[str] = mapped_column( + String(30), default="restricted", nullable=False + ) + acl_tokens: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) + mapped_data: Mapped[dict[str, Any]] = mapped_column( + JSON, default=dict, nullable=False + ) + provenance: Mapped[dict[str, Any]] = mapped_column( + JSON, default=dict, nullable=False + ) + change_cursor: Mapped[str | None] = mapped_column(String(500), index=True) + source_updated_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), index=True + ) + observed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, index=True + ) + resource_revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + + +class ConnectorKnowledgeSyncRun(Base, TimestampMixin): + __tablename__ = "connector_knowledge_sync_runs" + __table_args__ = ( + UniqueConstraint( + "tenant_id", + "profile_id", + "mode", + "idempotency_key", + name="uq_connector_knowledge_sync_run_idempotency", + ), + Index( + "ix_connector_knowledge_sync_runs_profile_started", + "tenant_id", + "profile_id", + "started_at", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + profile_id: Mapped[str] = mapped_column( + ForeignKey("connector_knowledge_profiles.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + mode: Mapped[str] = mapped_column(String(40), nullable=False, index=True) + idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False) + request_hash: Mapped[str] = mapped_column(String(64), nullable=False) + status: Mapped[str] = mapped_column(String(30), nullable=False, index=True) + cursor_before: Mapped[str | None] = mapped_column(String(500)) + cursor_after: Mapped[str | None] = mapped_column(String(500)) + high_watermark: Mapped[str | None] = mapped_column(String(500)) + counts: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + effects: Mapped[list[dict[str, Any]]] = mapped_column( + JSON, default=list, nullable=False + ) + diagnostics: Mapped[list[dict[str, Any]]] = mapped_column( + JSON, default=list, nullable=False + ) + provenance: Mapped[dict[str, Any]] = mapped_column( + JSON, default=dict, nullable=False + ) + created_by: Mapped[str | None] = mapped_column(String(255), index=True) + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, index=True + ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + __all__ = [ "ConnectorConfiguration", "ConnectorDefinition", "ConnectorDefinitionRevision", + "ConnectorKnowledgeObject", + "ConnectorKnowledgeProfile", + "ConnectorKnowledgeSyncRun", "ConnectorSanctionsAcquisitionRun", "ConnectorSanctionsSnapshot", "ConnectorSimulationRun", diff --git a/src/govoplan_connectors/backend/dsar_provider.py b/src/govoplan_connectors/backend/dsar_provider.py index dde7995..1277839 100644 --- a/src/govoplan_connectors/backend/dsar_provider.py +++ b/src/govoplan_connectors/backend/dsar_provider.py @@ -18,6 +18,8 @@ from govoplan_connectors.backend.db.models import ( ConnectorConfiguration, ConnectorDefinition, ConnectorDefinitionRevision, + ConnectorKnowledgeProfile, + ConnectorKnowledgeSyncRun, ConnectorSanctionsAcquisitionRun, ConnectorSimulationRun, ConnectorTabularSource, @@ -37,6 +39,8 @@ class _SubjectSelectors: definition_id: str | None configuration_id: str | None simulation_id: str | None + knowledge_profile_id: str | None + knowledge_run_id: str | None @property def narrowed(self) -> bool: @@ -47,6 +51,8 @@ class _SubjectSelectors: self.definition_id, self.configuration_id, self.simulation_id, + self.knowledge_profile_id, + self.knowledge_run_id, ) ) @@ -172,6 +178,44 @@ class ConnectorsDsarProvider: ) ) + if not selectors.narrowed or selectors.knowledge_profile_id: + query = db.query(ConnectorKnowledgeProfile).filter( + ConnectorKnowledgeProfile.tenant_id == tenant_id, + ConnectorKnowledgeProfile.updated_by == selectors.account_id, + ) + if selectors.knowledge_profile_id: + query = query.filter( + ConnectorKnowledgeProfile.id == selectors.knowledge_profile_id + ) + records.extend( + _knowledge_profile_attribution(row) + for row in _limited( + query, + ConnectorKnowledgeProfile.created_at, + ConnectorKnowledgeProfile.id, + label="knowledge profile attribution", + ) + ) + + if not selectors.narrowed or selectors.knowledge_run_id: + query = db.query(ConnectorKnowledgeSyncRun).filter( + ConnectorKnowledgeSyncRun.tenant_id == tenant_id, + ConnectorKnowledgeSyncRun.created_by == selectors.account_id, + ) + if selectors.knowledge_run_id: + query = query.filter( + ConnectorKnowledgeSyncRun.id == selectors.knowledge_run_id + ) + records.extend( + _knowledge_run_attribution(row) + for row in _limited( + query, + ConnectorKnowledgeSyncRun.started_at, + ConnectorKnowledgeSyncRun.id, + label="knowledge run attribution", + ) + ) + if len(records) > _MAX_RECORDS: raise ValueError("Connectors DSAR result limit exceeded; narrow selectors.") return tuple( @@ -270,6 +314,14 @@ def _subject_selectors(subject: DsarSubjectRef) -> _SubjectSelectors | None: references.get("connectors.simulation"), references.get("connectors.simulation_id"), ), + "knowledge_profile_id": _coalesce( + references.get("connectors.knowledge_profile"), + references.get("connectors.knowledge_profile_id"), + ), + "knowledge_run_id": _coalesce( + references.get("connectors.knowledge_run"), + references.get("connectors.knowledge_run_id"), + ), } if account is _CONFLICT or any(value is _CONFLICT for value in values.values()): return None @@ -283,6 +335,8 @@ def _subject_selectors(subject: DsarSubjectRef) -> _SubjectSelectors | None: definition_id=_optional_string(values["definition_id"]), configuration_id=_optional_string(values["configuration_id"]), simulation_id=_optional_string(values["simulation_id"]), + knowledge_profile_id=_optional_string(values["knowledge_profile_id"]), + knowledge_run_id=_optional_string(values["knowledge_run_id"]), ) @@ -396,6 +450,48 @@ def _simulation_attribution( ) +def _knowledge_profile_attribution( + row: ConnectorKnowledgeProfile, +) -> DsarRecordRef: + return _record( + resource_type="knowledge_profile_actor_attribution", + resource_id=row.id, + title="External knowledge profile actor attribution", + data={ + "knowledge_profile_id": row.id, + "configuration_id": row.configuration_id, + "status": row.status, + "desired_maturity": row.desired_maturity, + "source_authority_mode": row.source_authority_mode, + "resource_revision": row.resource_revision, + "activity": "updated_external_knowledge_profile", + "created_at": _iso(row.created_at), + "updated_at": _iso(row.updated_at), + }, + observed_at=row.updated_at, + ) + + +def _knowledge_run_attribution( + row: ConnectorKnowledgeSyncRun, +) -> DsarRecordRef: + return _record( + resource_type="knowledge_run_actor_attribution", + resource_id=row.id, + title="External knowledge operation actor attribution", + data={ + "knowledge_run_id": row.id, + "knowledge_profile_id": row.profile_id, + "mode": row.mode, + "status": row.status, + "started_at": _iso(row.started_at), + "finished_at": _iso(row.finished_at), + "activity": "started_external_knowledge_operation", + }, + observed_at=row.finished_at or row.started_at, + ) + + def _record( *, resource_type: str, @@ -462,6 +558,8 @@ _RESOURCE_TYPES = { "definition_actor_attribution", "configuration_actor_attribution", "simulation_actor_attribution", + "knowledge_profile_actor_attribution", + "knowledge_run_actor_attribution", } diff --git a/src/govoplan_connectors/backend/knowledge_connector.py b/src/govoplan_connectors/backend/knowledge_connector.py new file mode 100644 index 0000000..0797e58 --- /dev/null +++ b/src/govoplan_connectors/backend/knowledge_connector.py @@ -0,0 +1,2175 @@ +from __future__ import annotations + +import hashlib +import json +import re +from collections import Counter +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from typing import Any +from urllib.parse import quote + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from govoplan_core.audit.logging import audit_event +from govoplan_core.auth import ApiPrincipal, has_scope +from govoplan_core.core.external_references import ( + INTEGRATION_MATURITY_ORDER, + ExternalObjectReference, +) +from govoplan_core.core.runtime import get_registry +from govoplan_core.core.search import search_index_writer +from govoplan_core.security.credential_envelopes import ( + CredentialAccessContext, + CredentialEnvelopeError, + resolve_credential_envelope, +) +from govoplan_connectors.backend.db.models import ( + ConnectorConfiguration, + ConnectorDefinition, + ConnectorKnowledgeObject, + ConnectorKnowledgeProfile, + ConnectorKnowledgeSyncRun, +) +from govoplan_connectors.backend.knowledge_schemas import ( + KnowledgeDiagnostic, + KnowledgeDiscoveryResponse, + KnowledgeExternalReferenceResponse, + KnowledgeMigrationDryRunRequest, + KnowledgeMigrationDryRunResponse, + KnowledgeObjectItem, + KnowledgeProfileCreateRequest, + KnowledgeProfileItem, + KnowledgeProfileUpdateRequest, + KnowledgePublishRequest, + KnowledgePublishResponse, + KnowledgeSyncRequest, + KnowledgeSyncRunItem, +) +from govoplan_connectors.backend.mediawiki_transport import ( + HttpMediaWikiTransport, + MediaWikiPublishResult, + MediaWikiTransport, + MediaWikiTransportError, +) +from govoplan_connectors.backend.recovery import ( + ConnectorRecoveryError, + begin_connector_external_mutation, +) + + +KNOWLEDGE_READ_SCOPE = "connectors:knowledge:read" +KNOWLEDGE_ADMIN_SCOPE = "connectors:knowledge:admin" +KNOWLEDGE_SYNC_SCOPE = "connectors:knowledge:sync" +KNOWLEDGE_PUBLISH_SCOPE = "connectors:knowledge:publish" +KNOWLEDGE_MIGRATE_SCOPE = "connectors:knowledge:migrate" +KNOWLEDGE_CAPABILITY = "connectors.external_knowledge" +KNOWLEDGE_INTERFACE_VERSION = "1.0.0" +KNOWLEDGE_PROVIDER_ID = "connectors.mediawiki.pages" +KNOWLEDGE_RESOURCE_TYPE = "external_knowledge_page" + +_MACRO = re.compile(r"\{\{\s*([^{}|#]+)", re.MULTILINE) +_REDIRECT = re.compile(r"^\s*#redirect\s*\[\[([^\]]+)\]\]", re.IGNORECASE) + + +class KnowledgeConnectorError(RuntimeError): + def __init__(self, code: str, message: str, *, retryable: bool = False) -> None: + super().__init__(message) + self.code = code + self.retryable = retryable + + +def create_profile( + session: Session, + principal: ApiPrincipal, + payload: KnowledgeProfileCreateRequest, +) -> KnowledgeProfileItem: + _require_scope(principal, KNOWLEDGE_ADMIN_SCOPE) + configuration = _configuration( + session, principal.tenant_id, payload.configuration_id, active=False + ) + _assert_knowledge_configuration(session, configuration) + item = ConnectorKnowledgeProfile( + tenant_id=principal.tenant_id, + configuration_id=configuration.id, + desired_maturity=payload.desired_maturity, + source_authority_mode=payload.source_authority_mode, + default_visibility=payload.default_visibility, + default_acl_tokens=list(payload.default_acl_tokens), + namespace_mappings=[item.model_dump(mode="json") for item in payload.namespace_mappings], + updated_by=_actor_id(principal), + ) + session.add(item) + try: + session.flush() + except IntegrityError as exc: + raise KnowledgeConnectorError( + "profile_conflict", + "This connector configuration already has a knowledge profile.", + ) from exc + _audit( + session, + principal, + action="profile.created", + object_type="connector_knowledge_profile", + object_id=item.id, + details={ + "configuration_id": configuration.id, + "desired_maturity": item.desired_maturity, + "source_authority_mode": item.source_authority_mode, + }, + ) + response = _profile_item(item, configuration) + session.commit() + return response + + +def update_profile( + session: Session, + principal: ApiPrincipal, + *, + profile_id: str, + payload: KnowledgeProfileUpdateRequest, + registry: object | None = None, +) -> KnowledgeProfileItem: + _require_scope(principal, KNOWLEDGE_ADMIN_SCOPE) + item = _profile(session, principal.tenant_id, profile_id) + if item.resource_revision != payload.expected_resource_revision: + raise KnowledgeConnectorError( + "profile_conflict", "The profile changed; reload it before saving." + ) + for field_name in ( + "status", + "desired_maturity", + "source_authority_mode", + "default_visibility", + ): + value = getattr(payload, field_name) + if value is not None: + setattr(item, field_name, value) + if payload.default_acl_tokens is not None: + item.default_acl_tokens = list(payload.default_acl_tokens) + if payload.namespace_mappings is not None: + item.namespace_mappings = [ + mapping.model_dump(mode="json") for mapping in payload.namespace_mappings + ] + _validate_effective_acl(item.default_visibility, item.default_acl_tokens) + affected = tuple( + session.scalars( + select(ConnectorKnowledgeObject).where( + ConnectorKnowledgeObject.tenant_id == principal.tenant_id, + ConnectorKnowledgeObject.profile_id == item.id, + ConnectorKnowledgeObject.object_type == "page", + ConnectorKnowledgeObject.status != "deleted", + ) + ) + ) + for row in affected: + mapping = _namespace_mapping(item, row.namespace_id) + if mapping is None or not bool(mapping.get("include", True)): + continue + data = dict(row.mapped_data or {}) + data["target_space_ref"] = mapping.get("target_space_ref") + data["target_path"] = _target_path(mapping, row.title) + if data.get("permission_source") == "configured_fallback": + visibility = str(mapping.get("visibility") or item.default_visibility) + tokens = list(mapping.get("acl_tokens") or item.default_acl_tokens) + _validate_effective_acl(visibility, tokens) + row.visibility = visibility + row.acl_tokens = tokens + row.mapped_data = data + row.content_hash = _mapped_content_hash( + data, + visibility=row.visibility, + acl_tokens=row.acl_tokens, + ) + row.resource_revision += 1 + row.observed_at = _now() + item.resource_revision += 1 + item.updated_by = _actor_id(principal) + session.flush() + configuration = _configuration( + session, principal.tenant_id, item.configuration_id, active=False + ) + _audit( + session, + principal, + action="profile.updated", + object_type="connector_knowledge_profile", + object_id=item.id, + details={ + "resource_revision": item.resource_revision, + "status": item.status, + "desired_maturity": item.desired_maturity, + }, + ) + search_diagnostics: list[dict[str, Any]] = [] + _update_search( + session, + principal, + affected, + registry=registry or get_registry(), + diagnostics=search_diagnostics, + source_active=item.status == "active", + ) + if search_diagnostics: + item.health_details = { + **dict(item.health_details or {}), + "search_diagnostics": search_diagnostics, + } + response = _profile_item(item, configuration) + session.commit() + return response + + +def list_profiles( + session: Session, principal: ApiPrincipal +) -> tuple[KnowledgeProfileItem, ...]: + _require_any_scope(principal, KNOWLEDGE_READ_SCOPE, KNOWLEDGE_ADMIN_SCOPE) + rows = tuple( + session.scalars( + select(ConnectorKnowledgeProfile) + .where(ConnectorKnowledgeProfile.tenant_id == principal.tenant_id) + .order_by(ConnectorKnowledgeProfile.created_at, ConnectorKnowledgeProfile.id) + ) + ) + configurations = _configurations(session, principal.tenant_id, rows) + return tuple(_profile_item(row, configurations[row.configuration_id]) for row in rows) + + +def discover_profile( + session: Session, + principal: ApiPrincipal, + *, + profile_id: str, + transport: MediaWikiTransport | None = None, +) -> KnowledgeDiscoveryResponse: + _require_any_scope(principal, KNOWLEDGE_ADMIN_SCOPE, KNOWLEDGE_SYNC_SCOPE) + item = _profile(session, principal.tenant_id, profile_id) + configuration = _configuration( + session, principal.tenant_id, item.configuration_id, active=True + ) + _assert_knowledge_configuration(session, configuration) + endpoint = _endpoint(configuration) + credential = _credential(session, principal, configuration) + try: + payload = (transport or HttpMediaWikiTransport()).discover( + endpoint_url=endpoint, + credential=credential, + ) + except MediaWikiTransportError as exc: + item.health_status = "unavailable" + item.health_details = { + "code": exc.code, + "retryable": exc.retryable, + "summary": str(exc), + } + session.commit() + raise KnowledgeConnectorError( + "discovery_unavailable", str(exc), retryable=exc.retryable + ) from exc + discovery = _discovery(payload, credential_present=bool(configuration.credential_ref)) + item.product = discovery["product"] + item.product_version = discovery["product_version"] + item.capabilities = discovery["capabilities"] + item.discovered_maturity = discovery["maturity"] + item.discovery_revision = discovery["revision"] + item.discovery_evidence = { + "api_version": discovery["api_version"], + "extensions": discovery["extensions"], + "namespaces": discovery["namespaces"], + "capability_count": len(discovery["capabilities"]), + } + item.health_status = discovery["health_status"] + item.health_details = { + "diagnostics": [value.model_dump(mode="json") for value in discovery["diagnostics"]], + "checked_at": _now().isoformat(), + } + item.discovered_at = _now() + item.resource_revision += 1 + session.flush() + _audit( + session, + principal, + action="profile.discovered", + object_type="connector_knowledge_profile", + object_id=item.id, + details={ + "product": item.product, + "product_version": item.product_version, + "maturity": item.discovered_maturity, + "health": item.health_status, + "discovery_revision": item.discovery_revision, + }, + ) + response = KnowledgeDiscoveryResponse( + profile=_profile_item(item, configuration), + product=discovery["product"], + product_version=discovery["product_version"], + api_version=discovery["api_version"], + capabilities=discovery["capabilities"], + namespaces=discovery["namespaces"], + extensions=discovery["extensions"], + maturity=discovery["maturity"], + health_status=discovery["health_status"], + diagnostics=discovery["diagnostics"], + revision=discovery["revision"], + ) + session.commit() + return response + + +def synchronize_profile( + session: Session, + principal: ApiPrincipal, + *, + profile_id: str, + payload: KnowledgeSyncRequest, + transport: MediaWikiTransport | None = None, + registry: object | None = None, +) -> KnowledgeSyncRunItem: + _require_scope(principal, KNOWLEDGE_SYNC_SCOPE) + item = _profile(session, principal.tenant_id, profile_id) + if item.status != "active": + raise KnowledgeConnectorError("profile_paused", "The knowledge profile is paused.") + configuration = _configuration( + session, principal.tenant_id, item.configuration_id, active=True + ) + _assert_knowledge_configuration(session, configuration) + if item.discovered_at is None: + raise KnowledgeConnectorError( + "profile_not_discovered", "Discover the provider before synchronization." + ) + if not _supports(item.discovered_maturity, "synchronize") or not _supports( + item.desired_maturity, "synchronize" + ): + raise KnowledgeConnectorError( + "synchronization_unsupported", + "The discovered provider maturity does not support synchronization.", + ) + cursor = None if payload.force_full else payload.cursor or item.last_sync_cursor + mode = "backfill" if payload.force_full else "delta" + request_hash = _hash( + { + "profile_id": item.id, + "mode": mode, + "cursor": cursor, + "limit": payload.limit, + "discovery_revision": item.discovery_revision, + } + ) + replay = _run_replay( + session, + tenant_id=principal.tenant_id, + profile_id=item.id, + mode=mode, + idempotency_key=payload.idempotency_key, + request_hash=request_hash, + ) + if replay is not None: + return _run_item(replay) + run = ConnectorKnowledgeSyncRun( + tenant_id=principal.tenant_id, + profile_id=item.id, + mode=mode, + idempotency_key=payload.idempotency_key, + request_hash=request_hash, + status="running", + cursor_before=cursor, + counts={}, + effects=[], + diagnostics=[], + provenance={ + "configuration_revision": configuration.resource_revision, + "configuration_hash": configuration.effective_hash, + "discovery_revision": item.discovery_revision, + }, + created_by=_actor_id(principal), + started_at=_now(), + ) + session.add(run) + session.flush() + credential = _credential(session, principal, configuration) + try: + batch = (transport or HttpMediaWikiTransport()).changes( + endpoint_url=_endpoint(configuration), + credential=credential, + cursor=cursor, + limit=payload.limit, + force_full=payload.force_full, + ) + except MediaWikiTransportError as exc: + run.status = "failed" + run.diagnostics = [ + _diagnostic( + "error", + exc.code, + str(exc), + retryable=exc.retryable, + ) + ] + run.finished_at = _now() + item.health_status = "unavailable" + item.health_details = {"code": exc.code, "retryable": exc.retryable} + session.commit() + raise KnowledgeConnectorError( + "synchronization_unavailable", str(exc), retryable=exc.retryable + ) from exc + + effects: list[dict[str, Any]] = [] + diagnostics: list[dict[str, Any]] = [] + changed: list[ConnectorKnowledgeObject] = [] + for raw in batch.changes: + mapping = _namespace_mapping(item, raw.get("ns")) + change_kind = str(raw.get("change_kind") or "upsert") + external_id = ( + _optional_text(raw.get("pageid")) + or _optional_text(raw.get("title")) + or "unknown" + if change_kind == "delete" + else _external_page_id(raw) + ) + if mapping is None or not bool(mapping.get("include", True)): + effects.append( + { + "effect": "ignored", + "external_id": external_id, + "title": _optional_text(raw.get("title")) or "Unmapped page", + "reason_code": "namespace_not_mapped", + } + ) + diagnostics.append( + _diagnostic( + "warning", + "namespace_not_mapped", + "The source namespace is excluded or has no target mapping.", + object_ref=external_id, + ) + ) + continue + if change_kind == "delete": + stored, effect = _delete_object( + session, + item, + raw, + mapping=mapping, + cursor=raw.get("change_cursor") or batch.next_cursor, + ) + effects.append(effect) + if effect.get("reason_code"): + diagnostics.append( + _diagnostic( + "warning", + str(effect["reason_code"]), + "The deletion could not be matched to a synchronized stable page id; a separate tombstone was retained.", + object_ref=str(effect["external_id"]), + ) + ) + if stored is not None: + changed.append(stored) + continue + mapped, mapped_diagnostics = _map_page(item, raw, mapping=mapping) + stored, effect = _upsert_object( + session, + item, + mapped, + cursor=raw.get("change_cursor") or batch.next_cursor, + ) + effects.append(effect) + diagnostics.extend(mapped_diagnostics) + if effect["effect"] != "unchanged": + changed.append(stored) + + session.flush() + _update_search( + session, + principal, + changed, + registry=registry or get_registry(), + diagnostics=diagnostics, + ) + counts = dict(Counter(str(effect["effect"]) for effect in effects)) + run.status = "completed" + run.cursor_after = batch.next_cursor + run.high_watermark = batch.high_watermark + run.counts = counts + run.effects = effects + run.diagnostics = diagnostics + run.provenance = {**run.provenance, "transport": dict(batch.evidence)} + run.finished_at = _now() + item.last_sync_cursor = batch.next_cursor + item.last_high_watermark = batch.high_watermark + item.health_status = "healthy" + item.health_details = { + "last_sync_at": run.finished_at.isoformat(), + "complete": batch.complete, + "counts": counts, + } + item.resource_revision += 1 + session.flush() + _audit( + session, + principal, + action="profile.synchronized", + object_type="connector_knowledge_sync_run", + object_id=run.id, + details={ + "profile_id": item.id, + "mode": mode, + "counts": counts, + "complete": batch.complete, + "high_watermark": batch.high_watermark, + }, + ) + response = _run_item(run) + session.commit() + return response + + +def list_objects( + session: Session, + principal: ApiPrincipal, + *, + profile_id: str, + cursor: str | None = None, + limit: int = 100, +) -> tuple[tuple[KnowledgeObjectItem, ...], str | None]: + _require_any_scope(principal, KNOWLEDGE_READ_SCOPE, KNOWLEDGE_ADMIN_SCOPE) + item = _profile(session, principal.tenant_id, profile_id) + statement = select(ConnectorKnowledgeObject).where( + ConnectorKnowledgeObject.tenant_id == principal.tenant_id, + ConnectorKnowledgeObject.profile_id == item.id, + ) + if cursor: + statement = statement.where(ConnectorKnowledgeObject.id > cursor) + rows = tuple( + session.scalars( + statement.order_by(ConnectorKnowledgeObject.id.asc()).limit(limit + 1) + ) + ) + selected = rows[:limit] + next_cursor = selected[-1].id if len(rows) > limit and selected else None + return tuple(_object_item(row, item) for row in selected), next_cursor + + +def list_runs( + session: Session, + principal: ApiPrincipal, + *, + profile_id: str | None = None, + limit: int = 100, +) -> tuple[KnowledgeSyncRunItem, ...]: + _require_any_scope(principal, KNOWLEDGE_READ_SCOPE, KNOWLEDGE_ADMIN_SCOPE) + statement = select(ConnectorKnowledgeSyncRun).where( + ConnectorKnowledgeSyncRun.tenant_id == principal.tenant_id + ) + if profile_id: + _profile(session, principal.tenant_id, profile_id) + statement = statement.where(ConnectorKnowledgeSyncRun.profile_id == profile_id) + return tuple( + _run_item(row) + for row in session.scalars( + statement.order_by( + ConnectorKnowledgeSyncRun.started_at.desc(), + ConnectorKnowledgeSyncRun.id.desc(), + ).limit(limit) + ) + ) + + +def migration_dry_run( + session: Session, + principal: ApiPrincipal, + *, + profile_id: str, + payload: KnowledgeMigrationDryRunRequest, +) -> KnowledgeMigrationDryRunResponse: + _require_scope(principal, KNOWLEDGE_MIGRATE_SCOPE) + profile = _profile(session, principal.tenant_id, profile_id) + if ( + profile.discovered_at is None + or not _supports(profile.discovered_maturity, "migrate") + or not _supports(profile.desired_maturity, "migrate") + ): + raise KnowledgeConnectorError( + "migration_unsupported", + "The discovered provider maturity does not support migration.", + ) + request_hash = _hash(payload.model_dump(mode="json")) + replay = _run_replay( + session, + tenant_id=principal.tenant_id, + profile_id=profile.id, + mode="migration_dry_run", + idempotency_key=payload.idempotency_key, + request_hash=request_hash, + ) + if replay is not None: + return _migration_response(replay, payload.target_space_ref) + rows = tuple( + session.scalars( + select(ConnectorKnowledgeObject) + .where( + ConnectorKnowledgeObject.tenant_id == principal.tenant_id, + ConnectorKnowledgeObject.profile_id == profile.id, + ConnectorKnowledgeObject.object_type == "page", + ConnectorKnowledgeObject.status != "deleted", + ) + .order_by(ConnectorKnowledgeObject.id.asc()) + .limit(payload.max_items + 1) + ) + ) + selected = rows[: payload.max_items] + truncated = len(rows) > payload.max_items + targets = {item.path: item for item in payload.existing_targets} + supported_macros = {item.casefold() for item in payload.supported_macros} + effects: list[dict[str, Any]] = [] + diagnostics: list[dict[str, Any]] = [] + for row in selected: + data = dict(row.mapped_data or {}) + if data.get("target_space_ref") != payload.target_space_ref: + continue + path = str(data.get("target_path") or row.title) + target = targets.get(path) + effect = "create" + reason = None + if target is not None: + if target.source_external_id == row.external_id: + effect = "update" + else: + effect = "conflict" + reason = "target_path_conflict" + diagnostics.append( + _diagnostic( + "error", + "target_path_conflict", + "Another target page already uses the mapped path.", + object_ref=row.external_id, + field="path", + details={"target_path": path}, + ) + ) + unsupported = sorted( + macro + for macro in set(data.get("macros") or ()) + if str(macro).casefold() not in supported_macros + ) + for macro in unsupported: + diagnostics.append( + _diagnostic( + "warning", + "unsupported_macro", + "The source page uses a macro that is not declared as supported.", + object_ref=row.external_id, + field="body", + details={"macro": macro}, + ) + ) + attachment_names = { + str(value.get("name") or "") + for value in data.get("files") or () + if isinstance(value, Mapping) + } + conflicts = sorted( + attachment_names.intersection( + set(target.attachment_names if target is not None else ()) + ) + ) + for name in conflicts: + diagnostics.append( + _diagnostic( + "error", + "attachment_name_conflict", + "A target attachment already uses this source filename.", + object_ref=row.external_id, + field="files", + details={"name": name, "target_path": path}, + ) + ) + effect = "conflict" + reason = "attachment_name_conflict" + effects.append( + { + "effect": effect, + "external_id": row.external_id, + "source_revision": row.source_revision, + "target_space_ref": payload.target_space_ref, + "target_path": path, + "title": row.title, + "unsupported_macros": unsupported, + "attachment_count": len(attachment_names), + "reason_code": reason, + } + ) + if truncated: + diagnostics.append( + _diagnostic( + "warning", + "migration_truncated", + "The migration preview reached its configured item limit.", + ) + ) + counts = dict(Counter(str(effect["effect"]) for effect in effects)) + fingerprint = _hash( + [(row.external_id, row.source_revision, row.content_hash) for row in selected] + ) + run = ConnectorKnowledgeSyncRun( + tenant_id=principal.tenant_id, + profile_id=profile.id, + mode="migration_dry_run", + idempotency_key=payload.idempotency_key, + request_hash=request_hash, + status="completed", + cursor_before=None, + cursor_after=None, + high_watermark=profile.last_high_watermark, + counts=counts, + effects=effects, + diagnostics=diagnostics, + provenance={ + "target_space_ref": payload.target_space_ref, + "source_revision": profile.last_high_watermark or profile.discovery_revision, + "source_fingerprint": fingerprint, + "truncated": truncated, + }, + created_by=_actor_id(principal), + started_at=_now(), + finished_at=_now(), + ) + session.add(run) + session.flush() + _audit( + session, + principal, + action="migration.previewed", + object_type="connector_knowledge_sync_run", + object_id=run.id, + details={ + "profile_id": profile.id, + "target_space_ref": payload.target_space_ref, + "counts": counts, + "truncated": truncated, + }, + ) + response = _migration_response(run, payload.target_space_ref) + session.commit() + return response + + +def publish_page( + session: Session, + principal: ApiPrincipal, + *, + profile_id: str, + external_page_id: str, + payload: KnowledgePublishRequest, + transport: MediaWikiTransport | None = None, + registry: object | None = None, + durable_recovery: bool = True, +) -> KnowledgePublishResponse: + _require_scope(principal, KNOWLEDGE_PUBLISH_SCOPE) + profile = _profile(session, principal.tenant_id, profile_id) + if profile.status != "active": + raise KnowledgeConnectorError("profile_paused", "The knowledge profile is paused.") + if ( + profile.discovered_at is None + or "publish" not in profile.capabilities + or not _supports(profile.desired_maturity, "publish") + ): + raise KnowledgeConnectorError( + "publication_unsupported", + "The discovered provider maturity does not support publication.", + ) + configuration = _configuration( + session, principal.tenant_id, profile.configuration_id, active=True + ) + request_hash = _hash( + { + "profile_id": profile.id, + "external_page_id": external_page_id, + **payload.model_dump(mode="json"), + } + ) + replay = _run_replay( + session, + tenant_id=principal.tenant_id, + profile_id=profile.id, + mode="publish", + idempotency_key=payload.idempotency_key, + request_hash=request_hash, + ) + if replay is not None: + stored = _object_by_external_id( + session, profile.id, "page", external_page_id + ) + if stored is None: + raise KnowledgeConnectorError( + "publication_reconciliation_required", + "The publication evidence exists but the local reference is unavailable.", + ) + return KnowledgePublishResponse( + run=_run_item(replay), + external_reference=_external_reference_response(stored, profile), + accepted=replay.status == "completed", + outcome_unknown=replay.status == "outcome_unknown", + ) + run = ConnectorKnowledgeSyncRun( + tenant_id=principal.tenant_id, + profile_id=profile.id, + mode="publish", + idempotency_key=payload.idempotency_key, + request_hash=request_hash, + status="prepared", + counts={}, + effects=[], + diagnostics=[], + provenance={ + "configuration_revision": configuration.resource_revision, + "configuration_hash": configuration.effective_hash, + "expected_external_revision": payload.expected_external_revision, + }, + created_by=_actor_id(principal), + started_at=_now(), + ) + session.add(run) + session.flush() + recovery = None + if durable_recovery: + try: + recovery = begin_connector_external_mutation( + session, + tenant_id=principal.tenant_id, + provider_id=f"mediawiki:{profile.id}", + idempotency_key=payload.idempotency_key, + request_sha256=request_hash, + source_revision=payload.expected_external_revision, + cursor=profile.last_sync_cursor, + dry_run_evidence={ + "profile_revision": profile.resource_revision, + "configuration_revision": configuration.resource_revision, + "title": payload.title, + }, + resource_type="external_knowledge_page", + resource_id=external_page_id, + ) + except ConnectorRecoveryError as exc: + session.rollback() + raise KnowledgeConnectorError("recovery_unavailable", str(exc)) from exc + credential = _credential(session, principal, configuration) + try: + result = (transport or HttpMediaWikiTransport()).publish( + endpoint_url=_endpoint(configuration), + credential=credential, + title=payload.title, + body=payload.body, + summary=payload.summary, + expected_revision=payload.expected_external_revision, + minor=payload.minor, + ) + except MediaWikiTransportError as exc: + run.status = "outcome_unknown" if exc.outcome_unknown else "rejected" + run.diagnostics = [ + _diagnostic( + "error", exc.code, str(exc), retryable=exc.retryable + ) + ] + run.finished_at = _now() + session.commit() + if recovery is not None: + if exc.outcome_unknown: + recovery.outcome_unknown(summary=str(exc), provider_code=exc.code) + else: + recovery.reject(summary=str(exc), provider_code=exc.code) + if exc.outcome_unknown: + raise KnowledgeConnectorError( + "publication_outcome_unknown", + "Publication outcome is unknown; inspect the provider before retrying.", + ) from exc + raise KnowledgeConnectorError("publication_rejected", str(exc)) from exc + stored = _store_published_result( + session, + profile, + external_page_id=external_page_id, + payload=payload, + result=result, + ) + run.status = "completed" + run.counts = {"update": 1} + run.effects = [ + { + "effect": "update", + "external_id": stored.external_id, + "source_revision": stored.source_revision, + "title": stored.title, + } + ] + run.provenance = {**run.provenance, "provider": dict(result.evidence)} + run.finished_at = _now() + session.flush() + diagnostics: list[dict[str, Any]] = [] + _update_search( + session, + principal, + [stored], + registry=registry or get_registry(), + diagnostics=diagnostics, + ) + if diagnostics: + run.diagnostics = diagnostics + _audit( + session, + principal, + action="page.published", + object_type="external_knowledge_page", + object_id=stored.id, + details={ + "profile_id": profile.id, + "external_page_id": stored.external_id, + "external_revision_id": stored.external_revision_id, + "recovery_operation_id": recovery.operation_id if recovery else None, + }, + ) + response = KnowledgePublishResponse( + run=_run_item(run), + external_reference=_external_reference_response(stored, profile), + accepted=True, + outcome_unknown=False, + ) + if recovery is not None: + recovery.commit_success( + session, + provider_evidence={ + "verified": True, + "checks": { + "provider_result": "Success", + "stable_page_id": result.page_id, + "accepted_revision_id": result.revision_id, + }, + "page_id": result.page_id, + "revision_id": result.revision_id, + "canonical_url": result.canonical_url, + }, + ) + else: + session.commit() + return response + + +class ExternalKnowledgeCapability: + def list_references( + self, + session: object, + principal: object, + *, + profile_id: str, + limit: int = 100, + ) -> Sequence[ExternalObjectReference]: + if not isinstance(session, Session) or not isinstance(principal, ApiPrincipal): + raise TypeError("External knowledge references require platform context.") + rows, _cursor = list_objects( + session, principal, profile_id=profile_id, limit=min(limit, 500) + ) + profile = _profile(session, principal.tenant_id, profile_id) + return tuple( + ExternalObjectReference( + system=profile.product, + object_type=row.object_type, + object_id=row.external_id, + maturity=profile.discovered_maturity, + authority_mode=profile.source_authority_mode, + connector_id=profile.id, + canonical_url=row.canonical_url, + version=row.source_revision, + metadata={"title": row.title, "status": row.status}, + ) + for row in rows + ) + + +def _discovery( + payload: Mapping[str, Any], *, credential_present: bool +) -> dict[str, Any]: + query = payload.get("query") + if not isinstance(query, Mapping): + raise KnowledgeConnectorError( + "invalid_discovery", "MediaWiki site discovery returned no query metadata." + ) + general = query.get("general") + if not isinstance(general, Mapping): + raise KnowledgeConnectorError( + "invalid_discovery", "MediaWiki site discovery returned no general metadata." + ) + extensions = tuple( + _safe_extension(item) + for item in query.get("extensions") or () + if isinstance(item, Mapping) + ) + extension_names = { + str(item.get("name") or "").casefold() for item in extensions + } + blue_spice = any("bluespice" in name for name in extension_names) + generator = _optional_text(general.get("generator")) or "MediaWiki" + product = "bluespice" if blue_spice else "mediawiki" + product_version = _version( + next( + ( + str(item.get("version")) + for item in extensions + if "bluespice" in str(item.get("name") or "").casefold() + and item.get("version") + ), + None, + ) + or generator + ) + namespaces = tuple( + sorted( + ( + { + "id": int(value.get("id", key)), + "name": _optional_text(value.get("name") or value.get("*")) or "", + "canonical": _optional_text(value.get("canonical")), + "content": bool(value.get("content")), + } + for key, value in (query.get("namespaces") or {}).items() + if isinstance(value, Mapping) + ), + key=lambda value: value["id"], + ) + ) + userinfo = query.get("userinfo") + rights = { + str(value) + for value in ( + userinfo.get("rights") + if isinstance(userinfo, Mapping) + else () + ) + } + capabilities = [ + "discover", + "link", + "search", + "read", + "synchronize", + "migrate", + "namespaces", + "pages", + "revisions", + "users", + "categories", + "links", + "files", + "redirects", + "deletion_tombstones", + "permission_fallback", + ] + diagnostics: list[KnowledgeDiagnostic] = [] + if credential_present and (not rights or "edit" in rights): + capabilities.append("publish") + elif credential_present: + diagnostics.append( + KnowledgeDiagnostic( + severity="warning", + code="publish_right_missing", + message="The configured principal did not advertise the edit right.", + ) + ) + else: + diagnostics.append( + KnowledgeDiagnostic( + severity="info", + code="publish_credential_missing", + message="Read and synchronization are available; publication requires a governed credential.", + ) + ) + if blue_spice: + capabilities.extend(("bluespice", "permission_metadata", "discussions")) + else: + diagnostics.extend( + ( + KnowledgeDiagnostic( + severity="warning", + code="permission_mapping_defaulted", + message="The standard Action API does not expose complete page ACLs; configured namespace ACLs are used and every Search result is rechecked.", + ), + KnowledgeDiagnostic( + severity="warning", + code="discussion_mapping_lossy", + message="Discussion threads are mapped only when the provider supplies a supported discussion payload.", + ), + ) + ) + maturity = max( + (value for value in capabilities if value in INTEGRATION_MATURITY_ORDER), + key=INTEGRATION_MATURITY_ORDER.index, + ) + safe_payload = { + "generator": generator, + "product": product, + "product_version": product_version, + "api_version": _optional_text(general.get("phpversion")), + "namespaces": namespaces, + "extensions": extensions, + "capabilities": sorted(set(capabilities)), + } + return { + **safe_payload, + "maturity": maturity, + "health_status": "healthy", + "diagnostics": diagnostics, + "revision": _hash(safe_payload), + } + + +def _map_page( + profile: ConnectorKnowledgeProfile, + raw: Mapping[str, Any], + *, + mapping: Mapping[str, Any], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + external_id = _external_page_id(raw) + title = _optional_text(raw.get("title")) or f"Page {external_id}" + revisions = tuple( + item for item in raw.get("revisions") or () if isinstance(item, Mapping) + ) + revision = revisions[0] if revisions else {} + revision_id = _optional_text( + revision.get("revid") or raw.get("lastrevid") or raw.get("revid") + ) or _optional_text(raw.get("touched")) or "unknown" + body = _revision_content(revision) + timestamp = _parse_datetime( + revision.get("timestamp") or raw.get("touched") or raw.get("timestamp") + ) + visibility, acl_tokens, permission_diagnostics, permission_source = _mapped_acl( + profile, mapping=mapping, raw=raw, external_id=external_id + ) + categories = tuple( + _without_prefix(str(item.get("title") or ""), "Category:") + for item in raw.get("categories") or () + if isinstance(item, Mapping) and item.get("title") + ) + links = tuple( + { + "title": str(item.get("title") or ""), + "namespace_id": item.get("ns"), + } + for item in raw.get("links") or () + if isinstance(item, Mapping) and item.get("title") + ) + files = tuple( + _mapped_file(profile, item, observed_at=_now()) + for item in raw.get("images") or () + if isinstance(item, Mapping) and item.get("title") + ) + discussions = tuple( + _mapped_discussion(profile, item, external_id=external_id) + for item in raw.get("discussions") or () + if isinstance(item, Mapping) + ) + redirect_match = _REDIRECT.match(body) + redirect_target = ( + _optional_text(raw.get("redirect_target_pageid")) + or (redirect_match.group(1).strip() if redirect_match else None) + ) + target_path = _target_path(mapping, title) + canonical_url = _optional_text(raw.get("fullurl")) + macros = tuple(sorted(set(match.strip() for match in _MACRO.findall(body)))) + revision_author = _optional_text(revision.get("user")) + revision_author_reference = ( + ExternalObjectReference( + system=profile.product if profile.product != "unknown" else "mediawiki", + object_type="user", + object_id=revision_author, + maturity=profile.discovered_maturity, + authority_mode="linked_reference", + connector_id=profile.id, + observed_at=_now(), + metadata={"display_name": revision_author}, + ).to_dict() + if revision_author + else None + ) + reference = ExternalObjectReference( + system=profile.product if profile.product != "unknown" else "mediawiki", + object_type="page", + object_id=external_id, + maturity=profile.discovered_maturity, + authority_mode=profile.source_authority_mode, + connector_id=profile.id, + canonical_url=canonical_url, + version=revision_id, + etag=_optional_text(revision.get("sha1")), + observed_at=_now(), + metadata={ + "namespace_id": raw.get("ns"), + "title": title, + "redirect": bool(redirect_target), + }, + ) + mapped = { + "object_type": "page", + "external_id": external_id, + "external_page_id": external_id, + "external_revision_id": revision_id, + "namespace_id": _integer(raw.get("ns")), + "title": title, + "canonical_url": canonical_url, + "status": "redirect" if redirect_target else "active", + "redirect_target_external_id": redirect_target, + "source_revision": revision_id, + "source_updated_at": timestamp, + "visibility": visibility, + "acl_tokens": acl_tokens, + "mapped_data": { + "title": title, + "summary": _optional_text(revision.get("comment")) or "", + "body": body, + "content_model": _optional_text(revision.get("contentmodel")) or "wikitext", + "target_space_ref": mapping.get("target_space_ref"), + "target_path": target_path, + "namespace_id": _integer(raw.get("ns")), + "categories": list(categories), + "links": list(links), + "files": list(files), + "discussions": list(discussions), + "macros": list(macros), + "permission_source": permission_source, + "revision_author": revision_author, + "revision_author_reference": revision_author_reference, + "revision_minor": bool(revision.get("minor")), + "redirect_target_external_id": redirect_target, + "external_reference": reference.to_dict(), + }, + "provenance": { + "connector_profile_id": profile.id, + "source_system": reference.system, + "source_revision": revision_id, + "source_updated_at": timestamp.isoformat() if timestamp else None, + "observed_at": _now().isoformat(), + }, + } + mapped["content_hash"] = _mapped_content_hash( + mapped["mapped_data"], + visibility=visibility, + acl_tokens=acl_tokens, + ) + diagnostics = permission_diagnostics + if not discussions and "discussions" not in profile.capabilities: + diagnostics.append( + _diagnostic( + "warning", + "discussions_unavailable", + "No supported discussion payload was available for this page.", + object_ref=external_id, + field="discussions", + ) + ) + if files: + diagnostics.append( + _diagnostic( + "info", + "attachments_reference_only", + "Source files are stable external references; binary migration requires Files or DMS conflict resolution.", + object_ref=external_id, + field="files", + ) + ) + return mapped, diagnostics + + +def _upsert_object( + session: Session, + profile: ConnectorKnowledgeProfile, + mapped: Mapping[str, Any], + *, + cursor: object, +) -> tuple[ConnectorKnowledgeObject, dict[str, Any]]: + row = _object_by_external_id( + session, + profile.id, + str(mapped["object_type"]), + str(mapped["external_id"]), + ) + effect = "create" if row is None else "update" + if row is None: + row = ConnectorKnowledgeObject( + tenant_id=profile.tenant_id, + profile_id=profile.id, + object_type=str(mapped["object_type"]), + external_id=str(mapped["external_id"]), + external_page_id=_optional_text(mapped.get("external_page_id")), + external_revision_id=_optional_text(mapped.get("external_revision_id")), + namespace_id=_integer(mapped.get("namespace_id")), + title=str(mapped["title"]), + canonical_url=_optional_text(mapped.get("canonical_url")), + status=str(mapped["status"]), + redirect_target_external_id=_optional_text( + mapped.get("redirect_target_external_id") + ), + source_revision=str(mapped["source_revision"]), + content_hash=str(mapped["content_hash"]), + visibility=str(mapped["visibility"]), + acl_tokens=list(mapped["acl_tokens"]), + mapped_data=dict(mapped["mapped_data"]), + provenance=dict(mapped["provenance"]), + change_cursor=_optional_text(cursor), + source_updated_at=mapped.get("source_updated_at"), + observed_at=_now(), + ) + session.add(row) + elif ( + row.source_revision == mapped["source_revision"] + and row.content_hash == mapped["content_hash"] + and row.status == mapped["status"] + ): + effect = "unchanged" + row.change_cursor = _optional_text(cursor) or row.change_cursor + row.observed_at = _now() + else: + for field_name in ( + "external_page_id", + "external_revision_id", + "namespace_id", + "title", + "canonical_url", + "status", + "redirect_target_external_id", + "source_revision", + "content_hash", + "visibility", + "source_updated_at", + ): + setattr(row, field_name, mapped.get(field_name)) + row.acl_tokens = list(mapped["acl_tokens"]) + row.mapped_data = dict(mapped["mapped_data"]) + row.provenance = dict(mapped["provenance"]) + row.change_cursor = _optional_text(cursor) + row.observed_at = _now() + row.resource_revision += 1 + return row, { + "effect": effect, + "external_id": row.external_id, + "title": row.title, + "source_revision": row.source_revision, + "status": row.status, + } + + +def _delete_object( + session: Session, + profile: ConnectorKnowledgeProfile, + raw: Mapping[str, Any], + *, + mapping: Mapping[str, Any], + cursor: object, +) -> tuple[ConnectorKnowledgeObject | None, dict[str, Any]]: + supplied_id = _optional_text(raw.get("pageid")) + title = _optional_text(raw.get("title")) or "Deleted page" + external_id = supplied_id if supplied_id not in {None, "0"} else None + row = ( + _object_by_external_id(session, profile.id, "page", external_id) + if external_id + else session.scalar( + select(ConnectorKnowledgeObject).where( + ConnectorKnowledgeObject.profile_id == profile.id, + ConnectorKnowledgeObject.object_type == "page", + ConnectorKnowledgeObject.namespace_id == _integer(raw.get("ns")), + ConnectorKnowledgeObject.title == title, + ) + ) + ) + matched_by_title = row is not None and external_id is None + if row is not None: + external_id = row.external_id + if external_id is None: + external_id = f"deleted:{_hash({'title': title, 'ns': raw.get('ns')})[:32]}" + if row is None: + row = ConnectorKnowledgeObject( + tenant_id=profile.tenant_id, + profile_id=profile.id, + object_type="page", + external_id=external_id, + external_page_id=external_id, + namespace_id=_integer(raw.get("ns")), + title=title, + status="deleted", + source_revision=f"deleted:{_optional_text(raw.get('logid')) or cursor or 'unknown'}", + content_hash=_hash({"deleted": external_id, "cursor": cursor}), + visibility=profile.default_visibility, + acl_tokens=list(profile.default_acl_tokens), + mapped_data={ + "title": title, + "target_space_ref": mapping.get("target_space_ref"), + "target_path": _target_path(mapping, title), + "deleted": True, + }, + provenance={ + "connector_profile_id": profile.id, + "deletion_log_id": raw.get("logid"), + }, + change_cursor=_optional_text(cursor), + source_updated_at=_parse_datetime(raw.get("timestamp")), + observed_at=_now(), + ) + session.add(row) + effect = "delete" + elif row.status == "deleted": + effect = "unchanged" + else: + row.status = "deleted" + row.source_revision = ( + f"deleted:{_optional_text(raw.get('logid')) or cursor or row.source_revision}" + ) + row.content_hash = _hash({"deleted": external_id, "cursor": cursor}) + row.mapped_data = { + **dict(row.mapped_data or {}), + "body": "", + "deleted": True, + } + row.change_cursor = _optional_text(cursor) + row.observed_at = _now() + row.resource_revision += 1 + effect = "delete" + return row, { + "effect": effect, + "external_id": external_id, + "title": row.title, + "source_revision": row.source_revision, + "status": "deleted", + "matched_by": "title" if matched_by_title else "page_id", + "reason_code": ( + "unmatched_deletion_tombstone" + if row.external_id.startswith("deleted:") + else None + ), + } + + +def _store_published_result( + session: Session, + profile: ConnectorKnowledgeProfile, + *, + external_page_id: str, + payload: KnowledgePublishRequest, + result: MediaWikiPublishResult, +) -> ConnectorKnowledgeObject: + mapping = _namespace_mapping(profile, 0) or profile.namespace_mappings[0] + mapped = { + "object_type": "page", + "external_id": result.page_id or external_page_id, + "external_page_id": result.page_id or external_page_id, + "external_revision_id": result.revision_id, + "namespace_id": 0, + "title": result.title, + "canonical_url": result.canonical_url, + "status": "active", + "redirect_target_external_id": None, + "source_revision": result.revision_id, + "source_updated_at": _now(), + "visibility": profile.default_visibility, + "acl_tokens": list(profile.default_acl_tokens), + "mapped_data": { + "title": result.title, + "summary": payload.summary, + "body": payload.body, + "content_model": "wikitext", + "target_space_ref": mapping.get("target_space_ref"), + "target_path": _target_path(mapping, result.title), + "namespace_id": 0, + "categories": [], + "links": [], + "files": [], + "discussions": [], + "macros": sorted(set(match.strip() for match in _MACRO.findall(payload.body))), + "permission_source": "configured_fallback", + }, + "provenance": { + "connector_profile_id": profile.id, + "source_system": profile.product, + "source_revision": result.revision_id, + "published_by_govoplan": True, + "observed_at": _now().isoformat(), + }, + } + mapped["content_hash"] = _mapped_content_hash( + mapped["mapped_data"], + visibility=profile.default_visibility, + acl_tokens=profile.default_acl_tokens, + ) + row, _effect = _upsert_object( + session, + profile, + mapped, + cursor=result.revision_id, + ) + return row + + +def _mapped_acl( + profile: ConnectorKnowledgeProfile, + *, + mapping: Mapping[str, Any], + raw: Mapping[str, Any], + external_id: str, +) -> tuple[str, list[str], list[dict[str, Any]], str]: + permissions = raw.get("permissions") + diagnostics: list[dict[str, Any]] = [] + if isinstance(permissions, Mapping): + visibility = str(permissions.get("visibility") or "").strip() + tokens = [ + str(item).strip() + for item in permissions.get("acl_tokens") or () + if str(item).strip() + ] + if visibility in {"tenant", "restricted"}: + try: + _validate_effective_acl(visibility, tokens) + except KnowledgeConnectorError: + diagnostics.append( + _diagnostic( + "error", + "invalid_remote_acl", + "The remote permission mapping is invalid; the configured fallback ACL was applied.", + object_ref=external_id, + field="permissions", + ) + ) + else: + return visibility, tokens, diagnostics, "remote" + visibility = str(mapping.get("visibility") or profile.default_visibility) + tokens = list(mapping.get("acl_tokens") or profile.default_acl_tokens) + _validate_effective_acl(visibility, tokens) + diagnostics.append( + _diagnostic( + "warning", + "permission_mapping_defaulted", + "The configured namespace visibility and ACL were applied because authoritative page permissions were unavailable.", + object_ref=external_id, + field="permissions", + ) + ) + return visibility, tokens, diagnostics, "configured_fallback" + + +def _mapped_content_hash( + data: Mapping[str, Any], + *, + visibility: str, + acl_tokens: Sequence[str], +) -> str: + return _hash( + { + "title": data.get("title"), + "body": data.get("body"), + "categories": data.get("categories") or (), + "links": data.get("links") or (), + "files": data.get("files") or (), + "discussions": data.get("discussions") or (), + "redirect_target": data.get("redirect_target_external_id"), + "visibility": visibility, + "acl_tokens": tuple(acl_tokens), + } + ) + + +def _mapped_file( + profile: ConnectorKnowledgeProfile, + raw: Mapping[str, Any], + *, + observed_at: datetime, +) -> dict[str, Any]: + title = str(raw.get("title") or "") + external_id = _optional_text(raw.get("pageid")) or title + return { + "name": _without_prefix(title, "File:"), + "external_id": external_id, + "reference": ExternalObjectReference( + system=profile.product if profile.product != "unknown" else "mediawiki", + object_type="file", + object_id=external_id, + maturity=profile.discovered_maturity, + authority_mode=profile.source_authority_mode, + connector_id=profile.id, + canonical_url=_optional_text(raw.get("url")), + version=_optional_text(raw.get("sha1") or raw.get("timestamp")), + observed_at=observed_at, + metadata={"title": title}, + ).to_dict(), + } + + +def _mapped_discussion( + profile: ConnectorKnowledgeProfile, + raw: Mapping[str, Any], + *, + external_id: str, +) -> dict[str, Any]: + discussion_id = _optional_text(raw.get("id")) or _hash(raw)[:32] + return { + "external_id": discussion_id, + "author": _optional_text(raw.get("author")), + "body": str(raw.get("body") or "")[:20_000], + "created_at": _optional_text(raw.get("created_at")), + "reference": ExternalObjectReference( + system=profile.product if profile.product != "unknown" else "mediawiki", + object_type="discussion", + object_id=discussion_id, + maturity=profile.discovered_maturity, + authority_mode=profile.source_authority_mode, + connector_id=profile.id, + version=_optional_text(raw.get("revision")), + observed_at=_now(), + metadata={"page_id": external_id}, + ).to_dict(), + } + + +def _update_search( + session: Session, + principal: ApiPrincipal, + rows: Sequence[ConnectorKnowledgeObject], + *, + registry: object | None, + diagnostics: list[dict[str, Any]], + source_active: bool = True, +) -> None: + writer = search_index_writer(registry) + if writer is None: + diagnostics.append( + _diagnostic( + "info", + "search_unavailable", + "Search is not installed; synchronized pages remain available through the connector.", + ) + ) + return + from govoplan_connectors.backend.knowledge_search import search_document + + for row in rows: + try: + if row.status == "deleted" or not source_active: + writer.delete_document( + session, + principal, + tenant_id=row.tenant_id, + module_id="connectors", + resource_type=KNOWLEDGE_RESOURCE_TYPE, + resource_id=row.id, + ) + else: + document = search_document(session, row) + if document is not None: + writer.upsert_document(session, principal, document=document) + except Exception: + diagnostics.append( + _diagnostic( + "warning", + "search_update_deferred", + "The Search projection could not be updated immediately; authorization rechecks still fail closed and the next rebuild will reconcile it.", + object_ref=row.external_id, + retryable=True, + ) + ) + + +def _profile_item( + item: ConnectorKnowledgeProfile, configuration: ConnectorConfiguration +) -> KnowledgeProfileItem: + return KnowledgeProfileItem( + id=item.id, + tenant_id=item.tenant_id, + configuration_id=item.configuration_id, + status=item.status, + product=item.product, + product_version=item.product_version, + desired_maturity=item.desired_maturity, + discovered_maturity=item.discovered_maturity, + source_authority_mode=item.source_authority_mode, + default_visibility=item.default_visibility, + default_acl_tokens=list(item.default_acl_tokens or ()), + namespace_mappings=list(item.namespace_mappings or ()), + capabilities=list(item.capabilities or ()), + discovery_revision=item.discovery_revision, + health_status=item.health_status, + health_details=dict(item.health_details or {}), + discovered_at=item.discovered_at, + last_sync_cursor=item.last_sync_cursor, + last_high_watermark=item.last_high_watermark, + resource_revision=item.resource_revision, + credential_reference_present=bool(configuration.credential_ref), + endpoint_configured=bool(configuration.endpoint_url), + created_at=item.created_at, + updated_at=item.updated_at, + ) + + +def _object_item( + row: ConnectorKnowledgeObject, profile: ConnectorKnowledgeProfile +) -> KnowledgeObjectItem: + return KnowledgeObjectItem( + id=row.id, + profile_id=row.profile_id, + object_type=row.object_type, + external_id=row.external_id, + external_page_id=row.external_page_id, + external_revision_id=row.external_revision_id, + namespace_id=row.namespace_id, + title=row.title, + canonical_url=row.canonical_url, + status=row.status, + redirect_target_external_id=row.redirect_target_external_id, + source_revision=row.source_revision, + visibility=row.visibility, + acl_tokens=list(row.acl_tokens or ()), + mapped_data=dict(row.mapped_data or {}), + external_reference=_external_reference_response(row, profile), + source_updated_at=row.source_updated_at, + observed_at=row.observed_at, + resource_revision=row.resource_revision, + ) + + +def _external_reference_response( + row: ConnectorKnowledgeObject, profile: ConnectorKnowledgeProfile +) -> KnowledgeExternalReferenceResponse: + return KnowledgeExternalReferenceResponse.model_validate( + ExternalObjectReference( + system=profile.product if profile.product != "unknown" else "mediawiki", + object_type=row.object_type, + object_id=row.external_id, + maturity=profile.discovered_maturity, + authority_mode=profile.source_authority_mode, + connector_id=profile.id, + canonical_url=row.canonical_url, + version=row.source_revision, + etag=row.content_hash, + observed_at=row.observed_at, + metadata={ + "title": row.title, + "status": row.status, + "namespace_id": row.namespace_id, + "external_revision_id": row.external_revision_id, + }, + ).to_dict() + ) + + +def _run_item(row: ConnectorKnowledgeSyncRun) -> KnowledgeSyncRunItem: + return KnowledgeSyncRunItem( + id=row.id, + tenant_id=row.tenant_id, + profile_id=row.profile_id, + mode=row.mode, + idempotency_key=row.idempotency_key, + status=row.status, + cursor_before=row.cursor_before, + cursor_after=row.cursor_after, + high_watermark=row.high_watermark, + counts=dict(row.counts or {}), + effects=list(row.effects or ()), + diagnostics=[ + KnowledgeDiagnostic.model_validate(item) + for item in row.diagnostics or () + ], + provenance=dict(row.provenance or {}), + started_at=row.started_at, + finished_at=row.finished_at, + created_at=row.created_at, + ) + + +def _migration_response( + row: ConnectorKnowledgeSyncRun, target_space_ref: str +) -> KnowledgeMigrationDryRunResponse: + provenance = dict(row.provenance or {}) + diagnostics = [ + KnowledgeDiagnostic.model_validate(item) for item in row.diagnostics or () + ] + return KnowledgeMigrationDryRunResponse( + run=_run_item(row), + target_space_ref=target_space_ref, + source_revision=str(provenance.get("source_revision") or "unknown"), + source_fingerprint=str(provenance.get("source_fingerprint") or "unknown"), + summary={key: int(value) for key, value in dict(row.counts or {}).items()}, + effects=list(row.effects or ()), + diagnostics=diagnostics, + truncated=bool(provenance.get("truncated")), + can_apply=( + not bool(provenance.get("truncated")) + and not any(item.severity == "error" for item in diagnostics) + and int(dict(row.counts or {}).get("conflict", 0)) == 0 + ), + ) + + +def _configuration( + session: Session, tenant_id: str, configuration_id: str, *, active: bool +) -> ConnectorConfiguration: + row = session.scalar( + select(ConnectorConfiguration).where( + ConnectorConfiguration.tenant_id == tenant_id, + ConnectorConfiguration.id == configuration_id, + ) + ) + if row is None: + raise KnowledgeConnectorError( + "configuration_not_found", "Connector configuration not found." + ) + if active and row.status != "active": + raise KnowledgeConnectorError( + "configuration_inactive", "Connector configuration is not active." + ) + return row + + +def _profile( + session: Session, tenant_id: str, profile_id: str +) -> ConnectorKnowledgeProfile: + row = session.scalar( + select(ConnectorKnowledgeProfile).where( + ConnectorKnowledgeProfile.tenant_id == tenant_id, + ConnectorKnowledgeProfile.id == profile_id, + ) + ) + if row is None: + raise KnowledgeConnectorError( + "profile_not_found", "Knowledge connector profile not found." + ) + return row + + +def _configurations( + session: Session, + tenant_id: str, + profiles: Sequence[ConnectorKnowledgeProfile], +) -> dict[str, ConnectorConfiguration]: + ids = tuple(dict.fromkeys(item.configuration_id for item in profiles)) + if not ids: + return {} + return { + row.id: row + for row in session.scalars( + select(ConnectorConfiguration).where( + ConnectorConfiguration.tenant_id == tenant_id, + ConnectorConfiguration.id.in_(ids), + ) + ) + } + + +def _assert_knowledge_configuration( + session: Session, configuration: ConnectorConfiguration +) -> None: + definition = session.scalar( + select(ConnectorDefinition).where( + ConnectorDefinition.id == configuration.definition_id, + ConnectorDefinition.tenant_id == configuration.tenant_id, + ) + ) + specification = dict(configuration.effective_configuration or {}) + provider = str(specification.get("provider") or "").casefold() + protocol = str(specification.get("protocol") or "").casefold() + if provider not in {"mediawiki", "bluespice", "knowledge"} or protocol not in { + "mediawiki", + "action_api", + "mediawiki_action_api", + }: + raise KnowledgeConnectorError( + "configuration_type_invalid", + "The selected configuration is not a MediaWiki/BlueSpice Action API connector.", + ) + if definition is None or definition.status != "active": + raise KnowledgeConnectorError( + "definition_unavailable", "The connector definition is unavailable." + ) + + +def _credential( + session: Session, + principal: ApiPrincipal, + configuration: ConnectorConfiguration, +) -> dict[str, Any] | None: + if not configuration.credential_ref: + return None + try: + value = resolve_credential_envelope( + session, + credential_id=configuration.credential_ref, + context=CredentialAccessContext( + tenant_id=principal.tenant_id, + user_id=_actor_id(principal), + group_ids=frozenset(principal.principal.group_ids), + target_scope_type="tenant", + target_scope_id=principal.tenant_id, + module_id="connectors", + server_ref=_endpoint(configuration), + ), + ) + except CredentialEnvelopeError as exc: + raise KnowledgeConnectorError( + "credential_unavailable", + "The connector credential is inactive, unavailable, or outside its approved scope.", + ) from exc + return {**dict(value.public_data), **dict(value.secret_data)} + + +def _run_replay( + session: Session, + *, + tenant_id: str, + profile_id: str, + mode: str, + idempotency_key: str, + request_hash: str, +) -> ConnectorKnowledgeSyncRun | None: + row = session.scalar( + select(ConnectorKnowledgeSyncRun).where( + ConnectorKnowledgeSyncRun.tenant_id == tenant_id, + ConnectorKnowledgeSyncRun.profile_id == profile_id, + ConnectorKnowledgeSyncRun.mode == mode, + ConnectorKnowledgeSyncRun.idempotency_key == idempotency_key, + ) + ) + if row is None: + return None + if row.request_hash != request_hash: + raise KnowledgeConnectorError( + "idempotency_conflict", + "This idempotency key was already used with a different request.", + ) + if row.status in {"running", "prepared", "outcome_unknown"}: + raise KnowledgeConnectorError( + "operation_unresolved", + "This connector operation is active or requires reconciliation.", + ) + return row + + +def _object_by_external_id( + session: Session, profile_id: str, object_type: str, external_id: str +) -> ConnectorKnowledgeObject | None: + return session.scalar( + select(ConnectorKnowledgeObject).where( + ConnectorKnowledgeObject.profile_id == profile_id, + ConnectorKnowledgeObject.object_type == object_type, + ConnectorKnowledgeObject.external_id == external_id, + ) + ) + + +def _namespace_mapping( + profile: ConnectorKnowledgeProfile, namespace_id: object +) -> Mapping[str, Any] | None: + expected = _integer(namespace_id) + return next( + ( + item + for item in profile.namespace_mappings or () + if isinstance(item, Mapping) + and _integer(item.get("source_namespace_id")) == expected + ), + None, + ) + + +def _target_path(mapping: Mapping[str, Any], title: str) -> str: + source_name = str(mapping.get("source_name") or "").strip() + unqualified = _without_prefix(title, f"{source_name}:") if source_name else title + prefix = str(mapping.get("target_path_prefix") or "").strip(" /_") + slug = "/".join( + quote(part.strip().replace(" ", "-"), safe="-._~") + for part in unqualified.split("/") + if part.strip() + ) + return "/".join(value for value in (prefix, slug) if value) + + +def _validate_effective_acl(visibility: str, tokens: Sequence[str]) -> None: + if visibility not in {"tenant", "restricted"}: + raise KnowledgeConnectorError( + "visibility_invalid", "Knowledge visibility is invalid." + ) + if visibility == "restricted" and not tokens: + raise KnowledgeConnectorError( + "acl_required", "Restricted knowledge requires at least one ACL token." + ) + + +def _require_scope(principal: ApiPrincipal, scope: str) -> None: + if not has_scope(principal, scope): + raise KnowledgeConnectorError("access_denied", f"Missing required scope: {scope}") + + +def _require_any_scope(principal: ApiPrincipal, *scopes: str) -> None: + if not any(has_scope(principal, scope) for scope in scopes): + raise KnowledgeConnectorError( + "access_denied", f"Missing one of the required scopes: {', '.join(scopes)}" + ) + + +def _endpoint(configuration: ConnectorConfiguration) -> str: + value = _optional_text(configuration.endpoint_url) + if not value: + raise KnowledgeConnectorError( + "endpoint_missing", "The connector configuration has no endpoint URL." + ) + return value + + +def _external_page_id(raw: Mapping[str, Any]) -> str: + value = _optional_text(raw.get("pageid")) + if not value: + raise KnowledgeConnectorError( + "external_identity_missing", "A MediaWiki page change omitted its stable page id." + ) + return value + + +def _revision_content(revision: Mapping[str, Any]) -> str: + slots = revision.get("slots") + if isinstance(slots, Mapping): + main = slots.get("main") + if isinstance(main, Mapping): + for name in ("content", "*", "text"): + if main.get(name) is not None: + return str(main[name])[:200_000] + for name in ("content", "*", "text"): + if revision.get(name) is not None: + return str(revision[name])[:200_000] + return "" + + +def _safe_extension(value: Mapping[str, Any]) -> dict[str, Any]: + return { + key: value[key] + for key in ("name", "version", "type", "license-name") + if key in value and value[key] is not None + } + + +def _diagnostic( + severity: str, + code: str, + message: str, + *, + object_ref: str | None = None, + field: str | None = None, + retryable: bool = False, + details: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + return { + "severity": severity, + "code": code, + "message": message, + "object_ref": object_ref, + "field": field, + "retryable": retryable, + "details": dict(details or {}), + } + + +def _audit( + session: Session, + principal: ApiPrincipal, + *, + action: str, + object_type: str, + object_id: str, + details: Mapping[str, Any], +) -> None: + audit_event( + session, + tenant_id=principal.tenant_id, + user_id=getattr(principal.user, "id", None), + api_key_id=principal.api_key_id, + action=f"connectors.knowledge.{action}", + object_type=object_type, + object_id=object_id, + details=dict(details), + ) + + +def _actor_id(principal: ApiPrincipal) -> str | None: + return _optional_text( + getattr(principal.user, "id", None) + or getattr(principal.account, "id", None) + or principal.principal.account_id + ) + + +def _supports(current: str, required: str) -> bool: + try: + return INTEGRATION_MATURITY_ORDER.index(current) >= INTEGRATION_MATURITY_ORDER.index( + required # type: ignore[arg-type] + ) + except ValueError: + return False + + +def _parse_datetime(value: object) -> datetime | None: + text = _optional_text(value) + if not text: + return None + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def _version(value: object) -> str | None: + match = re.search(r"\d+(?:\.\d+){1,3}(?:[-+._a-zA-Z0-9]*)?", str(value or "")) + return match.group(0) if match else None + + +def _without_prefix(value: str, prefix: str) -> str: + return value[len(prefix) :] if value.casefold().startswith(prefix.casefold()) else value + + +def _hash(value: object) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode() + ).hexdigest() + + +def _optional_text(value: object) -> str | None: + normalized = str(value or "").strip() + return normalized or None + + +def _integer(value: object) -> int | None: + try: + return int(value) if value is not None else None + except (TypeError, ValueError): + return None + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +__all__ = [ + "ExternalKnowledgeCapability", + "KNOWLEDGE_ADMIN_SCOPE", + "KNOWLEDGE_CAPABILITY", + "KNOWLEDGE_INTERFACE_VERSION", + "KNOWLEDGE_MIGRATE_SCOPE", + "KNOWLEDGE_PROVIDER_ID", + "KNOWLEDGE_PUBLISH_SCOPE", + "KNOWLEDGE_READ_SCOPE", + "KNOWLEDGE_RESOURCE_TYPE", + "KNOWLEDGE_SYNC_SCOPE", + "KnowledgeConnectorError", + "create_profile", + "discover_profile", + "list_objects", + "list_profiles", + "list_runs", + "migration_dry_run", + "publish_page", + "synchronize_profile", + "update_profile", +] diff --git a/src/govoplan_connectors/backend/knowledge_schemas.py b/src/govoplan_connectors/backend/knowledge_schemas.py new file mode 100644 index 0000000..2acef2a --- /dev/null +++ b/src/govoplan_connectors/backend/knowledge_schemas.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +KnowledgeMaturity = Literal[ + "discover", + "link", + "search", + "read", + "publish", + "synchronize", + "migrate", +] +KnowledgeVisibility = Literal["tenant", "restricted"] + + +class KnowledgeNamespaceMapping(BaseModel): + model_config = ConfigDict(extra="forbid") + + source_namespace_id: int + source_name: str = Field(default="", max_length=200) + target_space_ref: str = Field(min_length=1, max_length=255) + target_path_prefix: str = Field(default="", max_length=500) + include: bool = True + visibility: KnowledgeVisibility | None = None + acl_tokens: list[str] = Field(default_factory=list, max_length=500) + + @field_validator("acl_tokens") + @classmethod + def normalize_acl_tokens(cls, values: list[str]) -> list[str]: + return _normalized_tokens(values) + + @model_validator(mode="after") + def restricted_mapping_requires_acl(self) -> "KnowledgeNamespaceMapping": + if self.visibility == "restricted" and not self.acl_tokens: + raise ValueError("Restricted namespace mappings require ACL tokens") + return self + + +class KnowledgeProfileCreateRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + configuration_id: str = Field(min_length=1, max_length=36) + desired_maturity: KnowledgeMaturity = "read" + source_authority_mode: Literal[ + "external_authoritative", + "external_mirror", + "governed_sync", + "linked_reference", + ] = "external_mirror" + default_visibility: KnowledgeVisibility = "restricted" + default_acl_tokens: list[str] = Field(default_factory=list, max_length=500) + namespace_mappings: list[KnowledgeNamespaceMapping] = Field( + min_length=1, max_length=500 + ) + + @field_validator("default_acl_tokens") + @classmethod + def normalize_acl_tokens(cls, values: list[str]) -> list[str]: + return _normalized_tokens(values) + + @model_validator(mode="after") + def restricted_profile_requires_acl(self) -> "KnowledgeProfileCreateRequest": + if self.default_visibility == "restricted" and not self.default_acl_tokens: + raise ValueError("Restricted knowledge profiles require default ACL tokens") + return self + + +class KnowledgeProfileUpdateRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + expected_resource_revision: int = Field(ge=1) + status: Literal["active", "paused"] | None = None + desired_maturity: KnowledgeMaturity | None = None + source_authority_mode: Literal[ + "external_authoritative", + "external_mirror", + "governed_sync", + "linked_reference", + ] | None = None + default_visibility: KnowledgeVisibility | None = None + default_acl_tokens: list[str] | None = Field(default=None, max_length=500) + namespace_mappings: list[KnowledgeNamespaceMapping] | None = Field( + default=None, min_length=1, max_length=500 + ) + + @field_validator("default_acl_tokens") + @classmethod + def normalize_acl_tokens(cls, values: list[str] | None) -> list[str] | None: + return _normalized_tokens(values) if values is not None else None + + +class KnowledgeProfileItem(BaseModel): + id: str + tenant_id: str + configuration_id: str + status: str + product: str + product_version: str | None = None + desired_maturity: str + discovered_maturity: str + source_authority_mode: str + default_visibility: str + default_acl_tokens: list[str] + namespace_mappings: list[dict[str, Any]] + capabilities: list[str] + discovery_revision: str | None = None + health_status: str + health_details: dict[str, Any] + discovered_at: datetime | None = None + last_sync_cursor: str | None = None + last_high_watermark: str | None = None + resource_revision: int + credential_reference_present: bool = False + endpoint_configured: bool = False + created_at: datetime + updated_at: datetime + + +class KnowledgeProfileListResponse(BaseModel): + items: list[KnowledgeProfileItem] + + +class KnowledgeDiagnostic(BaseModel): + severity: Literal["info", "warning", "error"] + code: str + message: str + object_ref: str | None = None + field: str | None = None + retryable: bool = False + details: dict[str, Any] = Field(default_factory=dict) + + +class KnowledgeDiscoveryResponse(BaseModel): + profile: KnowledgeProfileItem + product: str + product_version: str | None = None + api_version: str | None = None + capabilities: list[str] + namespaces: list[dict[str, Any]] + extensions: list[dict[str, Any]] + maturity: str + health_status: str + diagnostics: list[KnowledgeDiagnostic] + revision: str + + +class KnowledgeSyncRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + idempotency_key: str = Field(min_length=1, max_length=255) + cursor: str | None = Field(default=None, max_length=500) + force_full: bool = False + limit: int = Field(default=100, ge=1, le=500) + + +class KnowledgeSyncRunItem(BaseModel): + id: str + tenant_id: str + profile_id: str + mode: str + idempotency_key: str + status: str + cursor_before: str | None = None + cursor_after: str | None = None + high_watermark: str | None = None + counts: dict[str, Any] + effects: list[dict[str, Any]] + diagnostics: list[KnowledgeDiagnostic] + provenance: dict[str, Any] + started_at: datetime + finished_at: datetime | None = None + created_at: datetime + + +class KnowledgeSyncRunListResponse(BaseModel): + items: list[KnowledgeSyncRunItem] + + +class KnowledgeExternalReferenceResponse(BaseModel): + system: str + object_type: str + object_id: str + maturity: str + authority_mode: str + connector_id: str | None = None + canonical_url: str | None = None + version: str | None = None + etag: str | None = None + observed_at: str | None = None + metadata: dict[str, Any] + + +class KnowledgeObjectItem(BaseModel): + id: str + profile_id: str + object_type: str + external_id: str + external_page_id: str | None = None + external_revision_id: str | None = None + namespace_id: int | None = None + title: str + canonical_url: str | None = None + status: str + redirect_target_external_id: str | None = None + source_revision: str + visibility: str + acl_tokens: list[str] + mapped_data: dict[str, Any] + external_reference: KnowledgeExternalReferenceResponse + source_updated_at: datetime | None = None + observed_at: datetime + resource_revision: int + + +class KnowledgeObjectListResponse(BaseModel): + items: list[KnowledgeObjectItem] + next_cursor: str | None = None + + +class KnowledgeMigrationTargetState(BaseModel): + model_config = ConfigDict(extra="forbid") + + path: str = Field(min_length=1, max_length=500) + source_external_id: str | None = Field(default=None, max_length=255) + attachment_names: list[str] = Field(default_factory=list, max_length=500) + + +class KnowledgeMigrationDryRunRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + idempotency_key: str = Field(min_length=1, max_length=255) + target_space_ref: str = Field(min_length=1, max_length=255) + max_items: int = Field(default=100, ge=1, le=500) + supported_macros: list[str] = Field(default_factory=list, max_length=200) + existing_targets: list[KnowledgeMigrationTargetState] = Field( + default_factory=list, max_length=5_000 + ) + + +class KnowledgeMigrationDryRunResponse(BaseModel): + run: KnowledgeSyncRunItem + target_space_ref: str + source_revision: str + source_fingerprint: str + summary: dict[str, int] + effects: list[dict[str, Any]] + diagnostics: list[KnowledgeDiagnostic] + truncated: bool + can_apply: bool + + +class KnowledgePublishRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + idempotency_key: str = Field(min_length=1, max_length=255) + title: str = Field(min_length=1, max_length=500) + body: str = Field(max_length=200_000) + summary: str = Field(default="", max_length=500) + expected_external_revision: str | None = Field(default=None, max_length=255) + minor: bool = False + + +class KnowledgePublishResponse(BaseModel): + run: KnowledgeSyncRunItem + external_reference: KnowledgeExternalReferenceResponse + accepted: bool + outcome_unknown: bool = False + + +def _normalized_tokens(values: list[str]) -> list[str]: + normalized = [str(item).strip() for item in values] + if any(not item or len(item) > 500 for item in normalized): + raise ValueError("ACL tokens must contain 1 to 500 characters") + if len(normalized) != len(set(normalized)): + raise ValueError("ACL tokens must be unique") + return normalized + + +__all__ = [ + "KnowledgeDiagnostic", + "KnowledgeDiscoveryResponse", + "KnowledgeExternalReferenceResponse", + "KnowledgeMigrationDryRunRequest", + "KnowledgeMigrationDryRunResponse", + "KnowledgeMigrationTargetState", + "KnowledgeNamespaceMapping", + "KnowledgeObjectItem", + "KnowledgeObjectListResponse", + "KnowledgeProfileCreateRequest", + "KnowledgeProfileItem", + "KnowledgeProfileListResponse", + "KnowledgeProfileUpdateRequest", + "KnowledgePublishRequest", + "KnowledgePublishResponse", + "KnowledgeSyncRequest", + "KnowledgeSyncRunItem", + "KnowledgeSyncRunListResponse", +] diff --git a/src/govoplan_connectors/backend/knowledge_search.py b/src/govoplan_connectors/backend/knowledge_search.py new file mode 100644 index 0000000..3ba4312 --- /dev/null +++ b/src/govoplan_connectors/backend/knowledge_search.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from urllib.parse import quote + +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from govoplan_core.core.external_references import ExternalObjectReference +from govoplan_core.core.modules import ModuleContext +from govoplan_core.core.search import ( + SearchAuthorizationRequest, + SearchBackfillPage, + SearchBackfillRequest, + SearchDocument, + SearchResourceType, +) +from govoplan_connectors.backend.db.models import ( + ConnectorKnowledgeObject, + ConnectorKnowledgeProfile, +) +from govoplan_connectors.backend.knowledge_connector import ( + KNOWLEDGE_PROVIDER_ID, + KNOWLEDGE_READ_SCOPE, + KNOWLEDGE_RESOURCE_TYPE, +) + + +class ExternalKnowledgeSearchSource: + def resource_types(self) -> Sequence[SearchResourceType]: + return ( + SearchResourceType( + provider_id=KNOWLEDGE_PROVIDER_ID, + module_id="connectors", + resource_type=KNOWLEDGE_RESOURCE_TYPE, + label="External knowledge pages", + requires_authorization_recheck=True, + ), + ) + + def backfill( + self, session: object, *, request: SearchBackfillRequest + ) -> SearchBackfillPage: + _assert_source(request.provider_id, request.resource_type) + db = _session(session) + query = ( + select(ConnectorKnowledgeObject, ConnectorKnowledgeProfile) + .join( + ConnectorKnowledgeProfile, + ConnectorKnowledgeProfile.id == ConnectorKnowledgeObject.profile_id, + ) + .where( + ConnectorKnowledgeObject.tenant_id == request.tenant_id, + ConnectorKnowledgeObject.object_type == "page", + ConnectorKnowledgeObject.status != "deleted", + ConnectorKnowledgeProfile.tenant_id == request.tenant_id, + ConnectorKnowledgeProfile.status == "active", + ) + ) + if request.cursor: + query = query.where(ConnectorKnowledgeObject.id > request.cursor) + rows = tuple( + db.execute( + query.order_by(ConnectorKnowledgeObject.id.asc()).limit( + request.limit + 1 + ) + ) + ) + has_more = len(rows) > request.limit + selected = rows[: request.limit] + watermark = db.scalar( + select(func.max(ConnectorKnowledgeObject.updated_at)) + .join( + ConnectorKnowledgeProfile, + ConnectorKnowledgeProfile.id == ConnectorKnowledgeObject.profile_id, + ) + .where( + ConnectorKnowledgeObject.tenant_id == request.tenant_id, + ConnectorKnowledgeObject.object_type == "page", + ConnectorKnowledgeObject.status != "deleted", + ConnectorKnowledgeProfile.tenant_id == request.tenant_id, + ConnectorKnowledgeProfile.status == "active", + ) + ) + return SearchBackfillPage( + documents=tuple(search_document(db, row, profile) for row, profile in selected), + next_cursor=selected[-1][0].id if has_more and selected else None, + complete=not has_more, + high_watermark=watermark.isoformat() if watermark else None, + ) + + def authorize( + self, + session: object, + principal: object, + *, + requests: Sequence[SearchAuthorizationRequest], + ) -> Mapping[str, bool]: + db = _session(session) + tenant_id = str(getattr(principal, "tenant_id", "") or "") + can_read = _has_scope(principal, KNOWLEDGE_READ_SCOPE) + tokens = set(_principal_acl_tokens(principal)) + decisions = {item.reference.key: False for item in requests} + if not tenant_id or not can_read: + return decisions + for request in requests: + reference = request.reference + if ( + reference.tenant_id != tenant_id + or reference.module_id != "connectors" + or reference.resource_type != KNOWLEDGE_RESOURCE_TYPE + ): + continue + row = db.execute( + select(ConnectorKnowledgeObject, ConnectorKnowledgeProfile) + .join( + ConnectorKnowledgeProfile, + ConnectorKnowledgeProfile.id + == ConnectorKnowledgeObject.profile_id, + ) + .where( + ConnectorKnowledgeObject.tenant_id == tenant_id, + ConnectorKnowledgeObject.id == reference.resource_id, + ConnectorKnowledgeObject.object_type == "page", + ConnectorKnowledgeObject.status != "deleted", + ConnectorKnowledgeProfile.tenant_id == tenant_id, + ConnectorKnowledgeProfile.status == "active", + ) + ).first() + if row is None: + continue + page, _profile = row + decisions[reference.key] = page.visibility == "tenant" or bool( + tokens.intersection(str(value) for value in page.acl_tokens or ()) + ) + return decisions + + +def create_external_knowledge_search_source( + _context: ModuleContext, +) -> ExternalKnowledgeSearchSource: + return ExternalKnowledgeSearchSource() + + +def search_document( + session: Session, + row: ConnectorKnowledgeObject, + profile: ConnectorKnowledgeProfile | None = None, +) -> SearchDocument: + if profile is None: + profile = session.scalar( + select(ConnectorKnowledgeProfile).where( + ConnectorKnowledgeProfile.tenant_id == row.tenant_id, + ConnectorKnowledgeProfile.id == row.profile_id, + ) + ) + if profile is None: + raise ValueError("External knowledge profile is unavailable.") + data = dict(row.mapped_data or {}) + categories = tuple(str(value)[:200] for value in data.get("categories") or ()) + links = tuple( + str(value.get("title") or value.get("external_id") or "")[:200] + for value in data.get("links") or () + if isinstance(value, Mapping) + ) + external_reference = ExternalObjectReference( + system=profile.product if profile.product != "unknown" else "mediawiki", + object_type=row.object_type, + object_id=row.external_id, + maturity=profile.discovered_maturity, + authority_mode=profile.source_authority_mode, + connector_id=profile.id, + canonical_url=row.canonical_url, + version=row.source_revision, + etag=row.content_hash, + observed_at=row.observed_at, + metadata={ + "title": row.title, + "namespace_id": row.namespace_id, + "external_revision_id": row.external_revision_id, + }, + ) + return SearchDocument( + tenant_id=row.tenant_id, + module_id="connectors", + provider_id=KNOWLEDGE_PROVIDER_ID, + resource_type=KNOWLEDGE_RESOURCE_TYPE, + resource_id=row.id, + title=row.title, + url=( + "/connectors/knowledge?profileId=" + f"{quote(row.profile_id, safe='')}&objectId={quote(row.id, safe='')}" + ), + summary=str(data.get("summary") or data.get("body") or "")[:4_000] or None, + body=str(data.get("body") or "")[:200_000] or None, + keywords=tuple(dict.fromkeys((*categories, *links)))[:100], + visibility=row.visibility, + acl_tokens=( + tuple(str(value) for value in row.acl_tokens or ()) + if row.visibility == "restricted" + else () + ), + external_reference=external_reference, + metadata={ + "profile_id": row.profile_id, + "external_page_id": row.external_page_id, + "namespace_id": row.namespace_id, + "target_space_ref": data.get("target_space_ref"), + "target_path": data.get("target_path"), + "status": row.status, + "redirect_target_external_id": row.redirect_target_external_id, + }, + source_revision=row.source_revision, + change_cursor=row.change_cursor, + source_updated_at=row.source_updated_at or row.observed_at, + requires_authorization_recheck=True, + ) + + +def _principal_acl_tokens(principal: object) -> tuple[str, ...]: + values: list[str] = [] + for prefix, attribute in ( + ("account", "account_id"), + ("membership", "membership_id"), + ("identity", "identity_id"), + ): + value = getattr(principal, attribute, None) + if value: + values.append(f"{prefix}:{value}") + for prefix, attribute in ( + ("group", "group_ids"), + ("role", "role_ids"), + ("function", "function_assignment_ids"), + ("scope", "scopes"), + ): + values.extend( + f"{prefix}:{value}" + for value in getattr(principal, attribute, ()) + if value + ) + return tuple(dict.fromkeys(values))[:500] + + +def _has_scope(principal: object, required: str) -> bool: + check = getattr(principal, "has", None) + if callable(check): + return bool(check(required)) + return required in getattr(principal, "scopes", ()) + + +def _assert_source(provider_id: str, resource_type: str) -> None: + if ( + provider_id != KNOWLEDGE_PROVIDER_ID + or resource_type != KNOWLEDGE_RESOURCE_TYPE + ): + raise ValueError("Unsupported external knowledge search source.") + + +def _session(value: object) -> Session: + if not isinstance(value, Session): + raise TypeError("External knowledge Search requires a SQLAlchemy session.") + return value + + +__all__ = [ + "ExternalKnowledgeSearchSource", + "create_external_knowledge_search_source", + "search_document", +] diff --git a/src/govoplan_connectors/backend/manifest.py b/src/govoplan_connectors/backend/manifest.py index dc53a9b..5f54bc2 100644 --- a/src/govoplan_connectors/backend/manifest.py +++ b/src/govoplan_connectors/backend/manifest.py @@ -41,16 +41,34 @@ from govoplan_core.core.files import CAPABILITY_FILES_TABULAR_CONTENT from govoplan_core.core.sanctions import ( CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS, ) +from govoplan_core.core.search import SearchSourceProviderRegistration from govoplan_core.db.base import Base from govoplan_connectors.backend.db.models import ( ConnectorConfiguration, ConnectorDefinition, ConnectorDefinitionRevision, + ConnectorKnowledgeObject, + ConnectorKnowledgeProfile, + ConnectorKnowledgeSyncRun, ConnectorSanctionsAcquisitionRun, ConnectorSanctionsSnapshot, ConnectorSimulationRun, ConnectorTabularSource, ) +from govoplan_connectors.backend.knowledge_connector import ( + KNOWLEDGE_ADMIN_SCOPE, + KNOWLEDGE_CAPABILITY, + KNOWLEDGE_INTERFACE_VERSION, + KNOWLEDGE_MIGRATE_SCOPE, + KNOWLEDGE_PROVIDER_ID, + KNOWLEDGE_PUBLISH_SCOPE, + KNOWLEDGE_READ_SCOPE, + KNOWLEDGE_SYNC_SCOPE, + ExternalKnowledgeCapability, +) +from govoplan_connectors.backend.knowledge_search import ( + create_external_knowledge_search_source, +) from govoplan_connectors.backend.dsar_provider import ( CONNECTORS_DSAR_CAPABILITY, ConnectorsDsarProvider, @@ -77,13 +95,14 @@ from govoplan_connectors.backend.feeds import ( from govoplan_connectors.backend.provider_state import ( SANCTIONS_PROVIDER_ID, TABULAR_PROVIDER_ID, + knowledge_provider_states, sanctions_provider_states, tabular_provider_states, ) MODULE_ID = "connectors" -MODULE_VERSION = "0.1.20" +MODULE_VERSION = "0.1.21" TABULAR_SOURCE_INTERFACE_VERSION = "0.1.0" DATASOURCE_ORIGIN_INTERFACE_VERSION = "0.1.0" SANCTIONS_SNAPSHOT_INTERFACE_VERSION = "1.0.0" @@ -125,6 +144,11 @@ ARCHITECTURE = ModuleArchitectureDeclaration( reference="tests/test_governed_runtime.py", summary="Exercises immutable definition revisions, protected local overrides, idempotent simulations, and explicit ambiguity review.", ), + ModuleMaturityEvidence( + kind="test", + reference="tests/test_mediawiki_connector.py", + summary="Exercises deterministic MediaWiki/BlueSpice discovery, stable mapping, bounded deltas, ACL-safe Search, migration loss diagnostics, and publication recovery states.", + ), ModuleMaturityEvidence( kind="documentation", reference="docs/CONNECTOR_SOURCE_LIFECYCLE.md", @@ -134,11 +158,13 @@ ARCHITECTURE = ModuleArchitectureDeclaration( known_limits=( "Tabular origins support immutable snapshots, exact managed CSV/XLSX versions, and read-only PostgreSQL tables; arbitrary REST and other database adapters remain future providers.", "Feed publication renders a governed document but does not yet push it to an external publishing endpoint.", - "The generic governed runtime simulates deterministic mapping and validation; provider-specific live writes remain owned by explicit connector adapters.", + "The MediaWiki/BlueSpice adapter publishes revision-checked page edits; generic simulations and all other providers do not imply a live write capability.", + "Migration into native Wiki is preview-only; a target-side write worker and Wiki-owned mutation contract remain future work.", ), supported_authority_modes=( "external_authoritative", "external_mirror", + "governed_sync", "linked_reference", ), owned_concepts=( @@ -146,23 +172,38 @@ ARCHITECTURE = ModuleArchitectureDeclaration( "protocol interaction", "immutable connector snapshots", "connector acquisition health", + "external knowledge synchronization evidence", ), non_owned_concepts=( "datasource catalogue identity and lifecycle", "domain records and business semantics", "data transformations", "screening dispositions", + "native Wiki spaces, pages, and revision semantics", ), target_tested_providers=( TABULAR_PROVIDER_ID, SANCTIONS_PROVIDER_ID, + KNOWLEDGE_PROVIDER_ID, ), documentation=ModuleArchitectureDocumentation( migration=("src/govoplan_connectors/backend/migrations/versions",), - upgrade=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",), - recovery=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",), - security=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",), - operations=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",), + upgrade=( + "docs/CONNECTOR_SOURCE_LIFECYCLE.md", + "docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md", + ), + recovery=( + "docs/CONNECTOR_SOURCE_LIFECYCLE.md", + "docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md", + ), + security=( + "docs/CONNECTOR_SOURCE_LIFECYCLE.md", + "docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md", + ), + operations=( + "docs/CONNECTOR_SOURCE_LIFECYCLE.md", + "docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md", + ), ), ) @@ -261,6 +302,88 @@ EXTERNAL_PROVIDERS = ( "connectors.sanctions-snapshots", ), ), + ExternalProviderDeclaration( + id=KNOWLEDGE_PROVIDER_ID, + module_id=MODULE_ID, + label="MediaWiki and BlueSpice knowledge provider", + maturity="migrate", + operations=( + "discover", + "link", + "search", + "read", + "publish", + "synchronize", + "migrate", + "preview", + "dry_run", + ), + objects=( + ProviderObjectDeclaration( + object_type="external_knowledge_page", + field_groups=( + "stable_identity", + "revisions", + "namespaces", + "categories", + "links", + "files", + "discussions", + "permissions", + "source_provenance", + ), + authority_modes=( + "external_authoritative", + "external_mirror", + "governed_sync", + "linked_reference", + ), + default_authority_mode="external_mirror", + ), + ), + behavior=ProviderBehaviorDeclaration( + revision_tokens="Stable MediaWiki page ids, revision ids, discovery revisions, recent-change cursors, content digests, and provider timestamps are retained.", + concurrency="Profile changes use optimistic revisions; publication supplies an expected provider revision and a durable idempotency fence.", + freshness="The latest discovery time, sync high-watermark, recent-change cursor, source update time, and health state are exposed.", + health="Product discovery, authentication, transport, mapping loss, ACL degradation, Search deferral, and outcome-unknown publication are explicit without exposing credentials.", + max_read_items=500, + idempotency="Backfill, delta synchronization, migration previews, and publication require caller keys; exact replays return the committed evidence and mismatched reuse is rejected.", + retry="Read-only discovery and synchronization may be retried with a new key; a publication with an unknown outcome must be reconciled before retry.", + timeout_seconds=30, + conflicts="Namespace and path mappings are explicit; migration previews block target-path and attachment conflicts, while publication rejects stale expected revisions.", + outcome_unknown="A timed-out publication remains outcome-unknown behind a durable recovery fence until the institutional operator verifies the provider revision.", + outcome_unknown_supported=True, + evidence="Mapped snapshots retain stable external references, revision identity, source hashes, transport provenance, ACLs, and bounded loss diagnostics.", + audit_event_types=( + "connectors.knowledge.profile.created", + "connectors.knowledge.profile.updated", + "connectors.knowledge.profile.discovered", + "connectors.knowledge.profile.synchronized", + "connectors.knowledge.migration.previewed", + "connectors.knowledge.page.published", + ), + correction="A later provider revision or explicit publication creates a new connector snapshot revision while synchronization-run evidence remains retained.", + rollback="Local snapshot and terminal recovery evidence commit atomically; remote publication cannot be rolled back by a local transaction.", + reconciliation="Rediscover capabilities, compare stable page and revision ids, run a bounded delta or full backfill, and inspect unresolved publication evidence.", + outage="Existing snapshots remain visible only through current tenant and ACL authorization and are marked stale or unavailable; no provider freshness claim is made.", + classifications=("public", "internal", "confidential", "restricted"), + purposes=( + "external knowledge discovery", + "authorized federated search", + "knowledge synchronization", + "migration planning", + "governed publication", + ), + retention="The tenant's connector, Records, and target Wiki policies determine snapshot and operation-evidence retention.", + secret_handling="Credentials are resolved from a scoped Core credential envelope, never placed in endpoint URLs, persisted snapshots, diagnostics, or API responses.", + ), + capability_names=(KNOWLEDGE_CAPABILITY,), + interface_names=(KNOWLEDGE_CAPABILITY,), + documentation_topic_ids=( + "connectors.authority-and-effects", + "connectors.mediawiki-bluespice", + ), + ), ) @@ -314,6 +437,31 @@ PERMISSIONS = ( "Refresh sanctions sources", "Acquire a new immutable sanctions source snapshot.", ), + _permission( + KNOWLEDGE_READ_SCOPE, + "View external knowledge", + "View authorized MediaWiki and BlueSpice profiles, pages, and synchronization evidence.", + ), + _permission( + KNOWLEDGE_ADMIN_SCOPE, + "Administer external knowledge", + "Configure knowledge profiles, namespace mappings, authority, visibility, and discovery.", + ), + _permission( + KNOWLEDGE_SYNC_SCOPE, + "Synchronize external knowledge", + "Run bounded MediaWiki and BlueSpice backfills and change synchronization.", + ), + _permission( + KNOWLEDGE_PUBLISH_SCOPE, + "Publish external knowledge", + "Publish a governed page revision with concurrency and recovery evidence.", + ), + _permission( + KNOWLEDGE_MIGRATE_SCOPE, + "Preview knowledge migration", + "Dry-run a bounded migration into Wiki and inspect loss or conflict diagnostics.", + ), ) ROLE_TEMPLATES = ( @@ -329,6 +477,11 @@ ROLE_TEMPLATES = ( FEED_PRIVATE_PUBLISH_SCOPE, SANCTIONS_READ_SCOPE, SANCTIONS_REFRESH_SCOPE, + KNOWLEDGE_READ_SCOPE, + KNOWLEDGE_ADMIN_SCOPE, + KNOWLEDGE_SYNC_SCOPE, + KNOWLEDGE_PUBLISH_SCOPE, + KNOWLEDGE_MIGRATE_SCOPE, ), ), RoleTemplate( @@ -341,13 +494,16 @@ ROLE_TEMPLATES = ( FEED_PUBLISH_SCOPE, SANCTIONS_READ_SCOPE, SANCTIONS_REFRESH_SCOPE, + KNOWLEDGE_READ_SCOPE, + KNOWLEDGE_SYNC_SCOPE, + KNOWLEDGE_MIGRATE_SCOPE, ), ), RoleTemplate( slug="connector_source_reader", name="Connector source reader", description="Discover and preview tabular connector sources.", - permissions=(READ_SCOPE, SANCTIONS_READ_SCOPE), + permissions=(READ_SCOPE, SANCTIONS_READ_SCOPE, KNOWLEDGE_READ_SCOPE), ), ) @@ -382,6 +538,10 @@ def _dsar_provider(_context) -> ConnectorsDsarProvider: return ConnectorsDsarProvider() +def _knowledge_provider(_context) -> ExternalKnowledgeCapability: + return ExternalKnowledgeCapability() + + def _tenant_summary(session, tenant_id: str) -> dict[str, int]: return { "connector_definitions": ( @@ -417,6 +577,24 @@ def _tenant_summary(session, tenant_id: str) -> dict[str, int]: .filter(ConnectorSanctionsAcquisitionRun.tenant_id == tenant_id) .count() ), + "connector_knowledge_profiles": ( + session.query(ConnectorKnowledgeProfile) + .filter(ConnectorKnowledgeProfile.tenant_id == tenant_id) + .count() + ), + "connector_knowledge_objects": ( + session.query(ConnectorKnowledgeObject) + .filter( + ConnectorKnowledgeObject.tenant_id == tenant_id, + ConnectorKnowledgeObject.status != "deleted", + ) + .count() + ), + "connector_knowledge_runs": ( + session.query(ConnectorKnowledgeSyncRun) + .filter(ConnectorKnowledgeSyncRun.tenant_id == tenant_id) + .count() + ), } @@ -433,6 +611,8 @@ manifest = ModuleManifest( "portal", "reporting", "risk_compliance", + "search", + "wiki", ), required_capabilities=( CAPABILITY_AUTH_PRINCIPAL_RESOLVER, @@ -463,6 +643,10 @@ manifest = ModuleManifest( name="connectors.runtime_contract", version=CONNECTOR_RUNTIME_INTERFACE_VERSION, ), + ModuleInterfaceProvider( + name=KNOWLEDGE_CAPABILITY, + version=KNOWLEDGE_INTERFACE_VERSION, + ), ModuleInterfaceProvider(name=CONNECTORS_DSAR_CAPABILITY, version="0.1.0"), ), requires_interfaces=( @@ -495,6 +679,14 @@ manifest = ModuleManifest( parent_id="connectors.admin.governed-configurations", order=20, ), + ViewSurface( + id="connectors.admin.external-knowledge", + module_id=MODULE_ID, + kind="section", + label="External knowledge", + parent_id="connectors.admin.governed-configurations", + order=30, + ), ), ), capability_factories={ @@ -503,6 +695,7 @@ manifest = ModuleManifest( CAPABILITY_DATASOURCE_ORIGINS: _datasource_origin_provider, CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS: (_sanctions_snapshot_provider), CAPABILITY_CONNECTORS_FEEDS: _feed_provider, + KNOWLEDGE_CAPABILITY: _knowledge_provider, CONNECTORS_DSAR_CAPABILITY: _dsar_provider, }, capability_documentation={ @@ -516,6 +709,13 @@ manifest = ModuleManifest( ), }, tenant_summary_providers=(_tenant_summary,), + search_sources=( + SearchSourceProviderRegistration( + id=KNOWLEDGE_PROVIDER_ID, + factory=create_external_knowledge_search_source, + order=65, + ), + ), architecture=ARCHITECTURE, external_providers=EXTERNAL_PROVIDERS, external_provider_state_providers=( @@ -529,6 +729,11 @@ manifest = ModuleManifest( provider_id=SANCTIONS_PROVIDER_ID, provider=sanctions_provider_states, ), + ExternalProviderStateProviderRegistration( + module_id=MODULE_ID, + provider_id=KNOWLEDGE_PROVIDER_ID, + provider=knowledge_provider_states, + ), ), migration_spec=MigrationSpec( module_id=MODULE_ID, @@ -536,6 +741,9 @@ manifest = ModuleManifest( script_location=str(Path(__file__).with_name("migrations") / "versions"), retirement_supported=True, retirement_provider=drop_table_retirement_provider( + ConnectorKnowledgeSyncRun, + ConnectorKnowledgeObject, + ConnectorKnowledgeProfile, ConnectorSimulationRun, ConnectorConfiguration, ConnectorDefinitionRevision, @@ -552,6 +760,9 @@ manifest = ModuleManifest( ), uninstall_guard_providers=( persistent_table_uninstall_guard( + ConnectorKnowledgeSyncRun, + ConnectorKnowledgeObject, + ConnectorKnowledgeProfile, ConnectorSimulationRun, ConnectorConfiguration, ConnectorDefinitionRevision, @@ -573,8 +784,9 @@ manifest = ModuleManifest( body=( "Connectors correlates only an exact tenant account identifier and can " "narrow an already verified search to one source, acquisition, " - "definition, configuration, or simulation. The export identifies the " - "subject's configuration, acquisition, simulation, and review activity " + "definition, configuration, simulation, external-knowledge profile, or " + "knowledge operation. The export identifies the subject's configuration, " + "acquisition, simulation, knowledge-operation, and review activity " "using bounded lifecycle metadata. It never includes credential or " "endpoint references, source rows, external responses, request payloads, " "mapping and configuration documents, diagnostics, provenance, hashes, " @@ -709,6 +921,45 @@ manifest = ModuleManifest( related_modules=("datasources", "dataflow", "portal", "reporting"), order=42, ), + DocumentationTopic( + id="connectors.mediawiki-bluespice", + title="Connect MediaWiki and BlueSpice knowledge", + summary="Discover, synchronize, search, publish, and preview migration of external knowledge without weakening source permissions.", + body=( + "A connector administrator first creates a governed MediaWiki Action API configuration whose endpoint passes Core egress and peer validation; credentials remain in a scoped credential envelope. A knowledge profile then maps each included source namespace to a target Wiki space and path prefix, selects source authority, and sets a tenant or restricted fallback ACL. Discovery records product, version, extensions, namespaces, authentication state, capabilities, and loss diagnostics. Run a bounded full backfill once, then cursor-based deltas for revisions, moves, deletions, and permission changes. Stable page and revision identifiers, canonical links, redirects, files, categories, links, discussions, provenance, and current ACLs remain attached to the connector snapshot. Search indexes only active, non-deleted pages and rechecks the current profile, tenant, permission, and ACL before returning every result; disabling Connectors removes its Search projection. Publication requires an expected external revision, an idempotency key, and durable outcome evidence. An unknown provider outcome must be reconciled before retry. Migration into native Wiki is deliberately preview-only in this slice: the dry-run reports target-path collisions, attachment-name conflicts, unsupported macros, truncation, and source fingerprints; a successful preview does not write native Wiki pages. Existing authorized snapshots can remain visible during a provider outage, but health and freshness stay explicit and no current-source claim is made." + ), + layer="configured", + documentation_types=("admin", "user"), + audience=( + "operator", + "module_admin", + "integration_admin", + "knowledge_manager", + "auditor", + ), + related_modules=("core", "search", "wiki", "files", "audit", "policy"), + order=43, + translations={ + "de": { + "title": "Wissen aus MediaWiki und BlueSpice anbinden", + "summary": "Externes Wissen erkennen, synchronisieren, durchsuchen, veröffentlichen und eine Migration prüfen, ohne Quellberechtigungen abzuschwächen.", + "body": ( + "Eine Connector-Administration erstellt zuerst eine gesteuerte Konfiguration für die MediaWiki Action API. Der Endpunkt unterliegt der zentralen Ausgangs- und Gegenstellenprüfung; Zugangsdaten bleiben in einem zweckgebundenen Zugangsdaten-Umschlag. Das Wissensprofil ordnet jeden eingeschlossenen Quellnamensraum einem Zielbereich und Pfadpräfix in Wiki zu und legt Quellhoheit sowie eine mandantenweite oder eingeschränkte Ersatz-ACL fest. Die Erkennung dokumentiert Produkt, Version, Erweiterungen, Namensräume, Authentifizierungsstatus, Fähigkeiten und Verlustdiagnosen. Nach einem begrenzten Vollabgleich folgen cursorbasierte Änderungen für Revisionen, Verschiebungen, Löschungen und Berechtigungen. Stabile Seiten- und Revisionskennungen, kanonische Links, Weiterleitungen, Dateien, Kategorien, Links, Diskussionen, Herkunft und aktuelle ACLs bleiben am Snapshot. Search indiziert nur aktive, nicht gelöschte Seiten und prüft bei jedem Treffer Profilstatus, Mandant, Berechtigung und ACL erneut; bei Deaktivierung von Connectors wird dessen Suchprojektion entfernt. Veröffentlichungen erfordern die erwartete externe Revision, einen Idempotenzschlüssel und dauerhafte Ergebnisevidenz. Ein unbekanntes Ergebnis muss vor einem erneuten Versuch abgeglichen werden. Die Migration in das native Wiki ist in diesem Ausbauschritt ausschließlich eine Vorschau: Sie meldet Pfad- und Anhangskonflikte, nicht unterstützte Makros, Begrenzungen und Quellfingerabdrücke, schreibt aber keine Wiki-Seiten. Bei einem Ausfall dürfen bestehende Snapshots nur für weiterhin Berechtigte sichtbar bleiben; Zustand und Aktualität bleiben ausdrücklich erkennbar." + ), + } + }, + metadata={ + "kind": "guide", + "help_contexts": ["connectors.admin.external-knowledge"], + "prerequisites": [ + "An active governed MediaWiki Action API configuration exists.", + "Namespace targets and fallback ACLs have been reviewed.", + "The Search and Wiki modules are optional and remain capability-separated.", + ], + "outcome": "External knowledge remains identity-stable, loss-visible, ACL-safe, and migration-ready.", + "verification": "Rediscover the profile, run a keyed delta, inspect health and diagnostics, verify an allowed and denied Search principal, and run a migration dry-run before any target-side work.", + }, + ), DocumentationTopic( id="connectors.sanctions-snapshots", title="Sanctions source snapshots", diff --git a/src/govoplan_connectors/backend/mediawiki_transport.py b/src/govoplan_connectors/backend/mediawiki_transport.py new file mode 100644 index 0000000..4b9c872 --- /dev/null +++ b/src/govoplan_connectors/backend/mediawiki_transport.py @@ -0,0 +1,503 @@ +from __future__ import annotations + +import base64 +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Protocol +from urllib.parse import quote, urlencode, urljoin, urlsplit, urlunsplit + +from govoplan_core.security.http_fetch import fetch_http + + +MAX_MEDIAWIKI_RESPONSE_BYTES = 10_000_000 + + +class MediaWikiTransportError(RuntimeError): + def __init__( + self, + code: str, + message: str, + *, + retryable: bool = False, + outcome_unknown: bool = False, + ) -> None: + super().__init__(message) + self.code = code + self.retryable = retryable + self.outcome_unknown = outcome_unknown + + +@dataclass(frozen=True, slots=True) +class MediaWikiChangeBatch: + changes: tuple[Mapping[str, Any], ...] + next_cursor: str | None + complete: bool + high_watermark: str | None + evidence: Mapping[str, Any] + + +@dataclass(frozen=True, slots=True) +class MediaWikiPublishResult: + page_id: str + revision_id: str + title: str + canonical_url: str | None + evidence: Mapping[str, Any] + + +class MediaWikiTransport(Protocol): + def discover( + self, + *, + endpoint_url: str, + credential: Mapping[str, Any] | None, + ) -> Mapping[str, Any]: ... + + def changes( + self, + *, + endpoint_url: str, + credential: Mapping[str, Any] | None, + cursor: str | None, + limit: int, + force_full: bool, + ) -> MediaWikiChangeBatch: ... + + def publish( + self, + *, + endpoint_url: str, + credential: Mapping[str, Any] | None, + title: str, + body: str, + summary: str, + expected_revision: str | None, + minor: bool, + ) -> MediaWikiPublishResult: ... + + +class HttpMediaWikiTransport: + """Bounded MediaWiki Action API transport with Core outbound policy enforcement.""" + + def discover( + self, + *, + endpoint_url: str, + credential: Mapping[str, Any] | None, + ) -> Mapping[str, Any]: + return self._request( + endpoint_url, + credential=credential, + params={ + "action": "query", + "meta": "siteinfo|userinfo", + "siprop": "general|extensions|namespaces|namespacealiases|rightsinfo", + "uiprop": "rights|groups", + "format": "json", + "formatversion": "2", + "curtimestamp": "1", + }, + ) + + def changes( + self, + *, + endpoint_url: str, + credential: Mapping[str, Any] | None, + cursor: str | None, + limit: int, + force_full: bool, + ) -> MediaWikiChangeBatch: + if force_full: + listing = self._request( + endpoint_url, + credential=credential, + params={ + "action": "query", + "list": "allpages", + "aplimit": str(limit), + "apcontinue": cursor or "", + "apfilterredir": "all", + "format": "json", + "formatversion": "2", + "curtimestamp": "1", + }, + ) + rows = _list(listing, "query", "allpages") + page_ids = tuple( + str(item.get("pageid")) + for item in rows + if isinstance(item, Mapping) and item.get("pageid") is not None + ) + changes = self._page_details( + endpoint_url, + credential=credential, + page_ids=page_ids, + ) + next_cursor = _continue_token(listing, "apcontinue") + return MediaWikiChangeBatch( + changes=changes, + next_cursor=next_cursor, + complete=next_cursor is None, + high_watermark=_optional_text(listing.get("curtimestamp")), + evidence={ + "mode": "backfill", + "listed": len(rows), + "resolved": len(changes), + }, + ) + + listing = self._request( + endpoint_url, + credential=credential, + params={ + "action": "query", + "list": "recentchanges", + "rclimit": str(limit), + "rccontinue": cursor or "", + "rcdir": "newer", + "rcprop": "title|ids|sizes|flags|user|timestamp|loginfo|tags", + "rctype": "edit|new|log", + "format": "json", + "formatversion": "2", + "curtimestamp": "1", + }, + ) + recent = tuple( + item + for item in _list(listing, "query", "recentchanges") + if isinstance(item, Mapping) + ) + page_ids = tuple( + dict.fromkeys( + str(item.get("pageid")) + for item in recent + if item.get("pageid") is not None + and str(item.get("logtype") or "") != "delete" + ) + ) + resolved = { + str(item.get("pageid")): item + for item in self._page_details( + endpoint_url, + credential=credential, + page_ids=page_ids, + ) + } + changes: list[Mapping[str, Any]] = [] + for item in recent: + page_id = _optional_text(item.get("pageid")) + if str(item.get("logtype") or "") == "delete": + changes.append( + { + "change_kind": "delete", + "pageid": page_id or f"log:{item.get('logid')}", + "title": _optional_text(item.get("title")) or "Deleted page", + "ns": item.get("ns"), + "timestamp": item.get("timestamp"), + "logid": item.get("logid"), + "change_cursor": _change_cursor(item), + } + ) + continue + page = dict(resolved.get(page_id or "") or item) + page["change_kind"] = "upsert" + page["change_cursor"] = _change_cursor(item) + page["recent_change"] = _safe_recent_change(item) + changes.append(page) + next_cursor = _continue_token(listing, "rccontinue") + return MediaWikiChangeBatch( + changes=tuple(changes), + next_cursor=next_cursor, + complete=next_cursor is None, + high_watermark=_optional_text(listing.get("curtimestamp")), + evidence={ + "mode": "delta", + "listed": len(recent), + "resolved": len(resolved), + }, + ) + + def publish( + self, + *, + endpoint_url: str, + credential: Mapping[str, Any] | None, + title: str, + body: str, + summary: str, + expected_revision: str | None, + minor: bool, + ) -> MediaWikiPublishResult: + if not credential: + raise MediaWikiTransportError( + "credential_required", + "Publishing requires a governed credential envelope.", + ) + csrf_token = _optional_text(credential.get("csrf_token")) + if not csrf_token: + token_payload = self._request( + endpoint_url, + credential=credential, + params={ + "action": "query", + "meta": "tokens", + "type": "csrf", + "format": "json", + "formatversion": "2", + }, + ) + query = token_payload.get("query") + tokens = query.get("tokens") if isinstance(query, Mapping) else None + csrf_token = ( + _optional_text(tokens.get("csrftoken")) + if isinstance(tokens, Mapping) + else None + ) + if not csrf_token: + raise MediaWikiTransportError( + "csrf_token_unavailable", + "The provider did not issue a CSRF token for the configured credential.", + ) + parameters: dict[str, str] = { + "action": "edit", + "title": title, + "text": body, + "summary": summary, + "token": csrf_token, + "format": "json", + "formatversion": "2", + } + if minor: + parameters["minor"] = "1" + if expected_revision: + parameters["baserevid"] = expected_revision + payload = self._request( + endpoint_url, + credential=credential, + params=parameters, + method="POST", + ) + edit = payload.get("edit") + if not isinstance(edit, Mapping) or str(edit.get("result")) != "Success": + raise MediaWikiTransportError( + "publish_rejected", + "MediaWiki rejected the page publication.", + ) + page_id = _required_text(edit.get("pageid"), "MediaWiki omitted the page id") + revision_id = _required_text( + edit.get("newrevid"), "MediaWiki omitted the accepted revision id" + ) + canonical_url = _canonical_page_url(endpoint_url, title) + return MediaWikiPublishResult( + page_id=page_id, + revision_id=revision_id, + title=_optional_text(edit.get("title")) or title, + canonical_url=canonical_url, + evidence={ + "result": "Success", + "page_id": page_id, + "revision_id": revision_id, + "old_revision_id": _optional_text(edit.get("oldrevid")), + "new_page": bool(edit.get("new")), + }, + ) + + def _page_details( + self, + endpoint_url: str, + *, + credential: Mapping[str, Any] | None, + page_ids: Sequence[str], + ) -> tuple[Mapping[str, Any], ...]: + if not page_ids: + return () + payload = self._request( + endpoint_url, + credential=credential, + params={ + "action": "query", + "pageids": "|".join(page_ids), + "prop": "info|revisions|categories|links|images|pageprops", + "inprop": "url|displaytitle", + "rvlimit": "1", + "rvprop": "ids|timestamp|user|comment|content|contentmodel|sha1|flags", + "rvslots": "main", + "cllimit": "max", + "pllimit": "max", + "imlimit": "max", + "format": "json", + "formatversion": "2", + }, + ) + return tuple( + item + for item in _list(payload, "query", "pages") + if isinstance(item, Mapping) + ) + + def _request( + self, + endpoint_url: str, + *, + credential: Mapping[str, Any] | None, + params: Mapping[str, str], + method: str = "GET", + ) -> Mapping[str, Any]: + api_url = _api_url(endpoint_url) + headers = { + "Accept": "application/json", + "User-Agent": "GovOPlaN-Connectors/MediaWiki", + **_auth_headers(credential), + } + body = None + target_url = api_url + if method == "POST": + body = urlencode(params).encode("utf-8") + headers["Content-Type"] = "application/x-www-form-urlencoded" + else: + target_url = f"{api_url}?{urlencode({key: value for key, value in params.items() if value != ''})}" + try: + response = fetch_http( + target_url, + method=method, + headers=headers, + body=body, + max_bytes=MAX_MEDIAWIKI_RESPONSE_BYTES, + timeout=30, + label="MediaWiki Action API URL", + ) + except Exception as exc: + raise MediaWikiTransportError( + "transport_unavailable", + "MediaWiki transport failed before a valid response was received.", + retryable=True, + outcome_unknown=method == "POST", + ) from exc + if response.status < 200 or response.status >= 300: + raise MediaWikiTransportError( + "http_error", + f"MediaWiki returned HTTP {response.status}.", + retryable=response.status >= 500, + outcome_unknown=method == "POST" and response.status >= 500, + ) + try: + payload = json.loads(response.body) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise MediaWikiTransportError( + "invalid_json", + "MediaWiki returned an invalid JSON response.", + outcome_unknown=method == "POST", + ) from exc + if not isinstance(payload, Mapping): + raise MediaWikiTransportError( + "invalid_response", + "MediaWiki returned an unsupported response shape.", + outcome_unknown=method == "POST", + ) + error = payload.get("error") + if isinstance(error, Mapping): + code = _optional_text(error.get("code")) or "provider_error" + raise MediaWikiTransportError( + code, + _optional_text(error.get("info")) or "MediaWiki rejected the request.", + retryable=code in {"maxlag", "readonly", "ratelimited"}, + ) + return payload + + +def _api_url(endpoint_url: str) -> str: + normalized = endpoint_url.strip().rstrip("/") + parsed = urlsplit(normalized) + if parsed.path.endswith("/api.php"): + return normalized + path = f"{parsed.path.rstrip('/')}/api.php" + return urlunsplit((parsed.scheme, parsed.netloc, path, "", "")) + + +def _canonical_page_url(endpoint_url: str, title: str) -> str: + normalized = endpoint_url.strip().rstrip("/") + parsed = urlsplit(normalized) + root_path = parsed.path + if root_path.endswith("/api.php"): + root_path = root_path[: -len("/api.php")] + base = urlunsplit((parsed.scheme, parsed.netloc, f"{root_path.rstrip('/')}/", "", "")) + return urljoin(base, f"wiki/{quote(title.replace(' ', '_'), safe=':_-./~')}") + + +def _auth_headers(credential: Mapping[str, Any] | None) -> dict[str, str]: + if not credential: + return {} + token = _optional_text( + credential.get("bearer_token") + or credential.get("access_token") + or credential.get("token") + ) + if token: + return {"Authorization": f"Bearer {token}"} + username = _optional_text(credential.get("username") or credential.get("user")) + password = _optional_text(credential.get("password")) + if username and password: + encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + return {"Authorization": f"Basic {encoded}"} + return {} + + +def _list(payload: Mapping[str, Any], *path: str) -> list[Any]: + value: Any = payload + for part in path: + if not isinstance(value, Mapping): + return [] + value = value.get(part) + return list(value) if isinstance(value, list) else [] + + +def _continue_token(payload: Mapping[str, Any], name: str) -> str | None: + value = payload.get("continue") + return _optional_text(value.get(name)) if isinstance(value, Mapping) else None + + +def _change_cursor(change: Mapping[str, Any]) -> str: + for name in ("rcid", "logid", "revid", "old_revid"): + if change.get(name) is not None: + return f"{name}:{change[name]}" + return _optional_text(change.get("timestamp")) or "unknown" + + +def _safe_recent_change(change: Mapping[str, Any]) -> dict[str, Any]: + allowed = ( + "type", + "ns", + "title", + "pageid", + "revid", + "old_revid", + "timestamp", + "logtype", + "logaction", + "tags", + ) + return {key: change[key] for key in allowed if key in change} + + +def _required_text(value: object, message: str) -> str: + normalized = _optional_text(value) + if not normalized: + raise MediaWikiTransportError("invalid_response", message) + return normalized + + +def _optional_text(value: object) -> str | None: + normalized = str(value or "").strip() + return normalized or None + + +__all__ = [ + "HttpMediaWikiTransport", + "MAX_MEDIAWIKI_RESPONSE_BYTES", + "MediaWikiChangeBatch", + "MediaWikiPublishResult", + "MediaWikiTransport", + "MediaWikiTransportError", +] diff --git a/src/govoplan_connectors/backend/migrations/versions/b9e0f1a2c3d4_mediawiki_knowledge_connector.py b/src/govoplan_connectors/backend/migrations/versions/b9e0f1a2c3d4_mediawiki_knowledge_connector.py new file mode 100644 index 0000000..1090412 --- /dev/null +++ b/src/govoplan_connectors/backend/migrations/versions/b9e0f1a2c3d4_mediawiki_knowledge_connector.py @@ -0,0 +1,207 @@ +"""MediaWiki and BlueSpice knowledge connector state + +Revision ID: b9e0f1a2c3d4 +Revises: a8d9e0f1b2c3 +Create Date: 2026-08-22 14:00:00.000000 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "b9e0f1a2c3d4" +down_revision = "a8d9e0f1b2c3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "connector_knowledge_profiles", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("configuration_id", sa.String(length=36), nullable=False), + sa.Column("status", sa.String(length=30), nullable=False), + sa.Column("product", sa.String(length=50), nullable=False), + sa.Column("product_version", sa.String(length=100), nullable=True), + sa.Column("desired_maturity", sa.String(length=30), nullable=False), + sa.Column("discovered_maturity", sa.String(length=30), nullable=False), + sa.Column("source_authority_mode", sa.String(length=40), nullable=False), + sa.Column("default_visibility", sa.String(length=30), nullable=False), + sa.Column("default_acl_tokens", sa.JSON(), nullable=False), + sa.Column("namespace_mappings", sa.JSON(), nullable=False), + sa.Column("capabilities", sa.JSON(), nullable=False), + sa.Column("discovery_revision", sa.String(length=255), nullable=True), + sa.Column("discovery_evidence", sa.JSON(), nullable=False), + sa.Column("health_status", sa.String(length=30), nullable=False), + sa.Column("health_details", sa.JSON(), nullable=False), + sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_sync_cursor", sa.String(length=500), nullable=True), + sa.Column("last_high_watermark", sa.String(length=500), nullable=True), + sa.Column("resource_revision", sa.Integer(), nullable=False), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["configuration_id"], + ["connector_configurations.id"], + name=op.f( + "fk_connector_knowledge_profiles_configuration_id_connector_configurations" + ), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint( + "id", name=op.f("pk_connector_knowledge_profiles") + ), + sa.UniqueConstraint( + "tenant_id", + "configuration_id", + name="uq_connector_knowledge_profile_configuration", + ), + ) + for name, columns in ( + ("ix_connector_knowledge_profiles_tenant_id", ["tenant_id"]), + ("ix_connector_knowledge_profiles_configuration_id", ["configuration_id"]), + ("ix_connector_knowledge_profiles_status", ["status"]), + ("ix_connector_knowledge_profiles_product", ["product"]), + ("ix_connector_knowledge_profiles_discovery_revision", ["discovery_revision"]), + ("ix_connector_knowledge_profiles_health_status", ["health_status"]), + ("ix_connector_knowledge_profiles_updated_by", ["updated_by"]), + ): + op.create_index(op.f(name), "connector_knowledge_profiles", columns) + op.create_index( + "ix_connector_knowledge_profiles_tenant_status", + "connector_knowledge_profiles", + ["tenant_id", "status"], + ) + + op.create_table( + "connector_knowledge_objects", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("profile_id", sa.String(length=36), nullable=False), + sa.Column("object_type", sa.String(length=40), nullable=False), + sa.Column("external_id", sa.String(length=255), nullable=False), + sa.Column("external_page_id", sa.String(length=255), nullable=True), + sa.Column("external_revision_id", sa.String(length=255), nullable=True), + sa.Column("namespace_id", sa.Integer(), nullable=True), + sa.Column("title", sa.String(length=500), nullable=False), + sa.Column("canonical_url", sa.String(length=1500), nullable=True), + sa.Column("status", sa.String(length=30), nullable=False), + sa.Column("redirect_target_external_id", sa.String(length=255), nullable=True), + sa.Column("source_revision", sa.String(length=255), nullable=False), + sa.Column("content_hash", sa.String(length=64), nullable=False), + sa.Column("visibility", sa.String(length=30), nullable=False), + sa.Column("acl_tokens", sa.JSON(), nullable=False), + sa.Column("mapped_data", sa.JSON(), nullable=False), + sa.Column("provenance", sa.JSON(), nullable=False), + sa.Column("change_cursor", sa.String(length=500), nullable=True), + sa.Column("source_updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("resource_revision", sa.Integer(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["profile_id"], + ["connector_knowledge_profiles.id"], + name=op.f( + "fk_connector_knowledge_objects_profile_id_connector_knowledge_profiles" + ), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "id", name=op.f("pk_connector_knowledge_objects") + ), + sa.UniqueConstraint( + "profile_id", + "object_type", + "external_id", + name="uq_connector_knowledge_object_identity", + ), + ) + for name, columns in ( + ("ix_connector_knowledge_objects_tenant_id", ["tenant_id"]), + ("ix_connector_knowledge_objects_profile_id", ["profile_id"]), + ("ix_connector_knowledge_objects_object_type", ["object_type"]), + ("ix_connector_knowledge_objects_external_page_id", ["external_page_id"]), + ("ix_connector_knowledge_objects_external_revision_id", ["external_revision_id"]), + ("ix_connector_knowledge_objects_namespace_id", ["namespace_id"]), + ("ix_connector_knowledge_objects_status", ["status"]), + ("ix_connector_knowledge_objects_content_hash", ["content_hash"]), + ("ix_connector_knowledge_objects_change_cursor", ["change_cursor"]), + ("ix_connector_knowledge_objects_source_updated_at", ["source_updated_at"]), + ("ix_connector_knowledge_objects_observed_at", ["observed_at"]), + ): + op.create_index(op.f(name), "connector_knowledge_objects", columns) + op.create_index( + "ix_connector_knowledge_objects_tenant_profile_status", + "connector_knowledge_objects", + ["tenant_id", "profile_id", "status"], + ) + op.create_index( + "ix_connector_knowledge_objects_tenant_updated", + "connector_knowledge_objects", + ["tenant_id", "source_updated_at"], + ) + + op.create_table( + "connector_knowledge_sync_runs", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("profile_id", sa.String(length=36), nullable=False), + sa.Column("mode", sa.String(length=40), nullable=False), + sa.Column("idempotency_key", sa.String(length=255), nullable=False), + sa.Column("request_hash", sa.String(length=64), nullable=False), + sa.Column("status", sa.String(length=30), nullable=False), + sa.Column("cursor_before", sa.String(length=500), nullable=True), + sa.Column("cursor_after", sa.String(length=500), nullable=True), + sa.Column("high_watermark", sa.String(length=500), nullable=True), + sa.Column("counts", sa.JSON(), nullable=False), + sa.Column("effects", sa.JSON(), nullable=False), + sa.Column("diagnostics", sa.JSON(), nullable=False), + sa.Column("provenance", sa.JSON(), nullable=False), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["profile_id"], + ["connector_knowledge_profiles.id"], + name=op.f( + "fk_connector_knowledge_sync_runs_profile_id_connector_knowledge_profiles" + ), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "id", name=op.f("pk_connector_knowledge_sync_runs") + ), + sa.UniqueConstraint( + "tenant_id", + "profile_id", + "mode", + "idempotency_key", + name="uq_connector_knowledge_sync_run_idempotency", + ), + ) + for name, columns in ( + ("ix_connector_knowledge_sync_runs_tenant_id", ["tenant_id"]), + ("ix_connector_knowledge_sync_runs_profile_id", ["profile_id"]), + ("ix_connector_knowledge_sync_runs_mode", ["mode"]), + ("ix_connector_knowledge_sync_runs_status", ["status"]), + ("ix_connector_knowledge_sync_runs_created_by", ["created_by"]), + ("ix_connector_knowledge_sync_runs_started_at", ["started_at"]), + ): + op.create_index(op.f(name), "connector_knowledge_sync_runs", columns) + op.create_index( + "ix_connector_knowledge_sync_runs_profile_started", + "connector_knowledge_sync_runs", + ["tenant_id", "profile_id", "started_at"], + ) + + +def downgrade() -> None: + op.drop_table("connector_knowledge_sync_runs") + op.drop_table("connector_knowledge_objects") + op.drop_table("connector_knowledge_profiles") diff --git a/src/govoplan_connectors/backend/provider_state.py b/src/govoplan_connectors/backend/provider_state.py index 05dabae..4693a0d 100644 --- a/src/govoplan_connectors/backend/provider_state.py +++ b/src/govoplan_connectors/backend/provider_state.py @@ -7,6 +7,9 @@ from sqlalchemy import func, select from sqlalchemy.orm import Session from govoplan_connectors.backend.db.models import ( + ConnectorKnowledgeObject, + ConnectorKnowledgeProfile, + ConnectorKnowledgeSyncRun, ConnectorSanctionsAcquisitionRun, ConnectorSanctionsSnapshot, ConnectorTabularSource, @@ -19,6 +22,7 @@ from govoplan_core.core.provider_governance import ( TABULAR_PROVIDER_ID = "connectors.tabular_snapshot" SANCTIONS_PROVIDER_ID = "connectors.sanctions_snapshot" +KNOWLEDGE_PROVIDER_ID = "connectors.mediawiki.pages" def tabular_provider_states( @@ -85,6 +89,37 @@ def sanctions_provider_states( ) +def knowledge_provider_states( + context: ExternalProviderStateContext, +) -> tuple[ExternalProviderRuntimeState, ...]: + session = _session(context) + statement = select(ConnectorKnowledgeProfile) + if context.tenant_id is not None: + statement = statement.where( + ConnectorKnowledgeProfile.tenant_id == context.tenant_id + ) + profiles = tuple( + session.scalars( + statement.order_by( + ConnectorKnowledgeProfile.tenant_id, + ConnectorKnowledgeProfile.id, + ).limit(context.max_items + 1) + ) + )[: context.max_items] + counts = _knowledge_counts(session, profiles) + latest_runs = _latest_knowledge_runs(session, profiles) + observed_at = datetime.now(UTC) + return tuple( + _knowledge_state( + profile, + observed_at=observed_at, + object_count=counts.get(profile.id, 0), + latest_run=latest_runs.get(profile.id), + ) + for profile in profiles + ) + + def _session(context: ExternalProviderStateContext) -> Session: if not isinstance(context.session, Session): raise RuntimeError("Connectors provider state requires a database session.") @@ -161,6 +196,119 @@ def _snapshot_counts( } +def _knowledge_counts( + session: Session, + profiles: tuple[ConnectorKnowledgeProfile, ...], +) -> dict[str, int]: + profile_ids = tuple(item.id for item in profiles) + if not profile_ids: + return {} + return { + str(profile_id): int(count) + for profile_id, count in session.execute( + select( + ConnectorKnowledgeObject.profile_id, + func.count(ConnectorKnowledgeObject.id), + ) + .where( + ConnectorKnowledgeObject.profile_id.in_(profile_ids), + ConnectorKnowledgeObject.status != "deleted", + ) + .group_by(ConnectorKnowledgeObject.profile_id) + ) + } + + +def _latest_knowledge_runs( + session: Session, + profiles: tuple[ConnectorKnowledgeProfile, ...], +) -> dict[str, ConnectorKnowledgeSyncRun]: + profile_ids = tuple(item.id for item in profiles) + if not profile_ids: + return {} + rows = tuple( + session.scalars( + select(ConnectorKnowledgeSyncRun) + .where( + ConnectorKnowledgeSyncRun.profile_id.in_(profile_ids), + ConnectorKnowledgeSyncRun.mode.in_(("backfill", "delta")), + ) + .order_by( + ConnectorKnowledgeSyncRun.profile_id, + ConnectorKnowledgeSyncRun.started_at.desc(), + ConnectorKnowledgeSyncRun.id.desc(), + ) + ) + ) + latest: dict[str, ConnectorKnowledgeSyncRun] = {} + for row in rows: + latest.setdefault(row.profile_id, row) + return latest + + +def _knowledge_state( + profile: ConnectorKnowledgeProfile, + *, + observed_at: datetime, + object_count: int, + latest_run: ConnectorKnowledgeSyncRun | None, +) -> ExternalProviderRuntimeState: + active = profile.status == "active" + health = ( + "inactive" + if not active + else "healthy" + if profile.health_status == "healthy" + else "warning" + if profile.health_status in {"unknown", "degraded"} + else "error" + ) + last_success = ( + latest_run.finished_at + if latest_run is not None and latest_run.status == "completed" + else profile.discovered_at + ) + return ExternalProviderRuntimeState( + provider_id=KNOWLEDGE_PROVIDER_ID, + binding_ref=f"connectors:knowledge-profile:{profile.id}", + authority_mode=profile.source_authority_mode, + observed_at=observed_at, + configured=True, + active=active, + health=health, + freshness="unknown" if active else "not_applicable", + conflict=( + "pending" + if latest_run is not None and latest_run.status == "outcome_unknown" + else "not_applicable" + ), + recovery=( + "attention" + if latest_run is not None + and latest_run.status in {"failed", "outcome_unknown"} + else "ready" + if active + else "not_applicable" + ), + last_success_at=_aware(last_success), + detail=( + f"{profile.product} knowledge profile is synchronized and ACL-rechecked." + if active and profile.health_status == "healthy" + else "Knowledge profile requires discovery, synchronization, or health review." + if active + else "Knowledge profile is paused." + ), + metrics={ + "product": profile.product, + "product_version": profile.product_version, + "desired_maturity": profile.desired_maturity, + "discovered_maturity": profile.discovered_maturity, + "active_objects": int(object_count), + "last_run_status": latest_run.status if latest_run is not None else None, + }, + ) + + def _sanctions_state( run: ConnectorSanctionsAcquisitionRun, *, @@ -209,8 +357,10 @@ def _aware(value: datetime | None) -> datetime | None: __all__ = [ + "KNOWLEDGE_PROVIDER_ID", "SANCTIONS_PROVIDER_ID", "TABULAR_PROVIDER_ID", "sanctions_provider_states", + "knowledge_provider_states", "tabular_provider_states", ] diff --git a/src/govoplan_connectors/backend/recovery.py b/src/govoplan_connectors/backend/recovery.py index 70f2a6a..27eb3f0 100644 --- a/src/govoplan_connectors/backend/recovery.py +++ b/src/govoplan_connectors/backend/recovery.py @@ -70,7 +70,7 @@ CONNECTOR_RECOVERY_OPERATIONS = ( "unknown outcomes remain unresolved until provider-backed reconciliation", "never retry the same remote effect solely to reconstruct local state", ), - implemented=False, + implemented=True, ), ) @@ -262,6 +262,20 @@ class ConnectorExternalMutationRecovery: if self.operation is not None: self.operation.succeed(evidence=provider_evidence) + def commit_success( + self, + session: Session, + *, + provider_evidence: dict[str, Any], + ) -> None: + if self.operation is not None: + self.operation.commit_verified_success( + session, + evidence=provider_evidence, + ) + else: + session.commit() + def reject(self, *, summary: str, provider_code: str) -> None: if self.operation is not None: self.operation.reject( diff --git a/src/govoplan_connectors/backend/router.py b/src/govoplan_connectors/backend/router.py index 4e3b373..7b7249a 100644 --- a/src/govoplan_connectors/backend/router.py +++ b/src/govoplan_connectors/backend/router.py @@ -81,6 +81,33 @@ from govoplan_connectors.backend.governed_schemas import ( ConnectorRunListResponse, ConnectorRunRequest, ) +from govoplan_connectors.backend.knowledge_connector import ( + KnowledgeConnectorError, + create_profile, + discover_profile, + list_objects as list_knowledge_objects, + list_profiles as list_knowledge_profiles, + list_runs as list_knowledge_runs, + migration_dry_run, + publish_page, + synchronize_profile, + update_profile, +) +from govoplan_connectors.backend.knowledge_schemas import ( + KnowledgeDiscoveryResponse, + KnowledgeMigrationDryRunRequest, + KnowledgeMigrationDryRunResponse, + KnowledgeObjectListResponse, + KnowledgeProfileCreateRequest, + KnowledgeProfileItem, + KnowledgeProfileListResponse, + KnowledgeProfileUpdateRequest, + KnowledgePublishRequest, + KnowledgePublishResponse, + KnowledgeSyncRequest, + KnowledgeSyncRunItem, + KnowledgeSyncRunListResponse, +) from govoplan_connectors.backend.recovery import ( ConnectorRecoveryError, begin_connector_read_snapshot, @@ -188,6 +215,34 @@ def _governed_http_error(exc: GovernedConnectorError) -> HTTPException: ) +def _knowledge_http_error(exc: KnowledgeConnectorError) -> HTTPException: + if exc.code.endswith("_not_found"): + status_code = status.HTTP_404_NOT_FOUND + elif exc.code == "access_denied": + status_code = status.HTTP_403_FORBIDDEN + elif exc.code in { + "idempotency_conflict", + "operation_unresolved", + "profile_conflict", + "profile_paused", + "publication_conflict", + "publication_reconciliation_required", + }: + status_code = status.HTTP_409_CONFLICT + elif exc.retryable or exc.code.endswith("_unavailable"): + status_code = status.HTTP_503_SERVICE_UNAVAILABLE + else: + status_code = status.HTTP_422_UNPROCESSABLE_CONTENT + return HTTPException( + status_code=status_code, + detail={ + "code": exc.code, + "message": str(exc), + "retryable": exc.retryable, + }, + ) + + @router.post("/feeds/preview", response_model=FeedDocumentResponse) def api_preview_feed( payload: FeedAcquireRequest, @@ -986,6 +1041,197 @@ def api_review_governed_run( raise _governed_http_error(exc) from exc +@router.get( + "/knowledge/profiles", + response_model=KnowledgeProfileListResponse, +) +def api_list_knowledge_profiles( + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), +) -> KnowledgeProfileListResponse: + try: + return KnowledgeProfileListResponse( + items=list(list_knowledge_profiles(session, principal)) + ) + except KnowledgeConnectorError as exc: + raise _knowledge_http_error(exc) from exc + + +@router.post( + "/knowledge/profiles", + response_model=KnowledgeProfileItem, + status_code=status.HTTP_201_CREATED, +) +def api_create_knowledge_profile( + payload: KnowledgeProfileCreateRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), +) -> KnowledgeProfileItem: + try: + return create_profile(session, principal, payload) + except KnowledgeConnectorError as exc: + session.rollback() + raise _knowledge_http_error(exc) from exc + + +@router.put( + "/knowledge/profiles/{profile_id}", + response_model=KnowledgeProfileItem, +) +def api_update_knowledge_profile( + profile_id: str, + payload: KnowledgeProfileUpdateRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), +) -> KnowledgeProfileItem: + try: + return update_profile( + session, + principal, + profile_id=profile_id, + payload=payload, + ) + except KnowledgeConnectorError as exc: + session.rollback() + raise _knowledge_http_error(exc) from exc + + +@router.post( + "/knowledge/profiles/{profile_id}/discover", + response_model=KnowledgeDiscoveryResponse, +) +def api_discover_knowledge_profile( + profile_id: str, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), +) -> KnowledgeDiscoveryResponse: + try: + return discover_profile( + session, + principal, + profile_id=profile_id, + ) + except KnowledgeConnectorError as exc: + raise _knowledge_http_error(exc) from exc + + +@router.post( + "/knowledge/profiles/{profile_id}/sync", + response_model=KnowledgeSyncRunItem, +) +def api_synchronize_knowledge_profile( + profile_id: str, + payload: KnowledgeSyncRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), +) -> KnowledgeSyncRunItem: + try: + return synchronize_profile( + session, + principal, + profile_id=profile_id, + payload=payload, + ) + except KnowledgeConnectorError as exc: + raise _knowledge_http_error(exc) from exc + + +@router.get( + "/knowledge/profiles/{profile_id}/objects", + response_model=KnowledgeObjectListResponse, +) +def api_list_knowledge_objects( + profile_id: str, + cursor: str | None = Query(default=None, max_length=500), + limit: int = Query(default=100, ge=1, le=500), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), +) -> KnowledgeObjectListResponse: + try: + items, next_cursor = list_knowledge_objects( + session, + principal, + profile_id=profile_id, + cursor=cursor, + limit=limit, + ) + return KnowledgeObjectListResponse( + items=list(items), + next_cursor=next_cursor, + ) + except KnowledgeConnectorError as exc: + raise _knowledge_http_error(exc) from exc + + +@router.get( + "/knowledge/runs", + response_model=KnowledgeSyncRunListResponse, +) +def api_list_knowledge_runs( + profile_id: str | None = Query(default=None), + limit: int = Query(default=100, ge=1, le=500), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), +) -> KnowledgeSyncRunListResponse: + try: + return KnowledgeSyncRunListResponse( + items=list( + list_knowledge_runs( + session, + principal, + profile_id=profile_id, + limit=limit, + ) + ) + ) + except KnowledgeConnectorError as exc: + raise _knowledge_http_error(exc) from exc + + +@router.post( + "/knowledge/profiles/{profile_id}/migration-dry-runs", + response_model=KnowledgeMigrationDryRunResponse, +) +def api_preview_knowledge_migration( + profile_id: str, + payload: KnowledgeMigrationDryRunRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), +) -> KnowledgeMigrationDryRunResponse: + try: + return migration_dry_run( + session, + principal, + profile_id=profile_id, + payload=payload, + ) + except KnowledgeConnectorError as exc: + raise _knowledge_http_error(exc) from exc + + +@router.post( + "/knowledge/profiles/{profile_id}/pages/{external_page_id}/publish", + response_model=KnowledgePublishResponse, +) +def api_publish_knowledge_page( + profile_id: str, + external_page_id: str, + payload: KnowledgePublishRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(get_api_principal), +) -> KnowledgePublishResponse: + try: + return publish_page( + session, + principal, + profile_id=profile_id, + external_page_id=external_page_id, + payload=payload, + ) + except KnowledgeConnectorError as exc: + raise _knowledge_http_error(exc) from exc + + def _source_response(source: TabularSource) -> TabularSourceResponse: return TabularSourceResponse( ref=source.ref, diff --git a/tests/test_dsar_provider.py b/tests/test_dsar_provider.py index 26a8379..4696db6 100644 --- a/tests/test_dsar_provider.py +++ b/tests/test_dsar_provider.py @@ -17,6 +17,8 @@ from govoplan_connectors.backend.db.models import ( ConnectorConfiguration, ConnectorDefinition, ConnectorDefinitionRevision, + ConnectorKnowledgeProfile, + ConnectorKnowledgeSyncRun, ConnectorSanctionsAcquisitionRun, ConnectorSimulationRun, ConnectorTabularSource, @@ -201,6 +203,43 @@ class ConnectorsDsarProviderTests(unittest.TestCase): created_at=NOW, updated_at=NOW, ), + ConnectorKnowledgeProfile( + id="knowledge-profile-1", + tenant_id="tenant-1", + configuration_id="configuration-1", + status="active", + product="bluespice", + desired_maturity="migrate", + discovered_maturity="migrate", + source_authority_mode="external_mirror", + default_visibility="restricted", + default_acl_tokens=["group:secret-do-not-export"], + namespace_mappings=[{"secret": "mapping-do-not-export"}], + capabilities=["read", "synchronize", "migrate"], + discovery_evidence={"secret": "discovery-do-not-export"}, + health_details={"secret": "health-do-not-export"}, + updated_by="account-1", + created_at=NOW, + updated_at=NOW, + ), + ConnectorKnowledgeSyncRun( + id="knowledge-run-1", + tenant_id="tenant-1", + profile_id="knowledge-profile-1", + mode="delta", + idempotency_key="knowledge-idempotency-do-not-export", + request_hash="knowledge-request-hash-do-not-export", + status="completed", + counts={"update": 1}, + effects=[{"secret": "knowledge-effect-do-not-export"}], + diagnostics=[{"secret": "knowledge-diagnostic-do-not-export"}], + provenance={"secret": "knowledge-provenance-do-not-export"}, + created_by="account-1", + started_at=NOW, + finished_at=NOW, + created_at=NOW, + updated_at=NOW, + ), ) ) @@ -219,6 +258,8 @@ class ConnectorsDsarProviderTests(unittest.TestCase): "definition_actor_attribution", "configuration_actor_attribution", "simulation_actor_attribution", + "knowledge_profile_actor_attribution", + "knowledge_run_actor_attribution", }, {record.resource_type for record in records}, ) @@ -247,6 +288,15 @@ class ConnectorsDsarProviderTests(unittest.TestCase): "diagnostic-do-not-export", "provenance-do-not-export", "review-reason-do-not-export", + "secret-do-not-export", + "mapping-do-not-export", + "discovery-do-not-export", + "health-do-not-export", + "knowledge-idempotency-do-not-export", + "knowledge-request-hash-do-not-export", + "knowledge-effect-do-not-export", + "knowledge-diagnostic-do-not-export", + "knowledge-provenance-do-not-export", ): self.assertNotIn(excluded, exported) @@ -332,7 +382,7 @@ class ConnectorsDsarProviderTests(unittest.TestCase): expected_revision=row.resource_revision, ) self.assertEqual("searched", row.status) - self.assertEqual(5, row.search_result["record_count"]) + self.assertEqual(7, row.search_result["record_count"]) if __name__ == "__main__": diff --git a/tests/test_manifest.py b/tests/test_manifest.py index ce47da1..8fc9462 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -10,6 +10,10 @@ from govoplan_core.core.sanctions import ( CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS, ) from govoplan_connectors.backend.manifest import manifest +from govoplan_connectors.backend.knowledge_connector import ( + KNOWLEDGE_CAPABILITY, + KNOWLEDGE_PROVIDER_ID, +) class ConnectorsManifestTests(unittest.TestCase): @@ -41,6 +45,27 @@ class ConnectorsManifestTests(unittest.TestCase): "connectors.governed-configuration", {topic.id for topic in manifest.documentation}, ) + self.assertIn(KNOWLEDGE_CAPABILITY, manifest.capability_factories) + self.assertIn( + KNOWLEDGE_CAPABILITY, + {interface.name for interface in manifest.provides_interfaces}, + ) + self.assertIn("search", manifest.optional_dependencies) + self.assertIn("wiki", manifest.optional_dependencies) + self.assertIn( + KNOWLEDGE_PROVIDER_ID, + {provider.id for provider in manifest.external_providers}, + ) + self.assertIn( + KNOWLEDGE_PROVIDER_ID, + {registration.id for registration in manifest.search_sources}, + ) + topic = next( + item + for item in manifest.documentation + if item.id == "connectors.mediawiki-bluespice" + ) + self.assertIn("de", topic.translations) if __name__ == "__main__": diff --git a/tests/test_mediawiki_connector.py b/tests/test_mediawiki_connector.py new file mode 100644 index 0000000..0f3c03c --- /dev/null +++ b/tests/test_mediawiki_connector.py @@ -0,0 +1,659 @@ +from __future__ import annotations + +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session + +from govoplan_core.auth import ApiPrincipal +from govoplan_core.core.access import PrincipalRef +from govoplan_core.core.search import ( + SearchAuthorizationRequest, + SearchBackfillRequest, +) +from govoplan_core.core.recovery import RecoveryOperation, RecoveryStatus +from govoplan_core.core.runtime_coordination import ( + RuntimeIdentity, + bind_process_runtime_identity, +) +from govoplan_core.db.base import Base +from govoplan_connectors.backend.db.models import ( + ConnectorConfiguration, + ConnectorDefinition, + ConnectorDefinitionRevision, + ConnectorKnowledgeObject, + ConnectorKnowledgeProfile, + ConnectorKnowledgeSyncRun, +) +from govoplan_connectors.backend.knowledge_connector import ( + KNOWLEDGE_PROVIDER_ID, + KNOWLEDGE_RESOURCE_TYPE, + KnowledgeConnectorError, + create_profile, + discover_profile, + list_objects, + migration_dry_run, + publish_page, + synchronize_profile, + update_profile, +) +from govoplan_connectors.backend.knowledge_schemas import ( + KnowledgeMigrationDryRunRequest, + KnowledgeMigrationTargetState, + KnowledgeNamespaceMapping, + KnowledgeProfileCreateRequest, + KnowledgeProfileUpdateRequest, + KnowledgePublishRequest, + KnowledgeSyncRequest, +) +from govoplan_connectors.backend.knowledge_search import ( + ExternalKnowledgeSearchSource, +) +from govoplan_connectors.backend.mediawiki_transport import ( + MediaWikiChangeBatch, + MediaWikiPublishResult, + MediaWikiTransportError, +) + + +ALL_SCOPES = frozenset( + { + "connectors:knowledge:read", + "connectors:knowledge:admin", + "connectors:knowledge:sync", + "connectors:knowledge:publish", + "connectors:knowledge:migrate", + } +) + + +def principal( + tenant_id: str = "tenant-1", + *, + scopes: frozenset[str] = ALL_SCOPES, + groups: frozenset[str] = frozenset({"editors"}), +) -> ApiPrincipal: + return ApiPrincipal( + principal=PrincipalRef( + account_id="account-1", + membership_id="membership-1", + tenant_id=tenant_id, + scopes=scopes, + group_ids=groups, + ), + account=SimpleNamespace(id="account-1"), + user=SimpleNamespace(id="account-1"), + ) + + +class StaticTransport: + def __init__(self) -> None: + self.batches: list[MediaWikiChangeBatch] = [] + self.change_calls = 0 + self.publish_calls = 0 + self.publish_error: MediaWikiTransportError | None = None + + def discover(self, *, endpoint_url, credential): + del endpoint_url, credential + return { + "curtimestamp": "2026-08-22T10:00:00Z", + "query": { + "general": { + "generator": "MediaWiki 1.43.1", + "phpversion": "8.3.8", + }, + "extensions": [ + {"name": "BlueSpiceFoundation", "version": "4.5.2"}, + {"name": "BlueSpicePermissionManager", "version": "4.5.2"}, + ], + "namespaces": { + "0": {"id": 0, "name": "", "content": True}, + "4": {"id": 4, "name": "GovWiki", "content": True}, + }, + "userinfo": { + "id": 17, + "name": "govoplan", + "rights": ["read", "edit"], + "groups": ["bot"], + }, + }, + } + + def changes(self, **kwargs): + del kwargs + self.change_calls += 1 + if not self.batches: + raise AssertionError("No deterministic change batch remains") + return self.batches.pop(0) + + def publish(self, **kwargs): + self.publish_calls += 1 + if self.publish_error is not None: + raise self.publish_error + return MediaWikiPublishResult( + page_id=str(kwargs.get("expected_page_id") or "99"), + revision_id="901", + title=str(kwargs["title"]), + canonical_url="https://wiki.example.invalid/wiki/Published_Guide", + evidence={"result": "Success", "fixture": True}, + ) + + +def page( + *, + page_id: str = "42", + revision_id: str = "501", + title: str = "Citizen Guide", + acl_tokens: list[str] | None = None, +) -> dict[str, object]: + return { + "change_kind": "upsert", + "change_cursor": f"rcid:{revision_id}", + "pageid": page_id, + "ns": 0, + "title": title, + "fullurl": f"https://wiki.example.invalid/wiki/{title.replace(' ', '_')}", + "lastrevid": revision_id, + "revisions": [ + { + "revid": revision_id, + "timestamp": "2026-08-22T10:00:00Z", + "user": "Ada Admin", + "comment": "Reviewed guidance", + "contentmodel": "wikitext", + "sha1": f"sha-{revision_id}", + "slots": { + "main": { + "content": "Welcome {{UnsupportedBox|important}} [[Services]]" + } + }, + } + ], + "categories": [{"title": "Category:Citizen service"}], + "links": [{"ns": 0, "title": "Services"}], + "images": [{"ns": 6, "title": "File:guide.pdf", "pageid": 71}], + "discussions": [ + { + "id": "discussion-1", + "author": "Ada Admin", + "body": "Please verify this section.", + "created_at": "2026-08-20T09:00:00Z", + } + ], + "permissions": { + "visibility": "restricted", + "acl_tokens": acl_tokens or ["group:editors"], + }, + } + + +class MediaWikiConnectorTests(unittest.TestCase): + def setUp(self) -> None: + bind_process_runtime_identity( + RuntimeIdentity( + installation_id="mediawiki-connector-tests", + node_id="node-1", + incarnation="incarnation-1", + role="worker", + software_version="test", + composition_hash="a" * 64, + ) + ) + self.engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(self.engine) + self.session = Session(self.engine) + self.audit = patch( + "govoplan_connectors.backend.knowledge_connector.audit_event" + ) + self.audit.start() + self.credential = patch( + "govoplan_connectors.backend.knowledge_connector._credential", + return_value={"access_token": "fixture-token"}, + ) + self.credential.start() + self.transport = StaticTransport() + self.configuration_id = self._seed_configuration() + self.profile_id = self._profile() + + def tearDown(self) -> None: + bind_process_runtime_identity(None) + self.credential.stop() + self.audit.stop() + self.session.close() + self.engine.dispose() + + def _seed_configuration(self) -> str: + definition = ConnectorDefinition( + id="definition-1", + tenant_id="tenant-1", + definition_key="knowledge.mediawiki", + name="MediaWiki", + description="Knowledge transport", + status="active", + current_revision=1, + local_definition=True, + ) + self.session.add(definition) + self.session.add( + ConnectorDefinitionRevision( + id="definition-revision-1", + definition_id=definition.id, + revision=1, + specification={ + "provider": "mediawiki", + "protocol": "mediawiki_action_api", + }, + definition_hash="definition-hash", + origin="local", + created_by="account-1", + ) + ) + configuration = ConnectorConfiguration( + id="configuration-1", + tenant_id="tenant-1", + definition_id=definition.id, + name="Institutional knowledge", + status="active", + endpoint_url="https://wiki.example.invalid", + credential_ref="credential-envelope-1", + base_definition_revision=1, + local_overrides={}, + protected_paths=[], + effective_configuration={ + "provider": "mediawiki", + "protocol": "mediawiki_action_api", + }, + effective_hash="configuration-hash", + resource_revision=1, + ambiguity_policy="manual_review", + updated_by="account-1", + ) + self.session.add(configuration) + self.session.flush() + return configuration.id + + def _profile(self) -> str: + created = create_profile( + self.session, + principal(), + KnowledgeProfileCreateRequest( + configuration_id=self.configuration_id, + desired_maturity="migrate", + source_authority_mode="external_mirror", + default_visibility="restricted", + default_acl_tokens=["group:knowledge-managers"], + namespace_mappings=[ + KnowledgeNamespaceMapping( + source_namespace_id=0, + source_name="", + target_space_ref="service-guidance", + target_path_prefix="imported", + ) + ], + ), + ) + return created.id + + def _discover(self): + return discover_profile( + self.session, + principal(), + profile_id=self.profile_id, + transport=self.transport, + ) + + def _sync(self, raw_pages, *, key="sync-1", force_full=True): + self.transport.batches.append( + MediaWikiChangeBatch( + changes=tuple(raw_pages), + next_cursor=None, + complete=True, + high_watermark="2026-08-22T10:05:00Z", + evidence={"fixture": True}, + ) + ) + return synchronize_profile( + self.session, + principal(), + profile_id=self.profile_id, + payload=KnowledgeSyncRequest( + idempotency_key=key, + force_full=force_full, + ), + transport=self.transport, + registry=None, + ) + + def test_discovery_mapping_idempotency_and_migration_loss_diagnostics(self) -> None: + discovery = self._discover() + self.assertEqual("bluespice", discovery.product) + self.assertEqual("4.5.2", discovery.product_version) + self.assertEqual("migrate", discovery.maturity) + self.assertIn("publish", discovery.capabilities) + self.assertIn("permission_metadata", discovery.capabilities) + + run = self._sync([page()]) + self.assertEqual({"create": 1}, run.counts) + self.assertIn( + "attachments_reference_only", {item.code for item in run.diagnostics} + ) + objects, _cursor = list_objects( + self.session, principal(), profile_id=self.profile_id + ) + self.assertEqual(1, len(objects)) + item = objects[0] + self.assertEqual("42", item.external_reference.object_id) + self.assertEqual("501", item.external_reference.version) + self.assertEqual("imported/Citizen-Guide", item.mapped_data["target_path"]) + self.assertEqual("Ada Admin", item.mapped_data["revision_author"]) + self.assertEqual( + "user", item.mapped_data["revision_author_reference"]["object_type"] + ) + self.assertEqual("guide.pdf", item.mapped_data["files"][0]["name"]) + self.assertEqual("discussion-1", item.mapped_data["discussions"][0]["external_id"]) + + replay = synchronize_profile( + self.session, + principal(), + profile_id=self.profile_id, + payload=KnowledgeSyncRequest( + idempotency_key="sync-1", + force_full=True, + ), + transport=self.transport, + registry=None, + ) + self.assertEqual(run.id, replay.id) + self.assertEqual(1, self.transport.change_calls) + with self.assertRaisesRegex(KnowledgeConnectorError, "different request"): + synchronize_profile( + self.session, + principal(), + profile_id=self.profile_id, + payload=KnowledgeSyncRequest( + idempotency_key="sync-1", + force_full=True, + limit=25, + ), + transport=self.transport, + registry=None, + ) + + preview = migration_dry_run( + self.session, + principal(), + profile_id=self.profile_id, + payload=KnowledgeMigrationDryRunRequest( + idempotency_key="migration-1", + target_space_ref="service-guidance", + supported_macros=["SupportedBox"], + existing_targets=[ + KnowledgeMigrationTargetState( + path="imported/Citizen-Guide", + source_external_id="different-page", + attachment_names=["guide.pdf"], + ) + ], + ), + ) + self.assertFalse(preview.can_apply) + self.assertEqual("2026-08-22T10:05:00Z", preview.source_revision) + self.assertEqual( + {"attachment_name_conflict", "target_path_conflict", "unsupported_macro"}, + {item.code for item in preview.diagnostics}, + ) + + def test_acl_changes_moves_deletes_and_search_authorization_are_current(self) -> None: + self._discover() + self._sync([page()]) + source = ExternalKnowledgeSearchSource() + backfill = source.backfill( + self.session, + request=SearchBackfillRequest( + tenant_id="tenant-1", + provider_id=KNOWLEDGE_PROVIDER_ID, + resource_type=KNOWLEDGE_RESOURCE_TYPE, + rebuild_id="rebuild-1", + limit=100, + ), + ) + self.assertEqual(1, len(backfill.documents)) + document = backfill.documents[0] + request = SearchAuthorizationRequest( + reference=document.reference, + source_revision=document.source_revision, + ) + self.assertTrue( + source.authorize(self.session, principal(), requests=[request])[ + document.reference.key + ] + ) + denied = principal(groups=frozenset({"other"})) + self.assertFalse( + source.authorize(self.session, denied, requests=[request])[ + document.reference.key + ] + ) + + self._sync( + [ + page( + revision_id="502", + title="Resident Guide", + acl_tokens=["group:reviewers"], + ) + ], + key="sync-2", + force_full=False, + ) + self.assertFalse( + source.authorize(self.session, principal(), requests=[request])[ + document.reference.key + ] + ) + reviewers = principal(groups=frozenset({"reviewers"})) + self.assertTrue( + source.authorize(self.session, reviewers, requests=[request])[ + document.reference.key + ] + ) + stored = self.session.scalar( + select(ConnectorKnowledgeObject).where( + ConnectorKnowledgeObject.profile_id == self.profile_id, + ConnectorKnowledgeObject.external_id == "42", + ) + ) + self.assertEqual("Resident Guide", stored.title) + self.assertEqual("imported/Resident-Guide", stored.mapped_data["target_path"]) + + self._sync( + [ + { + "change_kind": "delete", + "change_cursor": "logid:700", + "pageid": 0, + "ns": 0, + "title": "Resident Guide", + "timestamp": "2026-08-22T11:00:00Z", + "logid": 700, + } + ], + key="sync-3", + force_full=False, + ) + self.assertEqual("deleted", stored.status) + self.assertFalse( + source.authorize(self.session, reviewers, requests=[request])[ + document.reference.key + ] + ) + after_delete = source.backfill( + self.session, + request=SearchBackfillRequest( + tenant_id="tenant-1", + provider_id=KNOWLEDGE_PROVIDER_ID, + resource_type=KNOWLEDGE_RESOURCE_TYPE, + rebuild_id="rebuild-2", + limit=100, + ), + ) + self.assertEqual((), after_delete.documents) + + def test_publication_replay_and_outcome_unknown_are_evidenced(self) -> None: + self._discover() + payload = KnowledgePublishRequest( + idempotency_key="publish-1", + title="Published Guide", + body="Reviewed body", + summary="Publish approved guidance", + expected_external_revision="900", + ) + result = publish_page( + self.session, + principal(), + profile_id=self.profile_id, + external_page_id="99", + payload=payload, + transport=self.transport, + registry=None, + durable_recovery=True, + ) + replay = publish_page( + self.session, + principal(), + profile_id=self.profile_id, + external_page_id="99", + payload=payload, + transport=self.transport, + registry=None, + durable_recovery=False, + ) + self.assertTrue(result.accepted) + self.assertEqual(result.run.id, replay.run.id) + self.assertEqual("901", result.external_reference.version) + self.assertEqual(1, self.transport.publish_calls) + recovery = self.session.scalar( + select(RecoveryOperation).where( + RecoveryOperation.resource_type == "external_knowledge_page", + RecoveryOperation.resource_id == "99", + ) + ) + self.assertEqual(RecoveryStatus.SUCCEEDED.value, recovery.status) + + self.transport.publish_error = MediaWikiTransportError( + "transport_timeout", + "Provider response timed out.", + retryable=True, + outcome_unknown=True, + ) + with self.assertRaisesRegex(KnowledgeConnectorError, "outcome is unknown"): + publish_page( + self.session, + principal(), + profile_id=self.profile_id, + external_page_id="100", + payload=KnowledgePublishRequest( + idempotency_key="publish-unknown", + title="Uncertain Guide", + body="Body", + ), + transport=self.transport, + registry=None, + durable_recovery=False, + ) + unresolved = self.session.scalar( + select(ConnectorKnowledgeSyncRun).where( + ConnectorKnowledgeSyncRun.profile_id == self.profile_id, + ConnectorKnowledgeSyncRun.idempotency_key == "publish-unknown", + ) + ) + self.assertEqual("outcome_unknown", unresolved.status) + + def test_fallback_acl_changes_and_profile_pause_fail_closed_immediately(self) -> None: + self._discover() + fallback_page = page(page_id="43", title="Fallback Guide") + fallback_page.pop("permissions") + self._sync([fallback_page]) + source = ExternalKnowledgeSearchSource() + document = source.backfill( + self.session, + request=SearchBackfillRequest( + tenant_id="tenant-1", + provider_id=KNOWLEDGE_PROVIDER_ID, + resource_type=KNOWLEDGE_RESOURCE_TYPE, + rebuild_id="fallback-rebuild", + ), + ).documents[0] + request = SearchAuthorizationRequest( + reference=document.reference, + source_revision=document.source_revision, + ) + managers = principal(groups=frozenset({"knowledge-managers"})) + self.assertTrue( + source.authorize(self.session, managers, requests=[request])[ + document.reference.key + ] + ) + profile = self.session.get(ConnectorKnowledgeProfile, self.profile_id) + before_hash = self.session.get( + ConnectorKnowledgeObject, document.resource_id + ).content_hash + updated = update_profile( + self.session, + principal(), + profile_id=self.profile_id, + payload=KnowledgeProfileUpdateRequest( + expected_resource_revision=profile.resource_revision, + default_acl_tokens=["group:reviewers"], + ), + registry=None, + ) + self.assertFalse( + source.authorize(self.session, managers, requests=[request])[ + document.reference.key + ] + ) + reviewers = principal(groups=frozenset({"reviewers"})) + self.assertTrue( + source.authorize(self.session, reviewers, requests=[request])[ + document.reference.key + ] + ) + self.assertNotEqual( + before_hash, + self.session.get(ConnectorKnowledgeObject, document.resource_id).content_hash, + ) + + update_profile( + self.session, + principal(), + profile_id=self.profile_id, + payload=KnowledgeProfileUpdateRequest( + expected_resource_revision=updated.resource_revision, + status="paused", + ), + registry=None, + ) + self.assertFalse( + source.authorize(self.session, reviewers, requests=[request])[ + document.reference.key + ] + ) + + def test_profiles_and_objects_are_tenant_isolated(self) -> None: + self._discover() + self._sync([page()]) + with self.assertRaisesRegex(KnowledgeConnectorError, "not found"): + list_objects( + self.session, + principal("tenant-2"), + profile_id=self.profile_id, + ) + profile = self.session.get(ConnectorKnowledgeProfile, self.profile_id) + self.assertEqual("tenant-1", profile.tenant_id) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mediawiki_transport.py b/tests/test_mediawiki_transport.py new file mode 100644 index 0000000..f4e95a1 --- /dev/null +++ b/tests/test_mediawiki_transport.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import json +from urllib.parse import parse_qs, urlsplit +import unittest +from unittest.mock import patch + +from govoplan_core.security.http_fetch import HttpFetchResponse +from govoplan_connectors.backend.mediawiki_transport import HttpMediaWikiTransport + + +def response(payload: dict[str, object]) -> HttpFetchResponse: + return HttpFetchResponse( + status=200, + headers={"Content-Type": "application/json"}, + body=json.dumps(payload).encode(), + ) + + +class MediaWikiHttpTransportTests(unittest.TestCase): + def setUp(self) -> None: + self.transport = HttpMediaWikiTransport() + + @patch("govoplan_connectors.backend.mediawiki_transport.fetch_http") + def test_discovery_uses_the_action_api_and_sanitized_auth_header(self, fetch) -> None: + fetch.return_value = response( + {"query": {"general": {"generator": "MediaWiki 1.43"}}} + ) + + self.transport.discover( + endpoint_url="https://wiki.example.test", + credential={"access_token": "secret-token"}, + ) + + call = fetch.call_args + self.assertEqual("https://wiki.example.test/api.php", urlsplit(call.args[0])._replace(query="").geturl()) + query = parse_qs(urlsplit(call.args[0]).query) + self.assertEqual(["query"], query["action"]) + self.assertEqual(["siteinfo|userinfo"], query["meta"]) + self.assertEqual("Bearer secret-token", call.kwargs["headers"]["Authorization"]) + self.assertNotIn("secret-token", call.args[0]) + self.assertEqual(10_000_000, call.kwargs["max_bytes"]) + + @patch("govoplan_connectors.backend.mediawiki_transport.fetch_http") + def test_full_backfill_resolves_page_details_and_cursor(self, fetch) -> None: + fetch.side_effect = ( + response( + { + "curtimestamp": "2026-08-22T10:00:00Z", + "continue": {"apcontinue": "Next_Page"}, + "query": { + "allpages": [ + {"pageid": 41, "ns": 0, "title": "First"}, + {"pageid": 42, "ns": 0, "title": "Second"}, + ] + }, + } + ), + response( + { + "query": { + "pages": [ + {"pageid": 41, "ns": 0, "title": "First"}, + {"pageid": 42, "ns": 0, "title": "Second"}, + ] + } + } + ), + ) + + batch = self.transport.changes( + endpoint_url="https://wiki.example.test/api.php", + credential=None, + cursor=None, + limit=2, + force_full=True, + ) + + self.assertEqual(2, len(batch.changes)) + self.assertEqual("Next_Page", batch.next_cursor) + self.assertFalse(batch.complete) + details = parse_qs(urlsplit(fetch.call_args_list[1].args[0]).query) + self.assertEqual(["41|42"], details["pageids"]) + self.assertIn("revisions", details["prop"][0]) + + @patch("govoplan_connectors.backend.mediawiki_transport.fetch_http") + def test_delta_preserves_deletion_log_tombstone_and_cursor(self, fetch) -> None: + fetch.return_value = response( + { + "curtimestamp": "2026-08-22T11:00:00Z", + "query": { + "recentchanges": [ + { + "type": "log", + "logtype": "delete", + "logid": 700, + "pageid": 0, + "ns": 0, + "title": "Deleted page", + "timestamp": "2026-08-22T10:59:00Z", + } + ] + }, + } + ) + + batch = self.transport.changes( + endpoint_url="https://wiki.example.test", + credential=None, + cursor="rccontinue-token", + limit=50, + force_full=False, + ) + + self.assertEqual("delete", batch.changes[0]["change_kind"]) + self.assertEqual("logid:700", batch.changes[0]["change_cursor"]) + self.assertTrue(batch.complete) + + @patch("govoplan_connectors.backend.mediawiki_transport.fetch_http") + def test_publish_uses_csrf_body_expected_revision_and_canonical_url(self, fetch) -> None: + fetch.return_value = response( + { + "edit": { + "result": "Success", + "pageid": 99, + "oldrevid": 900, + "newrevid": 901, + "title": "Published Guide", + } + } + ) + + result = self.transport.publish( + endpoint_url="https://wiki.example.test", + credential={"csrf_token": "csrf-secret", "access_token": "token"}, + title="Published Guide", + body="Reviewed body", + summary="Approved", + expected_revision="900", + minor=True, + ) + + call = fetch.call_args + self.assertEqual("POST", call.kwargs["method"]) + body = parse_qs(call.kwargs["body"].decode()) + self.assertEqual(["edit"], body["action"]) + self.assertEqual(["900"], body["baserevid"]) + self.assertEqual(["1"], body["minor"]) + self.assertEqual(["csrf-secret"], body["token"]) + self.assertEqual("99", result.page_id) + self.assertEqual( + "https://wiki.example.test/wiki/Published_Guide", + result.canonical_url, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_migrations.py b/tests/test_migrations.py index b4f694d..452c0f6 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -24,7 +24,7 @@ class ConnectorsMigrationTests(unittest.TestCase): try: with engine.connect() as connection: self.assertIn( - "a8d9e0f1b2c3", + "b9e0f1a2c3d4", set(MigrationContext.configure(connection).get_current_heads()), ) self.assertTrue( @@ -36,6 +36,9 @@ class ConnectorsMigrationTests(unittest.TestCase): "connector_definition_revisions", "connector_configurations", "connector_simulation_runs", + "connector_knowledge_profiles", + "connector_knowledge_objects", + "connector_knowledge_sync_runs", }.issubset(inspect(connection).get_table_names()) ) finally: diff --git a/tests/test_provider_state.py b/tests/test_provider_state.py index cf30a5d..a4eead7 100644 --- a/tests/test_provider_state.py +++ b/tests/test_provider_state.py @@ -7,14 +7,19 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from govoplan_connectors.backend.db.models import ( + ConnectorKnowledgeObject, + ConnectorKnowledgeProfile, + ConnectorKnowledgeSyncRun, ConnectorSanctionsAcquisitionRun, ConnectorSanctionsSnapshot, ConnectorTabularSource, ) from govoplan_connectors.backend.manifest import manifest from govoplan_connectors.backend.provider_state import ( + KNOWLEDGE_PROVIDER_ID, SANCTIONS_PROVIDER_ID, TABULAR_PROVIDER_ID, + knowledge_provider_states, sanctions_provider_states, tabular_provider_states, ) @@ -27,11 +32,6 @@ class ConnectorsProviderStateTests(unittest.TestCase): self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True) Base.metadata.create_all( self.engine, - tables=( - ConnectorTabularSource.__table__, - ConnectorSanctionsAcquisitionRun.__table__, - ConnectorSanctionsSnapshot.__table__, - ), ) self.session = sessionmaker(bind=self.engine, expire_on_commit=False)() @@ -109,10 +109,69 @@ class ConnectorsProviderStateTests(unittest.TestCase): self.assertNotIn("secret-source-name", rendered) self.assertNotIn("source.example.test", rendered) self.assertEqual( - {TABULAR_PROVIDER_ID, SANCTIONS_PROVIDER_ID}, + {TABULAR_PROVIDER_ID, SANCTIONS_PROVIDER_ID, KNOWLEDGE_PROVIDER_ID}, {item.provider_id for item in manifest.external_provider_state_providers}, ) + def test_external_knowledge_state_reports_health_without_acl_or_endpoint_data(self) -> None: + now = datetime.now(UTC) + profile = ConnectorKnowledgeProfile( + id="knowledge-profile-1", + tenant_id="tenant-1", + configuration_id="configuration-secret", + status="active", + product="bluespice", + product_version="4.5.2", + desired_maturity="migrate", + discovered_maturity="migrate", + source_authority_mode="external_mirror", + default_visibility="restricted", + default_acl_tokens=["group:secret-acl"], + namespace_mappings=[{"secret": "mapping"}], + capabilities=["read", "synchronize", "migrate"], + health_status="healthy", + discovered_at=now, + ) + page = ConnectorKnowledgeObject( + id="knowledge-object-1", + tenant_id="tenant-1", + profile_id=profile.id, + object_type="page", + external_id="42", + title="Secret page title", + status="active", + source_revision="501", + content_hash="c" * 64, + visibility="restricted", + acl_tokens=["group:secret-acl"], + observed_at=now, + ) + run = ConnectorKnowledgeSyncRun( + id="knowledge-run-1", + tenant_id="tenant-1", + profile_id=profile.id, + mode="delta", + idempotency_key="secret-key", + request_hash="d" * 64, + status="completed", + started_at=now, + finished_at=now, + ) + self.session.add_all((profile, page, run)) + self.session.commit() + + state = knowledge_provider_states( + ExternalProviderStateContext(session=self.session, tenant_id="tenant-1") + )[0] + + self.assertEqual("healthy", state.health) + self.assertEqual("ready", state.recovery) + self.assertEqual(1, state.metrics["active_objects"]) + rendered = str(state.to_dict()) + self.assertNotIn("Secret page title", rendered) + self.assertNotIn("group:secret-acl", rendered) + self.assertNotIn("configuration-secret", rendered) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_recovery.py b/tests/test_recovery.py index a2aabdd..2b90071 100644 --- a/tests/test_recovery.py +++ b/tests/test_recovery.py @@ -137,7 +137,7 @@ class ConnectorRecoveryTests(unittest.TestCase): self.assertFalse(declarations["read-snapshot"].provider_mutation) self.assertTrue(declarations["read-snapshot"].implemented) self.assertTrue(declarations["external-mutation"].provider_mutation) - self.assertFalse(declarations["external-mutation"].implemented) + self.assertTrue(declarations["external-mutation"].implemented) def test_stale_atomic_connector_fence_fails_without_claiming_an_effect(self) -> None: recovery = begin_connector_read_snapshot( diff --git a/webui/package.json b/webui/package.json index d300c19..f9eedf1 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/connectors-webui", - "version": "0.1.20", + "version": "0.1.21", "private": true, "type": "module", "main": "src/index.ts", diff --git a/webui/src/api/externalKnowledge.ts b/webui/src/api/externalKnowledge.ts new file mode 100644 index 0000000..a94d0ef --- /dev/null +++ b/webui/src/api/externalKnowledge.ts @@ -0,0 +1,212 @@ +import { apiFetch, apiPath, type ApiSettings } from "@govoplan/core-webui"; + +export type KnowledgeDiagnostic = { + severity: "info" | "warning" | "error"; + code: string; + message: string; + object_ref?: string | null; + field?: string | null; + retryable: boolean; + details: Record; +}; + +export type KnowledgeNamespaceMapping = { + source_namespace_id: number; + source_name: string; + target_space_ref: string; + target_path_prefix: string; + include: boolean; + visibility?: "tenant" | "restricted" | null; + acl_tokens: string[]; +}; + +export type KnowledgeProfile = { + id: string; + tenant_id: string; + configuration_id: string; + status: "active" | "paused"; + product: string; + product_version?: string | null; + desired_maturity: "discover" | "link" | "search" | "read" | "publish" | "synchronize" | "migrate"; + discovered_maturity: string; + source_authority_mode: "external_authoritative" | "external_mirror" | "governed_sync" | "linked_reference"; + default_visibility: "tenant" | "restricted"; + default_acl_tokens: string[]; + namespace_mappings: KnowledgeNamespaceMapping[]; + capabilities: string[]; + discovery_revision?: string | null; + health_status: string; + health_details: Record; + discovered_at?: string | null; + last_sync_cursor?: string | null; + last_high_watermark?: string | null; + resource_revision: number; + credential_reference_present: boolean; + endpoint_configured: boolean; + created_at: string; + updated_at: string; +}; + +export type KnowledgeObject = { + id: string; + profile_id: string; + object_type: string; + external_id: string; + external_page_id?: string | null; + external_revision_id?: string | null; + namespace_id?: number | null; + title: string; + canonical_url?: string | null; + status: string; + source_revision: string; + visibility: string; + acl_tokens: string[]; + mapped_data: Record; + observed_at: string; + resource_revision: number; +}; + +export type KnowledgeRun = { + id: string; + profile_id: string; + mode: string; + idempotency_key: string; + status: string; + cursor_before?: string | null; + cursor_after?: string | null; + high_watermark?: string | null; + counts: Record; + effects: Array>; + diagnostics: KnowledgeDiagnostic[]; + provenance: Record; + started_at: string; + finished_at?: string | null; + created_at: string; +}; + +export type KnowledgeDiscovery = { + profile: KnowledgeProfile; + product: string; + product_version?: string | null; + capabilities: string[]; + namespaces: Array>; + extensions: Array>; + maturity: string; + health_status: string; + diagnostics: KnowledgeDiagnostic[]; + revision: string; +}; + +export type KnowledgeMigrationPreview = { + run: KnowledgeRun; + target_space_ref: string; + source_revision: string; + source_fingerprint: string; + summary: Record; + effects: Array>; + diagnostics: KnowledgeDiagnostic[]; + truncated: boolean; + can_apply: boolean; +}; + +export type KnowledgePublishResult = { + run: KnowledgeRun; + external_reference: Record; + accepted: boolean; + outcome_unknown: boolean; +}; + +const ROOT = "/api/v1/connectors/knowledge"; + +export async function listKnowledgeProfiles(settings: ApiSettings): Promise { + const response = await apiFetch<{ items: KnowledgeProfile[] }>(settings, `${ROOT}/profiles`); + return response.items; +} + +export function createKnowledgeProfile( + settings: ApiSettings, + payload: Record +): Promise { + return apiFetch(settings, `${ROOT}/profiles`, { + method: "POST", + body: JSON.stringify(payload) + }); +} + +export function updateKnowledgeProfile( + settings: ApiSettings, + profileId: string, + payload: Record +): Promise { + return apiFetch(settings, `${ROOT}/profiles/${encodeURIComponent(profileId)}`, { + method: "PUT", + body: JSON.stringify(payload) + }); +} + +export function discoverKnowledgeProfile( + settings: ApiSettings, + profileId: string +): Promise { + return apiFetch(settings, `${ROOT}/profiles/${encodeURIComponent(profileId)}/discover`, { + method: "POST" + }); +} + +export function synchronizeKnowledgeProfile( + settings: ApiSettings, + profileId: string, + payload: Record +): Promise { + return apiFetch(settings, `${ROOT}/profiles/${encodeURIComponent(profileId)}/sync`, { + method: "POST", + body: JSON.stringify(payload) + }); +} + +export async function listKnowledgeObjects( + settings: ApiSettings, + profileId: string +): Promise { + const response = await apiFetch<{ items: KnowledgeObject[] }>( + settings, + apiPath(`${ROOT}/profiles/${encodeURIComponent(profileId)}/objects`, { limit: 100 }) + ); + return response.items; +} + +export async function listKnowledgeRuns( + settings: ApiSettings, + profileId?: string +): Promise { + const response = await apiFetch<{ items: KnowledgeRun[] }>( + settings, + apiPath(`${ROOT}/runs`, { profile_id: profileId || undefined, limit: 100 }) + ); + return response.items; +} + +export function previewKnowledgeMigration( + settings: ApiSettings, + profileId: string, + payload: Record +): Promise { + return apiFetch( + settings, + `${ROOT}/profiles/${encodeURIComponent(profileId)}/migration-dry-runs`, + { method: "POST", body: JSON.stringify(payload) } + ); +} + +export function publishKnowledgePage( + settings: ApiSettings, + profileId: string, + externalPageId: string, + payload: Record +): Promise { + return apiFetch( + settings, + `${ROOT}/profiles/${encodeURIComponent(profileId)}/pages/${encodeURIComponent(externalPageId)}/publish`, + { method: "POST", body: JSON.stringify(payload) } + ); +} diff --git a/webui/src/features/ExternalKnowledgePage.tsx b/webui/src/features/ExternalKnowledgePage.tsx new file mode 100644 index 0000000..3c10ada --- /dev/null +++ b/webui/src/features/ExternalKnowledgePage.tsx @@ -0,0 +1,561 @@ +import { useCallback, useEffect, useMemo, useState } from "react"; +import { + AdminPageLayout, + Button, + Card, + Dialog, + FilterBar, + FormField, + FormGrid, + MetricCard, + MetricGrid, + PageActionBar, + SelectionList, + SelectionListItem, + SelectionListItemContent, + StatePanel, + StatusBadge, + WorkspaceLayout, + formatDateTime, + hasScope, + useUnsavedChanges, + useUnsavedDraftGuard, + type ApiSettings, + type AuthInfo +} from "@govoplan/core-webui"; +import { + createKnowledgeProfile, + discoverKnowledgeProfile, + listKnowledgeObjects, + listKnowledgeProfiles, + listKnowledgeRuns, + previewKnowledgeMigration, + publishKnowledgePage, + synchronizeKnowledgeProfile, + updateKnowledgeProfile, + type KnowledgeMigrationPreview, + type KnowledgeObject, + type KnowledgeProfile, + type KnowledgeRun +} from "../api/externalKnowledge"; + +type Props = { + settings: ApiSettings; + auth: AuthInfo; +}; + +type ProfileDraft = { + status: "active" | "paused"; + desired_maturity: KnowledgeProfile["desired_maturity"]; + source_authority_mode: KnowledgeProfile["source_authority_mode"]; + default_visibility: KnowledgeProfile["default_visibility"]; + default_acl_tokens: string; + namespace_mappings: string; +}; + +const EMPTY_DRAFT: ProfileDraft = { + status: "active", + desired_maturity: "migrate", + source_authority_mode: "external_mirror", + default_visibility: "restricted", + default_acl_tokens: "scope:connectors:knowledge:read", + namespace_mappings: JSON.stringify([{ + source_namespace_id: 0, + source_name: "", + target_space_ref: "external-knowledge", + target_path_prefix: "", + include: true, + acl_tokens: [] + }], null, 2) +}; + +export default function ExternalKnowledgePage({ settings, auth }: Props) { + const [profiles, setProfiles] = useState([]); + const [objects, setObjects] = useState([]); + const [runs, setRuns] = useState([]); + const [selectedId, setSelectedId] = useState(""); + const [draft, setDraft] = useState(EMPTY_DRAFT); + const [savedKey, setSavedKey] = useState(""); + const [search, setSearch] = useState(""); + const [loading, setLoading] = useState(true); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const [success, setSuccess] = useState(""); + const [createOpen, setCreateOpen] = useState(false); + const [configurationId, setConfigurationId] = useState(""); + const [newDraft, setNewDraft] = useState(EMPTY_DRAFT); + const [migrationOpen, setMigrationOpen] = useState(false); + const [targetSpace, setTargetSpace] = useState("external-knowledge"); + const [supportedMacros, setSupportedMacros] = useState(""); + const [existingTargets, setExistingTargets] = useState("[]"); + const [migration, setMigration] = useState(null); + const [publishOpen, setPublishOpen] = useState(false); + const [externalPageId, setExternalPageId] = useState(""); + const [publishTitle, setPublishTitle] = useState(""); + const [publishBody, setPublishBody] = useState(""); + const [publishSummary, setPublishSummary] = useState(""); + const [publishRevision, setPublishRevision] = useState(""); + const { requestDiscard } = useUnsavedChanges(); + + const selected = profiles.find((item) => item.id === selectedId) ?? null; + const canAdmin = hasScope(auth, "connectors:knowledge:admin"); + const canSync = hasScope(auth, "connectors:knowledge:sync"); + const canMigrate = hasScope(auth, "connectors:knowledge:migrate"); + const canPublish = hasScope(auth, "connectors:knowledge:publish"); + const dirty = Boolean(selected && draftKey(draft) !== savedKey); + + const applyProfile = useCallback((profile: KnowledgeProfile | null) => { + const next = profile ? draftFromProfile(profile) : EMPTY_DRAFT; + setDraft(next); + setSavedKey(profile ? draftKey(next) : ""); + }, []); + + const reload = useCallback(async (preferredId?: string) => { + setLoading(true); + setError(""); + try { + const nextProfiles = await listKnowledgeProfiles(settings); + const nextId = preferredId && nextProfiles.some((item) => item.id === preferredId) + ? preferredId + : nextProfiles.some((item) => item.id === selectedId) + ? selectedId + : nextProfiles[0]?.id ?? ""; + const [nextObjects, nextRuns] = nextId + ? await Promise.all([ + listKnowledgeObjects(settings, nextId), + listKnowledgeRuns(settings, nextId) + ]) + : [[], []]; + setProfiles(nextProfiles); + setSelectedId(nextId); + setObjects(nextObjects); + setRuns(nextRuns); + applyProfile(nextProfiles.find((item) => item.id === nextId) ?? null); + } catch (caught) { + setError(errorMessage(caught)); + } finally { + setLoading(false); + } + }, [applyProfile, selectedId, settings]); + + useEffect(() => { + void reload(); + }, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]); + + const save = async (): Promise => { + if (!selected || !canAdmin) return false; + setBusy(true); + setError(""); + try { + const updated = await updateKnowledgeProfile(settings, selected.id, { + expected_resource_revision: selected.resource_revision, + status: draft.status, + desired_maturity: draft.desired_maturity, + source_authority_mode: draft.source_authority_mode, + default_visibility: draft.default_visibility, + default_acl_tokens: lines(draft.default_acl_tokens), + namespace_mappings: parseArray(draft.namespace_mappings, "Namespace mappings") + }); + setSuccess("External knowledge profile saved; fallback ACLs and Search projections were refreshed."); + await reload(updated.id); + return true; + } catch (caught) { + setError(errorMessage(caught)); + return false; + } finally { + setBusy(false); + } + }; + + useUnsavedDraftGuard({ + dirty, + onSave: save, + onDiscard: () => applyProfile(selected), + title: "Unsaved knowledge profile changes", + message: "Save or discard the profile changes before continuing." + }); + + const selectProfile = (profile: KnowledgeProfile) => { + if (profile.id === selectedId) return; + requestDiscard(() => { + setSelectedId(profile.id); + applyProfile(profile); + setObjects([]); + setRuns([]); + setMigration(null); + void Promise.all([ + listKnowledgeObjects(settings, profile.id), + listKnowledgeRuns(settings, profile.id) + ]).then(([nextObjects, nextRuns]) => { + setObjects(nextObjects); + setRuns(nextRuns); + }).catch((caught) => setError(errorMessage(caught))); + }); + }; + + const createProfile = async () => { + if (!configurationId.trim()) return; + setBusy(true); + setError(""); + try { + const created = await createKnowledgeProfile(settings, { + configuration_id: configurationId.trim(), + desired_maturity: newDraft.desired_maturity, + source_authority_mode: newDraft.source_authority_mode, + default_visibility: newDraft.default_visibility, + default_acl_tokens: lines(newDraft.default_acl_tokens), + namespace_mappings: parseArray(newDraft.namespace_mappings, "Namespace mappings") + }); + setCreateOpen(false); + setConfigurationId(""); + setNewDraft(EMPTY_DRAFT); + setSuccess("External knowledge profile created. Run discovery before synchronization."); + await reload(created.id); + } catch (caught) { + setError(errorMessage(caught)); + } finally { + setBusy(false); + } + }; + + const discover = async () => { + if (!selected || dirty) return; + setBusy(true); + setError(""); + try { + const result = await discoverKnowledgeProfile(settings, selected.id); + setSuccess(`Discovered ${result.product} ${result.product_version ?? ""} at ${result.maturity} maturity with ${result.diagnostics.length} diagnostics.`); + await reload(selected.id); + } catch (caught) { + setError(errorMessage(caught)); + } finally { + setBusy(false); + } + }; + + const sync = async (forceFull: boolean) => { + if (!selected || dirty) return; + setBusy(true); + setError(""); + try { + const run = await synchronizeKnowledgeProfile(settings, selected.id, { + idempotency_key: `knowledge-${forceFull ? "backfill" : "delta"}-${crypto.randomUUID()}`, + force_full: forceFull, + limit: 100 + }); + setSuccess(`${forceFull ? "Backfill" : "Delta"} completed with ${effectTotal(run)} effects.`); + await reload(selected.id); + } catch (caught) { + setError(errorMessage(caught)); + } finally { + setBusy(false); + } + }; + + const previewMigration = async () => { + if (!selected || !targetSpace.trim()) return; + setBusy(true); + setError(""); + try { + const result = await previewKnowledgeMigration(settings, selected.id, { + idempotency_key: `knowledge-migration-${crypto.randomUUID()}`, + target_space_ref: targetSpace.trim(), + max_items: 100, + supported_macros: lines(supportedMacros), + existing_targets: parseArray(existingTargets, "Existing targets") + }); + setMigration(result); + setMigrationOpen(false); + setSuccess(result.can_apply + ? "Migration preview is complete and contains no blocking conflict. It did not write Wiki pages." + : "Migration preview found conflicts, errors, or truncation. It did not write Wiki pages."); + await reload(selected.id); + } catch (caught) { + setError(errorMessage(caught)); + } finally { + setBusy(false); + } + }; + + const publish = async () => { + if (!selected || !externalPageId.trim() || !publishTitle.trim()) return; + setBusy(true); + setError(""); + try { + const result = await publishKnowledgePage( + settings, + selected.id, + externalPageId.trim(), + { + idempotency_key: `knowledge-publish-${crypto.randomUUID()}`, + title: publishTitle.trim(), + body: publishBody, + summary: publishSummary.trim(), + expected_external_revision: publishRevision.trim() || null, + minor: false + } + ); + setPublishOpen(false); + setSuccess(result.outcome_unknown + ? "Publication outcome is unknown. Reconcile the provider revision before retrying." + : "Provider accepted the page revision and durable recovery evidence was recorded."); + await reload(selected.id); + } catch (caught) { + setError(errorMessage(caught)); + } finally { + setBusy(false); + } + }; + + const visibleProfiles = useMemo(() => { + const needle = search.trim().toLocaleLowerCase(); + return profiles.filter((item) => !needle || + `${item.product} ${item.product_version ?? ""} ${item.health_status} ${item.configuration_id}` + .toLocaleLowerCase() + .includes(needle)); + }, [profiles, search]); + + const actionBar = void reload(selectedId), loading }} + primaryActions={<> + + + + + + + } + discardAction={{ + label: "Discard changes", + disabled: !selected, + onClick: () => applyProfile(selected) + }} + saveAction={{ + label: "Save", + disabled: !selected || !canAdmin || busy, + disabledReason: !canAdmin ? "External knowledge administration permission is required." : undefined, + onClick: () => void save() + }} + />; + + return + + + item.status !== "deleted").length} /> + !["healthy", "unknown"].includes(item.health_status)).length} tone="warning" /> + ["failed", "outcome_unknown"].includes(item.status)).length} tone="warning" /> + + + + + setSearch(event.target.value)} placeholder="Search profiles" aria-label="Search external knowledge profiles" /> + + + {visibleProfiles.map((profile) => selectProfile(profile)}> + + + )} + {!visibleProfiles.length ? : null} + + } + > + {!selected ? :
+ +
+ + + Discovered maturity: {selected.discovered_maturity} + r{selected.resource_revision} +
+ + + + + + + + + + + + + + +