Release Connectors v0.1.21 with external knowledge integration
Module Package Release / publish-packages (push) Successful in 12s
Module Package Release / publish-packages (push) Successful in 12s
This commit is contained in:
@@ -17,6 +17,8 @@ from govoplan_connectors.backend.db.models import (
|
||||
ConnectorConfiguration,
|
||||
ConnectorDefinition,
|
||||
ConnectorDefinitionRevision,
|
||||
ConnectorKnowledgeProfile,
|
||||
ConnectorKnowledgeSyncRun,
|
||||
ConnectorSanctionsAcquisitionRun,
|
||||
ConnectorSimulationRun,
|
||||
ConnectorTabularSource,
|
||||
@@ -201,6 +203,43 @@ class ConnectorsDsarProviderTests(unittest.TestCase):
|
||||
created_at=NOW,
|
||||
updated_at=NOW,
|
||||
),
|
||||
ConnectorKnowledgeProfile(
|
||||
id="knowledge-profile-1",
|
||||
tenant_id="tenant-1",
|
||||
configuration_id="configuration-1",
|
||||
status="active",
|
||||
product="bluespice",
|
||||
desired_maturity="migrate",
|
||||
discovered_maturity="migrate",
|
||||
source_authority_mode="external_mirror",
|
||||
default_visibility="restricted",
|
||||
default_acl_tokens=["group:secret-do-not-export"],
|
||||
namespace_mappings=[{"secret": "mapping-do-not-export"}],
|
||||
capabilities=["read", "synchronize", "migrate"],
|
||||
discovery_evidence={"secret": "discovery-do-not-export"},
|
||||
health_details={"secret": "health-do-not-export"},
|
||||
updated_by="account-1",
|
||||
created_at=NOW,
|
||||
updated_at=NOW,
|
||||
),
|
||||
ConnectorKnowledgeSyncRun(
|
||||
id="knowledge-run-1",
|
||||
tenant_id="tenant-1",
|
||||
profile_id="knowledge-profile-1",
|
||||
mode="delta",
|
||||
idempotency_key="knowledge-idempotency-do-not-export",
|
||||
request_hash="knowledge-request-hash-do-not-export",
|
||||
status="completed",
|
||||
counts={"update": 1},
|
||||
effects=[{"secret": "knowledge-effect-do-not-export"}],
|
||||
diagnostics=[{"secret": "knowledge-diagnostic-do-not-export"}],
|
||||
provenance={"secret": "knowledge-provenance-do-not-export"},
|
||||
created_by="account-1",
|
||||
started_at=NOW,
|
||||
finished_at=NOW,
|
||||
created_at=NOW,
|
||||
updated_at=NOW,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -219,6 +258,8 @@ class ConnectorsDsarProviderTests(unittest.TestCase):
|
||||
"definition_actor_attribution",
|
||||
"configuration_actor_attribution",
|
||||
"simulation_actor_attribution",
|
||||
"knowledge_profile_actor_attribution",
|
||||
"knowledge_run_actor_attribution",
|
||||
},
|
||||
{record.resource_type for record in records},
|
||||
)
|
||||
@@ -247,6 +288,15 @@ class ConnectorsDsarProviderTests(unittest.TestCase):
|
||||
"diagnostic-do-not-export",
|
||||
"provenance-do-not-export",
|
||||
"review-reason-do-not-export",
|
||||
"secret-do-not-export",
|
||||
"mapping-do-not-export",
|
||||
"discovery-do-not-export",
|
||||
"health-do-not-export",
|
||||
"knowledge-idempotency-do-not-export",
|
||||
"knowledge-request-hash-do-not-export",
|
||||
"knowledge-effect-do-not-export",
|
||||
"knowledge-diagnostic-do-not-export",
|
||||
"knowledge-provenance-do-not-export",
|
||||
):
|
||||
self.assertNotIn(excluded, exported)
|
||||
|
||||
@@ -332,7 +382,7 @@ class ConnectorsDsarProviderTests(unittest.TestCase):
|
||||
expected_revision=row.resource_revision,
|
||||
)
|
||||
self.assertEqual("searched", row.status)
|
||||
self.assertEqual(5, row.search_result["record_count"])
|
||||
self.assertEqual(7, row.search_result["record_count"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -10,6 +10,10 @@ from govoplan_core.core.sanctions import (
|
||||
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
|
||||
)
|
||||
from govoplan_connectors.backend.manifest import manifest
|
||||
from govoplan_connectors.backend.knowledge_connector import (
|
||||
KNOWLEDGE_CAPABILITY,
|
||||
KNOWLEDGE_PROVIDER_ID,
|
||||
)
|
||||
|
||||
|
||||
class ConnectorsManifestTests(unittest.TestCase):
|
||||
@@ -41,6 +45,27 @@ class ConnectorsManifestTests(unittest.TestCase):
|
||||
"connectors.governed-configuration",
|
||||
{topic.id for topic in manifest.documentation},
|
||||
)
|
||||
self.assertIn(KNOWLEDGE_CAPABILITY, manifest.capability_factories)
|
||||
self.assertIn(
|
||||
KNOWLEDGE_CAPABILITY,
|
||||
{interface.name for interface in manifest.provides_interfaces},
|
||||
)
|
||||
self.assertIn("search", manifest.optional_dependencies)
|
||||
self.assertIn("wiki", manifest.optional_dependencies)
|
||||
self.assertIn(
|
||||
KNOWLEDGE_PROVIDER_ID,
|
||||
{provider.id for provider in manifest.external_providers},
|
||||
)
|
||||
self.assertIn(
|
||||
KNOWLEDGE_PROVIDER_ID,
|
||||
{registration.id for registration in manifest.search_sources},
|
||||
)
|
||||
topic = next(
|
||||
item
|
||||
for item in manifest.documentation
|
||||
if item.id == "connectors.mediawiki-bluespice"
|
||||
)
|
||||
self.assertIn("de", topic.translations)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,659 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.core.access import PrincipalRef
|
||||
from govoplan_core.core.search import (
|
||||
SearchAuthorizationRequest,
|
||||
SearchBackfillRequest,
|
||||
)
|
||||
from govoplan_core.core.recovery import RecoveryOperation, RecoveryStatus
|
||||
from govoplan_core.core.runtime_coordination import (
|
||||
RuntimeIdentity,
|
||||
bind_process_runtime_identity,
|
||||
)
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_connectors.backend.db.models import (
|
||||
ConnectorConfiguration,
|
||||
ConnectorDefinition,
|
||||
ConnectorDefinitionRevision,
|
||||
ConnectorKnowledgeObject,
|
||||
ConnectorKnowledgeProfile,
|
||||
ConnectorKnowledgeSyncRun,
|
||||
)
|
||||
from govoplan_connectors.backend.knowledge_connector import (
|
||||
KNOWLEDGE_PROVIDER_ID,
|
||||
KNOWLEDGE_RESOURCE_TYPE,
|
||||
KnowledgeConnectorError,
|
||||
create_profile,
|
||||
discover_profile,
|
||||
list_objects,
|
||||
migration_dry_run,
|
||||
publish_page,
|
||||
synchronize_profile,
|
||||
update_profile,
|
||||
)
|
||||
from govoplan_connectors.backend.knowledge_schemas import (
|
||||
KnowledgeMigrationDryRunRequest,
|
||||
KnowledgeMigrationTargetState,
|
||||
KnowledgeNamespaceMapping,
|
||||
KnowledgeProfileCreateRequest,
|
||||
KnowledgeProfileUpdateRequest,
|
||||
KnowledgePublishRequest,
|
||||
KnowledgeSyncRequest,
|
||||
)
|
||||
from govoplan_connectors.backend.knowledge_search import (
|
||||
ExternalKnowledgeSearchSource,
|
||||
)
|
||||
from govoplan_connectors.backend.mediawiki_transport import (
|
||||
MediaWikiChangeBatch,
|
||||
MediaWikiPublishResult,
|
||||
MediaWikiTransportError,
|
||||
)
|
||||
|
||||
|
||||
ALL_SCOPES = frozenset(
|
||||
{
|
||||
"connectors:knowledge:read",
|
||||
"connectors:knowledge:admin",
|
||||
"connectors:knowledge:sync",
|
||||
"connectors:knowledge:publish",
|
||||
"connectors:knowledge:migrate",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def principal(
|
||||
tenant_id: str = "tenant-1",
|
||||
*,
|
||||
scopes: frozenset[str] = ALL_SCOPES,
|
||||
groups: frozenset[str] = frozenset({"editors"}),
|
||||
) -> ApiPrincipal:
|
||||
return ApiPrincipal(
|
||||
principal=PrincipalRef(
|
||||
account_id="account-1",
|
||||
membership_id="membership-1",
|
||||
tenant_id=tenant_id,
|
||||
scopes=scopes,
|
||||
group_ids=groups,
|
||||
),
|
||||
account=SimpleNamespace(id="account-1"),
|
||||
user=SimpleNamespace(id="account-1"),
|
||||
)
|
||||
|
||||
|
||||
class StaticTransport:
|
||||
def __init__(self) -> None:
|
||||
self.batches: list[MediaWikiChangeBatch] = []
|
||||
self.change_calls = 0
|
||||
self.publish_calls = 0
|
||||
self.publish_error: MediaWikiTransportError | None = None
|
||||
|
||||
def discover(self, *, endpoint_url, credential):
|
||||
del endpoint_url, credential
|
||||
return {
|
||||
"curtimestamp": "2026-08-22T10:00:00Z",
|
||||
"query": {
|
||||
"general": {
|
||||
"generator": "MediaWiki 1.43.1",
|
||||
"phpversion": "8.3.8",
|
||||
},
|
||||
"extensions": [
|
||||
{"name": "BlueSpiceFoundation", "version": "4.5.2"},
|
||||
{"name": "BlueSpicePermissionManager", "version": "4.5.2"},
|
||||
],
|
||||
"namespaces": {
|
||||
"0": {"id": 0, "name": "", "content": True},
|
||||
"4": {"id": 4, "name": "GovWiki", "content": True},
|
||||
},
|
||||
"userinfo": {
|
||||
"id": 17,
|
||||
"name": "govoplan",
|
||||
"rights": ["read", "edit"],
|
||||
"groups": ["bot"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def changes(self, **kwargs):
|
||||
del kwargs
|
||||
self.change_calls += 1
|
||||
if not self.batches:
|
||||
raise AssertionError("No deterministic change batch remains")
|
||||
return self.batches.pop(0)
|
||||
|
||||
def publish(self, **kwargs):
|
||||
self.publish_calls += 1
|
||||
if self.publish_error is not None:
|
||||
raise self.publish_error
|
||||
return MediaWikiPublishResult(
|
||||
page_id=str(kwargs.get("expected_page_id") or "99"),
|
||||
revision_id="901",
|
||||
title=str(kwargs["title"]),
|
||||
canonical_url="https://wiki.example.invalid/wiki/Published_Guide",
|
||||
evidence={"result": "Success", "fixture": True},
|
||||
)
|
||||
|
||||
|
||||
def page(
|
||||
*,
|
||||
page_id: str = "42",
|
||||
revision_id: str = "501",
|
||||
title: str = "Citizen Guide",
|
||||
acl_tokens: list[str] | None = None,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"change_kind": "upsert",
|
||||
"change_cursor": f"rcid:{revision_id}",
|
||||
"pageid": page_id,
|
||||
"ns": 0,
|
||||
"title": title,
|
||||
"fullurl": f"https://wiki.example.invalid/wiki/{title.replace(' ', '_')}",
|
||||
"lastrevid": revision_id,
|
||||
"revisions": [
|
||||
{
|
||||
"revid": revision_id,
|
||||
"timestamp": "2026-08-22T10:00:00Z",
|
||||
"user": "Ada Admin",
|
||||
"comment": "Reviewed guidance",
|
||||
"contentmodel": "wikitext",
|
||||
"sha1": f"sha-{revision_id}",
|
||||
"slots": {
|
||||
"main": {
|
||||
"content": "Welcome {{UnsupportedBox|important}} [[Services]]"
|
||||
}
|
||||
},
|
||||
}
|
||||
],
|
||||
"categories": [{"title": "Category:Citizen service"}],
|
||||
"links": [{"ns": 0, "title": "Services"}],
|
||||
"images": [{"ns": 6, "title": "File:guide.pdf", "pageid": 71}],
|
||||
"discussions": [
|
||||
{
|
||||
"id": "discussion-1",
|
||||
"author": "Ada Admin",
|
||||
"body": "Please verify this section.",
|
||||
"created_at": "2026-08-20T09:00:00Z",
|
||||
}
|
||||
],
|
||||
"permissions": {
|
||||
"visibility": "restricted",
|
||||
"acl_tokens": acl_tokens or ["group:editors"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class MediaWikiConnectorTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
bind_process_runtime_identity(
|
||||
RuntimeIdentity(
|
||||
installation_id="mediawiki-connector-tests",
|
||||
node_id="node-1",
|
||||
incarnation="incarnation-1",
|
||||
role="worker",
|
||||
software_version="test",
|
||||
composition_hash="a" * 64,
|
||||
)
|
||||
)
|
||||
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(self.engine)
|
||||
self.session = Session(self.engine)
|
||||
self.audit = patch(
|
||||
"govoplan_connectors.backend.knowledge_connector.audit_event"
|
||||
)
|
||||
self.audit.start()
|
||||
self.credential = patch(
|
||||
"govoplan_connectors.backend.knowledge_connector._credential",
|
||||
return_value={"access_token": "fixture-token"},
|
||||
)
|
||||
self.credential.start()
|
||||
self.transport = StaticTransport()
|
||||
self.configuration_id = self._seed_configuration()
|
||||
self.profile_id = self._profile()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
bind_process_runtime_identity(None)
|
||||
self.credential.stop()
|
||||
self.audit.stop()
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def _seed_configuration(self) -> str:
|
||||
definition = ConnectorDefinition(
|
||||
id="definition-1",
|
||||
tenant_id="tenant-1",
|
||||
definition_key="knowledge.mediawiki",
|
||||
name="MediaWiki",
|
||||
description="Knowledge transport",
|
||||
status="active",
|
||||
current_revision=1,
|
||||
local_definition=True,
|
||||
)
|
||||
self.session.add(definition)
|
||||
self.session.add(
|
||||
ConnectorDefinitionRevision(
|
||||
id="definition-revision-1",
|
||||
definition_id=definition.id,
|
||||
revision=1,
|
||||
specification={
|
||||
"provider": "mediawiki",
|
||||
"protocol": "mediawiki_action_api",
|
||||
},
|
||||
definition_hash="definition-hash",
|
||||
origin="local",
|
||||
created_by="account-1",
|
||||
)
|
||||
)
|
||||
configuration = ConnectorConfiguration(
|
||||
id="configuration-1",
|
||||
tenant_id="tenant-1",
|
||||
definition_id=definition.id,
|
||||
name="Institutional knowledge",
|
||||
status="active",
|
||||
endpoint_url="https://wiki.example.invalid",
|
||||
credential_ref="credential-envelope-1",
|
||||
base_definition_revision=1,
|
||||
local_overrides={},
|
||||
protected_paths=[],
|
||||
effective_configuration={
|
||||
"provider": "mediawiki",
|
||||
"protocol": "mediawiki_action_api",
|
||||
},
|
||||
effective_hash="configuration-hash",
|
||||
resource_revision=1,
|
||||
ambiguity_policy="manual_review",
|
||||
updated_by="account-1",
|
||||
)
|
||||
self.session.add(configuration)
|
||||
self.session.flush()
|
||||
return configuration.id
|
||||
|
||||
def _profile(self) -> str:
|
||||
created = create_profile(
|
||||
self.session,
|
||||
principal(),
|
||||
KnowledgeProfileCreateRequest(
|
||||
configuration_id=self.configuration_id,
|
||||
desired_maturity="migrate",
|
||||
source_authority_mode="external_mirror",
|
||||
default_visibility="restricted",
|
||||
default_acl_tokens=["group:knowledge-managers"],
|
||||
namespace_mappings=[
|
||||
KnowledgeNamespaceMapping(
|
||||
source_namespace_id=0,
|
||||
source_name="",
|
||||
target_space_ref="service-guidance",
|
||||
target_path_prefix="imported",
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
return created.id
|
||||
|
||||
def _discover(self):
|
||||
return discover_profile(
|
||||
self.session,
|
||||
principal(),
|
||||
profile_id=self.profile_id,
|
||||
transport=self.transport,
|
||||
)
|
||||
|
||||
def _sync(self, raw_pages, *, key="sync-1", force_full=True):
|
||||
self.transport.batches.append(
|
||||
MediaWikiChangeBatch(
|
||||
changes=tuple(raw_pages),
|
||||
next_cursor=None,
|
||||
complete=True,
|
||||
high_watermark="2026-08-22T10:05:00Z",
|
||||
evidence={"fixture": True},
|
||||
)
|
||||
)
|
||||
return synchronize_profile(
|
||||
self.session,
|
||||
principal(),
|
||||
profile_id=self.profile_id,
|
||||
payload=KnowledgeSyncRequest(
|
||||
idempotency_key=key,
|
||||
force_full=force_full,
|
||||
),
|
||||
transport=self.transport,
|
||||
registry=None,
|
||||
)
|
||||
|
||||
def test_discovery_mapping_idempotency_and_migration_loss_diagnostics(self) -> None:
|
||||
discovery = self._discover()
|
||||
self.assertEqual("bluespice", discovery.product)
|
||||
self.assertEqual("4.5.2", discovery.product_version)
|
||||
self.assertEqual("migrate", discovery.maturity)
|
||||
self.assertIn("publish", discovery.capabilities)
|
||||
self.assertIn("permission_metadata", discovery.capabilities)
|
||||
|
||||
run = self._sync([page()])
|
||||
self.assertEqual({"create": 1}, run.counts)
|
||||
self.assertIn(
|
||||
"attachments_reference_only", {item.code for item in run.diagnostics}
|
||||
)
|
||||
objects, _cursor = list_objects(
|
||||
self.session, principal(), profile_id=self.profile_id
|
||||
)
|
||||
self.assertEqual(1, len(objects))
|
||||
item = objects[0]
|
||||
self.assertEqual("42", item.external_reference.object_id)
|
||||
self.assertEqual("501", item.external_reference.version)
|
||||
self.assertEqual("imported/Citizen-Guide", item.mapped_data["target_path"])
|
||||
self.assertEqual("Ada Admin", item.mapped_data["revision_author"])
|
||||
self.assertEqual(
|
||||
"user", item.mapped_data["revision_author_reference"]["object_type"]
|
||||
)
|
||||
self.assertEqual("guide.pdf", item.mapped_data["files"][0]["name"])
|
||||
self.assertEqual("discussion-1", item.mapped_data["discussions"][0]["external_id"])
|
||||
|
||||
replay = synchronize_profile(
|
||||
self.session,
|
||||
principal(),
|
||||
profile_id=self.profile_id,
|
||||
payload=KnowledgeSyncRequest(
|
||||
idempotency_key="sync-1",
|
||||
force_full=True,
|
||||
),
|
||||
transport=self.transport,
|
||||
registry=None,
|
||||
)
|
||||
self.assertEqual(run.id, replay.id)
|
||||
self.assertEqual(1, self.transport.change_calls)
|
||||
with self.assertRaisesRegex(KnowledgeConnectorError, "different request"):
|
||||
synchronize_profile(
|
||||
self.session,
|
||||
principal(),
|
||||
profile_id=self.profile_id,
|
||||
payload=KnowledgeSyncRequest(
|
||||
idempotency_key="sync-1",
|
||||
force_full=True,
|
||||
limit=25,
|
||||
),
|
||||
transport=self.transport,
|
||||
registry=None,
|
||||
)
|
||||
|
||||
preview = migration_dry_run(
|
||||
self.session,
|
||||
principal(),
|
||||
profile_id=self.profile_id,
|
||||
payload=KnowledgeMigrationDryRunRequest(
|
||||
idempotency_key="migration-1",
|
||||
target_space_ref="service-guidance",
|
||||
supported_macros=["SupportedBox"],
|
||||
existing_targets=[
|
||||
KnowledgeMigrationTargetState(
|
||||
path="imported/Citizen-Guide",
|
||||
source_external_id="different-page",
|
||||
attachment_names=["guide.pdf"],
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
self.assertFalse(preview.can_apply)
|
||||
self.assertEqual("2026-08-22T10:05:00Z", preview.source_revision)
|
||||
self.assertEqual(
|
||||
{"attachment_name_conflict", "target_path_conflict", "unsupported_macro"},
|
||||
{item.code for item in preview.diagnostics},
|
||||
)
|
||||
|
||||
def test_acl_changes_moves_deletes_and_search_authorization_are_current(self) -> None:
|
||||
self._discover()
|
||||
self._sync([page()])
|
||||
source = ExternalKnowledgeSearchSource()
|
||||
backfill = source.backfill(
|
||||
self.session,
|
||||
request=SearchBackfillRequest(
|
||||
tenant_id="tenant-1",
|
||||
provider_id=KNOWLEDGE_PROVIDER_ID,
|
||||
resource_type=KNOWLEDGE_RESOURCE_TYPE,
|
||||
rebuild_id="rebuild-1",
|
||||
limit=100,
|
||||
),
|
||||
)
|
||||
self.assertEqual(1, len(backfill.documents))
|
||||
document = backfill.documents[0]
|
||||
request = SearchAuthorizationRequest(
|
||||
reference=document.reference,
|
||||
source_revision=document.source_revision,
|
||||
)
|
||||
self.assertTrue(
|
||||
source.authorize(self.session, principal(), requests=[request])[
|
||||
document.reference.key
|
||||
]
|
||||
)
|
||||
denied = principal(groups=frozenset({"other"}))
|
||||
self.assertFalse(
|
||||
source.authorize(self.session, denied, requests=[request])[
|
||||
document.reference.key
|
||||
]
|
||||
)
|
||||
|
||||
self._sync(
|
||||
[
|
||||
page(
|
||||
revision_id="502",
|
||||
title="Resident Guide",
|
||||
acl_tokens=["group:reviewers"],
|
||||
)
|
||||
],
|
||||
key="sync-2",
|
||||
force_full=False,
|
||||
)
|
||||
self.assertFalse(
|
||||
source.authorize(self.session, principal(), requests=[request])[
|
||||
document.reference.key
|
||||
]
|
||||
)
|
||||
reviewers = principal(groups=frozenset({"reviewers"}))
|
||||
self.assertTrue(
|
||||
source.authorize(self.session, reviewers, requests=[request])[
|
||||
document.reference.key
|
||||
]
|
||||
)
|
||||
stored = self.session.scalar(
|
||||
select(ConnectorKnowledgeObject).where(
|
||||
ConnectorKnowledgeObject.profile_id == self.profile_id,
|
||||
ConnectorKnowledgeObject.external_id == "42",
|
||||
)
|
||||
)
|
||||
self.assertEqual("Resident Guide", stored.title)
|
||||
self.assertEqual("imported/Resident-Guide", stored.mapped_data["target_path"])
|
||||
|
||||
self._sync(
|
||||
[
|
||||
{
|
||||
"change_kind": "delete",
|
||||
"change_cursor": "logid:700",
|
||||
"pageid": 0,
|
||||
"ns": 0,
|
||||
"title": "Resident Guide",
|
||||
"timestamp": "2026-08-22T11:00:00Z",
|
||||
"logid": 700,
|
||||
}
|
||||
],
|
||||
key="sync-3",
|
||||
force_full=False,
|
||||
)
|
||||
self.assertEqual("deleted", stored.status)
|
||||
self.assertFalse(
|
||||
source.authorize(self.session, reviewers, requests=[request])[
|
||||
document.reference.key
|
||||
]
|
||||
)
|
||||
after_delete = source.backfill(
|
||||
self.session,
|
||||
request=SearchBackfillRequest(
|
||||
tenant_id="tenant-1",
|
||||
provider_id=KNOWLEDGE_PROVIDER_ID,
|
||||
resource_type=KNOWLEDGE_RESOURCE_TYPE,
|
||||
rebuild_id="rebuild-2",
|
||||
limit=100,
|
||||
),
|
||||
)
|
||||
self.assertEqual((), after_delete.documents)
|
||||
|
||||
def test_publication_replay_and_outcome_unknown_are_evidenced(self) -> None:
|
||||
self._discover()
|
||||
payload = KnowledgePublishRequest(
|
||||
idempotency_key="publish-1",
|
||||
title="Published Guide",
|
||||
body="Reviewed body",
|
||||
summary="Publish approved guidance",
|
||||
expected_external_revision="900",
|
||||
)
|
||||
result = publish_page(
|
||||
self.session,
|
||||
principal(),
|
||||
profile_id=self.profile_id,
|
||||
external_page_id="99",
|
||||
payload=payload,
|
||||
transport=self.transport,
|
||||
registry=None,
|
||||
durable_recovery=True,
|
||||
)
|
||||
replay = publish_page(
|
||||
self.session,
|
||||
principal(),
|
||||
profile_id=self.profile_id,
|
||||
external_page_id="99",
|
||||
payload=payload,
|
||||
transport=self.transport,
|
||||
registry=None,
|
||||
durable_recovery=False,
|
||||
)
|
||||
self.assertTrue(result.accepted)
|
||||
self.assertEqual(result.run.id, replay.run.id)
|
||||
self.assertEqual("901", result.external_reference.version)
|
||||
self.assertEqual(1, self.transport.publish_calls)
|
||||
recovery = self.session.scalar(
|
||||
select(RecoveryOperation).where(
|
||||
RecoveryOperation.resource_type == "external_knowledge_page",
|
||||
RecoveryOperation.resource_id == "99",
|
||||
)
|
||||
)
|
||||
self.assertEqual(RecoveryStatus.SUCCEEDED.value, recovery.status)
|
||||
|
||||
self.transport.publish_error = MediaWikiTransportError(
|
||||
"transport_timeout",
|
||||
"Provider response timed out.",
|
||||
retryable=True,
|
||||
outcome_unknown=True,
|
||||
)
|
||||
with self.assertRaisesRegex(KnowledgeConnectorError, "outcome is unknown"):
|
||||
publish_page(
|
||||
self.session,
|
||||
principal(),
|
||||
profile_id=self.profile_id,
|
||||
external_page_id="100",
|
||||
payload=KnowledgePublishRequest(
|
||||
idempotency_key="publish-unknown",
|
||||
title="Uncertain Guide",
|
||||
body="Body",
|
||||
),
|
||||
transport=self.transport,
|
||||
registry=None,
|
||||
durable_recovery=False,
|
||||
)
|
||||
unresolved = self.session.scalar(
|
||||
select(ConnectorKnowledgeSyncRun).where(
|
||||
ConnectorKnowledgeSyncRun.profile_id == self.profile_id,
|
||||
ConnectorKnowledgeSyncRun.idempotency_key == "publish-unknown",
|
||||
)
|
||||
)
|
||||
self.assertEqual("outcome_unknown", unresolved.status)
|
||||
|
||||
def test_fallback_acl_changes_and_profile_pause_fail_closed_immediately(self) -> None:
|
||||
self._discover()
|
||||
fallback_page = page(page_id="43", title="Fallback Guide")
|
||||
fallback_page.pop("permissions")
|
||||
self._sync([fallback_page])
|
||||
source = ExternalKnowledgeSearchSource()
|
||||
document = source.backfill(
|
||||
self.session,
|
||||
request=SearchBackfillRequest(
|
||||
tenant_id="tenant-1",
|
||||
provider_id=KNOWLEDGE_PROVIDER_ID,
|
||||
resource_type=KNOWLEDGE_RESOURCE_TYPE,
|
||||
rebuild_id="fallback-rebuild",
|
||||
),
|
||||
).documents[0]
|
||||
request = SearchAuthorizationRequest(
|
||||
reference=document.reference,
|
||||
source_revision=document.source_revision,
|
||||
)
|
||||
managers = principal(groups=frozenset({"knowledge-managers"}))
|
||||
self.assertTrue(
|
||||
source.authorize(self.session, managers, requests=[request])[
|
||||
document.reference.key
|
||||
]
|
||||
)
|
||||
profile = self.session.get(ConnectorKnowledgeProfile, self.profile_id)
|
||||
before_hash = self.session.get(
|
||||
ConnectorKnowledgeObject, document.resource_id
|
||||
).content_hash
|
||||
updated = update_profile(
|
||||
self.session,
|
||||
principal(),
|
||||
profile_id=self.profile_id,
|
||||
payload=KnowledgeProfileUpdateRequest(
|
||||
expected_resource_revision=profile.resource_revision,
|
||||
default_acl_tokens=["group:reviewers"],
|
||||
),
|
||||
registry=None,
|
||||
)
|
||||
self.assertFalse(
|
||||
source.authorize(self.session, managers, requests=[request])[
|
||||
document.reference.key
|
||||
]
|
||||
)
|
||||
reviewers = principal(groups=frozenset({"reviewers"}))
|
||||
self.assertTrue(
|
||||
source.authorize(self.session, reviewers, requests=[request])[
|
||||
document.reference.key
|
||||
]
|
||||
)
|
||||
self.assertNotEqual(
|
||||
before_hash,
|
||||
self.session.get(ConnectorKnowledgeObject, document.resource_id).content_hash,
|
||||
)
|
||||
|
||||
update_profile(
|
||||
self.session,
|
||||
principal(),
|
||||
profile_id=self.profile_id,
|
||||
payload=KnowledgeProfileUpdateRequest(
|
||||
expected_resource_revision=updated.resource_revision,
|
||||
status="paused",
|
||||
),
|
||||
registry=None,
|
||||
)
|
||||
self.assertFalse(
|
||||
source.authorize(self.session, reviewers, requests=[request])[
|
||||
document.reference.key
|
||||
]
|
||||
)
|
||||
|
||||
def test_profiles_and_objects_are_tenant_isolated(self) -> None:
|
||||
self._discover()
|
||||
self._sync([page()])
|
||||
with self.assertRaisesRegex(KnowledgeConnectorError, "not found"):
|
||||
list_objects(
|
||||
self.session,
|
||||
principal("tenant-2"),
|
||||
profile_id=self.profile_id,
|
||||
)
|
||||
profile = self.session.get(ConnectorKnowledgeProfile, self.profile_id)
|
||||
self.assertEqual("tenant-1", profile.tenant_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,158 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from govoplan_core.security.http_fetch import HttpFetchResponse
|
||||
from govoplan_connectors.backend.mediawiki_transport import HttpMediaWikiTransport
|
||||
|
||||
|
||||
def response(payload: dict[str, object]) -> HttpFetchResponse:
|
||||
return HttpFetchResponse(
|
||||
status=200,
|
||||
headers={"Content-Type": "application/json"},
|
||||
body=json.dumps(payload).encode(),
|
||||
)
|
||||
|
||||
|
||||
class MediaWikiHttpTransportTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.transport = HttpMediaWikiTransport()
|
||||
|
||||
@patch("govoplan_connectors.backend.mediawiki_transport.fetch_http")
|
||||
def test_discovery_uses_the_action_api_and_sanitized_auth_header(self, fetch) -> None:
|
||||
fetch.return_value = response(
|
||||
{"query": {"general": {"generator": "MediaWiki 1.43"}}}
|
||||
)
|
||||
|
||||
self.transport.discover(
|
||||
endpoint_url="https://wiki.example.test",
|
||||
credential={"access_token": "secret-token"},
|
||||
)
|
||||
|
||||
call = fetch.call_args
|
||||
self.assertEqual("https://wiki.example.test/api.php", urlsplit(call.args[0])._replace(query="").geturl())
|
||||
query = parse_qs(urlsplit(call.args[0]).query)
|
||||
self.assertEqual(["query"], query["action"])
|
||||
self.assertEqual(["siteinfo|userinfo"], query["meta"])
|
||||
self.assertEqual("Bearer secret-token", call.kwargs["headers"]["Authorization"])
|
||||
self.assertNotIn("secret-token", call.args[0])
|
||||
self.assertEqual(10_000_000, call.kwargs["max_bytes"])
|
||||
|
||||
@patch("govoplan_connectors.backend.mediawiki_transport.fetch_http")
|
||||
def test_full_backfill_resolves_page_details_and_cursor(self, fetch) -> None:
|
||||
fetch.side_effect = (
|
||||
response(
|
||||
{
|
||||
"curtimestamp": "2026-08-22T10:00:00Z",
|
||||
"continue": {"apcontinue": "Next_Page"},
|
||||
"query": {
|
||||
"allpages": [
|
||||
{"pageid": 41, "ns": 0, "title": "First"},
|
||||
{"pageid": 42, "ns": 0, "title": "Second"},
|
||||
]
|
||||
},
|
||||
}
|
||||
),
|
||||
response(
|
||||
{
|
||||
"query": {
|
||||
"pages": [
|
||||
{"pageid": 41, "ns": 0, "title": "First"},
|
||||
{"pageid": 42, "ns": 0, "title": "Second"},
|
||||
]
|
||||
}
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
batch = self.transport.changes(
|
||||
endpoint_url="https://wiki.example.test/api.php",
|
||||
credential=None,
|
||||
cursor=None,
|
||||
limit=2,
|
||||
force_full=True,
|
||||
)
|
||||
|
||||
self.assertEqual(2, len(batch.changes))
|
||||
self.assertEqual("Next_Page", batch.next_cursor)
|
||||
self.assertFalse(batch.complete)
|
||||
details = parse_qs(urlsplit(fetch.call_args_list[1].args[0]).query)
|
||||
self.assertEqual(["41|42"], details["pageids"])
|
||||
self.assertIn("revisions", details["prop"][0])
|
||||
|
||||
@patch("govoplan_connectors.backend.mediawiki_transport.fetch_http")
|
||||
def test_delta_preserves_deletion_log_tombstone_and_cursor(self, fetch) -> None:
|
||||
fetch.return_value = response(
|
||||
{
|
||||
"curtimestamp": "2026-08-22T11:00:00Z",
|
||||
"query": {
|
||||
"recentchanges": [
|
||||
{
|
||||
"type": "log",
|
||||
"logtype": "delete",
|
||||
"logid": 700,
|
||||
"pageid": 0,
|
||||
"ns": 0,
|
||||
"title": "Deleted page",
|
||||
"timestamp": "2026-08-22T10:59:00Z",
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
batch = self.transport.changes(
|
||||
endpoint_url="https://wiki.example.test",
|
||||
credential=None,
|
||||
cursor="rccontinue-token",
|
||||
limit=50,
|
||||
force_full=False,
|
||||
)
|
||||
|
||||
self.assertEqual("delete", batch.changes[0]["change_kind"])
|
||||
self.assertEqual("logid:700", batch.changes[0]["change_cursor"])
|
||||
self.assertTrue(batch.complete)
|
||||
|
||||
@patch("govoplan_connectors.backend.mediawiki_transport.fetch_http")
|
||||
def test_publish_uses_csrf_body_expected_revision_and_canonical_url(self, fetch) -> None:
|
||||
fetch.return_value = response(
|
||||
{
|
||||
"edit": {
|
||||
"result": "Success",
|
||||
"pageid": 99,
|
||||
"oldrevid": 900,
|
||||
"newrevid": 901,
|
||||
"title": "Published Guide",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
result = self.transport.publish(
|
||||
endpoint_url="https://wiki.example.test",
|
||||
credential={"csrf_token": "csrf-secret", "access_token": "token"},
|
||||
title="Published Guide",
|
||||
body="Reviewed body",
|
||||
summary="Approved",
|
||||
expected_revision="900",
|
||||
minor=True,
|
||||
)
|
||||
|
||||
call = fetch.call_args
|
||||
self.assertEqual("POST", call.kwargs["method"])
|
||||
body = parse_qs(call.kwargs["body"].decode())
|
||||
self.assertEqual(["edit"], body["action"])
|
||||
self.assertEqual(["900"], body["baserevid"])
|
||||
self.assertEqual(["1"], body["minor"])
|
||||
self.assertEqual(["csrf-secret"], body["token"])
|
||||
self.assertEqual("99", result.page_id)
|
||||
self.assertEqual(
|
||||
"https://wiki.example.test/wiki/Published_Guide",
|
||||
result.canonical_url,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -24,7 +24,7 @@ class ConnectorsMigrationTests(unittest.TestCase):
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
self.assertIn(
|
||||
"a8d9e0f1b2c3",
|
||||
"b9e0f1a2c3d4",
|
||||
set(MigrationContext.configure(connection).get_current_heads()),
|
||||
)
|
||||
self.assertTrue(
|
||||
@@ -36,6 +36,9 @@ class ConnectorsMigrationTests(unittest.TestCase):
|
||||
"connector_definition_revisions",
|
||||
"connector_configurations",
|
||||
"connector_simulation_runs",
|
||||
"connector_knowledge_profiles",
|
||||
"connector_knowledge_objects",
|
||||
"connector_knowledge_sync_runs",
|
||||
}.issubset(inspect(connection).get_table_names())
|
||||
)
|
||||
finally:
|
||||
|
||||
@@ -7,14 +7,19 @@ from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from govoplan_connectors.backend.db.models import (
|
||||
ConnectorKnowledgeObject,
|
||||
ConnectorKnowledgeProfile,
|
||||
ConnectorKnowledgeSyncRun,
|
||||
ConnectorSanctionsAcquisitionRun,
|
||||
ConnectorSanctionsSnapshot,
|
||||
ConnectorTabularSource,
|
||||
)
|
||||
from govoplan_connectors.backend.manifest import manifest
|
||||
from govoplan_connectors.backend.provider_state import (
|
||||
KNOWLEDGE_PROVIDER_ID,
|
||||
SANCTIONS_PROVIDER_ID,
|
||||
TABULAR_PROVIDER_ID,
|
||||
knowledge_provider_states,
|
||||
sanctions_provider_states,
|
||||
tabular_provider_states,
|
||||
)
|
||||
@@ -27,11 +32,6 @@ class ConnectorsProviderStateTests(unittest.TestCase):
|
||||
self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
|
||||
Base.metadata.create_all(
|
||||
self.engine,
|
||||
tables=(
|
||||
ConnectorTabularSource.__table__,
|
||||
ConnectorSanctionsAcquisitionRun.__table__,
|
||||
ConnectorSanctionsSnapshot.__table__,
|
||||
),
|
||||
)
|
||||
self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
|
||||
|
||||
@@ -109,10 +109,69 @@ class ConnectorsProviderStateTests(unittest.TestCase):
|
||||
self.assertNotIn("secret-source-name", rendered)
|
||||
self.assertNotIn("source.example.test", rendered)
|
||||
self.assertEqual(
|
||||
{TABULAR_PROVIDER_ID, SANCTIONS_PROVIDER_ID},
|
||||
{TABULAR_PROVIDER_ID, SANCTIONS_PROVIDER_ID, KNOWLEDGE_PROVIDER_ID},
|
||||
{item.provider_id for item in manifest.external_provider_state_providers},
|
||||
)
|
||||
|
||||
def test_external_knowledge_state_reports_health_without_acl_or_endpoint_data(self) -> None:
|
||||
now = datetime.now(UTC)
|
||||
profile = ConnectorKnowledgeProfile(
|
||||
id="knowledge-profile-1",
|
||||
tenant_id="tenant-1",
|
||||
configuration_id="configuration-secret",
|
||||
status="active",
|
||||
product="bluespice",
|
||||
product_version="4.5.2",
|
||||
desired_maturity="migrate",
|
||||
discovered_maturity="migrate",
|
||||
source_authority_mode="external_mirror",
|
||||
default_visibility="restricted",
|
||||
default_acl_tokens=["group:secret-acl"],
|
||||
namespace_mappings=[{"secret": "mapping"}],
|
||||
capabilities=["read", "synchronize", "migrate"],
|
||||
health_status="healthy",
|
||||
discovered_at=now,
|
||||
)
|
||||
page = ConnectorKnowledgeObject(
|
||||
id="knowledge-object-1",
|
||||
tenant_id="tenant-1",
|
||||
profile_id=profile.id,
|
||||
object_type="page",
|
||||
external_id="42",
|
||||
title="Secret page title",
|
||||
status="active",
|
||||
source_revision="501",
|
||||
content_hash="c" * 64,
|
||||
visibility="restricted",
|
||||
acl_tokens=["group:secret-acl"],
|
||||
observed_at=now,
|
||||
)
|
||||
run = ConnectorKnowledgeSyncRun(
|
||||
id="knowledge-run-1",
|
||||
tenant_id="tenant-1",
|
||||
profile_id=profile.id,
|
||||
mode="delta",
|
||||
idempotency_key="secret-key",
|
||||
request_hash="d" * 64,
|
||||
status="completed",
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
)
|
||||
self.session.add_all((profile, page, run))
|
||||
self.session.commit()
|
||||
|
||||
state = knowledge_provider_states(
|
||||
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||
)[0]
|
||||
|
||||
self.assertEqual("healthy", state.health)
|
||||
self.assertEqual("ready", state.recovery)
|
||||
self.assertEqual(1, state.metrics["active_objects"])
|
||||
rendered = str(state.to_dict())
|
||||
self.assertNotIn("Secret page title", rendered)
|
||||
self.assertNotIn("group:secret-acl", rendered)
|
||||
self.assertNotIn("configuration-secret", rendered)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -137,7 +137,7 @@ class ConnectorRecoveryTests(unittest.TestCase):
|
||||
self.assertFalse(declarations["read-snapshot"].provider_mutation)
|
||||
self.assertTrue(declarations["read-snapshot"].implemented)
|
||||
self.assertTrue(declarations["external-mutation"].provider_mutation)
|
||||
self.assertFalse(declarations["external-mutation"].implemented)
|
||||
self.assertTrue(declarations["external-mutation"].implemented)
|
||||
|
||||
def test_stale_atomic_connector_fence_fails_without_claiming_an_effect(self) -> None:
|
||||
recovery = begin_connector_read_snapshot(
|
||||
|
||||
Reference in New Issue
Block a user