GovOPlaN/govoplan-connectors
2
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Release Connectors v0.1.21 with external knowledge integration
Module Package Release / publish-packages (push) Successful in 12s
Details

Browse Source
This commit is contained in:
zemion
2026-08-22 14:34:36 +02:00
parent 2c2b11f860
commit 79e2315e89
28 changed files with 6351 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_connectors/backend/manifest.py
+260 -9
View File
@@ -41,16 +41,34 @@ from govoplan_core.core.files import CAPABILITY_FILES_TABULAR_CONTENT
from govoplan_core.core.sanctions import (
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
)
from govoplan_core.core.search import SearchSourceProviderRegistration
from govoplan_core.db.base import Base
from govoplan_connectors.backend.db.models import (
ConnectorConfiguration,
ConnectorDefinition,
ConnectorDefinitionRevision,
ConnectorKnowledgeObject,
ConnectorKnowledgeProfile,
ConnectorKnowledgeSyncRun,
ConnectorSanctionsAcquisitionRun,
ConnectorSanctionsSnapshot,
ConnectorSimulationRun,
ConnectorTabularSource,
)
from govoplan_connectors.backend.knowledge_connector import (
KNOWLEDGE_ADMIN_SCOPE,
KNOWLEDGE_CAPABILITY,
KNOWLEDGE_INTERFACE_VERSION,
KNOWLEDGE_MIGRATE_SCOPE,
KNOWLEDGE_PROVIDER_ID,
KNOWLEDGE_PUBLISH_SCOPE,
KNOWLEDGE_READ_SCOPE,
KNOWLEDGE_SYNC_SCOPE,
ExternalKnowledgeCapability,
)
from govoplan_connectors.backend.knowledge_search import (
create_external_knowledge_search_source,
)
from govoplan_connectors.backend.dsar_provider import (
CONNECTORS_DSAR_CAPABILITY,
ConnectorsDsarProvider,
@@ -77,13 +95,14 @@ from govoplan_connectors.backend.feeds import (
from govoplan_connectors.backend.provider_state import (
SANCTIONS_PROVIDER_ID,
TABULAR_PROVIDER_ID,
knowledge_provider_states,
sanctions_provider_states,
tabular_provider_states,
)
MODULE_ID = "connectors"
MODULE_VERSION = "0.1.20"
MODULE_VERSION = "0.1.21"
TABULAR_SOURCE_INTERFACE_VERSION = "0.1.0"
DATASOURCE_ORIGIN_INTERFACE_VERSION = "0.1.0"
SANCTIONS_SNAPSHOT_INTERFACE_VERSION = "1.0.0"
@@ -125,6 +144,11 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
reference="tests/test_governed_runtime.py",
summary="Exercises immutable definition revisions, protected local overrides, idempotent simulations, and explicit ambiguity review.",
),
ModuleMaturityEvidence(
kind="test",
reference="tests/test_mediawiki_connector.py",
summary="Exercises deterministic MediaWiki/BlueSpice discovery, stable mapping, bounded deltas, ACL-safe Search, migration loss diagnostics, and publication recovery states.",
),
ModuleMaturityEvidence(
kind="documentation",
reference="docs/CONNECTOR_SOURCE_LIFECYCLE.md",
@@ -134,11 +158,13 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
known_limits=(
"Tabular origins support immutable snapshots, exact managed CSV/XLSX versions, and read-only PostgreSQL tables; arbitrary REST and other database adapters remain future providers.",
"Feed publication renders a governed document but does not yet push it to an external publishing endpoint.",
"The generic governed runtime simulates deterministic mapping and validation; provider-specific live writes remain owned by explicit connector adapters.",
"The MediaWiki/BlueSpice adapter publishes revision-checked page edits; generic simulations and all other providers do not imply a live write capability.",
"Migration into native Wiki is preview-only; a target-side write worker and Wiki-owned mutation contract remain future work.",
),
supported_authority_modes=(
"external_authoritative",
"external_mirror",
"governed_sync",
"linked_reference",
),
owned_concepts=(
@@ -146,23 +172,38 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
"protocol interaction",
"immutable connector snapshots",
"connector acquisition health",
"external knowledge synchronization evidence",
),
non_owned_concepts=(
"datasource catalogue identity and lifecycle",
"domain records and business semantics",
"data transformations",
"screening dispositions",
"native Wiki spaces, pages, and revision semantics",
),
target_tested_providers=(
TABULAR_PROVIDER_ID,
SANCTIONS_PROVIDER_ID,
KNOWLEDGE_PROVIDER_ID,
),
documentation=ModuleArchitectureDocumentation(
migration=("src/govoplan_connectors/backend/migrations/versions",),
upgrade=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
recovery=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
security=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
operations=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
upgrade=(
"docs/CONNECTOR_SOURCE_LIFECYCLE.md",
"docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md",
),
recovery=(
"docs/CONNECTOR_SOURCE_LIFECYCLE.md",
"docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md",
),
security=(
"docs/CONNECTOR_SOURCE_LIFECYCLE.md",
"docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md",
),
operations=(
"docs/CONNECTOR_SOURCE_LIFECYCLE.md",
"docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md",
),
),
)
@@ -261,6 +302,88 @@ EXTERNAL_PROVIDERS = (
"connectors.sanctions-snapshots",
),
),
ExternalProviderDeclaration(
id=KNOWLEDGE_PROVIDER_ID,
module_id=MODULE_ID,
label="MediaWiki and BlueSpice knowledge provider",
maturity="migrate",
operations=(
"discover",
"link",
"search",
"read",
"publish",
"synchronize",
"migrate",
"preview",
"dry_run",
),
objects=(
ProviderObjectDeclaration(
object_type="external_knowledge_page",
field_groups=(
"stable_identity",
"revisions",
"namespaces",
"categories",
"links",
"files",
"discussions",
"permissions",
"source_provenance",
),
authority_modes=(
"external_authoritative",
"external_mirror",
"governed_sync",
"linked_reference",
),
default_authority_mode="external_mirror",
),
),
behavior=ProviderBehaviorDeclaration(
revision_tokens="Stable MediaWiki page ids, revision ids, discovery revisions, recent-change cursors, content digests, and provider timestamps are retained.",
concurrency="Profile changes use optimistic revisions; publication supplies an expected provider revision and a durable idempotency fence.",
freshness="The latest discovery time, sync high-watermark, recent-change cursor, source update time, and health state are exposed.",
health="Product discovery, authentication, transport, mapping loss, ACL degradation, Search deferral, and outcome-unknown publication are explicit without exposing credentials.",
max_read_items=500,
idempotency="Backfill, delta synchronization, migration previews, and publication require caller keys; exact replays return the committed evidence and mismatched reuse is rejected.",
retry="Read-only discovery and synchronization may be retried with a new key; a publication with an unknown outcome must be reconciled before retry.",
timeout_seconds=30,
conflicts="Namespace and path mappings are explicit; migration previews block target-path and attachment conflicts, while publication rejects stale expected revisions.",
outcome_unknown="A timed-out publication remains outcome-unknown behind a durable recovery fence until the institutional operator verifies the provider revision.",
outcome_unknown_supported=True,
evidence="Mapped snapshots retain stable external references, revision identity, source hashes, transport provenance, ACLs, and bounded loss diagnostics.",
audit_event_types=(
"connectors.knowledge.profile.created",
"connectors.knowledge.profile.updated",
"connectors.knowledge.profile.discovered",
"connectors.knowledge.profile.synchronized",
"connectors.knowledge.migration.previewed",
"connectors.knowledge.page.published",
),
correction="A later provider revision or explicit publication creates a new connector snapshot revision while synchronization-run evidence remains retained.",
rollback="Local snapshot and terminal recovery evidence commit atomically; remote publication cannot be rolled back by a local transaction.",
reconciliation="Rediscover capabilities, compare stable page and revision ids, run a bounded delta or full backfill, and inspect unresolved publication evidence.",
outage="Existing snapshots remain visible only through current tenant and ACL authorization and are marked stale or unavailable; no provider freshness claim is made.",
classifications=("public", "internal", "confidential", "restricted"),
purposes=(
"external knowledge discovery",
"authorized federated search",
"knowledge synchronization",
"migration planning",
"governed publication",
),
retention="The tenant's connector, Records, and target Wiki policies determine snapshot and operation-evidence retention.",
secret_handling="Credentials are resolved from a scoped Core credential envelope, never placed in endpoint URLs, persisted snapshots, diagnostics, or API responses.",
),
capability_names=(KNOWLEDGE_CAPABILITY,),
interface_names=(KNOWLEDGE_CAPABILITY,),
documentation_topic_ids=(
"connectors.authority-and-effects",
"connectors.mediawiki-bluespice",
),
),
)
@@ -314,6 +437,31 @@ PERMISSIONS = (
"Refresh sanctions sources",
"Acquire a new immutable sanctions source snapshot.",
),
_permission(
KNOWLEDGE_READ_SCOPE,
"View external knowledge",
"View authorized MediaWiki and BlueSpice profiles, pages, and synchronization evidence.",
),
_permission(
KNOWLEDGE_ADMIN_SCOPE,
"Administer external knowledge",
"Configure knowledge profiles, namespace mappings, authority, visibility, and discovery.",
),
_permission(
KNOWLEDGE_SYNC_SCOPE,
"Synchronize external knowledge",
"Run bounded MediaWiki and BlueSpice backfills and change synchronization.",
),
_permission(
KNOWLEDGE_PUBLISH_SCOPE,
"Publish external knowledge",
"Publish a governed page revision with concurrency and recovery evidence.",
),
_permission(
KNOWLEDGE_MIGRATE_SCOPE,
"Preview knowledge migration",
"Dry-run a bounded migration into Wiki and inspect loss or conflict diagnostics.",
),
)
ROLE_TEMPLATES = (
@@ -329,6 +477,11 @@ ROLE_TEMPLATES = (
FEED_PRIVATE_PUBLISH_SCOPE,
SANCTIONS_READ_SCOPE,
SANCTIONS_REFRESH_SCOPE,
KNOWLEDGE_READ_SCOPE,
KNOWLEDGE_ADMIN_SCOPE,
KNOWLEDGE_SYNC_SCOPE,
KNOWLEDGE_PUBLISH_SCOPE,
KNOWLEDGE_MIGRATE_SCOPE,
),
),
RoleTemplate(
@@ -341,13 +494,16 @@ ROLE_TEMPLATES = (
FEED_PUBLISH_SCOPE,
SANCTIONS_READ_SCOPE,
SANCTIONS_REFRESH_SCOPE,
KNOWLEDGE_READ_SCOPE,
KNOWLEDGE_SYNC_SCOPE,
KNOWLEDGE_MIGRATE_SCOPE,
),
),
RoleTemplate(
slug="connector_source_reader",
name="Connector source reader",
description="Discover and preview tabular connector sources.",
permissions=(READ_SCOPE, SANCTIONS_READ_SCOPE),
permissions=(READ_SCOPE, SANCTIONS_READ_SCOPE, KNOWLEDGE_READ_SCOPE),
),
)
@@ -382,6 +538,10 @@ def _dsar_provider(_context) -> ConnectorsDsarProvider:
return ConnectorsDsarProvider()
def _knowledge_provider(_context) -> ExternalKnowledgeCapability:
return ExternalKnowledgeCapability()
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
return {
"connector_definitions": (
@@ -417,6 +577,24 @@ def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
.filter(ConnectorSanctionsAcquisitionRun.tenant_id == tenant_id)
.count()
),
"connector_knowledge_profiles": (
session.query(ConnectorKnowledgeProfile)
.filter(ConnectorKnowledgeProfile.tenant_id == tenant_id)
.count()
),
"connector_knowledge_objects": (
session.query(ConnectorKnowledgeObject)
.filter(
ConnectorKnowledgeObject.tenant_id == tenant_id,
ConnectorKnowledgeObject.status != "deleted",
)
.count()
),
"connector_knowledge_runs": (
session.query(ConnectorKnowledgeSyncRun)
.filter(ConnectorKnowledgeSyncRun.tenant_id == tenant_id)
.count()
),
}
@@ -433,6 +611,8 @@ manifest = ModuleManifest(
"portal",
"reporting",
"risk_compliance",
"search",
"wiki",
),
required_capabilities=(
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
@@ -463,6 +643,10 @@ manifest = ModuleManifest(
name="connectors.runtime_contract",
version=CONNECTOR_RUNTIME_INTERFACE_VERSION,
),
ModuleInterfaceProvider(
name=KNOWLEDGE_CAPABILITY,
version=KNOWLEDGE_INTERFACE_VERSION,
),
ModuleInterfaceProvider(name=CONNECTORS_DSAR_CAPABILITY, version="0.1.0"),
),
requires_interfaces=(
@@ -495,6 +679,14 @@ manifest = ModuleManifest(
parent_id="connectors.admin.governed-configurations",
order=20,
),
ViewSurface(
id="connectors.admin.external-knowledge",
module_id=MODULE_ID,
kind="section",
label="External knowledge",
parent_id="connectors.admin.governed-configurations",
order=30,
),
),
),
capability_factories={
@@ -503,6 +695,7 @@ manifest = ModuleManifest(
CAPABILITY_DATASOURCE_ORIGINS: _datasource_origin_provider,
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS: (_sanctions_snapshot_provider),
CAPABILITY_CONNECTORS_FEEDS: _feed_provider,
KNOWLEDGE_CAPABILITY: _knowledge_provider,
CONNECTORS_DSAR_CAPABILITY: _dsar_provider,
},
capability_documentation={
@@ -516,6 +709,13 @@ manifest = ModuleManifest(
),
},
tenant_summary_providers=(_tenant_summary,),
search_sources=(
SearchSourceProviderRegistration(
id=KNOWLEDGE_PROVIDER_ID,
factory=create_external_knowledge_search_source,
order=65,
),
),
architecture=ARCHITECTURE,
external_providers=EXTERNAL_PROVIDERS,
external_provider_state_providers=(
@@ -529,6 +729,11 @@ manifest = ModuleManifest(
provider_id=SANCTIONS_PROVIDER_ID,
provider=sanctions_provider_states,
),
ExternalProviderStateProviderRegistration(
module_id=MODULE_ID,
provider_id=KNOWLEDGE_PROVIDER_ID,
provider=knowledge_provider_states,
),
),
migration_spec=MigrationSpec(
module_id=MODULE_ID,
@@ -536,6 +741,9 @@ manifest = ModuleManifest(
script_location=str(Path(__file__).with_name("migrations") / "versions"),
retirement_supported=True,
retirement_provider=drop_table_retirement_provider(
ConnectorKnowledgeSyncRun,
ConnectorKnowledgeObject,
ConnectorKnowledgeProfile,
ConnectorSimulationRun,
ConnectorConfiguration,
ConnectorDefinitionRevision,
@@ -552,6 +760,9 @@ manifest = ModuleManifest(
),
uninstall_guard_providers=(
persistent_table_uninstall_guard(
ConnectorKnowledgeSyncRun,
ConnectorKnowledgeObject,
ConnectorKnowledgeProfile,
ConnectorSimulationRun,
ConnectorConfiguration,
ConnectorDefinitionRevision,
@@ -573,8 +784,9 @@ manifest = ModuleManifest(
body=(
"Connectors correlates only an exact tenant account identifier and can "
"narrow an already verified search to one source, acquisition, "
"definition, configuration, or simulation. The export identifies the "
"subject's configuration, acquisition, simulation, and review activity "
"definition, configuration, simulation, external-knowledge profile, or "
"knowledge operation. The export identifies the subject's configuration, "
"acquisition, simulation, knowledge-operation, and review activity "
"using bounded lifecycle metadata. It never includes credential or "
"endpoint references, source rows, external responses, request payloads, "
"mapping and configuration documents, diagnostics, provenance, hashes, "
@@ -709,6 +921,45 @@ manifest = ModuleManifest(
related_modules=("datasources", "dataflow", "portal", "reporting"),
order=42,
),
DocumentationTopic(
id="connectors.mediawiki-bluespice",
title="Connect MediaWiki and BlueSpice knowledge",
summary="Discover, synchronize, search, publish, and preview migration of external knowledge without weakening source permissions.",
body=(
"A connector administrator first creates a governed MediaWiki Action API configuration whose endpoint passes Core egress and peer validation; credentials remain in a scoped credential envelope. A knowledge profile then maps each included source namespace to a target Wiki space and path prefix, selects source authority, and sets a tenant or restricted fallback ACL. Discovery records product, version, extensions, namespaces, authentication state, capabilities, and loss diagnostics. Run a bounded full backfill once, then cursor-based deltas for revisions, moves, deletions, and permission changes. Stable page and revision identifiers, canonical links, redirects, files, categories, links, discussions, provenance, and current ACLs remain attached to the connector snapshot. Search indexes only active, non-deleted pages and rechecks the current profile, tenant, permission, and ACL before returning every result; disabling Connectors removes its Search projection. Publication requires an expected external revision, an idempotency key, and durable outcome evidence. An unknown provider outcome must be reconciled before retry. Migration into native Wiki is deliberately preview-only in this slice: the dry-run reports target-path collisions, attachment-name conflicts, unsupported macros, truncation, and source fingerprints; a successful preview does not write native Wiki pages. Existing authorized snapshots can remain visible during a provider outage, but health and freshness stay explicit and no current-source claim is made."
),
layer="configured",
documentation_types=("admin", "user"),
audience=(
"operator",
"module_admin",
"integration_admin",
"knowledge_manager",
"auditor",
),
related_modules=("core", "search", "wiki", "files", "audit", "policy"),
order=43,
translations={
"de": {
"title": "Wissen aus MediaWiki und BlueSpice anbinden",
"summary": "Externes Wissen erkennen, synchronisieren, durchsuchen, veröffentlichen und eine Migration prüfen, ohne Quellberechtigungen abzuschwächen.",
"body": (
"Eine Connector-Administration erstellt zuerst eine gesteuerte Konfiguration für die MediaWiki Action API. Der Endpunkt unterliegt der zentralen Ausgangs- und Gegenstellenprüfung; Zugangsdaten bleiben in einem zweckgebundenen Zugangsdaten-Umschlag. Das Wissensprofil ordnet jeden eingeschlossenen Quellnamensraum einem Zielbereich und Pfadpräfix in Wiki zu und legt Quellhoheit sowie eine mandantenweite oder eingeschränkte Ersatz-ACL fest. Die Erkennung dokumentiert Produkt, Version, Erweiterungen, Namensräume, Authentifizierungsstatus, Fähigkeiten und Verlustdiagnosen. Nach einem begrenzten Vollabgleich folgen cursorbasierte Änderungen für Revisionen, Verschiebungen, Löschungen und Berechtigungen. Stabile Seiten- und Revisionskennungen, kanonische Links, Weiterleitungen, Dateien, Kategorien, Links, Diskussionen, Herkunft und aktuelle ACLs bleiben am Snapshot. Search indiziert nur aktive, nicht gelöschte Seiten und prüft bei jedem Treffer Profilstatus, Mandant, Berechtigung und ACL erneut; bei Deaktivierung von Connectors wird dessen Suchprojektion entfernt. Veröffentlichungen erfordern die erwartete externe Revision, einen Idempotenzschlüssel und dauerhafte Ergebnisevidenz. Ein unbekanntes Ergebnis muss vor einem erneuten Versuch abgeglichen werden. Die Migration in das native Wiki ist in diesem Ausbauschritt ausschließlich eine Vorschau: Sie meldet Pfad- und Anhangskonflikte, nicht unterstützte Makros, Begrenzungen und Quellfingerabdrücke, schreibt aber keine Wiki-Seiten. Bei einem Ausfall dürfen bestehende Snapshots nur für weiterhin Berechtigte sichtbar bleiben; Zustand und Aktualität bleiben ausdrücklich erkennbar."
),
}
},
metadata={
"kind": "guide",
"help_contexts": ["connectors.admin.external-knowledge"],
"prerequisites": [
"An active governed MediaWiki Action API configuration exists.",
"Namespace targets and fallback ACLs have been reviewed.",
"The Search and Wiki modules are optional and remain capability-separated.",
],
"outcome": "External knowledge remains identity-stable, loss-visible, ACL-safe, and migration-ready.",
"verification": "Rediscover the profile, run a keyed delta, inspect health and diagnostics, verify an allowed and denied Search principal, and run a migration dry-run before any target-side work.",
},
),
DocumentationTopic(
id="connectors.sanctions-snapshots",
title="Sanctions source snapshots",