From b5d3983844f912937814a4f8f87aa91a6899d3c0 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Fri, 7 Aug 2026 14:53:14 +0200 Subject: [PATCH] feat: export manifest documentation for public sites --- README.md | 18 ++ pyproject.toml | 3 + src/govoplan_docs/backend/manifest.py | 39 ++- src/govoplan_docs/public_export.py | 424 ++++++++++++++++++++++++++ tests/test_public_export.py | 153 ++++++++++ 5 files changed, 635 insertions(+), 2 deletions(-) create mode 100644 src/govoplan_docs/public_export.py create mode 100644 tests/test_public_export.py diff --git a/README.md b/README.md index 8aad9fb..181f7c2 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,24 @@ cd /mnt/DATA/git/govoplan Feature content remains in the owning module. Docs indexes and renders the contributions without importing feature implementations. +## Public documentation export + +The public website is generated from the same static `DocumentationTopic` +contributions that power the in-product documentation. Export every installed +module, or every module checkout in a workspace, with: + +```bash +govoplan-docs-export-public \ + --workspace-root /mnt/DATA/git \ + --output public/docs/v1/catalog.json \ + --coverage-output docs/DOCUMENTATION_COVERAGE.md +``` + +Use `--check` in publication CI to reject a stale checked-in catalog. Dynamic +`documentation_providers` remain instance-only because their output can depend +on permissions, policy, configuration, and live provider state; the export +records which modules have such additional documentation. + Pressing `F1` resolves the focused field or action first, then its containing dialog or section, current page, and owning module. The shell sends the focused context together with `fallback_context` and `module`; Docs selects the first diff --git a/pyproject.toml b/pyproject.toml index 54e3d42..eb27169 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,3 +22,6 @@ govoplan_docs = ["py.typed"] [project.entry-points."govoplan.modules"] docs = "govoplan_docs.backend.manifest:get_manifest" + +[project.scripts] +govoplan-docs-export-public = "govoplan_docs.public_export:main" diff --git a/src/govoplan_docs/backend/manifest.py b/src/govoplan_docs/backend/manifest.py index 660e8d6..b548a67 100644 --- a/src/govoplan_docs/backend/manifest.py +++ b/src/govoplan_docs/backend/manifest.py @@ -168,8 +168,8 @@ manifest = ModuleManifest( }, links=( DocumentationLink( - label="Public GovOPlaN module documentation", - href="https://govplan.add-ideas.de/", + label="Public GovOPlaN documentation", + href="https://govoplan.add-ideas.de/docs", kind="public", ), DocumentationLink( @@ -185,6 +185,41 @@ manifest = ModuleManifest( ), metadata={"kind": "system"}, ), + DocumentationTopic( + id="docs.public-manifest-export", + title="Public documentation from module manifests", + summary="Publish the static documentation baseline from every module without maintaining a second content source.", + body=( + "The public exporter reads DocumentationTopic contributions from all installed packages or sibling module checkouts, " + "projects German and English content, and records documentation coverage. Runtime providers remain in the authenticated " + "Docs surface because their output can depend on the current actor, policy, configuration, and live service state. " + "Publication CI should run the export check and reject a stale source digest." + ), + layer="always", + documentation_types=("admin", "user"), + audience=("user", "tenant_admin", "operator", "module_admin", "publisher"), + order=12, + translations={ + "de": { + "title": "Öffentliche Dokumentation aus Modulmanifesten", + "summary": "Die statische Dokumentationsbasis aller Module veröffentlichen, ohne eine zweite Inhaltsquelle zu pflegen.", + "body": ( + "Der öffentliche Export liest die DocumentationTopic-Beiträge aus allen installierten Paketen oder benachbarten Modulquellen, " + "projiziert deutsche und englische Inhalte und weist Dokumentationslücken aus. Laufzeit-Provider verbleiben in der authentifizierten " + "Dokumentationsoberfläche, da ihre Ausgabe von Rolle, Richtlinie, Konfiguration und Dienstzustand abhängen kann. " + "Die Veröffentlichungs-CI soll den Quelldigest prüfen und veraltete Exporte ablehnen." + ), + } + }, + links=( + DocumentationLink( + label="Public documentation", + href="https://govoplan.add-ideas.de/docs", + kind="public", + ), + ), + metadata={"kind": "system"}, + ), DocumentationTopic( id="docs.reference.institutional-governance-architecture", title="Institutional governance architecture", diff --git a/src/govoplan_docs/public_export.py b/src/govoplan_docs/public_export.py new file mode 100644 index 0000000..330a794 --- /dev/null +++ b/src/govoplan_docs/public_export.py @@ -0,0 +1,424 @@ +from __future__ import annotations + +import argparse +import hashlib +import importlib +import json +import os +import sys +import tomllib +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from govoplan_core.core.discovery import discover_module_manifests +from govoplan_core.core.modules import ( + DocumentationCondition, + DocumentationLink, + DocumentationTopic, + ModuleManifest, +) + + +SCHEMA_VERSION = "1" +DEFAULT_LOCALE = "de" +SUPPORTED_LOCALES = ("de", "en") +TOPIC_KINDS = ( + "workflow", + "operator-workflow", + "guide", + "runbook", + "reference", + "pattern", + "system", +) + + +@dataclass(frozen=True, slots=True) +class ManifestSource: + manifest: ModuleManifest + repository: str | None = None + + +def collect_manifest_sources( + *, + workspace_root: Path | None = None, +) -> tuple[ManifestSource, ...]: + """Collect manifests without requiring every optional module to be installed.""" + + by_id = { + manifest.id: ManifestSource(manifest=manifest) + for manifest in discover_module_manifests(ignore_load_errors=True) + } + if workspace_root is not None: + for source in _workspace_manifest_sources(workspace_root): + by_id[source.manifest.id] = source + return tuple(by_id[module_id] for module_id in sorted(by_id)) + + +def build_public_catalog( + sources: Sequence[ManifestSource], + *, + generated_at: datetime | None = None, +) -> dict[str, Any]: + modules = [_module_payload(source) for source in sources] + digest_input = { + "schema_version": SCHEMA_VERSION, + "default_locale": DEFAULT_LOCALE, + "supported_locales": list(SUPPORTED_LOCALES), + "modules": modules, + } + source_digest = _sha256(digest_input) + timestamp = generated_at or _generated_at() + return { + **digest_input, + "generated_at": timestamp.astimezone(UTC).isoformat().replace("+00:00", "Z"), + "source_digest": source_digest, + "summary": _coverage_summary(modules), + } + + +def documentation_coverage_markdown(catalog: Mapping[str, Any]) -> str: + summary = _mapping(catalog.get("summary")) + modules = [item for item in catalog.get("modules", ()) if isinstance(item, Mapping)] + gaps = [item for item in summary.get("gaps", ()) if isinstance(item, Mapping)] + lines = [ + "# Public documentation coverage", + "", + "This report is generated from the static `DocumentationTopic` entries, including their structured workflow and field metadata, in every module manifest.", + "Configured-instance topics from runtime providers remain in the authenticated Docs module because they may depend on permissions, policy, and live state.", + "", + f"- Modules: {summary.get('module_count', 0)}", + f"- Static topics: {summary.get('topic_count', 0)}", + f"- Topics with complete German title, summary, and body: {summary.get('german_complete_topic_count', 0)}", + f"- Modules with runtime documentation providers: {summary.get('runtime_provider_module_count', 0)}", + f"- Source digest: `{catalog.get('source_digest', '')}`", + "", + "## Expansion priorities", + "", + "1. Add German title, summary, and body translations to every public topic; German is the reference target.", + "2. Give every user-facing module at least one scope-conditioned workflow topic and one field/consequence reference.", + "3. Give every configurable module an administrator topic covering permissions, policy provenance, retention, and operational consequences.", + "4. Keep live provider-state and instance-specific limitations in `documentation_providers`; do not publish them as generic facts.", + "5. Add a versioned localization contract for structured metadata such as steps, fields, limitations, and verification; these values currently retain their manifest source language.", + "", + "## Module gaps", + "", + "| Module | Topics | Missing German | Missing coverage |", + "| --- | ---: | ---: | --- |", + ] + gaps_by_id = {str(item.get("module_id")): item for item in gaps} + for module in modules: + module_id = str(module.get("id", "")) + coverage = _mapping(module.get("coverage")) + gap = gaps_by_id.get(module_id, {}) + missing = ", ".join(str(item) for item in gap.get("missing", ())) or "-" + lines.append( + f"| `{module_id}` | {coverage.get('topic_count', 0)} | " + f"{coverage.get('missing_german_topic_count', 0)} | {missing} |" + ) + lines.extend(("", "Generated file. Edit module manifests, then regenerate this report.", "")) + return "\n".join(lines) + + +def write_public_catalog( + output: Path, + catalog: Mapping[str, Any], + *, + coverage_output: Path | None = None, +) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(catalog, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + if coverage_output is not None: + coverage_output.parent.mkdir(parents=True, exist_ok=True) + coverage_output.write_text( + documentation_coverage_markdown(catalog), + encoding="utf-8", + ) + + +def catalog_matches_sources(output: Path, catalog: Mapping[str, Any]) -> bool: + if not output.is_file(): + return False + try: + current = json.loads(output.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return False + return current.get("source_digest") == catalog.get("source_digest") + + +def _workspace_manifest_sources(workspace_root: Path) -> tuple[ManifestSource, ...]: + declarations: list[tuple[Path, Mapping[str, str]]] = [] + for repository in sorted(workspace_root.glob("govoplan-*")): + pyproject = repository / "pyproject.toml" + if not pyproject.is_file(): + continue + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + raw = data.get("project", {}).get("entry-points", {}).get("govoplan.modules", {}) + if not isinstance(raw, Mapping) or not raw: + continue + declarations.append( + (repository, {str(key): str(value) for key, value in raw.items()}) + ) + + # Add every module source before imports so optional contracts resolve + # independently of package installation order. + for repository, _entry_points in reversed(declarations): + source_root = str(repository / "src") + if source_root not in sys.path: + sys.path.insert(0, source_root) + + sources: list[ManifestSource] = [] + for repository, entry_points in declarations: + for target in entry_points.values(): + module_name, separator, attribute = target.partition(":") + if not separator or not module_name or not attribute: + raise ValueError(f"Invalid GovOPlaN module entry point: {target!r}") + loaded = getattr(importlib.import_module(module_name), attribute) + manifest = loaded() if callable(loaded) else loaded + if not isinstance(manifest, ModuleManifest): + raise TypeError(f"Entry point {target!r} did not return ModuleManifest") + sources.append( + ManifestSource( + manifest=manifest, + repository=f"https://git.add-ideas.de/GovOPlaN/{repository.name}", + ) + ) + return tuple(sources) + + +def _module_payload(source: ManifestSource) -> dict[str, Any]: + manifest = source.manifest + topics = [ + _topic_payload(manifest.id, topic) + for topic in sorted(manifest.documentation, key=lambda item: (item.order, item.id)) + ] + return { + "id": manifest.id, + "name": manifest.name, + "version": manifest.version, + "repository": source.repository or _default_repository(manifest.id), + "dependencies": list(manifest.dependencies), + "optional_dependencies": list(manifest.optional_dependencies), + "runtime_documentation_provider_count": len(manifest.documentation_providers), + "topics": topics, + "coverage": _module_coverage(manifest, topics), + } + + +def _topic_payload(module_id: str, topic: DocumentationTopic) -> dict[str, Any]: + return { + "id": topic.id, + "source_module_id": topic.source_module_id or module_id, + "kind": _topic_kind(topic), + "layer": topic.layer, + "documentation_types": list(topic.documentation_types), + "audience": list(topic.audience), + "order": topic.order, + "localizations": { + locale: _localized_topic(topic, locale) for locale in SUPPORTED_LOCALES + }, + "links": [_link_payload(link) for link in topic.links], + "related_modules": list(topic.related_modules), + "unlocks": list(topic.unlocks), + "version_min": topic.version_min, + "version_max_exclusive": topic.version_max_exclusive, + "is_conditioned": bool(topic.conditions or topic.configuration_keys), + "conditions": [_condition_payload(item) for item in topic.conditions], + "configuration_keys": list(topic.configuration_keys), + "content": _public_value( + { + key: value + for key, value in topic.metadata.items() + if key != "kind" + } + ), + } + + +def _localized_topic(topic: DocumentationTopic, locale: str) -> dict[str, Any]: + translation = topic.translations.get(locale, {}) + translated_fields = [ + field + for field in ("title", "summary", "body") + if str(translation.get(field, "")).strip() + ] + return { + "title": str(translation.get("title") or topic.title), + "summary": str(translation.get("summary") or topic.summary), + "body": str(translation.get("body") or topic.body), + "source_locale": locale if translated_fields else "en", + "translated_fields": translated_fields, + "complete": len(translated_fields) == 3, + } + + +def _module_coverage( + manifest: ModuleManifest, + topics: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + kinds = {str(topic.get("kind")) for topic in topics} + has_user = any("user" in topic.get("documentation_types", ()) for topic in topics) + has_admin = any("admin" in topic.get("documentation_types", ()) for topic in topics) + missing_german = [ + str(topic.get("id")) + for topic in topics + if not _mapping(_mapping(topic.get("localizations")).get("de")).get("complete") + ] + missing: list[str] = [] + if not has_user: + missing.append("user documentation") + if not has_admin: + missing.append("administrator documentation") + if manifest.frontend is not None and "workflow" not in kinds: + missing.append("user workflow") + if "reference" not in kinds: + missing.append("field/consequence reference") + if missing_german: + missing.append("complete German localization") + return { + "topic_count": len(topics), + "user_topic_count": sum( + "user" in topic.get("documentation_types", ()) for topic in topics + ), + "admin_topic_count": sum( + "admin" in topic.get("documentation_types", ()) for topic in topics + ), + "missing_german_topic_count": len(missing_german), + "missing_german_topic_ids": missing_german, + "kinds": sorted(kinds), + "missing": missing, + } + + +def _coverage_summary(modules: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + topic_count = sum(len(module.get("topics", ())) for module in modules) + missing_german = sum( + int(_mapping(module.get("coverage")).get("missing_german_topic_count", 0)) + for module in modules + ) + gaps = [ + { + "module_id": str(module.get("id")), + "missing": list(_mapping(module.get("coverage")).get("missing", ())), + } + for module in modules + if _mapping(module.get("coverage")).get("missing") + ] + return { + "module_count": len(modules), + "topic_count": topic_count, + "german_complete_topic_count": topic_count - missing_german, + "runtime_provider_module_count": sum( + int(module.get("runtime_documentation_provider_count", 0)) > 0 + for module in modules + ), + "gap_module_count": len(gaps), + "gaps": gaps, + } + + +def _topic_kind(topic: DocumentationTopic) -> str: + raw = topic.metadata.get("kind") + if isinstance(raw, str): + normalized = raw.strip().lower().replace("_", "-") + if normalized in TOPIC_KINDS: + return normalized + return "system" + + +def _link_payload(link: DocumentationLink) -> dict[str, str]: + return {"label": link.label, "href": link.href, "kind": link.kind} + + +def _condition_payload(condition: DocumentationCondition) -> dict[str, list[str]]: + return { + "required_modules": list(condition.required_modules), + "any_modules": list(condition.any_modules), + "missing_modules": list(condition.missing_modules), + "required_capabilities": list(condition.required_capabilities), + "required_scopes": list(condition.required_scopes), + "any_scopes": list(condition.any_scopes), + "configuration_keys": list(condition.configuration_keys), + } + + +def _public_value(value: object) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Mapping): + return {str(key): _public_value(item) for key, item in value.items()} + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [_public_value(item) for item in value] + raise TypeError( + "Static documentation metadata must contain only JSON-compatible values; " + f"received {type(value).__name__}." + ) + + +def _default_repository(module_id: str) -> str: + slug = {"campaigns": "campaign"}.get(module_id, module_id.replace("_", "-")) + return f"https://git.add-ideas.de/GovOPlaN/govoplan-{slug}" + + +def _sha256(value: object) -> str: + encoded = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _generated_at() -> datetime: + raw_epoch = os.environ.get("SOURCE_DATE_EPOCH") + if raw_epoch: + return datetime.fromtimestamp(int(raw_epoch), tz=UTC) + return datetime.now(UTC) + + +def _mapping(value: object) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Export all static GovOPlaN module documentation for a public site." + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--coverage-output", type=Path) + parser.add_argument("--workspace-root", type=Path) + parser.add_argument( + "--check", + action="store_true", + help="Fail when the existing output does not match current manifest content.", + ) + return parser + + +def main(argv: Iterable[str] | None = None) -> int: + args = _parser().parse_args(list(argv) if argv is not None else None) + catalog = build_public_catalog( + collect_manifest_sources(workspace_root=args.workspace_root) + ) + if args.check: + if catalog_matches_sources(args.output, catalog): + return 0 + print(f"Public documentation catalog is stale: {args.output}", file=sys.stderr) + return 1 + write_public_catalog( + args.output, + catalog, + coverage_output=args.coverage_output, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_public_export.py b/tests/test_public_export.py new file mode 100644 index 0000000..b0b914e --- /dev/null +++ b/tests/test_public_export.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from govoplan_core.core.modules import DocumentationTopic, ModuleManifest +from govoplan_docs.public_export import ( + ManifestSource, + build_public_catalog, + catalog_matches_sources, + documentation_coverage_markdown, + write_public_catalog, +) + + +class PublicDocumentationExportTests(unittest.TestCase): + def test_catalog_projects_localized_manifest_topics_and_gaps(self) -> None: + manifest = ModuleManifest( + id="example", + name="Example", + version="1.2.3", + documentation=( + DocumentationTopic( + id="example.workflow", + title="Use example", + summary="Perform the example workflow.", + body="Open and finish it.", + documentation_types=("user", "admin"), + metadata={"kind": "workflow"}, + translations={ + "de": { + "title": "Beispiel verwenden", + "summary": "Den Beispielablauf durchführen.", + "body": "Öffnen und abschließen.", + } + }, + ), + ), + ) + + catalog = build_public_catalog( + (ManifestSource(manifest, "https://example.invalid/repository"),) + ) + + topic = catalog["modules"][0]["topics"][0] + self.assertEqual("Beispiel verwenden", topic["localizations"]["de"]["title"]) + self.assertTrue(topic["localizations"]["de"]["complete"]) + self.assertEqual("workflow", topic["kind"]) + self.assertEqual({}, topic["content"]) + self.assertIn( + "field/consequence reference", + catalog["modules"][0]["coverage"]["missing"], + ) + + def test_digest_check_ignores_generation_timestamp(self) -> None: + source = ManifestSource( + ModuleManifest( + id="example", + name="Example", + version="1.0.0", + documentation=( + DocumentationTopic( + id="example.reference", + title="Reference", + summary="Reference summary", + metadata={"kind": "reference"}, + ), + ), + ) + ) + first = build_public_catalog((source,)) + second = build_public_catalog((source,)) + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "catalog.json" + write_public_catalog(output, first) + self.assertTrue(catalog_matches_sources(output, second)) + parsed = json.loads(output.read_text(encoding="utf-8")) + parsed["source_digest"] = "stale" + output.write_text(json.dumps(parsed), encoding="utf-8") + self.assertFalse(catalog_matches_sources(output, second)) + + def test_coverage_report_identifies_generated_source(self) -> None: + catalog = build_public_catalog( + ( + ManifestSource( + ModuleManifest( + id="example", + name="Example", + version="1.0.0", + documentation=(), + ) + ), + ) + ) + report = documentation_coverage_markdown(catalog) + self.assertIn("Public documentation coverage", report) + self.assertIn("`example`", report) + self.assertIn("Edit module manifests", report) + + def test_catalog_preserves_structured_static_documentation(self) -> None: + manifest = ModuleManifest( + id="example", + name="Example", + version="1.0.0", + documentation=( + DocumentationTopic( + id="example.workflow", + title="Run example", + summary="Run it safely.", + metadata={ + "kind": "workflow", + "prerequisites": ["Approved input"], + "steps": ["Review", "Execute"], + "fields": [ + { + "label": "Mode", + "user_description": "Selected behavior.", + } + ], + }, + ), + ), + ) + + topic = build_public_catalog((ManifestSource(manifest),))["modules"][0]["topics"][0] + + self.assertEqual(["Review", "Execute"], topic["content"]["steps"]) + self.assertEqual("Mode", topic["content"]["fields"][0]["label"]) + + def test_catalog_preserves_specialized_documentation_kinds(self) -> None: + manifest = ModuleManifest( + id="example", + name="Example", + version="1.0.0", + documentation=( + DocumentationTopic( + id="example.runbook", + title="Recover example", + summary="Restore the example safely.", + metadata={"kind": "operator_workflow"}, + ), + ), + ) + + topic = build_public_catalog((ManifestSource(manifest),))["modules"][0]["topics"][0] + + self.assertEqual("operator-workflow", topic["kind"]) + + +if __name__ == "__main__": + unittest.main()