From 6c8a8c655da2939cd988fc82f696d25f17b4abd4 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Tue, 21 Jul 2026 16:31:49 +0200 Subject: [PATCH] docs(files): expose adaptive handbook topics --- README.md | 2 + src/govoplan_files/backend/manifest.py | 220 ++++++++++++++++++++----- tests/test_manifest_documentation.py | 93 ++++++++++- 3 files changed, 264 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 7b8cf23..dcbde56 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,8 @@ manual interoperability testing. Connector and collaboration ownership boundaries are documented in `docs/CONNECTOR_BOUNDARY.md` and `docs/DOCUMENT_COLLABORATION_BOUNDARY.md`. +The role-adaptive user, administration, integration, and operator guide is the +[Files handbook](docs/FILES_HANDBOOK.md). ZIP uploads are processed without buffering the whole archive in memory. The API spools incoming ZIP request bodies to a bounded temporary file, then extracts diff --git a/src/govoplan_files/backend/manifest.py b/src/govoplan_files/backend/manifest.py index 1117714..230b709 100644 --- a/src/govoplan_files/backend/manifest.py +++ b/src/govoplan_files/backend/manifest.py @@ -195,72 +195,206 @@ manifest = ModuleManifest( ), documentation=( DocumentationTopic( - id="files.governed-connectors-and-provenance", - title="Governed file connectors and source provenance", - summary="Connector profiles control who may browse an external source, which destinations are permitted, and how imported files retain verifiable source context.", + id="files.workflow.import-managed-snapshot", + title="Import an external file as a governed snapshot", + summary="Browse an authorized connection read-only, import one selected file into managed storage, and use the frozen version in another GovOPlaN task.", body=( - "Users select only connector profiles visible in their current scope and import remote content into managed Files storage before another module uses it. " - "Administrators define profiles, separate credential references, and ordered system/tenant/owner policies; deny rules win and profile responses never expose secrets. " - "Deleting a database-managed profile or credential immediately scrubs Files-owned encrypted material and records a non-secret audit event in the same transaction; legacy non-owned external references are detached without provider calls. " - "Operators control private-network access deployment-wide and must keep every remote connection pinned to a policy-validated DNS/IP answer. " - "The built-in HTTP transport pins each connection and refuses redirects. Live S3 and SMB SDK access fails closed until S3 redirects and SMB DFS referrals can be revalidated and pinned. " - "Successful imports store the connector id, provider, remote path and identity, source revision, and selected metadata as provenance on the managed file and its audit events." + "Choose a visible file connection in Files, browse only the permitted remote path, and import the selected content into your personal or group space. " + "The managed copy records source provenance and remains unchanged until an explicit manual sync. Browse, import, and sync never mutate the remote source." ), layer="configured", - documentation_types=("admin", "user"), - audience=("file_user", "file_admin", "tenant_admin", "operator"), + documentation_types=("user",), + audience=("file_user", "campaign_manager", "report_author"), + order=40, + conditions=( + DocumentationCondition( + required_modules=("files",), + any_scopes=("files:file:read", "files:file:upload"), + ), + ), + links=( + DocumentationLink(label="Files", href="/files", kind="runtime"), + DocumentationLink(label="Visible connector profiles", href="/api/v1/files/connectors/profiles", kind="api"), + DocumentationLink( + label="Connector import API", + href="/api/v1/files/connectors/profiles/{profile_id}/import", + kind="api", + ), + DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + ), + related_modules=("campaigns",), + unlocks=("Campaigns, reports, and workflows can consume a managed snapshot with stable source evidence.",), + metadata={ + "kind": "workflow", + "route": "/files", + "screen": "Files", + "prerequisites": [ + "Files is installed and you may read and upload files.", + "An administrator has configured a connector profile visible in your current user, group, tenant, or campaign scope.", + ], + "steps": [ + "Open Files and choose a managed destination space.", + "Choose Sync from connection, select a visible profile, and browse to the permitted remote file.", + "Import or sync the selected file and resolve any destination conflict explicitly.", + "Review the managed file's source and current-version details before using it in another task.", + ], + "outcome": "The external content is a tenant-managed snapshot with a checksum, exact version, and recorded source context.", + "verification": "Reopen the managed file and confirm its connector/provider, remote identity or path, source revision when available, checksum, and current version.", + "related_topic_ids": [ + "files.governed-connectors-and-provenance", + "files.reference.integrity-recovery-and-fail-closed-transports", + "files.reference.snapshot-provenance-and-capabilities", + ], + }, + ), + DocumentationTopic( + id="files.governed-connectors-and-provenance", + title="Govern file connections and credential deletion", + summary="Keep endpoint profiles, reusable credentials, and inherited connector policy separate, and understand what DELETE removes immediately.", + body=( + "System, tenant, and one user/group/campaign leaf form the effective policy chain: deny rules win and every configured allow rule must match. " + "Responses redact secret values and deployment references. Deleting a database-managed credential or profile immediately scrubs Files-owned encrypted material and private metadata in the same transaction as a non-secret audit event; dependent profiles are disabled, while legacy non-owned references are only detached and audited." + ), + layer="configured", + documentation_types=("admin",), + audience=("file_admin", "tenant_admin", "system_admin", "security_auditor"), + order=41, + conditions=( + DocumentationCondition( + required_modules=("files",), + any_scopes=( + "files:file:admin", + "admin:settings:read", + "admin:settings:write", + "system:settings:read", + "system:settings:write", + ), + ), + ), + links=( + DocumentationLink(label="Tenant connector administration", href="/admin?section=tenant-file-connectors", kind="runtime"), + DocumentationLink(label="System connector administration", href="/admin?section=system-file-connectors", kind="runtime"), + DocumentationLink(label="Tenant connector policy API", href="/api/v1/files/connectors/policies/tenant", kind="api"), + DocumentationLink(label="Connector credentials API", href="/api/v1/files/connectors/credentials", kind="api"), + DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + ), + related_modules=("access", "audit", "mail"), + unlocks=("Scoped, explainable external-file access without exposing credentials to consuming modules.",), + configuration_keys=( + "GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON", + "GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE", + "GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST", + "GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST", + "MASTER_KEY_B64", + ), + metadata={ + "kind": "reference", + "route": "/admin?section=tenant-file-connectors", + "screen": "File connections", + "section": "Profiles, credentials, and effective connector policy", + "security_invariants": [ + "New API-managed external secret references fail closed until Files can prove ownership and provider-side deletion.", + "Deletion and destructive retirement scrub Files-owned encrypted connector material before completion and emit non-secret audit evidence.", + "Legacy non-owned external references are detached and audited, never sent to an arbitrary provider delete operation.", + ], + "related_topic_ids": [ + "files.workflow.import-managed-snapshot", + "files.reference.integrity-recovery-and-fail-closed-transports", + "mail.profiles-and-policy", + ], + }, + ), + DocumentationTopic( + id="files.reference.integrity-recovery-and-fail-closed-transports", + title="Operate Files integrity, recovery, and connector transport safety", + summary="Back up database evidence, blob bytes, and the encryption key as one recovery unit, and keep unsupported SDK transports fail-closed.", + body=( + "Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the original master key, then verify representative checksums and access paths. " + "Live S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Destructive module retirement drops database tables but does not remove backend blob objects." + ), + layer="configured", + documentation_types=("admin",), + audience=("operator", "system_admin", "security_auditor"), order=42, conditions=( DocumentationCondition( required_modules=("files",), any_scopes=( - "files:file:read", - "files:file:upload", "files:file:admin", "admin:settings:read", "system:settings:read", + "system:audit:read", ), ), ), links=( - DocumentationLink(label="Files", href="/files", kind="runtime"), - DocumentationLink(label="Tenant connector administration", href="/admin?section=tenant-file-connectors", kind="runtime"), - DocumentationLink(label="Connector profiles API", href="/api/v1/files/connectors/profiles", kind="api"), - DocumentationLink(label="Connector providers API", href="/api/v1/files/connectors/providers", kind="api"), - DocumentationLink(label="Tenant connector policy API", href="/api/v1/files/connectors/policies/tenant", kind="api"), + DocumentationLink(label="System file connections", href="/admin?section=system-file-connectors", kind="runtime"), + DocumentationLink(label="Connector provider status", href="/api/v1/files/connectors/providers", kind="api"), + DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), ), - related_modules=("campaigns",), - unlocks=("Campaigns can consume managed attachments while Files preserves frozen source provenance.",), + related_modules=("audit", "ops"), + unlocks=("Recoverable managed-file evidence without weakening outbound peer validation.",), configuration_keys=( + "FILE_STORAGE_BACKEND", + "FILE_STORAGE_LOCAL_ROOT", + "FILE_STORAGE_LOCAL_FALLBACK_ROOTS", + "FILE_UPLOAD_MAX_BYTES", + "FILE_UPLOAD_ZIP_MAX_BYTES", + "MASTER_KEY_B64", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", - "GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON", - "GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE", - "GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST", + "GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES", + "GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES", ), metadata={ "kind": "reference", - "perspectives": { - "user": "Choose a visible connection, browse it read-only, and import a selected file into managed storage before using it elsewhere.", - "admin": "Govern profile visibility, separate credential references, provider allow/deny policy, and permitted connector operations.", - "operator": "Control private-network access deployment-wide, provide credential and CA allowlists, and investigate fail-closed transport errors without weakening peer validation.", - }, - "security_invariants": [ - "Every network peer must be policy-validated and pinned at connection time.", - "Redirects and protocol referrals must be rejected or independently revalidated and pinned.", - "SDK transports that cannot provide those guarantees fail before client construction.", - "New API-managed external secret references fail closed until Files can prove ownership and provider-side deletion.", - "Deletion and destructive retirement scrub Files-owned encrypted connector material before completion and emit non-secret audit evidence.", - "Legacy non-owned external references are detached and audited, never sent to an arbitrary provider delete operation.", + "route": "/admin?section=system-file-connectors", + "screen": "System file connections and deployment operations", + "section": "Storage integrity, backup/recovery, and fail-closed transports", + "recovery_unit": ["Files database rows", "managed blob namespace", "MASTER_KEY_B64", "deployment-owned connector configuration"], + "verification": "After restore, download representative files, compare their bytes with recorded SHA-256 values, verify authorized and denied access, and test one permitted pinned HTTP connector.", + "related_topic_ids": [ + "files.governed-connectors-and-provenance", + "files.reference.snapshot-provenance-and-capabilities", ], - "provenance_fields": [ - "connector_id", - "provider", - "external_id", - "external_path", - "revision", - "metadata", + }, + ), + DocumentationTopic( + id="files.reference.snapshot-provenance-and-capabilities", + title="Integrate through managed snapshots and Files capabilities", + summary="Other modules consume stable Files capabilities or HTTP contracts and retain exact version evidence instead of importing Files internals.", + body=( + "Use files.access to explain resource access and files.campaign_attachments to freeze campaign inputs at an exact asset, version, blob, checksum, and source revision. " + "Import external content before a governed use, preserve provenance on derived snapshots, and keep collaboration, provider sync, OAuth, remote mutation, and domain workflow state in their owning modules." + ), + layer="available", + documentation_types=("admin", "user"), + audience=("module_integrator", "file_admin", "campaign_admin", "process_designer"), + order=43, + conditions=( + DocumentationCondition( + required_modules=("files",), + any_scopes=("files:file:read", "files:file:upload", "files:file:admin"), + ), + ), + links=( + DocumentationLink(label="Files", href="/files", kind="runtime"), + DocumentationLink(label="Files API", href="/api/v1/files", kind="api"), + DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + ), + related_modules=("campaigns", "docs"), + unlocks=("Campaign, report, template, workflow, and document modules can exchange governed input/output snapshots.",), + metadata={ + "kind": "reference", + "route": "/files", + "screen": "Files and module integration", + "section": "Managed snapshot provenance and capability boundaries", + "provided_interfaces": ["files.access@0.1.6", "files.campaign_attachments@0.1.6"], + "provenance_fields": ["connector_id", "provider", "external_id", "external_path", "revision", "metadata"], + "related_topic_ids": [ + "files.workflow.import-managed-snapshot", + "files.governed-connectors-and-provenance", + "files.reference.integrity-recovery-and-fail-closed-transports", ], - "related_topic_ids": ["mail.profiles-and-policy"], }, ), ), diff --git a/tests/test_manifest_documentation.py b/tests/test_manifest_documentation.py index 1ce6141..2d8cae0 100644 --- a/tests/test_manifest_documentation.py +++ b/tests/test_manifest_documentation.py @@ -3,20 +3,97 @@ from __future__ import annotations import unittest +TOPIC_IDS = { + "files.workflow.import-managed-snapshot", + "files.governed-connectors-and-provenance", + "files.reference.integrity-recovery-and-fail-closed-transports", + "files.reference.snapshot-provenance-and-capabilities", +} +HANDBOOK_HREF = "govoplan-files/docs/FILES_HANDBOOK.md" + + class FilesManifestDocumentationTests(unittest.TestCase): - def test_connector_topic_covers_governance_pinning_and_provenance_perspectives(self) -> None: + @classmethod + def setUpClass(cls) -> None: from govoplan_files.backend.manifest import manifest - topic = next(item for item in manifest.documentation if item.id == "files.governed-connectors-and-provenance") + cls.topics = {topic.id: topic for topic in manifest.documentation} + + def topic(self, topic_id: str): + return self.topics[topic_id] + + def test_adaptive_topics_have_role_scope_module_and_link_contracts(self) -> None: + self.assertEqual(TOPIC_IDS, set(self.topics)) + self.assertEqual({"admin", "user"}, {item for topic in self.topics.values() for item in topic.documentation_types}) + + for topic in self.topics.values(): + with self.subTest(topic=topic.id): + self.assertTrue(topic.audience) + self.assertTrue(topic.conditions) + self.assertTrue(any("files" in condition.required_modules for condition in topic.conditions)) + self.assertTrue(any(condition.any_scopes or condition.required_scopes for condition in topic.conditions)) + self.assertIn("runtime", {link.kind for link in topic.links}) + self.assertIn("api", {link.kind for link in topic.links}) + self.assertIn(HANDBOOK_HREF, {link.href for link in topic.links if link.kind == "repository"}) + + def test_import_topic_is_a_complete_user_workflow(self) -> None: + topic = self.topic("files.workflow.import-managed-snapshot") + + self.assertEqual(("user",), topic.documentation_types) + self.assertEqual("workflow", topic.metadata["kind"]) + self.assertEqual("/files", topic.metadata["route"]) + self.assertEqual("Files", topic.metadata["screen"]) + for key in ("prerequisites", "steps", "outcome", "verification"): + self.assertTrue(topic.metadata[key]) + self.assertIn("never mutate the remote source", topic.body) + self.assertIn("/api/v1/files/connectors/profiles/{profile_id}/import", {link.href for link in topic.links}) + + def test_admin_topic_covers_policy_redaction_and_atomic_credential_deletion(self) -> None: + topic = self.topic("files.governed-connectors-and-provenance") + + self.assertEqual(("admin",), topic.documentation_types) + self.assertEqual("reference", topic.metadata["kind"]) + self.assertEqual("File connections", topic.metadata["screen"]) + self.assertTrue(topic.metadata["section"]) + self.assertIn("deny rules win", topic.body) + self.assertIn("redact secret values", topic.body) + self.assertIn("same transaction", topic.body) + self.assertTrue(any("non-owned external references" in item for item in topic.metadata["security_invariants"])) + self.assertIn("/api/v1/files/connectors/policies/tenant", {link.href for link in topic.links}) + self.assertIn("/api/v1/files/connectors/credentials", {link.href for link in topic.links}) + + def test_operator_topic_covers_recovery_and_fail_closed_s3_smb(self) -> None: + topic = self.topic("files.reference.integrity-recovery-and-fail-closed-transports") + + self.assertEqual(("admin",), topic.documentation_types) + self.assertEqual("reference", topic.metadata["kind"]) + self.assertIn("S3", topic.body) + self.assertIn("SMB", topic.body) + self.assertIn("fail closed", topic.body) + self.assertIn("DFS referrals", topic.body) + self.assertIn("does not remove backend blob objects", topic.body) + self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"]) + self.assertTrue(topic.metadata["verification"]) + self.assertIn("GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", topic.configuration_keys) + + def test_integrator_topic_exposes_capability_and_provenance_boundary(self) -> None: + topic = self.topic("files.reference.snapshot-provenance-and-capabilities") self.assertEqual(("admin", "user"), topic.documentation_types) - self.assertEqual({"user", "admin", "operator"}, set(topic.metadata["perspectives"])) - self.assertIn("fails closed", topic.body) - self.assertIn("DFS referrals", topic.body) - self.assertIn("immediately scrubs", topic.body) - self.assertTrue(any("non-owned external references" in item for item in topic.metadata["security_invariants"])) + self.assertEqual("reference", topic.metadata["kind"]) + self.assertEqual( + ["files.access@0.1.6", "files.campaign_attachments@0.1.6"], + topic.metadata["provided_interfaces"], + ) self.assertIn("revision", topic.metadata["provenance_fields"]) - self.assertIn("/api/v1/files/connectors/profiles", {link.href for link in topic.links}) + self.assertIn("exact asset, version, blob, checksum", topic.body) + self.assertIn("collaboration", topic.body) + + def test_internal_related_topic_ids_resolve(self) -> None: + for topic in self.topics.values(): + for related_id in topic.metadata.get("related_topic_ids", []): + if related_id.startswith("files."): + self.assertIn(related_id, TOPIC_IDS, f"{topic.id} refers to missing topic {related_id}") if __name__ == "__main__":