Document governed file connector boundaries

README.md

@@ -107,7 +107,17 @@ uploads. The Seafile provider uses account-token auth and the native file
 download-link API; Nextcloud and generic WebDAV profiles use authenticated `GET`
 requests against the configured WebDAV endpoint. SMB profiles use
 `smb://server[:port]/share[/path]` endpoints and deployment-owned or encrypted
-stored credentials through `smbprotocol`.
+stored credentials through `smbprotocol`. Because that SDK cannot accept a
+preconnected socket and may follow DFS referrals to additional hosts, live SMB
+access fails closed in both public-only and private-network deployments. It will
+remain disabled until every initial connection and referral target can be
+policy-validated and pinned.
+
+The S3 browse/import implementation and S3 managed-storage backend currently
+fail closed before creating a `boto3` client. Botocore does not yet use the
+GovOPlaN pinned HTTP transport and may manage redirects itself, so neither an
+explicit endpoint nor SDK endpoint discovery is allowed until both peer pinning
+and redirect revalidation are enforced.
 
 Local connector development assets live in `dev/connectors/`. The compose stack
 boots Nextcloud, Seafile, WebDAV, and SMB endpoints for provider development and

dev/connectors/README.md

@@ -138,12 +138,13 @@ cd /mnt/DATA/git/govoplan-files/dev/connectors
 /mnt/DATA/git/govoplan-core/.venv/bin/python smoke.py
 ```
 
-The script reads `.env` from this directory when present, seeds tiny WebDAV,
-Nextcloud, SMB, and MinIO fixtures, then browses and imports them through the
-connector helper layer. Pass `--require-smb` after recreating the SMB container
-to fail on SMB access errors instead of reporting them as an optional skip. Pass
-`--require-s3` after starting MinIO and installing `govoplan-files[s3]` to fail
-on S3 access errors instead of reporting them as an optional skip.
+The script reads `.env` from this directory when present and seeds tiny WebDAV,
+Nextcloud, SMB, and MinIO fixtures. WebDAV and Nextcloud are browsed and imported
+through the pinned connector transport. SMB and S3 product access deliberately
+fails closed until their SDK transports support peer pinning, so their fixtures
+are retained for transport development and reported as expected optional skips.
+The `--require-smb` and `--require-s3` switches are useful only while developing
+that transport support and currently make the smoke check fail by design.
 
 SMB smoke checks need the optional Python dependency in the environment running
 the script:

docs/CONNECTOR_BOUNDARY.md

@@ -57,6 +57,14 @@ Every provider must:
 - preserve source provenance and revision metadata
 - emit connector audit events for imported or accessed files
 - treat remote ACLs as upstream checks, not as a replacement for GovOPlaN policy
+- use a transport that pins every connection to a policy-validated DNS/IP answer
+  and revalidates redirects; SDK transports without that guarantee fail closed
+
+Live SMB access is disabled in all modes until `smbprotocol` initial connections
+and DFS referral targets can be pinned and policy-validated. An explicit IP is
+not sufficient because the server may still issue a referral to another peer.
+Live S3 access is likewise disabled until `boto3`/botocore can be bound to the
+pinned transport, including SDK-managed redirects and endpoint discovery.
 
 ## Non-Goals For Files
 

src/govoplan_files/backend/manifest.py

@@ -6,6 +6,9 @@ from govoplan_core.core.access import CAPABILITY_AUTH_PERMISSION_EVALUATOR, CAPA
 from govoplan_core.core.files import CAPABILITY_FILES_ACCESS
 from govoplan_core.core.module_guards import drop_table_retirement_provider, persistent_table_uninstall_guard
 from govoplan_core.core.modules import (
+    DocumentationCondition,
+    DocumentationLink,
+    DocumentationTopic,
     FrontendModule,
     MigrationSpec,
     ModuleContext,
@@ -138,6 +141,73 @@ manifest = ModuleManifest(
         package_name="@govoplan/files-webui",
         nav_items=(NavItem(path="/files", label="Files", icon="folder", required_any=("files:file:read",), order=40),),
     ),
+    documentation=(
+        DocumentationTopic(
+            id="files.governed-connectors-and-provenance",
+            title="Governed file connectors and source provenance",
+            summary="Connector profiles control who may browse an external source, which destinations are permitted, and how imported files retain verifiable source context.",
+            body=(
+                "Users select only connector profiles visible in their current scope and import remote content into managed Files storage before another module uses it. "
+                "Administrators define profiles, separate credential references, and ordered system/tenant/owner policies; deny rules win and profile responses never expose secrets. "
+                "Operators control private-network access deployment-wide and must keep every remote connection pinned to a policy-validated DNS/IP answer. "
+                "The built-in HTTP transport pins each connection and refuses redirects. Live S3 and SMB SDK access fails closed until S3 redirects and SMB DFS referrals can be revalidated and pinned. "
+                "Successful imports store the connector id, provider, remote path and identity, source revision, and selected metadata as provenance on the managed file and its audit events."
+            ),
+            layer="configured",
+            documentation_types=("admin", "user"),
+            audience=("file_user", "file_admin", "tenant_admin", "operator"),
+            order=42,
+            conditions=(
+                DocumentationCondition(
+                    required_modules=("files",),
+                    any_scopes=(
+                        "files:file:read",
+                        "files:file:upload",
+                        "files:file:admin",
+                        "admin:settings:read",
+                        "system:settings:read",
+                    ),
+                ),
+            ),
+            links=(
+                DocumentationLink(label="Files", href="/files", kind="runtime"),
+                DocumentationLink(label="Tenant connector administration", href="/admin?section=tenant-file-connectors", kind="runtime"),
+                DocumentationLink(label="Connector profiles API", href="/api/v1/files/connectors/profiles", kind="api"),
+                DocumentationLink(label="Connector providers API", href="/api/v1/files/connectors/providers", kind="api"),
+                DocumentationLink(label="Tenant connector policy API", href="/api/v1/files/connectors/policies/tenant", kind="api"),
+            ),
+            related_modules=("campaigns",),
+            unlocks=("Campaigns can consume managed attachments while Files preserves frozen source provenance.",),
+            configuration_keys=(
+                "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
+                "GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON",
+                "GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE",
+                "GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST",
+            ),
+            metadata={
+                "kind": "reference",
+                "perspectives": {
+                    "user": "Choose a visible connection, browse it read-only, and import a selected file into managed storage before using it elsewhere.",
+                    "admin": "Govern profile visibility, separate credential references, provider allow/deny policy, and permitted connector operations.",
+                    "operator": "Control private-network access deployment-wide, provide credential and CA allowlists, and investigate fail-closed transport errors without weakening peer validation.",
+                },
+                "security_invariants": [
+                    "Every network peer must be policy-validated and pinned at connection time.",
+                    "Redirects and protocol referrals must be rejected or independently revalidated and pinned.",
+                    "SDK transports that cannot provide those guarantees fail before client construction.",
+                ],
+                "provenance_fields": [
+                    "connector_id",
+                    "provider",
+                    "external_id",
+                    "external_path",
+                    "revision",
+                    "metadata",
+                ],
+                "related_topic_ids": ["mail.profiles-and-policy"],
+            },
+        ),
+    ),
     migration_spec=MigrationSpec(
         module_id="files",
         metadata=Base.metadata,

src/govoplan_files/backend/storage/backends.py

@@ -98,24 +98,24 @@ class S3StorageBackend:
 
     @property
     def client(self):
-        try:
-            import boto3
-        except ModuleNotFoundError as exc:
-            raise StorageBackendError("boto3 is required for the S3 storage backend") from exc
         try:
             endpoint_url = validate_unpinned_sdk_http_url(
                 self.endpoint_url,
                 label="File storage S3 endpoint",
             )
-            return boto3.client(
-                "s3",
-                endpoint_url=endpoint_url,
-                region_name=self.region_name,
-                aws_access_key_id=self.access_key_id,
-                aws_secret_access_key=self.secret_access_key,
-            )
         except OutboundHttpError as exc:
             raise StorageBackendError(str(exc)) from exc
+        try:
+            import boto3
+        except ModuleNotFoundError as exc:
+            raise StorageBackendError("boto3 is required for the S3 storage backend") from exc
+        return boto3.client(
+            "s3",
+            endpoint_url=endpoint_url,
+            region_name=self.region_name,
+            aws_access_key_id=self.access_key_id,
+            aws_secret_access_key=self.secret_access_key,
+        )
 
     def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
         max_bytes = response_limit("file")

src/govoplan_files/backend/storage/connector_browse.py

@@ -15,8 +15,7 @@ from defusedxml import ElementTree as SafeElementTree
 
 from govoplan_core.security.outbound_http import (
     OutboundHttpError,
-    outbound_http_policy,
-    pinned_outbound_hostname,
+    validate_unpinned_sdk_host,
     validate_unpinned_sdk_http_url,
 )
 
@@ -312,24 +311,25 @@ def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None,
 def _s3_client(profile: ConnectorProfile) -> Any:
     if profile.secret_ref:
         raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing")
-    try:
-        boto3 = import_module("boto3")
-        config_module = import_module("botocore.config")
-    except ImportError as exc:
-        raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
-    kwargs: dict[str, object] = {}
     if profile.endpoint_url:
         try:
-            kwargs["endpoint_url"] = validate_unpinned_sdk_http_url(
+            endpoint_url = validate_unpinned_sdk_http_url(
                 profile.endpoint_url,
                 label="S3 connector endpoint",
             )
         except OutboundHttpError as exc:
             raise ConnectorBrowseError(str(exc)) from exc
-    elif not outbound_http_policy().allow_private_networks:
+    else:
         raise ConnectorBrowseError(
-            "S3 connector endpoint discovery cannot pin the SDK connection address while private-network access is disabled"
+            "S3 connector endpoint discovery uses an SDK transport that cannot guarantee connection-time DNS/IP "
+            "pinning; live S3 access is disabled until that transport supports pinning"
         )
+    try:
+        boto3 = import_module("boto3")
+        config_module = import_module("botocore.config")
+    except ImportError as exc:
+        raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
+    kwargs: dict[str, object] = {"endpoint_url": endpoint_url}
     region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region")
     if region:
         kwargs["region_name"] = region
@@ -783,7 +783,7 @@ def _smb_location(profile: ConnectorProfile) -> _SmbLocation:
         raise ConnectorBrowseError("SMB connector endpoint_url must include a server")
     port = parsed.port or _int(profile.metadata.get("port")) or 445
     try:
-        server = pinned_outbound_hostname(server, port=port, label="SMB connector endpoint")
+        validate_unpinned_sdk_host(server, port=port, label="SMB connector endpoint")
     except OutboundHttpError as exc:
         raise ConnectorBrowseError(str(exc)) from exc
     path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part]

src/govoplan_files/backend/storage/connector_imports.py

@@ -194,12 +194,12 @@ def _read_webdav_file(profile: ConnectorProfile, *, path: str, max_bytes: int) -
 
 
 def _read_smb_file(profile: ConnectorProfile, *, path: str, max_bytes: int) -> ConnectorDownloadedFile:
-    location = _smb_location(profile)
     file_path = normalize_connector_browse_path(path)
     if not file_path:
         raise ConnectorImportError("SMB import requires a file path")
-    unc_path = _smb_unc_path(location, file_path)
     try:
+        location = _smb_location(profile)
+        unc_path = _smb_unc_path(location, file_path)
         smbclient = _smbclient_module()
         kwargs = _smb_client_kwargs(profile, location)
         stat_result = smbclient.stat(unc_path, **kwargs)

src/govoplan_files/backend/storage/connector_providers.py

@@ -110,7 +110,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
             conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after download.",
             preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the share.",
             audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
-            notes="Requires the optional smbprotocol dependency and an smb://server[:port]/share[/path] profile endpoint.",
+            notes="Browse/import logic is implemented, but live smbprotocol access fails closed until the SDK supports connection-time DNS/IP pinning for initial connections and DFS referrals.",
         ),
         ConnectorProviderDescriptor(
             provider="s3",
@@ -125,7 +125,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
             conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.",
             preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.",
             audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
-            notes="Configure endpoint_url for MinIO or other S3-compatible stores; use profile metadata for bucket, region, addressing_style, verify_tls, and ca_bundle.",
+            notes="Browse/import logic is implemented, but live boto3 access fails closed until its HTTP transport supports connection-time DNS/IP pinning and redirect revalidation.",
         ),
         ConnectorProviderDescriptor(
             provider="sharepoint",

tests/test_connector_providers.py

@@ -4,9 +4,8 @@ import unittest
 from datetime import UTC, datetime
 from unittest.mock import patch
 
-from govoplan_files.backend.storage.connector_browse import _smb_location, browse_connector_profile
-from govoplan_files.backend.storage.connector_browse import ConnectorBrowseUnsupported
-from govoplan_files.backend.storage.connector_imports import read_connector_file
+from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _smb_location, browse_connector_profile
+from govoplan_files.backend.storage.connector_imports import ConnectorImportError, read_connector_file
 from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload
 from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors
 
@@ -76,23 +75,55 @@ def s3_profile(**overrides: object) -> ConnectorProfile:
 
 
 class ConnectorProviderTests(unittest.TestCase):
-    def test_smb_uses_validated_numeric_target_when_private_networks_are_disabled(self) -> None:
+    def test_smb_sdk_transport_fails_closed_before_client_creation_in_all_modes(self) -> None:
         profile = ConnectorProfile(
-            id="public-smb",
-            label="Public SMB",
+            id="smb",
+            label="SMB",
             provider="smb",
             endpoint_url="smb://files.example.test/share",
         )
+        for allow_private, address in ((False, "93.184.216.34"), (True, "10.0.0.5")):
+            with self.subTest(allow_private=allow_private), patch.dict(
+                "os.environ",
+                {
+                    "APP_ENV": "production",
+                    "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": str(allow_private).lower(),
+                },
+            ), patch(
+                "govoplan_core.security.outbound_http.socket.getaddrinfo",
+                return_value=[(2, 1, 6, "", (address, 445))],
+            ), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk, self.assertRaisesRegex(
+                ConnectorBrowseError,
+                "redirects/referrals.*DNS/IP pinning",
+            ):
+                browse_connector_profile(profile, path="")
+            sdk.assert_not_called()
+
+    def test_smb_explicit_ip_still_fails_closed_because_the_sdk_may_follow_referrals(self) -> None:
+        profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
         with patch.dict(
             "os.environ",
-            {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
+            {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
         ), patch(
             "govoplan_core.security.outbound_http.socket.getaddrinfo",
-            return_value=[(2, 1, 6, "", ("93.184.216.34", 445))],
-        ):
-            location = _smb_location(profile)
+            return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
+        ), self.assertRaisesRegex(ConnectorBrowseError, "redirects/referrals.*DNS/IP pinning"):
+            _smb_location(profile)
 
-        self.assertEqual("93.184.216.34", location.server)
+    def test_smb_import_surfaces_fail_closed_policy_as_an_import_error(self) -> None:
+        profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
+        with patch.dict(
+            "os.environ",
+            {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
+        ), patch(
+            "govoplan_core.security.outbound_http.socket.getaddrinfo",
+            return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
+        ), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk, self.assertRaisesRegex(
+            ConnectorImportError,
+            "redirects/referrals.*DNS/IP pinning",
+        ):
+            read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024)
+        sdk.assert_not_called()
 
     def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None:
         descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()}
@@ -150,10 +181,27 @@ class ConnectorProviderTests(unittest.TestCase):
 
         self.assertEqual(["archive"], [item.path for item in items])
 
-    def test_s3_browse_reports_missing_optional_dependency(self) -> None:
-        with patch("govoplan_files.backend.storage.connector_browse.import_module", side_effect=ImportError):
-            with self.assertRaisesRegex(ConnectorBrowseUnsupported, "boto3"):
-                browse_connector_profile(s3_profile(), path="")
+    def test_s3_sdk_transport_fails_closed_before_client_creation_in_private_mode(self) -> None:
+        with patch.dict(
+            "os.environ",
+            {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
+        ), patch(
+            "govoplan_core.security.outbound_http.socket.getaddrinfo",
+            return_value=[(2, 1, 6, "", ("127.0.0.1", 9000))],
+        ), patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex(
+            ConnectorBrowseError,
+            "until that transport supports.*DNS/IP pinning",
+        ):
+            browse_connector_profile(s3_profile(), path="")
+        importer.assert_not_called()
+
+    def test_s3_sdk_endpoint_discovery_fails_closed(self) -> None:
+        with patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex(
+            ConnectorBrowseError,
+            "endpoint discovery.*cannot guarantee.*DNS/IP pinning",
+        ):
+            browse_connector_profile(s3_profile(endpoint_url=None), path="")
+        importer.assert_not_called()
 
     def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None:
         client = FakeS3Client()

tests/test_http_client.py

@@ -58,7 +58,10 @@ class ConnectorHttpClientTests(unittest.TestCase):
         context = MagicMock()
         context.__enter__.return_value = response
 
-        with patch("govoplan_files.backend.storage.http_client._stream_connector_request", return_value=context):
+        with patch(
+            "govoplan_core.security.outbound_http.socket.getaddrinfo",
+            return_value=[(2, 1, 6, "", ("93.184.216.34", 443))],
+        ), patch("govoplan_files.backend.storage.http_client._stream_connector_request", return_value=context):
             result = request_connector_bytes("GET", "https://example.test/object", max_bytes=10)
 
         self.assertEqual(b"abcdef", result.content)
@@ -72,6 +75,9 @@ class ConnectorHttpClientTests(unittest.TestCase):
         context.__enter__.return_value = response
 
         with patch(
+            "govoplan_core.security.outbound_http.socket.getaddrinfo",
+            return_value=[(2, 1, 6, "", ("93.184.216.34", 443))],
+        ), patch(
             "govoplan_files.backend.storage.http_client._stream_connector_request",
             return_value=context,
         ), self.assertRaisesRegex(

tests/test_manifest_documentation.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import unittest
+
+
+class FilesManifestDocumentationTests(unittest.TestCase):
+    def test_connector_topic_covers_governance_pinning_and_provenance_perspectives(self) -> None:
+        from govoplan_files.backend.manifest import manifest
+
+        topic = next(item for item in manifest.documentation if item.id == "files.governed-connectors-and-provenance")
+
+        self.assertEqual(("admin", "user"), topic.documentation_types)
+        self.assertEqual({"user", "admin", "operator"}, set(topic.metadata["perspectives"]))
+        self.assertIn("fails closed", topic.body)
+        self.assertIn("DFS referrals", topic.body)
+        self.assertIn("revision", topic.metadata["provenance_fields"])
+        self.assertIn("/api/v1/files/connectors/profiles", {link.href for link in topic.links})
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_storage_backends.py

@@ -18,6 +18,20 @@ def _backend() -> S3StorageBackend:
 
 
 class S3StorageBackendTests(unittest.TestCase):
+    def test_sdk_transport_fails_closed_in_public_and_private_modes(self) -> None:
+        for allow_private, address in ((False, "93.184.216.34"), (True, "10.0.0.5")):
+            with self.subTest(allow_private=allow_private), patch.dict(
+                "os.environ",
+                {
+                    "APP_ENV": "production",
+                    "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": str(allow_private).lower(),
+                },
+            ), patch(
+                "govoplan_core.security.outbound_http.socket.getaddrinfo",
+                return_value=[(2, 1, 6, "", (address, 443))],
+            ), self.assertRaisesRegex(StorageBackendError, "until that transport supports.*DNS/IP pinning"):
+                _backend().client
+
     def test_get_bytes_rejects_declared_oversize_object_without_reading(self) -> None:
         body = MagicMock()
         client = MagicMock()