Harden external file connector boundaries

tests/test_connector_deployment.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+from unittest.mock import MagicMock
+
+from fastapi import HTTPException
+
+from govoplan_files.backend.router import discover_connector_endpoint
+from govoplan_files.backend.schemas import FileConnectorDiscoveryRequest
+from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _profile_password, _s3_verify
+from govoplan_files.backend.storage.connector_deployment import (
+    ConnectorDeploymentConfigurationError,
+    connector_effective_endpoint_url,
+    connector_secret_env_value,
+    reject_api_controlled_deployment_references,
+)
+from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
+from govoplan_files.backend.storage.connector_profile_store import create_connector_profile_row
+
+
+class ConnectorDeploymentBoundaryTests(unittest.TestCase):
+    def test_database_profile_cannot_read_arbitrary_process_environment(self) -> None:
+        profile = ConnectorProfile(
+            id="tenant-webdav",
+            label="Tenant WebDAV",
+            provider="webdav",
+            password_env="MASTER_KEY_B64",
+            source_kind="database",
+        )
+        with patch.dict(
+            os.environ,
+            {
+                "MASTER_KEY_B64": "must-not-leave-process",
+                "GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST": "MASTER_KEY_B64",
+            },
+            clear=False,
+        ), self.assertRaisesRegex(ConnectorBrowseError, "deployment-owned"):
+            _profile_password(profile)
+
+    def test_deployment_profile_requires_exact_secret_allowlist(self) -> None:
+        with patch.dict(
+            os.environ,
+            {
+                "GOVOPLAN_FILES_WEBDAV_PASSWORD": "deployment-secret",
+                "GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST": "GOVOPLAN_FILES_WEBDAV_PASSWORD",
+            },
+            clear=False,
+        ):
+            self.assertEqual(
+                "deployment-secret",
+                connector_secret_env_value(
+                    "GOVOPLAN_FILES_WEBDAV_PASSWORD",
+                    source_kind="settings",
+                ),
+            )
+            with self.assertRaisesRegex(ConnectorDeploymentConfigurationError, "not listed"):
+                connector_secret_env_value("MASTER_KEY_B64", source_kind="settings")
+
+    def test_api_profiles_cannot_select_secret_environment_names(self) -> None:
+        with self.assertRaisesRegex(ConnectorDeploymentConfigurationError, "deployment-owned"):
+            reject_api_controlled_deployment_references(password_env="DATABASE_URL")
+        with self.assertRaisesRegex(ConnectorDeploymentConfigurationError, "deployment-owned"):
+            reject_api_controlled_deployment_references(metadata={"secret_access_key_env": "MASTER_KEY_B64"})
+
+        session = MagicMock()
+        with self.assertRaisesRegex(ConnectorDeploymentConfigurationError, "deployment-owned"):
+            create_connector_profile_row(
+                session,
+                tenant_id="tenant-1",
+                user_id="user-1",
+                profile_id="unsafe",
+                label="Unsafe",
+                provider="webdav",
+                scope_type="tenant",
+                password_env="DATABASE_URL",
+            )
+        session.get.assert_not_called()
+
+    def test_profile_responses_hide_environment_and_local_ca_references(self) -> None:
+        profile = ConnectorProfile(
+            id="deployment-s3",
+            label="Deployment S3",
+            provider="s3",
+            metadata={
+                "bucket": "documents",
+                "secret_access_key_env": "GOVOPLAN_FILES_S3_SECRET",
+                "ca_bundle": "/etc/govoplan/connector-ca.pem",
+            },
+        )
+
+        self.assertEqual({"bucket": "documents"}, profile.to_response()["metadata"])
+
+    def test_ca_bundle_must_be_an_exact_deployment_allowlisted_file(self) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            allowed = Path(temp_dir, "connector-ca.pem")
+            other = Path(temp_dir, "other-ca.pem")
+            allowed.write_text("test CA", encoding="utf-8")
+            other.write_text("other CA", encoding="utf-8")
+            with patch.dict(
+                os.environ,
+                {
+                    "APP_ENV": "production",
+                    "GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST": str(allowed),
+                },
+                clear=False,
+            ):
+                profile = ConnectorProfile(
+                    id="s3",
+                    label="S3",
+                    provider="s3",
+                    metadata={"ca_bundle": str(allowed)},
+                )
+                self.assertEqual(str(allowed.resolve()), _s3_verify(profile))
+                with self.assertRaisesRegex(ConnectorDeploymentConfigurationError, "not listed"):
+                    reject_api_controlled_deployment_references(metadata={"ca_bundle": str(other)})
+
+    def test_tls_verification_can_only_be_disabled_in_development(self) -> None:
+        with patch.dict(os.environ, {"APP_ENV": "production"}, clear=False), self.assertRaisesRegex(
+            ConnectorDeploymentConfigurationError,
+            "dev/test",
+        ):
+            reject_api_controlled_deployment_references(metadata={"verify_tls": False})
+        with patch.dict(os.environ, {"APP_ENV": "test"}, clear=False):
+            reject_api_controlled_deployment_references(metadata={"verify_tls": False})
+
+    def test_effective_endpoint_uses_webdav_override(self) -> None:
+        self.assertEqual(
+            "https://dav.example.test/root",
+            connector_effective_endpoint_url(
+                provider="seafile",
+                endpoint_url="https://seafile.example.test",
+                metadata={"webdav_endpoint_url": "https://dav.example.test/root"},
+            ),
+        )
+
+
+class ConnectorDiscoveryBoundaryTests(unittest.TestCase):
+    @staticmethod
+    def _principal() -> SimpleNamespace:
+        return SimpleNamespace(tenant_id="tenant-1", user=SimpleNamespace(id="user-1"), api_key=None)
+
+    def test_discovery_rejects_environment_credentials_before_io(self) -> None:
+        payload = FileConnectorDiscoveryRequest(
+            provider="webdav",
+            endpoint_url="https://dav.example.test",
+            credential_mode="basic",
+            credentials={"username": "admin", "password_env": "MASTER_KEY_B64"},
+        )
+        with patch("govoplan_files.backend.router.browse_connector_profile") as browse, self.assertRaises(
+            HTTPException
+        ) as raised:
+            discover_connector_endpoint(payload, session=object(), principal=self._principal())  # type: ignore[arg-type]
+        self.assertEqual(400, raised.exception.status_code)
+        browse.assert_not_called()
+
+    def test_discovery_applies_policy_and_audit_before_each_io_candidate(self) -> None:
+        payload = FileConnectorDiscoveryRequest(
+            provider="webdav",
+            endpoint_url="https://dav.example.test/root",
+            metadata={
+                "webdav_endpoint_url": "https://bypass.example.test",
+                "static_listing": {"": []},
+            },
+        )
+        events: list[str] = []
+
+        def ensure(*_args: object, **kwargs: object) -> None:
+            self.assertEqual("https://dav.example.test/root/", kwargs["endpoint_url"])
+            events.append("policy")
+
+        def audit(*_args: object, **_kwargs: object) -> None:
+            events.append("audit")
+
+        def browse(profile: ConnectorProfile, **_kwargs: object) -> list[object]:
+            self.assertEqual({}, profile.metadata)
+            events.append("io")
+            return []
+
+        with patch("govoplan_files.backend.router._ensure_connector_configuration_allowed", side_effect=ensure), patch(
+            "govoplan_files.backend.router._audit_connector_discovery_attempt",
+            side_effect=audit,
+        ), patch("govoplan_files.backend.router.browse_connector_profile", side_effect=browse):
+            response = discover_connector_endpoint(payload, session=object(), principal=self._principal())  # type: ignore[arg-type]
+
+        self.assertEqual("usable", response.status)
+        self.assertEqual(["policy", "audit", "io"], events)
+        self.assertEqual({"discovered_by": "webdav-propfind"}, response.metadata)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_connector_providers.py

@@ -4,7 +4,7 @@ import unittest
 from datetime import UTC, datetime
 from unittest.mock import patch
 
-from govoplan_files.backend.storage.connector_browse import browse_connector_profile
+from govoplan_files.backend.storage.connector_browse import _smb_location, browse_connector_profile
 from govoplan_files.backend.storage.connector_browse import ConnectorBrowseUnsupported
 from govoplan_files.backend.storage.connector_imports import read_connector_file
 from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload
@@ -76,6 +76,24 @@ def s3_profile(**overrides: object) -> ConnectorProfile:
 
 
 class ConnectorProviderTests(unittest.TestCase):
+    def test_smb_uses_validated_numeric_target_when_private_networks_are_disabled(self) -> None:
+        profile = ConnectorProfile(
+            id="public-smb",
+            label="Public SMB",
+            provider="smb",
+            endpoint_url="smb://files.example.test/share",
+        )
+        with patch.dict(
+            "os.environ",
+            {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
+        ), patch(
+            "govoplan_core.security.outbound_http.socket.getaddrinfo",
+            return_value=[(2, 1, 6, "", ("93.184.216.34", 445))],
+        ):
+            location = _smb_location(profile)
+
+        self.assertEqual("93.184.216.34", location.server)
+
     def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None:
         descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()}
 

tests/test_http_client.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import unittest
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from threading import Thread
+from unittest.mock import MagicMock, patch
+
+import httpcore
+
+from govoplan_core.security.outbound_http import outbound_http_policy, validate_outbound_http_url
+from govoplan_files.backend.storage.http_client import (
+    ConnectorHttpError,
+    _OutboundPolicyNetworkBackend,
+    _SocketNetworkStream,
+    request_connector_bytes,
+)
+
+
+class _TestHandler(BaseHTTPRequestHandler):
+    def do_GET(self) -> None:  # noqa: N802 - stdlib handler contract
+        body = b"connector-ok"
+        self.send_response(200)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, _format: str, *_args: object) -> None:
+        return
+
+
+class ConnectorHttpClientTests(unittest.TestCase):
+    def test_public_transport_adapter_performs_a_real_http_request(self) -> None:
+        server = ThreadingHTTPServer(("127.0.0.1", 0), _TestHandler)
+        thread = Thread(target=server.serve_forever, daemon=True)
+        thread.start()
+        try:
+            with patch.dict(
+                "os.environ",
+                {"APP_ENV": "test", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
+                clear=False,
+            ):
+                result = request_connector_bytes(
+                    "GET",
+                    f"http://127.0.0.1:{server.server_port}/object",
+                )
+        finally:
+            server.shutdown()
+            server.server_close()
+            thread.join(timeout=2)
+        self.assertEqual(200, result.status_code)
+        self.assertEqual(b"connector-ok", result.content)
+
+    def test_connector_responses_are_streamed_without_redirects(self) -> None:
+        response = MagicMock()
+        response.status_code = 200
+        response.headers = {"content-length": "6"}
+        response.iter_bytes.return_value = iter((b"abc", b"def"))
+        context = MagicMock()
+        context.__enter__.return_value = response
+
+        with patch("govoplan_files.backend.storage.http_client._stream_connector_request", return_value=context):
+            result = request_connector_bytes("GET", "https://example.test/object", max_bytes=10)
+
+        self.assertEqual(b"abcdef", result.content)
+
+    def test_connector_response_limit_is_enforced_while_streaming(self) -> None:
+        response = MagicMock()
+        response.status_code = 200
+        response.headers = {}
+        response.iter_bytes.return_value = iter((b"12345", b"67890", b"!"))
+        context = MagicMock()
+        context.__enter__.return_value = response
+
+        with patch(
+            "govoplan_files.backend.storage.http_client._stream_connector_request",
+            return_value=context,
+        ), self.assertRaisesRegex(
+            ConnectorHttpError,
+            "configured limit",
+        ):
+            request_connector_bytes("GET", "https://example.test/object", max_bytes=10)
+
+    def test_private_destination_is_blocked_before_http_connection(self) -> None:
+        with patch.dict(
+            "os.environ",
+            {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
+        ), patch(
+            "govoplan_core.security.outbound_http.socket.getaddrinfo",
+            return_value=[(2, 1, 6, "", ("10.0.0.5", 443))],
+        ), patch("govoplan_files.backend.storage.http_client._stream_connector_request") as stream, self.assertRaisesRegex(
+            ConnectorHttpError,
+            "non-public network",
+        ):
+            request_connector_bytes("GET", "https://connector.example.test/object")
+        stream.assert_not_called()
+
+    def test_connection_backend_rejects_private_rebinding_before_socket_open(self) -> None:
+        policy = outbound_http_policy({"APP_ENV": "production"})
+        public = [(2, 1, 6, "", ("93.184.216.34", 443))]
+        private = [(2, 1, 6, "", ("127.0.0.1", 443))]
+        with patch.dict(
+            "os.environ",
+            {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
+        ), patch(
+            "govoplan_core.security.outbound_http.socket.getaddrinfo",
+            side_effect=(public, private),
+        ), patch("govoplan_core.security.outbound_http.socket.socket") as socket_factory:
+            validate_outbound_http_url("https://connector.example.test/object", policy=policy)
+            with self.assertRaises(httpcore.ConnectError):
+                _OutboundPolicyNetworkBackend().connect_tcp("connector.example.test", 443)
+        socket_factory.assert_not_called()
+
+    def test_network_backend_returns_public_network_stream_adapter(self) -> None:
+        sock = MagicMock()
+        with patch(
+            "govoplan_files.backend.storage.http_client.create_outbound_connection",
+            return_value=sock,
+        ):
+            stream = _OutboundPolicyNetworkBackend().connect_tcp("connector.example.test", 443)
+
+        self.assertIsInstance(stream, _SocketNetworkStream)
+        self.assertIs(stream.get_extra_info("socket"), sock)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_storage_backends.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import io
+import unittest
+from unittest.mock import MagicMock, PropertyMock, patch
+
+from govoplan_files.backend.storage.backends import S3StorageBackend, StorageBackendError
+
+
+def _backend() -> S3StorageBackend:
+    return S3StorageBackend(
+        bucket="files",
+        endpoint_url="https://objects.example.test",
+        region_name="test",
+        access_key_id="access",
+        secret_access_key="secret",
+    )
+
+
+class S3StorageBackendTests(unittest.TestCase):
+    def test_get_bytes_rejects_declared_oversize_object_without_reading(self) -> None:
+        body = MagicMock()
+        client = MagicMock()
+        client.get_object.return_value = {"ContentLength": 6, "Body": body}
+        with patch.dict("os.environ", {"GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES": "5"}), patch.object(
+            S3StorageBackend,
+            "client",
+            new_callable=PropertyMock,
+            return_value=client,
+        ), self.assertRaisesRegex(StorageBackendError, "deployment limit"):
+            _backend().get_bytes("large.bin")
+        body.read.assert_not_called()
+        body.close.assert_called_once_with()
+
+    def test_iter_bytes_rejects_undeclared_oversize_object(self) -> None:
+        client = MagicMock()
+        client.get_object.return_value = {"Body": io.BytesIO(b"123456")}
+        with patch.dict("os.environ", {"GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES": "5"}), patch.object(
+            S3StorageBackend,
+            "client",
+            new_callable=PropertyMock,
+            return_value=client,
+        ), self.assertRaisesRegex(StorageBackendError, "deployment limit"):
+            list(_backend().iter_bytes("large.bin", chunk_size=3))
+
+    def test_iter_bytes_closes_streaming_body_when_consumer_stops_early(self) -> None:
+        body = MagicMock()
+        body.read.side_effect = (b"123", b"456", b"")
+        client = MagicMock()
+        client.get_object.return_value = {"ContentLength": 6, "Body": body}
+        with patch.dict("os.environ", {"GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES": "10"}), patch.object(
+            S3StorageBackend,
+            "client",
+            new_callable=PropertyMock,
+            return_value=client,
+        ):
+            chunks = _backend().iter_bytes("object.bin", chunk_size=3)
+            self.assertEqual(b"123", next(chunks))
+            chunks.close()
+        body.close.assert_called_once_with()
+
+
+if __name__ == "__main__":
+    unittest.main()