From 92e649477f6654d20e64566ed14349f26f773f9e Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Tue, 4 Aug 2026 10:40:46 +0200 Subject: [PATCH] Pin S3 and SMB connector peers --- README.md | 27 +- docs/CONNECTOR_BOUNDARY.md | 14 +- docs/FILES_HANDBOOK.md | 46 ++- src/govoplan_files/backend/manifest.py | 25 +- .../backend/storage/connector_browse.py | 67 +++- .../backend/storage/connector_imports.py | 23 +- .../backend/storage/connector_providers.py | 4 +- .../backend/storage/connector_visibility.py | 5 - .../backend/storage/sdk_peer_pinning.py | 331 ++++++++++++++++++ tests/test_connector_providers.py | 90 ++--- tests/test_connector_visibility.py | 19 +- tests/test_documentation.py | 1 + tests/test_manifest_documentation.py | 4 +- tests/test_sdk_peer_pinning.py | 294 ++++++++++++++++ tests/test_search_source.py | 2 + 15 files changed, 840 insertions(+), 112 deletions(-) create mode 100644 src/govoplan_files/backend/storage/sdk_peer_pinning.py create mode 100644 tests/test_sdk_peer_pinning.py diff --git a/README.md b/README.md index 23c2950..60c428f 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,8 @@ run profile policy with the `browse` operation, and support Seafile, WebDAV/Nextcloud, and SMB when the optional `smb` extra is installed. Seafile profiles browse libraries and directories via Seafile's read-only API; profiles can still opt into WebDAV browsing by setting `metadata.webdav_endpoint_url` or -`metadata.browse_protocol` to `webdav`. +`metadata.browse_protocol` to `webdav`. S3-compatible profiles support bucket +and prefix browsing when the optional `s3` extra is installed. `POST /api/v1/files/connectors/profiles/{profile_id}/import` imports a Seafile or WebDAV/Nextcloud/SMB file into managed storage through the same governance, conflict handling, source provenance, and connector audit path as direct @@ -113,17 +114,21 @@ uploads. The Seafile provider uses account-token auth and the native file download-link API; Nextcloud and generic WebDAV profiles use authenticated `GET` requests against the configured WebDAV endpoint. SMB profiles use `smb://server[:port]/share[/path]` endpoints and deployment-owned or encrypted -stored credentials through `smbprotocol`. Because that SDK cannot accept a -preconnected socket and may follow DFS referrals to additional hosts, live SMB -access fails closed in both public-only and private-network deployments. It will -remain disabled until every initial connection and referral target can be -policy-validated and pinned. +stored credentials through `smbprotocol`. Files installs a pinned transport for +every initial session, reconnect, alias, and DFS referral target. The deployment +private-network policy is re-evaluated immediately before each socket opens. -S3 connector browse/import remains fail-closed because a user-governed connector -endpoint must use the GovOPlaN pinned HTTP trust model. Durable platform storage -has a separate deployment boundary: installer-owned Garage is accepted only at -its exact generated endpoint, while an operator-selected external backend -requires a clean HTTPS origin and +S3 connector browse/import binds botocore HTTP and HTTPS pools to the same pinned +socket policy. Retries, redirects, endpoint discovery, bucket aliases, and new +connections are therefore revalidated while TLS keeps the configured hostname +for SNI and certificate checks. Connector clients do not use outbound proxies or +ambient AWS credential discovery; configure credentials on the governed profile, +or explicitly use an anonymous profile for public objects. An incompatible SDK +upgrade fails closed before a usable client/session is returned. + +Durable platform storage has a separate deployment boundary: installer-owned +Garage is accepted only at its exact generated endpoint, while an +operator-selected external backend requires a clean HTTPS origin and `FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. That flag is deployment configuration, cannot be supplied through a Files connector profile, and does not replace operator responsibility for DNS, certificates, egress, bucket policy, diff --git a/docs/CONNECTOR_BOUNDARY.md b/docs/CONNECTOR_BOUNDARY.md index 929da7e..e1dd061 100644 --- a/docs/CONNECTOR_BOUNDARY.md +++ b/docs/CONNECTOR_BOUNDARY.md @@ -19,6 +19,7 @@ normal product workflows and can share the same governance model: - Nextcloud through WebDAV - generic WebDAV - SMB through the optional `smb` extra +- S3-compatible stores through the optional `s3` extra These providers are surfaced through connector descriptors at `GET /api/v1/files/connectors/providers`. Provider descriptors declare whether @@ -60,11 +61,14 @@ Every provider must: - use a transport that pins every connection to a policy-validated DNS/IP answer and revalidates redirects; SDK transports without that guarantee fail closed -Live SMB access is disabled in all modes until `smbprotocol` initial connections -and DFS referral targets can be pinned and policy-validated. An explicit IP is -not sufficient because the server may still issue a referral to another peer. -Live S3 access is likewise disabled until `boto3`/botocore can be bound to the -pinned transport, including SDK-managed redirects and endpoint discovery. +SMB initial connections, reconnects, aliases, and DFS referral targets are +created through a Files-owned pinned `smbprotocol` transport. S3 HTTP/HTTPS pools +use the equivalent botocore adapter for every connection selected by retries, +redirects, endpoint discovery, and virtual-host addressing. Both adapters apply +the deployment-wide private-network policy at socket creation. S3 keeps the +configured hostname for TLS SNI and certificate verification, but does not use +outbound proxies or ambient AWS credential discovery. An incompatible optional +SDK release fails closed before a usable session or client is returned. ## Non-Goals For Files diff --git a/docs/FILES_HANDBOOK.md b/docs/FILES_HANDBOOK.md index 7de475b..71ff7d7 100644 --- a/docs/FILES_HANDBOOK.md +++ b/docs/FILES_HANDBOOK.md @@ -405,15 +405,15 @@ in a development/test runtime. | Seafile | Read-only native API browse/download-link import and manual sync implemented using the pinned HTTP transport; WebDAV opt-in supported | | Nextcloud | Read-only WebDAV browse/import/manual sync implemented using the pinned HTTP transport | | Generic WebDAV | Read-only browse/import/manual sync implemented using the pinned HTTP transport | -| SMB | Browse/import code and descriptor implemented, but every live connection fails closed until initial connections and DFS referrals can be policy-validated and pinned | -| S3 connector | Browse/import code and descriptor implemented, but every live SDK connection fails closed until botocore connections, redirects, and endpoint discovery can be validated and pinned | +| SMB | Read-only browse/import/manual sync implemented through a pinned smbprotocol transport for initial peers, reconnects, aliases, and DFS referral targets | +| S3 connector | Read-only bucket/prefix browse, import, and manual sync implemented through pinned botocore pools covering retries, redirects, endpoint discovery, and provider aliases | | SharePoint and OneDrive | Provider keys/descriptors reserved; live Microsoft Graph browse/import is planned | | NFS and local connector | Described as optional future providers; the local managed-storage backend is a different feature | Provider descriptors are available from `GET /api/v1/files/connectors/providers`. Use their `implemented`, `installed`, -and support fields for display, but expect a fail-closed transport error where -the table above says live access is disabled. +and support fields for display. An incompatible optional SDK release fails closed +before it can return a usable client or session. ## Operator runbook @@ -478,17 +478,29 @@ The built-in HTTP transport: - bounds structured responses to 16 MiB and file transfers to 512 MiB by default. +The S3 SDK adapter applies the same socket rule to every botocore pool selected +for a retry, redirect, discovered endpoint, or virtual-host bucket alias. The +original authority remains in the request and TLS SNI/certificate check. S3 +connector clients use no outbound proxy and never discover ambient AWS +credentials: configure both access and secret keys on the governed profile, or +use an anonymous profile for a public source. + +The SMB adapter owns a separate connection cache and replaces smbprotocol's TCP +factory process-wide with the stricter pinned socket. Initial peers, reconnects, +server aliases, domain-controller connections, and DFS referral targets therefore +pass the same policy at connection time. Signing is required by default; enable +SMB encryption on the profile where the server supports it. + Override the connector response limits with `GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES` and `GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES`. The smaller applicable limit wins when an import is also subject to `FILE_UPLOAD_MAX_BYTES`. Never work around a connector pinning failure by adding a raw IP, disabling TLS, -or enabling private networks. SMB may redirect through DFS, and user-configured -S3 connectors may perform their own redirects or endpoint discovery; those -connector transports remain disabled until every connection peer can be -governed. The separately configured platform S3 backend is trusted only by the -deployment owner and is not selectable by a user or connector profile. +or enabling private networks. A failure means the peer policy rejected an actual +connection destination or the installed SDK no longer exposes the verified +transport seam. The separately configured platform S3 backend is trusted only by +the deployment owner and is not selectable by a user or connector profile. ### Backup and restore @@ -620,7 +632,7 @@ Use these symptoms as routing hints: | `Stored object does not exist` | Database/blob restore mismatch, wrong root, or missing shared storage | | `Stored secret cannot be decrypted` | Wrong or rotated `MASTER_KEY_B64` | | Private/non-public endpoint blocked | Deployment-wide egress policy is public-only or DNS returned a forbidden answer | -| SDK cannot pin redirects/referrals | Expected fail-closed S3/SMB boundary, not a transient connector outage | +| SDK peer-pinning seam is unavailable | Optional S3/SMB SDK is incompatible; keep access fail-closed and validate the supported dependency range before upgrade | | Connector response exceeds limit | Remote payload exceeds connector or upload limit | | Profile is not visible | Scope, disabled state, campaign access, or policy mismatch | | Group removal is vetoed | The group still owns a file/folder/connector space or is a share target | @@ -824,9 +836,11 @@ bypass the same rule. ### Pinned connector transport Given public-only mode and DNS returns any private address, the connection must -be rejected before a socket opens. Given private mode, the HTTP connection must -still use the validated address and reject redirects. SMB and S3 must fail -before their SDK clients connect until all SDK-managed peers can be pinned. +be rejected before a socket opens. Given private mode, every HTTP, S3, and SMB +connection must still use an address from the answer validated for that exact +attempt. Botocore retries, redirects, endpoint discovery, and aliases, plus SMB +reconnects and DFS referrals, must pass through the pinned factories. A changed +or unsupported SDK seam must fail before a usable client/session is returned. ### Imported evidence and sync @@ -867,7 +881,7 @@ returning different content or credentials. | Deletion/retention | Soft-delete assets/folders/spaces; immediate audited connector-secret scrubbing | File restore API, hard purge, retention policy, legal hold, and blob GC ([#38](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/38)) | | Connector governance | Scoped profiles/credentials/policies, effective source explanation, separate credentials, linked user/group spaces | Provider-owned external secret lifecycle; API `secret_ref` remains rejected | | HTTP connectors | Pinned, bounded, no-redirect Seafile and WebDAV/Nextcloud browse/import/manual sync | Background/folder sync, remote mutation, long-running transfer workers | -| SMB and S3 connectors | Provider descriptors and browse/import logic | Live access only after SDK connections plus DFS referrals/redirects are validated and pinned ([SMB #35](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/35), [S3 #34](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/34)) | +| SMB and S3 connectors | Provider descriptors, browse/import/manual sync, pinned SDK transports, and redirect/retry/referral transport-contract tests | Live topology smoke evidence, provider-specific OAuth, remote writes, and background indexing remain separate deployment or connector-module concerns | | Other providers | Reserved SharePoint/OneDrive keys and NFS/local descriptors | Graph/OAuth/provider paging, NFS deployment integration, DMS connectors | | Connector spaces | User/group link, browse, manual selected-file sync, edit/disable/delete | Background sync, remote writes/deletes, full conflict-reporting jobs | | Profile capabilities | Stored and displayed | Enforce capability flags as an independent operation gate | @@ -890,8 +904,8 @@ Before releasing Files: 6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion. 7. Exercise connector policy explanation and one pinned HTTP provider where configured; verify the deployment-managed or trusted external S3 backend if - selected, and verify user-configured S3 and SMB connector peers still fail - closed. + selected, and verify one configured S3 and SMB connector while recording the + actual target topology and private-network policy. 8. Verify credential deletion scrubs dependents and produces audit evidence. 9. Verify a campaign attachment snapshot still identifies its exact version and checksum after the current file changes. diff --git a/src/govoplan_files/backend/manifest.py b/src/govoplan_files/backend/manifest.py index 9c4ac9a..5f084d9 100644 --- a/src/govoplan_files/backend/manifest.py +++ b/src/govoplan_files/backend/manifest.py @@ -359,6 +359,23 @@ manifest = ModuleManifest( audience=("file_user", "file_manager", "administrator"), related_modules=("search",), order=41, + conditions=( + DocumentationCondition( + required_modules=("files",), + any_scopes=("files:file:read", "files:file:admin"), + ), + ), + links=( + DocumentationLink(label="Files", href="/files", kind="runtime"), + DocumentationLink(label="Search", href="/search", kind="runtime"), + DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + ), + metadata={ + "kind": "reference", + "route": "/files", + "screen": "Files search contribution", + "help_contexts": ["files.list"], + }, ), DocumentationTopic( id="files.workflow.organize-managed-files", @@ -618,10 +635,10 @@ manifest = ModuleManifest( DocumentationTopic( id="files.reference.integrity-recovery-and-fail-closed-transports", title="Operate Files integrity, recovery, and connector transport safety", - summary="Back up database evidence, blob ciphertext, and Encryption custody as one recovery unit, and keep unsupported SDK transports fail-closed.", + summary="Back up database evidence, blob ciphertext, and Encryption custody as one recovery unit, and pin every SDK-managed connector peer.", body=( "Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the matching Encryption tables and original deployment master key, then run the bounded resumable integrity scan from Administration and verify representative protected and unprotected access paths. Each scan batch and finding action requires the revision shown to the operator, so a stale screen cannot recheck or delete after concurrent reconciliation. Protected scans verify stored ciphertext before decryption and then verify plaintext semantic evidence. Managed blob creation/repair and applied orphan cleanup commit lease-fenced Core recovery intent before object effects; success, compensation, and forward completion require independent database and object checks, while mismatch is quarantined and unresolved work remains visible in Ops. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. " - "Arbitrary external S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Installer-owned Garage storage is supported only at the exact deployment service endpoint with its explicit trust marker. Destructive module retirement drops database tables but does not remove backend blob objects." + "S3 connector pools pin every retry, redirect, discovered endpoint, and provider alias while retaining the configured TLS authority; outbound proxies and ambient credential discovery are disabled. SMB initial connections, reconnects, aliases, and DFS referrals use a Files-owned pinned transport and cache. Both apply the deployment private-network policy immediately before each socket opens and fail closed if an SDK no longer exposes the verified transport seam. Installer-owned Garage storage is supported only at the exact deployment service endpoint with its explicit trust marker. Destructive module retirement drops database tables but does not remove backend blob objects." ), layer="configured", documentation_types=("admin",), @@ -669,7 +686,7 @@ manifest = ModuleManifest( "screen": "System file connections and deployment operations", "section": "Storage integrity, backup/recovery, and fail-closed transports", "recovery_unit": ["Files database rows", "Encryption envelope and wrapped-key rows", "managed blob namespace", "MASTER_KEY_B64", "deployment-owned connector configuration"], - "verification": "After restore, complete a checksum-enabled integrity scan, resolve every missing/corrupt finding, approve or retain every reported orphan, inspect Files recovery operations in Ops, verify authorized and denied access, and test one permitted pinned HTTP connector.", + "verification": "After restore, complete a checksum-enabled integrity scan, resolve every missing/corrupt finding, approve or retain every reported orphan, inspect Files recovery operations in Ops, verify authorized and denied access, and test configured pinned HTTP, S3, and SMB connectors against their recorded target topology.", "related_topic_ids": [ "files.governed-connectors-and-provenance", "files.reference.snapshot-provenance-and-capabilities", @@ -774,7 +791,7 @@ manifest = ModuleManifest( "Confirm version alignment, apply migrations on clean and upgrade databases, and run the repository plus meta-repository security and static-analysis gates.", "Exercise allowed and denied personal, group, and share access with representative accounts.", "Exercise bounded archive preview and confirmation, every required conflict strategy, organization, download, and soft deletion.", - "Where connectors are configured, verify policy explanation and one pinned HTTP provider; verify installer-owned Garage when selected, and verify arbitrary external S3 and SMB access still fails closed.", + "Where connectors are configured, verify policy explanation and one pinned HTTP provider; verify installer-owned Garage when selected, then exercise configured S3 retries/aliases and SMB reconnect/referral targets under the deployment private-network policy, confirming an incompatible SDK transport seam fails closed.", "Restore a coordinated database/blob/key backup and compare representative downloaded bytes with their recorded SHA-256 checksums.", "Record the tested versions, results, known limitations, evidence locations, residual risks, owner, and approval decision.", ], diff --git a/src/govoplan_files/backend/storage/connector_browse.py b/src/govoplan_files/backend/storage/connector_browse.py index 3aa0ed8..a62bcf4 100644 --- a/src/govoplan_files/backend/storage/connector_browse.py +++ b/src/govoplan_files/backend/storage/connector_browse.py @@ -15,8 +15,8 @@ from defusedxml import ElementTree as SafeElementTree from govoplan_core.security.outbound_http import ( OutboundHttpError, - validate_unpinned_sdk_host, - validate_unpinned_sdk_http_url, + validate_outbound_host, + validate_outbound_http_url, ) from govoplan_files.backend.storage.http_client import ConnectorHttpError, request_connector_bytes @@ -27,6 +27,12 @@ from govoplan_files.backend.storage.connector_deployment import ( validate_connector_tls_metadata, ) from govoplan_files.backend.storage.connector_profiles import ConnectorProfile +from govoplan_files.backend.storage.sdk_peer_pinning import ( + SdkPeerPinningError, + create_pinned_s3_client, + install_pinned_smb_transport, + pinned_smb_connection_cache, +) class ConnectorBrowseError(RuntimeError): @@ -246,6 +252,28 @@ def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowse def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]: client = _s3_client(profile) + try: + return _browse_s3_with_client( + client, + profile=profile, + path=path, + library_id=library_id, + continuation_token=continuation_token, + ) + finally: + close = getattr(client, "close", None) + if callable(close): + close() + + +def _browse_s3_with_client( + client: Any, + *, + profile: ConnectorProfile, + path: str, + library_id: str | None, + continuation_token: str | None, +) -> list[ConnectorBrowseItem]: bucket = _s3_bucket(profile, library_id) if not bucket: try: @@ -313,23 +341,22 @@ def _s3_client(profile: ConnectorProfile) -> Any: raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing") if profile.endpoint_url: try: - endpoint_url = validate_unpinned_sdk_http_url( + endpoint_url = validate_outbound_http_url( profile.endpoint_url, label="S3 connector endpoint", ) except OutboundHttpError as exc: raise ConnectorBrowseError(str(exc)) from exc else: - raise ConnectorBrowseError( - "S3 connector endpoint discovery uses an SDK transport that cannot guarantee connection-time DNS/IP " - "pinning; live S3 access is disabled until that transport supports pinning" - ) + endpoint_url = None try: - boto3 = import_module("boto3") config_module = import_module("botocore.config") + unsigned = import_module("botocore").UNSIGNED except ImportError as exc: raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc - kwargs: dict[str, object] = {"endpoint_url": endpoint_url} + kwargs: dict[str, object] = {} + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region") if region: kwargs["region_name"] = region @@ -346,10 +373,21 @@ def _s3_client(profile: ConnectorProfile) -> Any: if verify is not None: kwargs["verify"] = verify addressing_style = _s3_addressing_style(profile) + config_values: dict[str, object] = { + "proxies": {}, + "retries": {"mode": "standard", "max_attempts": 4}, + } if addressing_style: - kwargs["config"] = config_module.Config(s3={"addressing_style": addressing_style}) + config_values["s3"] = {"addressing_style": addressing_style} + if bool(access_key) != bool(secret_key): + raise ConnectorBrowseError("S3 connectors require both an access key and a secret key") + if not access_key: + config_values["signature_version"] = unsigned + kwargs["config"] = config_module.Config(**config_values) try: - return boto3.client("s3", **kwargs) + return create_pinned_s3_client(**kwargs) + except SdkPeerPinningError as exc: + raise ConnectorBrowseError(str(exc)) from exc except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc @@ -783,7 +821,7 @@ def _smb_location(profile: ConnectorProfile) -> _SmbLocation: raise ConnectorBrowseError("SMB connector endpoint_url must include a server") port = parsed.port or _int(profile.metadata.get("port")) or 445 try: - validate_unpinned_sdk_host(server, port=port, label="SMB connector endpoint") + validate_outbound_host(server, port=port, label="SMB connector endpoint") except OutboundHttpError as exc: raise ConnectorBrowseError(str(exc)) from exc path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part] @@ -810,6 +848,7 @@ def _smb_unc_path(location: _SmbLocation, path: str) -> str: def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]: kwargs: dict[str, object] = { "port": location.port, + "connection_cache": pinned_smb_connection_cache(), "require_signing": _metadata_bool(profile, "require_signing", default=True), "auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm", } @@ -845,9 +884,11 @@ def _profile_token(profile: ConnectorProfile) -> str | None: def _smbclient_module() -> Any: try: - return import_module("smbclient") + return install_pinned_smb_transport(import_module("smbclient")) except ImportError as exc: raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc + except SdkPeerPinningError as exc: + raise ConnectorBrowseUnsupported(str(exc)) from exc def _smb_entry_stat(entry: object) -> object | None: diff --git a/src/govoplan_files/backend/storage/connector_imports.py b/src/govoplan_files/backend/storage/connector_imports.py index 09edd2d..600fb44 100644 --- a/src/govoplan_files/backend/storage/connector_imports.py +++ b/src/govoplan_files/backend/storage/connector_imports.py @@ -314,15 +314,20 @@ def _read_s3_file(profile: ConnectorProfile, *, library_id: str, path: str, max_ if not key: raise ConnectorImportError("S3 import requires an object key") client = _s3_import_client(profile) - detail = _s3_object_detail(client, bucket=bucket, key=key, max_bytes=max_bytes) - version_id = _clean(detail.get("VersionId")) - response, data = _download_s3_object( - client, - bucket=bucket, - key=key, - version_id=version_id, - max_bytes=max_bytes, - ) + try: + detail = _s3_object_detail(client, bucket=bucket, key=key, max_bytes=max_bytes) + version_id = _clean(detail.get("VersionId")) + response, data = _download_s3_object( + client, + bucket=bucket, + key=key, + version_id=version_id, + max_bytes=max_bytes, + ) + finally: + close = getattr(client, "close", None) + if callable(close): + close() content_type = _clean(response.get("ContentType") if isinstance(response, dict) else None) or _clean(detail.get("ContentType")) or mimetypes.guess_type(key)[0] etag = _clean(response.get("ETag") if isinstance(response, dict) else None) or _clean(detail.get("ETag")) filename = filename_from_path(key) diff --git a/src/govoplan_files/backend/storage/connector_providers.py b/src/govoplan_files/backend/storage/connector_providers.py index d625235..be3cec2 100644 --- a/src/govoplan_files/backend/storage/connector_providers.py +++ b/src/govoplan_files/backend/storage/connector_providers.py @@ -110,7 +110,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]: conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after download.", preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the share.", audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), - notes="Browse/import logic is implemented, but live smbprotocol access fails closed until the SDK supports connection-time DNS/IP pinning for initial connections and DFS referrals.", + notes="Initial sessions, reconnects, aliases, and DFS referral targets use the Files-owned pinned smbprotocol transport and the deployment-wide private-network policy.", ), ConnectorProviderDescriptor( provider="s3", @@ -125,7 +125,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]: conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.", preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.", audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), - notes="Browse/import logic is implemented, but live boto3 access fails closed until its HTTP transport supports connection-time DNS/IP pinning and redirect revalidation.", + notes="Every botocore connection pool uses the Files-owned pinned transport, including retries, redirects, endpoint discovery, and virtual-host aliases; outbound proxies and ambient credential discovery are disabled.", ), ConnectorProviderDescriptor( provider="sharepoint", diff --git a/src/govoplan_files/backend/storage/connector_visibility.py b/src/govoplan_files/backend/storage/connector_visibility.py index d28960b..118fea4 100644 --- a/src/govoplan_files/backend/storage/connector_visibility.py +++ b/src/govoplan_files/backend/storage/connector_visibility.py @@ -136,11 +136,6 @@ def connector_profile_usable_for_import(profile: ConnectorProfile) -> bool: and descriptor.import_supported ): return False - # These SDK paths intentionally fail closed until every connection peer and - # SDK-managed redirect/referral can be pinned and revalidated. - if profile.provider in {"s3", "smb"}: - return False - # Prove that the initial root browse performed by the current Files UI is # policy-allowed. A later selected remote path/item is checked again. return connector_policy_decision( diff --git a/src/govoplan_files/backend/storage/sdk_peer_pinning.py b/src/govoplan_files/backend/storage/sdk_peer_pinning.py new file mode 100644 index 0000000..f2901a7 --- /dev/null +++ b/src/govoplan_files/backend/storage/sdk_peer_pinning.py @@ -0,0 +1,331 @@ +from __future__ import annotations + +import inspect +import socket +import threading +from functools import lru_cache +from importlib import import_module +from typing import Any + +from govoplan_core.security.outbound_http import ( + OutboundHttpError, + create_outbound_connection, +) + + +class SdkPeerPinningError(RuntimeError): + """Raised when an optional SDK cannot be bound to the pinned transport.""" + + +_BOTOCORE_CLIENT_CREATION_LOCK = threading.Lock() + + +def create_pinned_s3_client(**kwargs: Any) -> Any: + """Construct an S3 client whose first and every later socket is pinned. + + Botocore does not expose the HTTP-session class through boto3's public + client API. Its endpoint creator does expose that seam, so this function + replaces the creator only for the bounded client-construction operation. + The lock avoids an unsafe interleaving with another GovOPlaN client + construction. Any unrelated botocore client created during the short + replacement window also receives the stricter transport. + """ + + try: + boto3 = import_module("boto3") + botocore_args = import_module("botocore.args") + except ImportError as exc: + raise SdkPeerPinningError( + "S3 connector browsing requires the optional boto3 dependency" + ) from exc + + pinned_session_cls = _botocore_transport_types()[0] + original_creator = getattr(botocore_args, "EndpointCreator", None) + if original_creator is None or not hasattr(original_creator, "create_endpoint"): + raise SdkPeerPinningError( + "The installed botocore release does not expose the endpoint transport seam required for peer pinning" + ) + + class PinnedEndpointCreator(original_creator): # type: ignore[misc, valid-type] + def create_endpoint(self, *args: Any, **endpoint_kwargs: Any) -> Any: + endpoint_kwargs["http_session_cls"] = pinned_session_cls + return super().create_endpoint(*args, **endpoint_kwargs) + + with _BOTOCORE_CLIENT_CREATION_LOCK: + current_creator = getattr(botocore_args, "EndpointCreator", None) + if current_creator is not original_creator: + raise SdkPeerPinningError( + "Botocore endpoint construction changed concurrently; refusing to create an unproven S3 client" + ) + botocore_args.EndpointCreator = PinnedEndpointCreator + try: + session_type = getattr(getattr(boto3, "session", None), "Session", None) + if session_type is None: + raise SdkPeerPinningError( + "The installed boto3 release does not expose its isolated session constructor" + ) + client = session_type().client("s3", **kwargs) + finally: + if getattr(botocore_args, "EndpointCreator", None) is PinnedEndpointCreator: + botocore_args.EndpointCreator = original_creator + + http_session = getattr(getattr(client, "_endpoint", None), "http_session", None) + if http_session is None or not isinstance(http_session, pinned_session_cls): + close = getattr(client, "close", None) + if callable(close): + close() + raise SdkPeerPinningError( + "Botocore did not install the required pinned HTTP transport; the S3 client was discarded" + ) + return client + + +@lru_cache(maxsize=1) +def _botocore_transport_types() -> tuple[type[Any], type[Any], type[Any]]: + try: + awsrequest = import_module("botocore.awsrequest") + httpsession = import_module("botocore.httpsession") + urllib3_exceptions = import_module("urllib3.exceptions") + except ImportError as exc: + raise SdkPeerPinningError( + "S3 connector browsing requires a compatible botocore HTTP transport" + ) from exc + + required = ( + "AWSHTTPConnection", + "AWSHTTPSConnection", + "AWSHTTPConnectionPool", + "AWSHTTPSConnectionPool", + ) + if any(not hasattr(awsrequest, name) for name in required) or not hasattr( + httpsession, "URLLib3Session" + ): + raise SdkPeerPinningError( + "The installed botocore release is missing the connection classes required for peer pinning" + ) + + def pinned_new_connection(connection: Any) -> socket.socket: + hostname = str(getattr(connection, "_dns_host", "") or "").strip() + port = int(getattr(connection, "port", 0) or 0) + if not hostname or not port: + raise urllib3_exceptions.NewConnectionError( + connection, "Pinned S3 connection is missing its target authority" + ) + try: + return create_outbound_connection( + hostname, + port, + timeout=getattr(connection, "timeout", None), + source_address=getattr(connection, "source_address", None), + socket_options=getattr(connection, "socket_options", None), + label="S3 connector peer", + ) + except socket.timeout as exc: + raise urllib3_exceptions.ConnectTimeoutError( + connection, + f"Connection to {hostname} timed out while selecting an approved peer", + ) from exc + except (OSError, OutboundHttpError, ValueError) as exc: + raise urllib3_exceptions.NewConnectionError( + connection, + f"S3 connector peer was rejected: {exc}", + ) from exc + + class PinnedAWSHTTPConnection(awsrequest.AWSHTTPConnection): + _new_conn = pinned_new_connection + + class PinnedAWSHTTPSConnection(awsrequest.AWSHTTPSConnection): + _new_conn = pinned_new_connection + + class PinnedAWSHTTPConnectionPool(awsrequest.AWSHTTPConnectionPool): + ConnectionCls = PinnedAWSHTTPConnection + + class PinnedAWSHTTPSConnectionPool(awsrequest.AWSHTTPSConnectionPool): + ConnectionCls = PinnedAWSHTTPSConnection + + class PinnedURLLib3Session(httpsession.URLLib3Session): + def __init__(self, *args: Any, **kwargs: Any) -> None: + # A proxy would select the final target outside this process. Until + # proxy peer delegation is modeled, connector traffic is direct. + kwargs["proxies"] = {} + super().__init__(*args, **kwargs) + self._pool_classes_by_scheme = { + "http": PinnedAWSHTTPConnectionPool, + "https": PinnedAWSHTTPSConnectionPool, + } + manager = getattr(self, "_manager", None) + if manager is None or not hasattr(manager, "pool_classes_by_scheme"): + raise SdkPeerPinningError( + "The installed botocore pool manager cannot enforce pinned connection classes" + ) + manager.pool_classes_by_scheme = self._pool_classes_by_scheme + + return PinnedURLLib3Session, PinnedAWSHTTPConnection, PinnedAWSHTTPSConnection + + +@lru_cache(maxsize=1) +def pinned_smb_connection_cache() -> dict[str, Any]: + """Return the Files-owned cache; unproven process-global sessions are never reused.""" + + return {} + + +def install_pinned_smb_transport(smbclient: Any) -> Any: + """Install a process-wide fail-closed smbclient session factory. + + smbclient routes initial connections, reconnects, and DFS targets through + ``smbclient._pool.register_session``. Replacing that single factory with a + behavior-compatible implementation ensures every target uses a socket + selected by Core's deployment-wide outbound policy. + """ + + try: + pool = import_module("smbclient._pool") + connection_module = import_module("smbprotocol.connection") + session_module = import_module("smbprotocol.session") + transport_module = import_module("smbprotocol.transport") + except ImportError as exc: + raise SdkPeerPinningError( + "SMB connector browsing requires the optional smbprotocol dependency" + ) from exc + + current = getattr(pool, "register_session", None) + if getattr(current, "__govoplan_peer_pinned__", False): + expected_tcp = getattr(current, "__govoplan_pinned_tcp__", None) + if expected_tcp is None or getattr(connection_module, "Tcp", None) is not expected_tcp: + raise SdkPeerPinningError( + "The installed SMB transport changed after peer pinning; refusing to reuse the session factory" + ) + return smbclient + required_parameters = { + "server", + "username", + "password", + "port", + "encrypt", + "connection_timeout", + "connection_cache", + "auth_protocol", + "require_signing", + } + if current is None or not required_parameters.issubset( + inspect.signature(current).parameters + ): + raise SdkPeerPinningError( + "The installed smbprotocol release does not expose the session seam required for peer pinning" + ) + + class PinnedTcp(transport_module.Tcp): + def connect(self) -> None: + with self._sock_lock: + if self.connected: + return + try: + self._sock = create_outbound_connection( + self.server, + int(self.port), + timeout=self.timeout, + label="SMB connector peer", + ) + except (OSError, OutboundHttpError, ValueError) as exc: + raise ValueError( + f"SMB connector peer '{self.server}:{self.port}' was rejected: {exc}" + ) from exc + self._sock.settimeout(None) + self.connected = True + + # Connection.connect() instantiates the module-level Tcp symbol on every + # reconnect. Replacing it process-wide is deliberate: an SMB connection + # created by another code path must become stricter, never bypass Files' + # peer boundary. + if not hasattr(connection_module, "Tcp"): + raise SdkPeerPinningError( + "The installed smbprotocol release cannot install the pinned TCP transport" + ) + connection_module.Tcp = PinnedTcp + + def pinned_register_session( + server: str, + username: str | None = None, + password: str | None = None, + port: int = 445, + encrypt: bool | None = None, + connection_timeout: float = 60, + connection_cache: dict[str, Any] | None = None, + auth_protocol: str = "negotiate", + require_signing: bool = True, + ) -> Any: + cache = pinned_smb_connection_cache() if connection_cache is None else connection_cache + connection_key = f"{server.lower()}:{port}" + connection = cache.get(connection_key) + transport = getattr(connection, "transport", None) + if connection is not None and not isinstance(transport, PinnedTcp): + disconnect = getattr(connection, "disconnect", None) + if callable(disconnect): + try: + disconnect(close=True) + except Exception: + pass + cache.pop(connection_key, None) + connection = None + if connection is None or not getattr(connection.transport, "connected", False): + connection = connection_module.Connection( + pool.ClientConfig().client_guid, + server, + port, + require_signing=require_signing, + ) + connection.transport = PinnedTcp(server, port) + connection.connect(timeout=connection_timeout) + if not isinstance(connection.transport, PinnedTcp): + disconnect = getattr(connection, "disconnect", None) + if callable(disconnect): + try: + disconnect(close=True) + except Exception: + pass + raise SdkPeerPinningError( + "smbprotocol replaced the required pinned TCP transport during connection setup" + ) + cache[connection_key] = connection + + session = next( + ( + item + for item in connection.session_table.values() + if username is None or item.username == username + ), + None, + ) + if session is None: + session = session_module.Session( + connection, + username=username, + password=password, + require_encryption=(encrypt is True), + auth_protocol=auth_protocol, + ) + session.connect() + elif encrypt is not None: + if session.encrypt_data and not encrypt: + raise ValueError( + "Cannot disable encryption on an already negotiated session." + ) + if not session.encrypt_data and encrypt: + session.encrypt = True + return session + + pinned_register_session.__govoplan_peer_pinned__ = True # type: ignore[attr-defined] + pinned_register_session.__govoplan_pinned_tcp__ = PinnedTcp # type: ignore[attr-defined] + pool.register_session = pinned_register_session + if hasattr(smbclient, "register_session"): + smbclient.register_session = pinned_register_session + return smbclient + + +__all__ = [ + "SdkPeerPinningError", + "create_pinned_s3_client", + "install_pinned_smb_transport", + "pinned_smb_connection_cache", +] diff --git a/tests/test_connector_providers.py b/tests/test_connector_providers.py index 4c74433..f33b133 100644 --- a/tests/test_connector_providers.py +++ b/tests/test_connector_providers.py @@ -5,7 +5,7 @@ from datetime import UTC, datetime from unittest.mock import patch from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _smb_location, browse_connector_profile -from govoplan_files.backend.storage.connector_imports import ConnectorImportError, read_connector_file +from govoplan_files.backend.storage.connector_imports import read_connector_file from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors @@ -23,6 +23,10 @@ class FakeS3Client: self.list_objects_request: dict[str, object] | None = None self.head_request: dict[str, object] | None = None self.get_request: dict[str, object] | None = None + self.closed = False + + def close(self) -> None: + self.closed = True def list_objects_v2(self, **kwargs: object) -> dict[str, object]: self.list_objects_request = dict(kwargs) @@ -75,42 +79,39 @@ def s3_profile(**overrides: object) -> ConnectorProfile: class ConnectorProviderTests(unittest.TestCase): - def test_smb_sdk_transport_fails_closed_before_client_creation_in_all_modes(self) -> None: + def test_smb_browse_uses_the_files_owned_pinned_connection_cache(self) -> None: profile = ConnectorProfile( id="smb", label="SMB", provider="smb", endpoint_url="smb://files.example.test/share", ) - for allow_private, address in ((False, "93.184.216.34"), (True, "10.0.0.5")): - with self.subTest(allow_private=allow_private), patch.dict( - "os.environ", - { - "APP_ENV": "production", - "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": str(allow_private).lower(), - }, - ), patch( - "govoplan_core.security.outbound_http.socket.getaddrinfo", - return_value=[(2, 1, 6, "", (address, 445))], - ), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk, self.assertRaisesRegex( - ConnectorBrowseError, - "redirects/referrals.*DNS/IP pinning", - ): - browse_connector_profile(profile, path="") - sdk.assert_not_called() + with patch.dict( + "os.environ", + {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"}, + ), patch( + "govoplan_core.security.outbound_http.socket.getaddrinfo", + return_value=[(2, 1, 6, "", ("93.184.216.34", 445))], + ), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk: + sdk.return_value.scandir.return_value.__enter__.return_value = iter(()) + self.assertEqual([], browse_connector_profile(profile, path="")) - def test_smb_explicit_ip_still_fails_closed_because_the_sdk_may_follow_referrals(self) -> None: + kwargs = sdk.return_value.scandir.call_args.kwargs + self.assertIsInstance(kwargs["connection_cache"], dict) + self.assertTrue(kwargs["require_signing"]) + + def test_smb_endpoint_preflight_applies_private_network_policy(self) -> None: profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share") with patch.dict( "os.environ", - {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"}, + {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"}, ), patch( "govoplan_core.security.outbound_http.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("10.0.0.5", 445))], - ), self.assertRaisesRegex(ConnectorBrowseError, "redirects/referrals.*DNS/IP pinning"): + ), self.assertRaisesRegex(ConnectorBrowseError, "non-public network"): _smb_location(profile) - def test_smb_import_surfaces_fail_closed_policy_as_an_import_error(self) -> None: + def test_smb_import_uses_the_same_pinned_connection_cache(self) -> None: profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share") with patch.dict( "os.environ", @@ -118,12 +119,13 @@ class ConnectorProviderTests(unittest.TestCase): ), patch( "govoplan_core.security.outbound_http.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("10.0.0.5", 445))], - ), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk, self.assertRaisesRegex( - ConnectorImportError, - "redirects/referrals.*DNS/IP pinning", - ): - read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024) - sdk.assert_not_called() + ), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk: + sdk.return_value.stat.return_value.st_size = 4 + sdk.return_value.open_file.return_value.__enter__.return_value.read.return_value = b"test" + downloaded = read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024) + + self.assertEqual(b"test", downloaded.data) + self.assertIsInstance(sdk.return_value.stat.call_args.kwargs["connection_cache"], dict) def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None: descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()} @@ -172,6 +174,7 @@ class ConnectorProviderTests(unittest.TestCase): self.assertEqual("govoplan:root/report.xlsx", items[1].external_id) self.assertEqual("root/report.xlsx", items[1].metadata["key"]) self.assertEqual("next-page", items[1].metadata["next_continuation_token"]) + self.assertTrue(client.closed) def test_s3_browse_lists_buckets_when_profile_has_no_bucket(self) -> None: client = FakeS3Client() @@ -181,27 +184,33 @@ class ConnectorProviderTests(unittest.TestCase): self.assertEqual(["archive"], [item.path for item in items]) - def test_s3_sdk_transport_fails_closed_before_client_creation_in_private_mode(self) -> None: + def test_s3_client_is_constructed_through_the_pinned_transport(self) -> None: with patch.dict( "os.environ", {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"}, ), patch( "govoplan_core.security.outbound_http.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("127.0.0.1", 9000))], - ), patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex( - ConnectorBrowseError, - "until that transport supports.*DNS/IP pinning", - ): + ), patch( + "govoplan_files.backend.storage.connector_browse.create_pinned_s3_client", + return_value=FakeS3Client(), + ) as factory: browse_connector_profile(s3_profile(), path="") - importer.assert_not_called() - def test_s3_sdk_endpoint_discovery_fails_closed(self) -> None: - with patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex( - ConnectorBrowseError, - "endpoint discovery.*cannot guarantee.*DNS/IP pinning", - ): + kwargs = factory.call_args.kwargs + self.assertEqual("http://127.0.0.1:9000", kwargs["endpoint_url"]) + self.assertEqual("access-key", kwargs["aws_access_key_id"]) + self.assertEqual("secret-key", kwargs["aws_secret_access_key"]) + self.assertEqual({}, kwargs["config"].proxies) + + def test_s3_endpoint_discovery_uses_the_same_pinned_transport(self) -> None: + with patch( + "govoplan_files.backend.storage.connector_browse.create_pinned_s3_client", + return_value=FakeS3Client(), + ) as factory: browse_connector_profile(s3_profile(endpoint_url=None), path="") - importer.assert_not_called() + + self.assertNotIn("endpoint_url", factory.call_args.kwargs) def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None: client = FakeS3Client() @@ -217,6 +226,7 @@ class ConnectorProviderTests(unittest.TestCase): self.assertEqual("govoplan:root/report.txt", downloaded.external_id) self.assertEqual("s3://govoplan/root/report.txt", downloaded.external_url) self.assertEqual("checksum", downloaded.metadata["checksum_sha256"]) + self.assertTrue(client.closed) if __name__ == "__main__": diff --git a/tests/test_connector_visibility.py b/tests/test_connector_visibility.py index bd78133..7e27373 100644 --- a/tests/test_connector_visibility.py +++ b/tests/test_connector_visibility.py @@ -10,6 +10,9 @@ from govoplan_files.backend.storage.connector_visibility import ( connector_profile_usable_for_import, visible_connector_profiles_for_actor, ) +from govoplan_files.backend.storage.connector_providers import ( + connector_provider_descriptors, +) def _profile( @@ -168,12 +171,16 @@ class ConnectorVisibilityTests(unittest.TestCase): self, ) -> None: self.assertTrue(connector_profile_usable_for_import(_profile("webdav"))) - self.assertFalse( - connector_profile_usable_for_import(_profile("s3", provider="s3")) - ) - self.assertFalse( - connector_profile_usable_for_import(_profile("smb", provider="smb")) - ) + descriptors = { + item.provider: item for item in connector_provider_descriptors() + } + for provider in ("s3", "smb"): + self.assertEqual( + descriptors[provider].installed, + connector_profile_usable_for_import( + _profile(provider, provider=provider) + ), + ) self.assertFalse( connector_profile_usable_for_import( _profile("sharepoint", provider="sharepoint") diff --git a/tests/test_documentation.py b/tests/test_documentation.py index 2d33a64..46507c4 100644 --- a/tests/test_documentation.py +++ b/tests/test_documentation.py @@ -142,6 +142,7 @@ class FilesRuntimeDocumentationTests(unittest.TestCase): base_path="/classified", credential_mode="basic", password_value="credential-secret", + secret_ref="runtime/connector-secret", ) with patch( "govoplan_files.backend.documentation.visible_connector_profiles_for_actor", diff --git a/tests/test_manifest_documentation.py b/tests/test_manifest_documentation.py index 0800de7..267b092 100644 --- a/tests/test_manifest_documentation.py +++ b/tests/test_manifest_documentation.py @@ -4,6 +4,7 @@ import unittest STATIC_TOPIC_IDS = { + "files.search.managed-content", "files.workflow.organize-managed-files", "files.workflow.find-and-download-files", "files.workflow.share-managed-files", @@ -152,7 +153,7 @@ class FilesManifestDocumentationTests(unittest.TestCase): "/api/v1/files/connectors/credentials", {link.href for link in topic.links} ) - def test_operator_topic_covers_recovery_and_fail_closed_s3_smb(self) -> None: + def test_operator_topic_covers_recovery_and_pinned_s3_smb(self) -> None: topic = self.topic( "files.reference.integrity-recovery-and-fail-closed-transports" ) @@ -163,6 +164,7 @@ class FilesManifestDocumentationTests(unittest.TestCase): self.assertIn("SMB", topic.body) self.assertIn("fail closed", topic.body) self.assertIn("DFS referrals", topic.body) + self.assertIn("ambient credential discovery", topic.body) self.assertIn("does not remove backend blob objects", topic.body) self.assertIn("bounded resumable integrity scan", topic.body) self.assertIn("quarantined", topic.body) diff --git a/tests/test_sdk_peer_pinning.py b/tests/test_sdk_peer_pinning.py new file mode 100644 index 0000000..13598f5 --- /dev/null +++ b/tests/test_sdk_peer_pinning.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +import socket +import sys +import threading +import types +import unittest +from importlib.util import find_spec +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import patch + +from govoplan_core.security.outbound_http import ( + OutboundHttpBlocked, + create_outbound_connection as core_create_outbound_connection, +) +from govoplan_files.backend.storage.sdk_peer_pinning import ( + SdkPeerPinningError, + _botocore_transport_types, + create_pinned_s3_client, + install_pinned_smb_transport, + pinned_smb_connection_cache, +) + + +class _Socket: + def __init__(self) -> None: + self.connected_to: object | None = None + + def settimeout(self, _value: object) -> None: + return None + + def setsockopt(self, *_args: object) -> None: + return None + + def connect(self, value: object) -> None: + self.connected_to = value + + def close(self) -> None: + return None + + +class _S3Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 - stdlib handler contract + body = ( + b'' + b'' + b'govoplanGovOPlaN' + b'evidence2026-08-04T00:00:00Z' + b'' + ) + self.send_response(200) + self.send_header("Content-Type", "application/xml") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format: str, *_args: object) -> None: + return None + + +@unittest.skipUnless(find_spec("botocore") and find_spec("boto3"), "boto3 extra is not installed") +class S3PeerPinningTests(unittest.TestCase): + def test_client_is_born_with_the_pinned_http_session(self) -> None: + client = create_pinned_s3_client( + endpoint_url="http://127.0.0.1:9000", + region_name="eu-central-1", + aws_access_key_id="access", + aws_secret_access_key="secret", + ) + try: + session_type, _, _ = _botocore_transport_types() + self.assertIsInstance(client._endpoint.http_session, session_type) + self.assertEqual({}, client._endpoint.http_session._proxy_config._proxies) + finally: + client.close() + + def test_each_connection_attempt_resolves_and_pins_again(self) -> None: + _, http_connection, _ = _botocore_transport_types() + connection = http_connection(host="objects.example.test", port=80) + sockets = [_Socket(), _Socket()] + with patch( + "govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection", + side_effect=sockets, + ) as connector: + self.assertIs(sockets[0], connection._new_conn()) + self.assertIs(sockets[1], connection._new_conn()) + + self.assertEqual(2, connector.call_count) + self.assertEqual("objects.example.test", connector.call_args.args[0]) + + def test_real_sdk_request_uses_the_pinned_socket_path(self) -> None: + server = ThreadingHTTPServer(("127.0.0.1", 0), _S3Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + endpoint = f"http://127.0.0.1:{server.server_port}" + try: + with patch.dict( + "os.environ", + {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"}, + ), patch( + "govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection", + wraps=core_create_outbound_connection, + ) as connector: + client = create_pinned_s3_client( + endpoint_url=endpoint, + region_name="eu-central-1", + aws_access_key_id="access", + aws_secret_access_key="secret", + ) + try: + response = client.list_buckets() + finally: + client.close() + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + self.assertEqual("evidence", response["Buckets"][0]["Name"]) + self.assertEqual("127.0.0.1", connector.call_args.args[0]) + + def test_every_sdk_selected_origin_uses_a_pinned_pool(self) -> None: + session_type, http_connection, https_connection = _botocore_transport_types() + session = session_type(proxies={}) + try: + first = session._manager.connection_from_url("http://one.example.test/root") + redirected = session._manager.connection_from_url("https://two.example.test/root") + self.assertIs(first.ConnectionCls, http_connection) + self.assertIs(redirected.ConnectionCls, https_connection) + finally: + session.close() + + def test_tls_connection_retains_the_configured_authority_for_sni(self) -> None: + _, _, https_connection = _botocore_transport_types() + connection = https_connection(host="objects.example.test", port=443) + with patch( + "govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection", + return_value=_Socket(), + ) as connector: + connection._new_conn() + + self.assertEqual("objects.example.test", connection.host) + self.assertEqual("objects.example.test", connection._dns_host) + self.assertEqual("objects.example.test", connector.call_args.args[0]) + + def test_mixed_answer_and_peer_change_fail_closed(self) -> None: + _, http_connection, _ = _botocore_transport_types() + connection = http_connection(host="objects.example.test", port=80) + records = [ + [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80))], + [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80)), + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 80)), + ], + ] + with patch.dict( + "os.environ", + {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"}, + ), patch( + "govoplan_core.security.outbound_http.socket.getaddrinfo", + side_effect=records, + ), patch( + "govoplan_core.security.outbound_http.socket.socket", + return_value=_Socket(), + ): + connection._new_conn() + with self.assertRaisesRegex(Exception, "non-public network"): + connection._new_conn() + + +class _FakeTcp: + def __init__(self, server: str, port: int, timeout: float | None = None) -> None: + self.server = server + self.port = port + self.timeout = timeout + self.connected = False + self._sock = None + self._sock_lock = threading.Lock() + + +class _FakeConnection: + def __init__(self, _guid: object, server: str, port: int, *, require_signing: bool) -> None: + self.server = server + self.port = port + self.require_signing = require_signing + self.transport = _FakeTcp(server, port) + self.session_table: dict[str, object] = {} + + def connect(self, *, timeout: float) -> None: + self.transport.timeout = timeout + self.transport.connect() + + +class _FakeSession: + def __init__( + self, + connection: _FakeConnection, + *, + username: str | None, + password: str | None, + require_encryption: bool, + auth_protocol: str, + ) -> None: + self.connection = connection + self.username = username + self.password = password + self.encrypt_data = require_encryption + self.auth_protocol = auth_protocol + self.encrypt = require_encryption + + def connect(self) -> None: + self.connection.session_table[self.username or "anonymous"] = self + + +class SmbPeerPinningTests(unittest.TestCase): + def setUp(self) -> None: + pinned_smb_connection_cache.cache_clear() + + def _modules(self) -> tuple[types.ModuleType, dict[str, types.ModuleType]]: + smbclient = types.ModuleType("smbclient") + pool = types.ModuleType("smbclient._pool") + + def original_register_session( + server: str, + username: str | None = None, + password: str | None = None, + port: int = 445, + encrypt: bool | None = None, + connection_timeout: float = 60, + connection_cache: dict[str, object] | None = None, + auth_protocol: str = "negotiate", + require_signing: bool = True, + ) -> None: + del server, username, password, port, encrypt, connection_timeout + del connection_cache, auth_protocol, require_signing + + pool.register_session = original_register_session + pool.ClientConfig = lambda: types.SimpleNamespace(client_guid="guid") + connection = types.ModuleType("smbprotocol.connection") + connection.Connection = _FakeConnection + connection.Tcp = _FakeTcp + session = types.ModuleType("smbprotocol.session") + session.Session = _FakeSession + transport = types.ModuleType("smbprotocol.transport") + transport.Tcp = _FakeTcp + return smbclient, { + "smbclient._pool": pool, + "smbprotocol.connection": connection, + "smbprotocol.session": session, + "smbprotocol.transport": transport, + } + + def test_initial_reconnect_and_referral_hosts_are_each_pinned(self) -> None: + smbclient, modules = self._modules() + sockets = [_Socket(), _Socket(), _Socket()] + with patch.dict(sys.modules, modules), patch( + "govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection", + side_effect=sockets, + ) as connector: + install_pinned_smb_transport(smbclient) + register = modules["smbclient._pool"].register_session + cache: dict[str, object] = {} + first = register("files.example.test", connection_cache=cache) + first.connection.transport.connected = False + register("files.example.test", connection_cache=cache) + register("dfs-target.example.test", connection_cache=cache) + + self.assertEqual( + ["files.example.test", "files.example.test", "dfs-target.example.test"], + [call.args[0] for call in connector.call_args_list], + ) + + def test_referral_to_disallowed_peer_fails_before_session_creation(self) -> None: + smbclient, modules = self._modules() + with patch.dict(sys.modules, modules), patch( + "govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection", + side_effect=OutboundHttpBlocked("non-public network"), + ): + install_pinned_smb_transport(smbclient) + register = modules["smbclient._pool"].register_session + with self.assertRaisesRegex(ValueError, "non-public network"): + register("private-referral.example.test", connection_cache={}) + + def test_transport_tampering_after_installation_fails_closed(self) -> None: + smbclient, modules = self._modules() + with patch.dict(sys.modules, modules): + install_pinned_smb_transport(smbclient) + modules["smbprotocol.connection"].Tcp = _FakeTcp + with self.assertRaisesRegex(SdkPeerPinningError, "changed after peer pinning"): + install_pinned_smb_transport(smbclient) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search_source.py b/tests/test_search_source.py index 6c47778..0ec28c3 100644 --- a/tests/test_search_source.py +++ b/tests/test_search_source.py @@ -9,6 +9,7 @@ from sqlalchemy.orm import Session from govoplan_access.backend.db.models import Account, Group, User from govoplan_core.auth import ApiPrincipal from govoplan_core.core.access import PrincipalRef +from govoplan_core.core.change_sequence import ChangeSequenceEntry from govoplan_core.core.events import EventObjectRef, EventTenantRef, PlatformEvent from govoplan_core.core.search import ( SearchAuthorizationRequest, @@ -35,6 +36,7 @@ class FilesSearchSourceTests(unittest.TestCase): FileAsset.__table__, FileFolder.__table__, FileShare.__table__, + ChangeSequenceEntry.__table__, ), ) self.session = Session(self.engine)