security(files): scrub connector credentials on deletion

src/govoplan_files/backend/storage/connector_credential_deletion.py

@@ -0,0 +1,337 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from sqlalchemy import inspect
+from sqlalchemy.orm import Session
+
+from govoplan_core.audit.logging import audit_event
+from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile
+
+
+@dataclass(frozen=True, slots=True)
+class ConnectorCredentialDeletionResult:
+    changed: bool
+    affected_profiles: tuple[FileConnectorProfile, ...] = ()
+
+
+def delete_connector_credential_row(
+    session: Session,
+    row: FileConnectorCredential,
+    *,
+    deletion_reason: str,
+    user_id: str | None = None,
+    api_key_id: str | None = None,
+) -> ConnectorCredentialDeletionResult:
+    """Scrub a credential tombstone and disable every profile that used it.
+
+    ``secret_ref`` predates a Files-owned secret-provider contract.  It may be
+    shared or deployment-owned, so it is detached locally and explicitly
+    audited without ever being passed to a provider delete operation.
+    """
+
+    dependent_profiles = (
+        tuple(
+            session.query(FileConnectorProfile)
+            .filter(FileConnectorProfile.credential_profile_id == row.id)
+            .order_by(FileConnectorProfile.id.asc())
+            .all()
+        )
+        if inspect(session.get_bind()).has_table(FileConnectorProfile.__tablename__)
+        else ()
+    )
+    affected_profiles: list[FileConnectorProfile] = []
+    for profile in dependent_profiles:
+        if _delete_connector_profile_row(
+            session,
+            profile,
+            deletion_reason="credential_deleted",
+            user_id=user_id,
+            api_key_id=api_key_id,
+        ):
+            affected_profiles.append(profile)
+
+    deleted_secret_kinds = _encrypted_secret_kinds(row)
+    removed_reference_kinds = _credential_reference_kinds(row)
+    removed_metadata = bool(row.metadata_)
+    changed = _credential_row_requires_deletion(row)
+    if changed:
+        _scrub_credential_row(row, user_id=user_id)
+        session.add(row)
+        _audit_credential_deletion(
+            session,
+            row,
+            deletion_reason=deletion_reason,
+            deleted_secret_kinds=deleted_secret_kinds,
+            removed_reference_kinds=removed_reference_kinds,
+            removed_metadata=removed_metadata,
+            affected_profile_count=len(affected_profiles),
+            user_id=user_id,
+            api_key_id=api_key_id,
+        )
+    session.flush()
+    return ConnectorCredentialDeletionResult(
+        changed=changed or bool(affected_profiles),
+        affected_profiles=tuple(affected_profiles),
+    )
+
+
+def delete_connector_profile_row(
+    session: Session,
+    row: FileConnectorProfile,
+    *,
+    deletion_reason: str,
+    user_id: str | None = None,
+    api_key_id: str | None = None,
+) -> bool:
+    changed = _delete_connector_profile_row(
+        session,
+        row,
+        deletion_reason=deletion_reason,
+        user_id=user_id,
+        api_key_id=api_key_id,
+    )
+    session.flush()
+    return changed
+
+
+def delete_connector_credentials_for_retirement(session: Session) -> int:
+    """Scrub and audit stored connector material before destructive retirement.
+
+    Legacy external references are detached and audited as non-owned. Files
+    never sends those arbitrary references to a provider delete operation.
+    """
+
+    inspector = inspect(session.get_bind())
+    profiles = (
+        session.query(FileConnectorProfile).order_by(FileConnectorProfile.id.asc()).all()
+        if inspector.has_table(FileConnectorProfile.__tablename__)
+        else []
+    )
+    credentials = (
+        session.query(FileConnectorCredential).order_by(FileConnectorCredential.id.asc()).all()
+        if inspector.has_table(FileConnectorCredential.__tablename__)
+        else []
+    )
+    deleted = 0
+    for profile in profiles:
+        if not _profile_row_has_credential_material(profile):
+            continue
+        if _delete_connector_profile_row(
+            session,
+            profile,
+            deletion_reason="module_data_retired",
+        ):
+            deleted += 1
+    for credential in credentials:
+        if not _credential_row_has_material(credential):
+            continue
+        result = delete_connector_credential_row(
+            session,
+            credential,
+            deletion_reason="module_data_retired",
+        )
+        if result.changed:
+            deleted += 1
+    session.flush()
+    return deleted
+
+
+def _delete_connector_profile_row(
+    session: Session,
+    row: FileConnectorProfile,
+    *,
+    deletion_reason: str,
+    user_id: str | None = None,
+    api_key_id: str | None = None,
+) -> bool:
+    deleted_secret_kinds = _encrypted_secret_kinds(row)
+    removed_reference_kinds = _profile_reference_kinds(row)
+    removed_metadata = bool(row.metadata_)
+    changed = _profile_row_requires_deletion(row)
+    if not changed:
+        return False
+    _scrub_profile_row(row, user_id=user_id)
+    session.add(row)
+    _audit_profile_deletion(
+        session,
+        row,
+        deletion_reason=deletion_reason,
+        deleted_secret_kinds=deleted_secret_kinds,
+        removed_reference_kinds=removed_reference_kinds,
+        removed_metadata=removed_metadata,
+        user_id=user_id,
+        api_key_id=api_key_id,
+    )
+    return True
+
+
+def _scrub_credential_row(row: FileConnectorCredential, *, user_id: str | None) -> None:
+    row.enabled = False
+    row.credential_mode = "none"
+    row.username = None
+    row.password_encrypted = None
+    row.token_encrypted = None
+    row.password_env = None
+    row.token_env = None
+    row.secret_ref = None
+    row.metadata_ = {}
+    row.updated_by_user_id = user_id
+
+
+def _scrub_profile_row(row: FileConnectorProfile, *, user_id: str | None) -> None:
+    row.enabled = False
+    row.credential_profile_id = None
+    row.credential_mode = "none"
+    row.username = None
+    row.password_encrypted = None
+    row.token_encrypted = None
+    row.password_env = None
+    row.token_env = None
+    row.secret_ref = None
+    row.metadata_ = {}
+    row.updated_by_user_id = user_id
+
+
+def _credential_row_requires_deletion(row: FileConnectorCredential) -> bool:
+    return bool(row.enabled or _credential_row_has_material(row))
+
+
+def _profile_row_requires_deletion(row: FileConnectorProfile) -> bool:
+    return bool(row.enabled or _profile_row_has_credential_material(row))
+
+
+def _credential_row_has_material(row: FileConnectorCredential) -> bool:
+    return bool(
+        row.username
+        or row.password_encrypted
+        or row.token_encrypted
+        or row.password_env
+        or row.token_env
+        or row.secret_ref
+        or row.metadata_
+        or row.credential_mode not in {"", "none", "anonymous"}
+    )
+
+
+def _profile_row_has_credential_material(row: FileConnectorProfile) -> bool:
+    return bool(
+        row.credential_profile_id
+        or row.username
+        or row.password_encrypted
+        or row.token_encrypted
+        or row.password_env
+        or row.token_env
+        or row.secret_ref
+        or row.metadata_
+        or row.credential_mode not in {"", "none", "anonymous"}
+    )
+
+
+def _encrypted_secret_kinds(row: FileConnectorCredential | FileConnectorProfile) -> list[str]:
+    kinds: list[str] = []
+    if row.password_encrypted:
+        kinds.append("password")
+    if row.token_encrypted:
+        kinds.append("token")
+    return kinds
+
+
+def _credential_reference_kinds(row: FileConnectorCredential | FileConnectorProfile) -> list[str]:
+    kinds: list[str] = []
+    if row.password_env:
+        kinds.append("password_env")
+    if row.token_env:
+        kinds.append("token_env")
+    if row.secret_ref:
+        kinds.append("unowned_external_secret_ref")
+    return kinds
+
+
+def _profile_reference_kinds(row: FileConnectorProfile) -> list[str]:
+    kinds = _credential_reference_kinds(row)
+    if row.credential_profile_id:
+        kinds.append("credential_profile")
+    return kinds
+
+
+def _storage_backend(deleted_secret_kinds: list[str], removed_reference_kinds: list[str]) -> str:
+    if "unowned_external_secret_ref" in removed_reference_kinds:
+        return "unowned_external_reference_detached"
+    if deleted_secret_kinds:
+        return "encrypted_database"
+    if removed_reference_kinds:
+        return "reference_only"
+    return "none"
+
+
+def _audit_credential_deletion(
+    session: Session,
+    row: FileConnectorCredential,
+    *,
+    deletion_reason: str,
+    deleted_secret_kinds: list[str],
+    removed_reference_kinds: list[str],
+    removed_metadata: bool,
+    affected_profile_count: int,
+    user_id: str | None,
+    api_key_id: str | None,
+) -> None:
+    audit_event(
+        session,
+        tenant_id=row.tenant_id,
+        scope=_audit_scope(row.scope_type),
+        user_id=user_id,
+        api_key_id=api_key_id,
+        action="files.connector_credential_deleted",
+        object_type="file_connector_credential",
+        object_id=row.id,
+        details={
+            "scope_type": row.scope_type,
+            "scope_id": row.scope_id,
+            "provider": row.provider,
+            "storage_backend": _storage_backend(deleted_secret_kinds, removed_reference_kinds),
+            "deleted_secret_kinds": deleted_secret_kinds,
+            "removed_reference_kinds": removed_reference_kinds,
+            "removed_metadata": removed_metadata,
+            "affected_profile_count": affected_profile_count,
+            "deletion_reason": deletion_reason,
+        },
+    )
+
+
+def _audit_profile_deletion(
+    session: Session,
+    row: FileConnectorProfile,
+    *,
+    deletion_reason: str,
+    deleted_secret_kinds: list[str],
+    removed_reference_kinds: list[str],
+    removed_metadata: bool,
+    user_id: str | None,
+    api_key_id: str | None,
+) -> None:
+    audit_event(
+        session,
+        tenant_id=row.tenant_id,
+        scope=_audit_scope(row.scope_type),
+        user_id=user_id,
+        api_key_id=api_key_id,
+        action="files.connector_profile_deleted",
+        object_type="file_connector_profile",
+        object_id=row.id,
+        details={
+            "scope_type": row.scope_type,
+            "scope_id": row.scope_id,
+            "provider": row.provider,
+            "storage_backend": _storage_backend(deleted_secret_kinds, removed_reference_kinds),
+            "deleted_secret_kinds": deleted_secret_kinds,
+            "removed_reference_kinds": removed_reference_kinds,
+            "removed_metadata": removed_metadata,
+            "deletion_reason": deletion_reason,
+        },
+    )
+
+
+def _audit_scope(scope_type: str) -> str:
+    return "system" if scope_type == "system" else "tenant"

src/govoplan_files/backend/storage/connector_credential_store.py

@@ -52,6 +52,8 @@ class ConnectorCredential:
 
     @property
     def credentials_configured(self) -> bool:
+        if not self.enabled:
+            return False
         if self.credential_mode.casefold() in {"", "none", "anonymous"}:
             return True
         # Environment references are deliberately unavailable to API-managed
@@ -188,6 +190,7 @@ def create_connector_credential_row(
     reject_api_controlled_deployment_references(
         password_env=password_env,
         token_env=token_env,
+        secret_ref=secret_ref,
         metadata=metadata,
     )
     clean_id = _normalize_id(credential_id)
@@ -242,8 +245,15 @@ def update_connector_credential_row(
     reject_api_controlled_deployment_references(
         password_env=password_env,
         token_env=token_env,
+        secret_ref=secret_ref,
         metadata=metadata,
     )
+    if secret_ref is not None and _clean(secret_ref) != _clean(row.secret_ref):
+        if _clean(row.secret_ref):
+            raise FileStorageError(
+                "An existing external secret reference cannot be replaced or cleared until Files can prove "
+                "provider ownership and confirm provider-side deletion"
+            )
     if label is not None:
         row.label = _normalize_label(label)
     if provider is not None:
@@ -278,14 +288,6 @@ def update_connector_credential_row(
     return row
 
 
-def deactivate_connector_credential_row(session: Session, row: FileConnectorCredential, *, user_id: str | None) -> FileConnectorCredential:
-    row.enabled = False
-    row.updated_by_user_id = user_id
-    session.add(row)
-    session.flush()
-    return row
-
-
 def _normalize_scope(*, tenant_id: str, scope_type: str, scope_id: str | None) -> tuple[str, str | None, str | None]:
     clean_scope_type = normalize_policy_scope_type(scope_type)
     clean_scope_id = _clean(scope_id)

src/govoplan_files/backend/storage/connector_deployment.py

@@ -17,6 +17,22 @@ _SECRET_ENV_METADATA_KEYS = frozenset(
         "session_token_env",
     }
 )
+_SECRET_VALUE_METADATA_KEYS = frozenset(
+    {
+        "password",
+        "token",
+        "access_token",
+        "auth_token",
+        "bearer_token",
+        "refresh_token",
+        "api_key",
+        "access_key",
+        "access_key_id",
+        "secret_key",
+        "secret_access_key",
+        "session_token",
+    }
+)
 _DEVELOPMENT_ENVIRONMENTS = frozenset({"dev", "development", "local", "test", "testing"})
 
 
@@ -65,6 +81,7 @@ def reject_api_controlled_deployment_references(
     *,
     password_env: str | None = None,
     token_env: str | None = None,
+    secret_ref: str | None = None,
     metadata: Mapping[str, Any] | None = None,
 ) -> None:
     """Reject process-secret selectors controlled through tenant-facing APIs."""
@@ -73,14 +90,39 @@ def reject_api_controlled_deployment_references(
         raise ConnectorDeploymentConfigurationError(
             "Environment-backed credentials may only be declared in deployment-owned connector configuration"
         )
-    for key, value in (metadata or {}).items():
-        if _clean(value) and (str(key).strip().casefold() in _SECRET_ENV_METADATA_KEYS or str(key).strip().casefold().endswith("_env")):
+    if _clean(secret_ref):
+        raise ConnectorDeploymentConfigurationError(
+            "External secret references cannot be managed through the Files API until Files can prove "
+            "provider ownership and confirm provider-side deletion"
+        )
+    for key, value in _metadata_entries(metadata or {}):
+        clean_key = str(key).strip().casefold()
+        if _clean(value) and (clean_key in _SECRET_ENV_METADATA_KEYS or clean_key.endswith("_env")):
             raise ConnectorDeploymentConfigurationError(
                 "Environment-backed credentials may only be declared in deployment-owned connector configuration"
             )
+        if _clean(value) and (
+            clean_key in _SECRET_VALUE_METADATA_KEYS
+            or clean_key.endswith(("_password", "_secret", "_api_key"))
+        ):
+            raise ConnectorDeploymentConfigurationError(
+                "Connector credential values must use the dedicated encrypted credential fields, not metadata"
+            )
     validate_connector_tls_metadata(metadata)
 
 
+def _metadata_entries(value: object) -> list[tuple[str, object]]:
+    entries: list[tuple[str, object]] = []
+    if isinstance(value, Mapping):
+        for key, item in value.items():
+            entries.append((str(key), item))
+            entries.extend(_metadata_entries(item))
+    elif isinstance(value, (list, tuple)):
+        for item in value:
+            entries.extend(_metadata_entries(item))
+    return entries
+
+
 def connector_ca_bundle_path(value: str) -> str:
     clean_value = _clean(value)
     if not clean_value:

src/govoplan_files/backend/storage/connector_profile_store.py

@@ -128,6 +128,7 @@ def create_connector_profile_row(
     reject_api_controlled_deployment_references(
         password_env=password_env,
         token_env=token_env,
+        secret_ref=secret_ref,
         metadata=metadata,
     )
     clean_id = _normalize_profile_id(profile_id)
@@ -190,8 +191,15 @@ def update_connector_profile_row(
     reject_api_controlled_deployment_references(
         password_env=password_env,
         token_env=token_env,
+        secret_ref=secret_ref,
         metadata=metadata,
     )
+    if secret_ref is not None and _clean(secret_ref) != _clean(row.secret_ref):
+        if _clean(row.secret_ref):
+            raise FileStorageError(
+                "An existing external secret reference cannot be replaced or cleared until Files can prove "
+                "provider ownership and confirm provider-side deletion"
+            )
     if label is not None:
         row.label = _normalize_label(label)
     if provider is not None:
@@ -234,14 +242,6 @@ def update_connector_profile_row(
     return row
 
 
-def deactivate_connector_profile_row(session: Session, row: FileConnectorProfile, *, user_id: str | None) -> FileConnectorProfile:
-    row.enabled = False
-    row.updated_by_user_id = user_id
-    session.add(row)
-    session.flush()
-    return row
-
-
 def _normalize_scope(*, tenant_id: str, scope_type: str, scope_id: str | None) -> tuple[str, str | None, str | None]:
     clean_scope_type = normalize_policy_scope_type(scope_type)
     clean_scope_id = _clean(scope_id)

src/govoplan_files/backend/storage/connector_profiles.py

@@ -80,6 +80,8 @@ class ConnectorProfile:
 
     @property
     def credentials_configured(self) -> bool:
+        if not self.enabled:
+            return False
         mode = self.credential_mode.casefold()
         if mode in {"", "none", "anonymous"}:
             return True