add-ideas/govoplan-files
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

chore: sync GovOPlaN module split state

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-10 12:51:21 +02:00
parent 18e6c3eb9b
commit 13fd7fc3bd
50 changed files with 12678 additions and 1109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
src/govoplan_files/backend/change_tracking.py Normal file
View File

@@ -0,0 +1,199 @@
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import event, inspect
from sqlalchemy.orm import Session as OrmSession
from govoplan_core.core.change_sequence import record_change
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare, new_uuid
FILES_MODULE_ID = "files"
FILES_ASSETS_COLLECTION = "files.assets"
FILES_FOLDERS_COLLECTION = "files.folders"
FILES_CONNECTOR_PROFILES_COLLECTION = "files.connector_profiles"
FILES_CONNECTOR_CREDENTIALS_COLLECTION = "files.connector_credentials"
FILES_CONNECTOR_POLICIES_COLLECTION = "files.connector_policies"
FILES_CONNECTOR_SPACES_COLLECTION = "files.connector_spaces"
_REGISTERED = False
def register_files_change_tracking() -> None:
global _REGISTERED
if _REGISTERED:
return
event.listen(OrmSession, "before_flush", _record_files_changes)
_REGISTERED = True
def _record_files_changes(session: OrmSession, _flush_context: object, _instances: object) -> None:
for obj in tuple(session.new) + tuple(session.dirty):
if isinstance(obj, FileAsset):
_record_asset_change(session, obj)
elif isinstance(obj, FileFolder):
_record_folder_change(session, obj)
elif isinstance(obj, FileShare):
_record_share_visibility_change(session, obj)
def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
operation = _operation_for_soft_deletable(
asset,
changed_attrs=(
"owner_type",
"owner_user_id",
"owner_group_id",
"current_version_id",
"display_path",
"filename",
"description",
"deleted_at",
"metadata_",
),
)
if operation is None:
return
resource_id = _ensure_id(asset)
record_change(
session,
module_id=FILES_MODULE_ID,
collection=FILES_ASSETS_COLLECTION,
resource_type="file",
resource_id=resource_id,
operation=operation,
tenant_id=asset.tenant_id,
actor_type="user" if asset.created_by_user_id else None,
actor_id=asset.created_by_user_id,
payload={
"owner_type": asset.owner_type,
"owner_id": _owner_id(asset.owner_type, asset.owner_user_id, asset.owner_group_id),
"path": asset.display_path,
"previous_path": _previous_value(asset, "display_path"),
"filename": asset.filename,
"deleted_at": _isoformat(asset.deleted_at),
},
)
def _record_folder_change(session: OrmSession, folder: FileFolder) -> None:
operation = _operation_for_soft_deletable(
folder,
changed_attrs=("owner_type", "owner_user_id", "owner_group_id", "path", "deleted_at", "metadata_"),
)
if operation is None:
return
resource_id = _ensure_id(folder)
record_change(
session,
module_id=FILES_MODULE_ID,
collection=FILES_FOLDERS_COLLECTION,
resource_type="folder",
resource_id=resource_id,
operation=operation,
tenant_id=folder.tenant_id,
actor_type="user" if folder.created_by_user_id else None,
actor_id=folder.created_by_user_id,
payload={
"owner_type": folder.owner_type,
"owner_id": _owner_id(folder.owner_type, folder.owner_user_id, folder.owner_group_id),
"path": folder.path,
"previous_path": _previous_value(folder, "path"),
"deleted_at": _isoformat(folder.deleted_at),
},
)
def _record_share_visibility_change(session: OrmSession, share: FileShare) -> None:
state = inspect(share)
if not share.file_asset_id:
return
if not state.pending and not _has_attr_changes(
state,
("file_asset_id", "target_type", "target_id", "permission", "revoked_at"),
):
return
_ensure_id(share)
record_change(
session,
module_id=FILES_MODULE_ID,
collection=FILES_ASSETS_COLLECTION,
resource_type="file",
resource_id=share.file_asset_id,
operation="updated",
tenant_id=share.tenant_id,
actor_type="user" if share.created_by_user_id else None,
actor_id=share.created_by_user_id,
payload={
"share_id": share.id,
"share_target_type": share.target_type,
"share_target_id": share.target_id,
"share_permission": share.permission,
"share_revoked_at": _isoformat(share.revoked_at),
},
)
def _operation_for_soft_deletable(obj: object, *, changed_attrs: tuple[str, ...]) -> str | None:
state = inspect(obj)
if state.pending:
return "created"
if not _has_attr_changes(state, changed_attrs):
return None
if "deleted_at" in state.attrs:
history = state.attrs.deleted_at.history
if history.has_changes():
if any(value is not None for value in history.added):
return "deleted"
if any(value is not None for value in history.deleted):
return "created"
return "updated"
def _has_attr_changes(state: Any, attrs: tuple[str, ...]) -> bool:
return any(name in state.attrs and state.attrs[name].history.has_changes() for name in attrs)
def _previous_value(obj: object, attr_name: str) -> str | None:
state = inspect(obj)
if attr_name not in state.attrs:
return None
history = state.attrs[attr_name].history
if not history.has_changes() or not history.deleted:
return None
value = history.deleted[0]
return str(value) if value is not None else None
def _ensure_id(obj: object) -> str:
resource_id = getattr(obj, "id", None)
if resource_id:
return str(resource_id)
resource_id = new_uuid()
setattr(obj, "id", resource_id)
return resource_id
def _owner_id(owner_type: str, owner_user_id: str | None, owner_group_id: str | None) -> str | None:
if owner_type == "user":
return owner_user_id
if owner_type == "group":
return owner_group_id
return None
def _isoformat(value: datetime | None) -> str | None:
return value.isoformat() if value else None
__all__ = [
"FILES_ASSETS_COLLECTION",
"FILES_CONNECTOR_CREDENTIALS_COLLECTION",
"FILES_CONNECTOR_POLICIES_COLLECTION",
"FILES_CONNECTOR_PROFILES_COLLECTION",
"FILES_CONNECTOR_SPACES_COLLECTION",
"FILES_FOLDERS_COLLECTION",
"FILES_MODULE_ID",
"register_files_change_tracking",
]

144
src/govoplan_files/backend/db/models.py
View File

@@ -4,7 +4,7 @@ import uuid
from datetime import datetime
from typing import Any
from sqlalchemy import DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint, text
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint, text
from sqlalchemy.orm import Mapped, mapped_column
from govoplan_core.db.base import Base, TimestampMixin
@@ -19,7 +19,7 @@ class FileBlob(Base, TimestampMixin):
__table_args__ = (UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True)
storage_backend: Mapped[str] = mapped_column(String(50), nullable=False)
storage_bucket: Mapped[str | None] = mapped_column(String(255))
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
@@ -50,12 +50,12 @@ class FileFolder(Base, TimestampMixin):
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True)
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("groups.id", ondelete="SET NULL"), nullable=True, index=True)
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
@@ -64,15 +64,15 @@ class FileAsset(Base, TimestampMixin):
__tablename__ = "file_assets"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True)
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("groups.id", ondelete="SET NULL"), nullable=True, index=True)
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
current_version_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
display_path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
filename: Mapped[str] = mapped_column(String(500), nullable=False, index=True)
description: Mapped[str | None] = mapped_column(Text)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
@@ -82,7 +82,7 @@ class FileVersion(Base, TimestampMixin):
__table_args__ = (UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True)
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True)
blob_id: Mapped[str] = mapped_column(ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True)
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
@@ -91,7 +91,7 @@ class FileVersion(Base, TimestampMixin):
content_type: Mapped[str | None] = mapped_column(String(255))
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
class FileShare(Base, TimestampMixin):
@@ -99,21 +99,131 @@ class FileShare(Base, TimestampMixin):
__table_args__ = (UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True)
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True)
target_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
target_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
permission: Mapped[str] = mapped_column(String(20), default="read", nullable=False)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
class FileConnectorProfile(Base, TimestampMixin):
__tablename__ = "file_connector_profiles"
__table_args__ = (
Index("ix_file_connector_profiles_scope", "scope_type", "scope_id"),
)
id: Mapped[str] = mapped_column(String(255), primary_key=True)
tenant_id: Mapped[str | None] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=True, index=True)
scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True)
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
label: Mapped[str] = mapped_column(String(255), nullable=False)
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
endpoint_url: Mapped[str | None] = mapped_column(String(1000), nullable=True)
base_path: Mapped[str | None] = mapped_column(String(1000), nullable=True)
enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
credential_profile_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False)
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
password_env: Mapped[str | None] = mapped_column(String(255), nullable=True)
token_env: Mapped[str | None] = mapped_column(String(255), nullable=True)
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
capabilities: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
class FileConnectorCredential(Base, TimestampMixin):
__tablename__ = "file_connector_credentials"
__table_args__ = (
Index("ix_file_connector_credentials_scope", "scope_type", "scope_id"),
)
id: Mapped[str] = mapped_column(String(255), primary_key=True)
tenant_id: Mapped[str | None] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=True, index=True)
scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True)
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
label: Mapped[str] = mapped_column(String(255), nullable=False)
provider: Mapped[str | None] = mapped_column(String(50), nullable=True, index=True)
enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False)
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
password_env: Mapped[str | None] = mapped_column(String(255), nullable=True)
token_env: Mapped[str | None] = mapped_column(String(255), nullable=True)
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
class FileConnectorPolicy(Base, TimestampMixin):
__tablename__ = "file_connector_policies"
__table_args__ = (
UniqueConstraint("tenant_id", "scope_type", "scope_id", name="uq_file_connector_policies_scope"),
Index("ix_file_connector_policies_scope", "scope_type", "scope_id"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str | None] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=True, index=True)
scope_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
class FileConnectorSpace(Base, TimestampMixin):
__tablename__ = "file_connector_spaces"
__table_args__ = (
Index("ix_file_connector_spaces_owner", "tenant_id", "owner_type", "owner_user_id", "owner_group_id"),
Index(
"uq_file_connector_spaces_active_user_label",
"tenant_id", "owner_user_id", "label",
unique=True,
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
),
Index(
"uq_file_connector_spaces_active_group_label",
"tenant_id", "owner_group_id", "label",
unique=True,
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True)
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
label: Mapped[str] = mapped_column(String(255), nullable=False)
connector_profile_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
library_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
remote_path: Mapped[str] = mapped_column(String(1000), default="", nullable=False)
sync_mode: Mapped[str] = mapped_column(String(30), default="manual", nullable=False)
read_only: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
class CampaignAttachmentUse(Base, TimestampMixin):
__tablename__ = "campaign_attachment_uses"
__table_args__ = (UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True)
campaign_id: Mapped[str] = mapped_column(ForeignKey("campaigns.id", ondelete="CASCADE"), nullable=False, index=True)
campaign_version_id: Mapped[str] = mapped_column(ForeignKey("campaign_versions.id", ondelete="CASCADE"), nullable=False, index=True)
campaign_job_id: Mapped[str | None] = mapped_column(ForeignKey("campaign_jobs.id", ondelete="SET NULL"), nullable=True, index=True)
@@ -134,6 +244,10 @@ __all__ = [
"CampaignAttachmentUse",
"FileAsset",
"FileBlob",
"FileConnectorCredential",
"FileConnectorPolicy",
"FileConnectorProfile",
"FileConnectorSpace",
"FileFolder",
"FileShare",
"FileVersion",

31
src/govoplan_files/backend/manifest.py
View File

@@ -5,8 +5,11 @@ from pathlib import Path
from govoplan_core.core.module_guards import drop_table_retirement_provider, persistent_table_uninstall_guard
from govoplan_core.core.modules import FrontendModule, MigrationSpec, ModuleContext, ModuleManifest, NavItem, PermissionDefinition, RoleTemplate
from govoplan_core.db.base import Base
from govoplan_files.backend.change_tracking import register_files_change_tracking
from govoplan_files.backend.db import models as file_models # noqa: F401 - populate Files ORM metadata
register_files_change_tracking()
def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
module_id, resource, action = scope.split(":", 2)
@@ -56,23 +59,33 @@ ROLE_TEMPLATES = (
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
from govoplan_files.backend.db.models import FileAsset
from govoplan_files.backend.db.models import FileAsset, FileConnectorCredential, FileConnectorPolicy, FileConnectorProfile, FileConnectorSpace
return {"files": session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id).count()}
return {
"files": session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id).count(),
"connector_credentials": session.query(FileConnectorCredential).filter(FileConnectorCredential.tenant_id == tenant_id).count(),
"connector_policies": session.query(FileConnectorPolicy).filter(FileConnectorPolicy.tenant_id == tenant_id).count(),
"connector_profiles": session.query(FileConnectorProfile).filter(FileConnectorProfile.tenant_id == tenant_id).count(),
"connector_spaces": session.query(FileConnectorSpace).filter(FileConnectorSpace.tenant_id == tenant_id).count(),
}
def _veto_group_delete(session, tenant_id: str, group_id: str) -> None:
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
from govoplan_files.backend.db.models import FileAsset, FileConnectorSpace, FileFolder, FileShare
owned_asset_count = session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id, FileAsset.owner_group_id == group_id).count()
owned_folder_count = session.query(FileFolder).filter(FileFolder.tenant_id == tenant_id, FileFolder.owner_group_id == group_id).count()
owned_connector_space_count = session.query(FileConnectorSpace).filter(
FileConnectorSpace.tenant_id == tenant_id,
FileConnectorSpace.owner_group_id == group_id,
).count()
shared_asset_count = session.query(FileShare).filter(
FileShare.tenant_id == tenant_id,
FileShare.target_type == "group",
FileShare.target_id == group_id,
).count()
if owned_asset_count or owned_folder_count or shared_asset_count:
raise ValueError("Cannot remove the group while it owns files, folders or file shares.")
if owned_asset_count or owned_folder_count or owned_connector_space_count or shared_asset_count:
raise ValueError("Cannot remove the group while it owns files, folders, connector spaces or file shares.")
def _files_router(context: ModuleContext):
@@ -112,6 +125,10 @@ manifest = ModuleManifest(
file_models.FileAsset,
file_models.FileVersion,
file_models.FileShare,
file_models.FileConnectorCredential,
file_models.FileConnectorPolicy,
file_models.FileConnectorProfile,
file_models.FileConnectorSpace,
file_models.CampaignAttachmentUse,
label="Files",
),
@@ -124,6 +141,10 @@ manifest = ModuleManifest(
file_models.FileAsset,
file_models.FileVersion,
file_models.FileShare,
file_models.FileConnectorCredential,
file_models.FileConnectorPolicy,
file_models.FileConnectorProfile,
file_models.FileConnectorSpace,
file_models.CampaignAttachmentUse,
label="Files",
),

114
src/govoplan_files/backend/migrations/versions/4f5a6b7c8d9e_file_connector_spaces.py Normal file
View File

@@ -0,0 +1,114 @@
"""file connector spaces
Revision ID: 4f5a6b7c8d9e
Revises: 2e3f4a5b6c7d
Create Date: 2026-07-08 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "4f5a6b7c8d9e"
down_revision = "2e3f4a5b6c7d"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "file_connector_spaces" not in tables:
op.create_table(
"file_connector_spaces",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("owner_type", sa.String(length=20), nullable=False),
sa.Column("owner_user_id", sa.String(length=36), nullable=True),
sa.Column("owner_group_id", sa.String(length=36), nullable=True),
sa.Column("label", sa.String(length=255), nullable=False),
sa.Column("connector_profile_id", sa.String(length=255), nullable=False),
sa.Column("provider", sa.String(length=50), nullable=False),
sa.Column("library_id", sa.String(length=255), nullable=True),
sa.Column("remote_path", sa.String(length=1000), nullable=False),
sa.Column("sync_mode", sa.String(length=30), nullable=False),
sa.Column("read_only", sa.Boolean(), nullable=False),
sa.Column("is_active", sa.Boolean(), nullable=False),
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("metadata", sa.JSON(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_spaces_created_by_user_id_users"), ondelete="SET NULL"),
sa.ForeignKeyConstraint(["owner_group_id"], ["access_groups.id"], name=op.f("fk_file_connector_spaces_owner_group_id_groups"), ondelete="SET NULL"),
sa.ForeignKeyConstraint(["owner_user_id"], ["access_users.id"], name=op.f("fk_file_connector_spaces_owner_user_id_users"), ondelete="SET NULL"),
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], name=op.f("fk_file_connector_spaces_tenant_id_tenants"), ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_file_connector_spaces")),
)
inspector = sa.inspect(op.get_bind())
indexes = {item["name"] for item in inspector.get_indexes("file_connector_spaces")}
for column in (
"tenant_id",
"owner_type",
"owner_user_id",
"owner_group_id",
"connector_profile_id",
"provider",
"is_active",
"created_by_user_id",
"deleted_at",
):
name = op.f(f"ix_file_connector_spaces_{column}")
if name not in indexes:
op.create_index(name, "file_connector_spaces", [column], unique=False)
if "ix_file_connector_spaces_owner" not in indexes:
op.create_index(
"ix_file_connector_spaces_owner",
"file_connector_spaces",
["tenant_id", "owner_type", "owner_user_id", "owner_group_id"],
unique=False,
)
if "uq_file_connector_spaces_active_user_label" not in indexes:
op.create_index(
"uq_file_connector_spaces_active_user_label",
"file_connector_spaces",
["tenant_id", "owner_user_id", "label"],
unique=True,
sqlite_where=sa.text("owner_type = 'user' AND deleted_at IS NULL"),
postgresql_where=sa.text("owner_type = 'user' AND deleted_at IS NULL"),
)
if "uq_file_connector_spaces_active_group_label" not in indexes:
op.create_index(
"uq_file_connector_spaces_active_group_label",
"file_connector_spaces",
["tenant_id", "owner_group_id", "label"],
unique=True,
sqlite_where=sa.text("owner_type = 'group' AND deleted_at IS NULL"),
postgresql_where=sa.text("owner_type = 'group' AND deleted_at IS NULL"),
)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "file_connector_spaces" not in inspector.get_table_names():
return
indexes = {item["name"] for item in inspector.get_indexes("file_connector_spaces")}
for name in (
"uq_file_connector_spaces_active_group_label",
"uq_file_connector_spaces_active_user_label",
"ix_file_connector_spaces_owner",
op.f("ix_file_connector_spaces_deleted_at"),
op.f("ix_file_connector_spaces_created_by_user_id"),
op.f("ix_file_connector_spaces_is_active"),
op.f("ix_file_connector_spaces_provider"),
op.f("ix_file_connector_spaces_connector_profile_id"),
op.f("ix_file_connector_spaces_owner_group_id"),
op.f("ix_file_connector_spaces_owner_user_id"),
op.f("ix_file_connector_spaces_owner_type"),
op.f("ix_file_connector_spaces_tenant_id"),
):
if name in indexes:
op.drop_index(name, table_name="file_connector_spaces")
op.drop_table("file_connector_spaces")

88
src/govoplan_files/backend/migrations/versions/5a6b7c8d9e0f_file_connector_profiles.py Normal file
View File

@@ -0,0 +1,88 @@
"""file connector profiles
Revision ID: 5a6b7c8d9e0f
Revises: 4f5a6b7c8d9e
Create Date: 2026-07-08 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "5a6b7c8d9e0f"
down_revision = "4f5a6b7c8d9e"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "file_connector_profiles" not in inspector.get_table_names():
op.create_table(
"file_connector_profiles",
sa.Column("id", sa.String(length=255), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=True),
sa.Column("scope_type", sa.String(length=20), nullable=False),
sa.Column("scope_id", sa.String(length=36), nullable=True),
sa.Column("label", sa.String(length=255), nullable=False),
sa.Column("provider", sa.String(length=50), nullable=False),
sa.Column("endpoint_url", sa.String(length=1000), nullable=True),
sa.Column("base_path", sa.String(length=1000), nullable=True),
sa.Column("enabled", sa.Boolean(), nullable=False),
sa.Column("credential_mode", sa.String(length=30), nullable=False),
sa.Column("username", sa.String(length=320), nullable=True),
sa.Column("password_encrypted", sa.Text(), nullable=True),
sa.Column("token_encrypted", sa.Text(), nullable=True),
sa.Column("password_env", sa.String(length=255), nullable=True),
sa.Column("token_env", sa.String(length=255), nullable=True),
sa.Column("secret_ref", sa.String(length=1000), nullable=True),
sa.Column("capabilities", sa.JSON(), nullable=True),
sa.Column("policy", sa.JSON(), nullable=True),
sa.Column("metadata", sa.JSON(), nullable=True),
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
sa.Column("updated_by_user_id", sa.String(length=36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_profiles_created_by_user_id_users"), ondelete="SET NULL"),
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], name=op.f("fk_file_connector_profiles_tenant_id_tenants"), ondelete="CASCADE"),
sa.ForeignKeyConstraint(["updated_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_profiles_updated_by_user_id_users"), ondelete="SET NULL"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_file_connector_profiles")),
)
inspector = sa.inspect(op.get_bind())
indexes = {item["name"] for item in inspector.get_indexes("file_connector_profiles")}
for column in (
"tenant_id",
"scope_type",
"scope_id",
"provider",
"enabled",
"created_by_user_id",
"updated_by_user_id",
):
name = op.f(f"ix_file_connector_profiles_{column}")
if name not in indexes:
op.create_index(name, "file_connector_profiles", [column], unique=False)
if "ix_file_connector_profiles_scope" not in indexes:
op.create_index("ix_file_connector_profiles_scope", "file_connector_profiles", ["scope_type", "scope_id"], unique=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "file_connector_profiles" not in inspector.get_table_names():
return
indexes = {item["name"] for item in inspector.get_indexes("file_connector_profiles")}
for name in (
"ix_file_connector_profiles_scope",
op.f("ix_file_connector_profiles_updated_by_user_id"),
op.f("ix_file_connector_profiles_created_by_user_id"),
op.f("ix_file_connector_profiles_enabled"),
op.f("ix_file_connector_profiles_provider"),
op.f("ix_file_connector_profiles_scope_id"),
op.f("ix_file_connector_profiles_scope_type"),
op.f("ix_file_connector_profiles_tenant_id"),
):
if name in indexes:
op.drop_index(name, table_name="file_connector_profiles")
op.drop_table("file_connector_profiles")

106
src/govoplan_files/backend/migrations/versions/6b7c8d9e0f1a_file_connector_credentials.py Normal file
View File

@@ -0,0 +1,106 @@
"""file connector credentials
Revision ID: 6b7c8d9e0f1a
Revises: 5a6b7c8d9e0f
Create Date: 2026-07-08 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "6b7c8d9e0f1a"
down_revision = "5a6b7c8d9e0f"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
table_names = inspector.get_table_names()
if "file_connector_profiles" in table_names:
columns = {item["name"] for item in inspector.get_columns("file_connector_profiles")}
if "credential_profile_id" not in columns:
op.add_column("file_connector_profiles", sa.Column("credential_profile_id", sa.String(length=255), nullable=True))
indexes = {item["name"] for item in inspector.get_indexes("file_connector_profiles")}
index_name = op.f("ix_file_connector_profiles_credential_profile_id")
if index_name not in indexes:
op.create_index(index_name, "file_connector_profiles", ["credential_profile_id"], unique=False)
if "file_connector_credentials" not in table_names:
op.create_table(
"file_connector_credentials",
sa.Column("id", sa.String(length=255), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=True),
sa.Column("scope_type", sa.String(length=20), nullable=False),
sa.Column("scope_id", sa.String(length=36), nullable=True),
sa.Column("label", sa.String(length=255), nullable=False),
sa.Column("provider", sa.String(length=50), nullable=True),
sa.Column("enabled", sa.Boolean(), nullable=False),
sa.Column("credential_mode", sa.String(length=30), nullable=False),
sa.Column("username", sa.String(length=320), nullable=True),
sa.Column("password_encrypted", sa.Text(), nullable=True),
sa.Column("token_encrypted", sa.Text(), nullable=True),
sa.Column("password_env", sa.String(length=255), nullable=True),
sa.Column("token_env", sa.String(length=255), nullable=True),
sa.Column("secret_ref", sa.String(length=1000), nullable=True),
sa.Column("policy", sa.JSON(), nullable=True),
sa.Column("metadata", sa.JSON(), nullable=True),
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
sa.Column("updated_by_user_id", sa.String(length=36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_credentials_created_by_user_id_users"), ondelete="SET NULL"),
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], name=op.f("fk_file_connector_credentials_tenant_id_tenants"), ondelete="CASCADE"),
sa.ForeignKeyConstraint(["updated_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_credentials_updated_by_user_id_users"), ondelete="SET NULL"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_file_connector_credentials")),
)
inspector = sa.inspect(op.get_bind())
indexes = {item["name"] for item in inspector.get_indexes("file_connector_credentials")}
for column in (
"tenant_id",
"scope_type",
"scope_id",
"provider",
"enabled",
"created_by_user_id",
"updated_by_user_id",
):
name = op.f(f"ix_file_connector_credentials_{column}")
if name not in indexes:
op.create_index(name, "file_connector_credentials", [column], unique=False)
if "ix_file_connector_credentials_scope" not in indexes:
op.create_index("ix_file_connector_credentials_scope", "file_connector_credentials", ["scope_type", "scope_id"], unique=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "file_connector_credentials" in inspector.get_table_names():
indexes = {item["name"] for item in inspector.get_indexes("file_connector_credentials")}
for name in (
"ix_file_connector_credentials_scope",
op.f("ix_file_connector_credentials_updated_by_user_id"),
op.f("ix_file_connector_credentials_created_by_user_id"),
op.f("ix_file_connector_credentials_enabled"),
op.f("ix_file_connector_credentials_provider"),
op.f("ix_file_connector_credentials_scope_id"),
op.f("ix_file_connector_credentials_scope_type"),
op.f("ix_file_connector_credentials_tenant_id"),
):
if name in indexes:
op.drop_index(name, table_name="file_connector_credentials")
op.drop_table("file_connector_credentials")
inspector = sa.inspect(op.get_bind())
if "file_connector_profiles" not in inspector.get_table_names():
return
profile_indexes = {item["name"] for item in inspector.get_indexes("file_connector_profiles")}
profile_index_name = op.f("ix_file_connector_profiles_credential_profile_id")
if profile_index_name in profile_indexes:
op.drop_index(profile_index_name, table_name="file_connector_profiles")
profile_columns = {item["name"] for item in inspector.get_columns("file_connector_profiles")}
if "credential_profile_id" in profile_columns:
op.drop_column("file_connector_profiles", "credential_profile_id")

71
src/govoplan_files/backend/migrations/versions/a7b8c9d0e1f3_file_connector_policies.py Normal file
View File

@@ -0,0 +1,71 @@
"""file connector policies
Revision ID: a7b8c9d0e1f3
Revises: 6b7c8d9e0f1a
Create Date: 2026-07-08 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "a7b8c9d0e1f3"
down_revision = "6b7c8d9e0f1a"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "file_connector_policies" not in inspector.get_table_names():
op.create_table(
"file_connector_policies",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=True),
sa.Column("scope_type", sa.String(length=20), nullable=False),
sa.Column("scope_id", sa.String(length=36), nullable=True),
sa.Column("policy", sa.JSON(), nullable=False),
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
sa.Column("updated_by_user_id", sa.String(length=36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_policies_created_by_user_id_users"), ondelete="SET NULL"),
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], name=op.f("fk_file_connector_policies_tenant_id_tenants"), ondelete="CASCADE"),
sa.ForeignKeyConstraint(["updated_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_policies_updated_by_user_id_users"), ondelete="SET NULL"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_file_connector_policies")),
sa.UniqueConstraint("tenant_id", "scope_type", "scope_id", name="uq_file_connector_policies_scope"),
)
inspector = sa.inspect(op.get_bind())
indexes = {item["name"] for item in inspector.get_indexes("file_connector_policies")}
for column in (
"tenant_id",
"scope_type",
"scope_id",
"created_by_user_id",
"updated_by_user_id",
):
name = op.f(f"ix_file_connector_policies_{column}")
if name not in indexes:
op.create_index(name, "file_connector_policies", [column], unique=False)
if "ix_file_connector_policies_scope" not in indexes:
op.create_index("ix_file_connector_policies_scope", "file_connector_policies", ["scope_type", "scope_id"], unique=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "file_connector_policies" not in inspector.get_table_names():
return
indexes = {item["name"] for item in inspector.get_indexes("file_connector_policies")}
for name in (
"ix_file_connector_policies_scope",
op.f("ix_file_connector_policies_updated_by_user_id"),
op.f("ix_file_connector_policies_created_by_user_id"),
op.f("ix_file_connector_policies_scope_id"),
op.f("ix_file_connector_policies_scope_type"),
op.f("ix_file_connector_policies_tenant_id"),
):
if name in indexes:
op.drop_index(name, table_name="file_connector_policies")
op.drop_table("file_connector_policies")

2544
src/govoplan_files/backend/router.py
View File

File diff suppressed because it is too large Load Diff

343
src/govoplan_files/backend/schemas.py
View File

@@ -4,6 +4,7 @@ from typing import Any, Literal
from pydantic import BaseModel, Field
from govoplan_core.api.v1.schemas import DeltaDeletedItem
from govoplan_files.backend.storage.common import FileConflictResolution
@@ -13,12 +14,63 @@ class FileSpaceResponse(BaseModel):
owner_type: Literal["user", "group"]
owner_id: str
description: str | None = None
space_type: Literal["managed", "connector"] = "managed"
connector_space_id: str | None = None
connector_profile_id: str | None = None
provider: str | None = None
library_id: str | None = None
remote_path: str | None = None
sync_mode: str | None = None
read_only: bool = False
class FileSpacesResponse(BaseModel):
spaces: list[FileSpaceResponse]
class FileConnectorSpaceCreateRequest(BaseModel):
owner_type: Literal["user", "group"] = "user"
owner_id: str | None = None
label: str
connector_profile_id: str
library_id: str | None = None
remote_path: str = ""
sync_mode: Literal["manual"] = "manual"
metadata: dict[str, Any] = Field(default_factory=dict)
class FileConnectorSpaceUpdateRequest(BaseModel):
label: str | None = None
library_id: str | None = None
remote_path: str | None = None
sync_mode: Literal["manual"] | None = None
is_active: bool | None = None
metadata: dict[str, Any] | None = None
class FileConnectorSpaceResponse(BaseModel):
id: str
tenant_id: str
owner_type: Literal["user", "group"]
owner_id: str
label: str
connector_profile_id: str
provider: str
library_id: str | None = None
remote_path: str = ""
sync_mode: str
read_only: bool
is_active: bool
created_at: str
updated_at: str
deleted_at: str | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class FileConnectorSpacesResponse(BaseModel):
spaces: list[FileConnectorSpaceResponse] = Field(default_factory=list)
class FileShareResponse(BaseModel):
id: str
target_type: str
@@ -28,6 +80,20 @@ class FileShareResponse(BaseModel):
revoked_at: str | None = None
class FileSourceProvenance(BaseModel):
source_type: str | None = None
connector_id: str | None = None
provider: str | None = None
external_id: str | None = None
external_path: str | None = None
external_url: str | None = None
revision: str | None = None
revision_label: str | None = None
observed_at: str | None = None
imported_at: str | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class FileAssetResponse(BaseModel):
id: str
tenant_id: str
@@ -45,6 +111,8 @@ class FileAssetResponse(BaseModel):
deleted_at: str | None = None
audit_relevant: bool = False
metadata: dict[str, Any] | None = None
source_provenance: FileSourceProvenance | None = None
source_revision: str | None = None
shares: list[FileShareResponse] = Field(default_factory=list)
@@ -61,6 +129,9 @@ class FileFolderResponse(BaseModel):
class FileFoldersResponse(BaseModel):
folders: list[FileFolderResponse]
cursor: str | None = None
next_cursor: str | None = None
watermark: str | None = None
class FileFolderCreateRequest(BaseModel):
@@ -83,12 +154,280 @@ class FileFolderDeleteResponse(BaseModel):
class FileListResponse(BaseModel):
files: list[FileAssetResponse]
cursor: str | None = None
next_cursor: str | None = None
watermark: str | None = None
class FileDeltaResponse(BaseModel):
files: list[FileAssetResponse] = Field(default_factory=list)
folders: list[FileFolderResponse] = Field(default_factory=list)
deleted: list[DeltaDeletedItem] = Field(default_factory=list)
watermark: str | None = None
has_more: bool = False
full: bool = False
class FileUploadResponse(BaseModel):
files: list[FileAssetResponse]
class FileConnectorPolicySource(BaseModel):
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system"
scope_id: str | None = None
label: str | None = None
policy: dict[str, Any] = Field(default_factory=dict)
class FileConnectorPolicyEvaluateRequest(BaseModel):
source_provenance: FileSourceProvenance
operation: str = "access"
policy_sources: list[FileConnectorPolicySource] = Field(default_factory=list)
class FileConnectorPolicyEvaluateResponse(BaseModel):
decision: dict[str, Any]
class FileConnectorPolicyUpdateRequest(BaseModel):
policy: dict[str, Any] = Field(default_factory=dict)
class FileConnectorPolicyStepResponse(BaseModel):
scope_type: str
scope_id: str | None = None
path: str
label: str
applied_fields: list[str] = Field(default_factory=list)
policy: dict[str, Any] = Field(default_factory=dict)
class FileConnectorPolicyResponse(BaseModel):
scope_type: Literal["system", "tenant", "user", "group", "campaign"]
scope_id: str | None = None
policy: dict[str, Any] = Field(default_factory=dict)
effective_policy: dict[str, Any] | None = None
parent_policy: dict[str, Any] | None = None
effective_policy_sources: list[FileConnectorPolicyStepResponse] = Field(default_factory=list)
parent_policy_sources: list[FileConnectorPolicyStepResponse] = Field(default_factory=list)
class FileConnectorProfileResponse(BaseModel):
id: str
label: str
provider: str
endpoint_url: str | None = None
base_path: str | None = None
enabled: bool = True
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system"
scope_id: str | None = None
source_path: str
credential_profile_id: str | None = None
credential_profile_label: str | None = None
credential_mode: str = "none"
credential_source: str | None = None
credentials_configured: bool = False
username: str | None = None
capabilities: list[str] = Field(default_factory=list)
policy_sources: list[FileConnectorPolicySource] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
source_kind: str = "settings"
class FileConnectorProfilesResponse(BaseModel):
profiles: list[FileConnectorProfileResponse]
class FileConnectorCredentialResponse(BaseModel):
id: str
label: str
provider: str | None = None
enabled: bool = True
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system"
scope_id: str | None = None
source_path: str
credential_mode: str = "none"
credential_secret_source: str | None = None
credentials_configured: bool = False
username: str | None = None
policy_sources: list[FileConnectorPolicySource] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
source_kind: str = "database"
class FileConnectorCredentialsResponse(BaseModel):
credentials: list[FileConnectorCredentialResponse]
class FileConnectorSettingsDeltaResponse(BaseModel):
profiles: list[FileConnectorProfileResponse] = Field(default_factory=list)
credentials: list[FileConnectorCredentialResponse] = Field(default_factory=list)
spaces: list[FileConnectorSpaceResponse] = Field(default_factory=list)
policy: FileConnectorPolicyResponse | None = None
changed_sections: list[str] = Field(default_factory=list)
deleted: list[DeltaDeletedItem] = Field(default_factory=list)
watermark: str | None = None
has_more: bool = False
full: bool = False
class FileConnectorProfileCredentialsRequest(BaseModel):
username: str | None = None
password: str | None = None
token: str | None = None
password_env: str | None = None
token_env: str | None = None
secret_ref: str | None = None
class FileConnectorDiscoveryCandidate(BaseModel):
endpoint_url: str
status: Literal["failed", "usable", "found", "credentials_rejected"]
message: str
class FileConnectorDiscoveryRequest(BaseModel):
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"]
endpoint_url: str
base_path: str | None = None
credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] = "none"
credentials: FileConnectorProfileCredentialsRequest = Field(default_factory=FileConnectorProfileCredentialsRequest)
metadata: dict[str, Any] = Field(default_factory=dict)
require_valid_credentials: bool = False
class FileConnectorDiscoveryResponse(BaseModel):
provider: str
endpoint_url: str | None = None
base_path: str | None = None
status: Literal["usable", "found", "credentials_rejected", "not_found", "unsupported"]
message: str
candidates: list[FileConnectorDiscoveryCandidate] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
class FileConnectorProfileCreateRequest(BaseModel):
id: str
label: str
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"]
endpoint_url: str | None = None
base_path: str | None = None
enabled: bool = True
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "tenant"
scope_id: str | None = None
credential_profile_id: str | None = None
credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] = "none"
credentials: FileConnectorProfileCredentialsRequest = Field(default_factory=FileConnectorProfileCredentialsRequest)
capabilities: list[str] = Field(default_factory=lambda: ["browse", "import", "sync"])
policy: dict[str, Any] = Field(default_factory=dict)
metadata: dict[str, Any] = Field(default_factory=dict)
class FileConnectorProfileUpdateRequest(BaseModel):
label: str | None = None
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None
endpoint_url: str | None = None
base_path: str | None = None
enabled: bool | None = None
credential_profile_id: str | None = None
credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] | None = None
credentials: FileConnectorProfileCredentialsRequest | None = None
clear_password: bool = False
clear_token: bool = False
capabilities: list[str] | None = None
policy: dict[str, Any] | None = None
metadata: dict[str, Any] | None = None
class FileConnectorCredentialCreateRequest(BaseModel):
id: str
label: str
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None
enabled: bool = True
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "tenant"
scope_id: str | None = None
credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] = "none"
credentials: FileConnectorProfileCredentialsRequest = Field(default_factory=FileConnectorProfileCredentialsRequest)
policy: dict[str, Any] = Field(default_factory=dict)
metadata: dict[str, Any] = Field(default_factory=dict)
class FileConnectorCredentialUpdateRequest(BaseModel):
label: str | None = None
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None
enabled: bool | None = None
credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] | None = None
credentials: FileConnectorProfileCredentialsRequest | None = None
clear_password: bool = False
clear_token: bool = False
policy: dict[str, Any] | None = None
metadata: dict[str, Any] | None = None
class FileConnectorProviderResponse(BaseModel):
provider: str
label: str
protocol: str
implemented: bool
installed: bool
browse_supported: bool
import_supported: bool
optional_dependency: str | None = None
permission_model: str
sync_strategy: str
conflict_strategy: str
preview_strategy: str
audit_events: list[str] = Field(default_factory=list)
notes: str | None = None
class FileConnectorProvidersResponse(BaseModel):
providers: list[FileConnectorProviderResponse]
class FileConnectorBrowseItem(BaseModel):
kind: Literal["library", "folder", "file"]
name: str
path: str
external_id: str | None = None
external_url: str | None = None
size_bytes: int | None = None
content_type: str | None = None
modified_at: str | None = None
etag: str | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class FileConnectorBrowseResponse(BaseModel):
profile_id: str
provider: str
path: str = ""
library_id: str | None = None
read_only: bool = True
decision: dict[str, Any]
items: list[FileConnectorBrowseItem]
class FileConnectorImportRequest(BaseModel):
library_id: str
path: str
owner_type: Literal["user", "group"] = "user"
owner_id: str | None = None
target_folder: str | None = None
target_path: str | None = None
campaign_id: str | None = None
conflict_strategy: Literal["reject", "overwrite", "rename"] = "reject"
source_revision: str | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class FileConnectorSyncResponse(BaseModel):
file: FileAssetResponse
action: Literal["created", "updated", "unchanged"]
previous_version_id: str | None = None
current_version_id: str
class BulkDeleteRequest(BaseModel):
file_ids: list[str]
@@ -128,8 +467,8 @@ class BulkFileShareResponse(BaseModel):
class RenameRequest(BaseModel):
file_ids: list[str] = Field(default_factory=list)
folder_paths: list[str] = Field(default_factory=list)
owner_type: Literal["user", "group"] | None = None
owner_id: str | None = None
owner_type: Literal["user", "group"]
owner_id: str
mode: Literal["direct", "prefix", "suffix", "replace"]
new_name: str | None = None
find: str | None = None

10
src/govoplan_files/backend/storage/archives.py
View File

@@ -3,8 +3,9 @@ from __future__ import annotations
import mimetypes
import zipfile
from io import BytesIO
from os import PathLike
from pathlib import Path
from typing import Iterable
from typing import Any, Iterable
from sqlalchemy.orm import Session
@@ -68,11 +69,12 @@ def extract_zip_upload(
owner_type: str,
owner_id: str,
user_id: str,
zip_data: bytes,
zip_data: bytes | str | PathLike[str],
folder: str | None,
campaign_id: str | None,
conflict_strategy: str = "reject",
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
metadata: dict[str, Any] | None = None,
is_admin: bool = False,
max_files: int = 1000,
max_file_bytes: int = 50 * 1024 * 1024,
@@ -82,7 +84,8 @@ def extract_zip_upload(
total = 0
base_folder = normalize_folder(folder)
try:
with zipfile.ZipFile(BytesIO(zip_data)) as archive:
source = BytesIO(zip_data) if isinstance(zip_data, bytes) else zip_data
with zipfile.ZipFile(source) as archive:
infos = [info for info in archive.infolist() if not info.is_dir()]
if len(infos) > max_files:
raise FileStorageError(f"ZIP contains too many files (limit {max_files})")
@@ -115,6 +118,7 @@ def extract_zip_upload(
data=data,
display_path=target_path,
content_type=mimetypes.guess_type(inner_path)[0] or "application/octet-stream",
metadata=metadata,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,

9
src/govoplan_files/backend/storage/campaign_attachments.py
View File

@@ -17,6 +17,7 @@ from govoplan_files.backend.storage.backends import StorageBackendError, get_sto
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.files import current_versions_and_blobs, list_assets_for_user
from govoplan_files.backend.storage.paths import normalize_folder, normalize_logical_path, safe_storage_component
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata, source_revision_from_metadata
MANAGED_SOURCE_PREFIX = "managed:"
@@ -36,10 +37,16 @@ class ManagedAttachmentFile:
checksum_sha256: str
size_bytes: int
content_type: str | None
source_provenance: dict[str, Any] | None = None
source_revision: str | None = None
def as_dict(self) -> dict[str, Any]:
payload = asdict(self)
payload.pop("local_path", None)
if payload.get("source_provenance") is None:
payload.pop("source_provenance", None)
if payload.get("source_revision") is None:
payload.pop("source_revision", None)
return payload
@@ -233,6 +240,8 @@ def prepare_campaign_snapshot(
checksum_sha256=blob.checksum_sha256,
size_bytes=blob.size_bytes,
content_type=blob.content_type,
source_provenance=source_provenance_from_metadata(asset.metadata_),
source_revision=source_revision_from_metadata(asset.metadata_),
)
for rule in _iter_rule_dicts(attachments, prepared_json):

600
src/govoplan_files/backend/storage/connector_browse.py Normal file
View File

@@ -0,0 +1,600 @@
from __future__ import annotations
import os
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from importlib import import_module
import mimetypes
from typing import Any
from urllib.parse import quote, unquote, urljoin, urlsplit
import xml.etree.ElementTree as ET
import httpx
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
class ConnectorBrowseError(RuntimeError):
pass
class ConnectorBrowseUnsupported(ConnectorBrowseError):
pass
@dataclass(frozen=True, slots=True)
class ConnectorBrowseItem:
kind: str
name: str
path: str
external_id: str | None = None
external_url: str | None = None
size_bytes: int | None = None
content_type: str | None = None
modified_at: str | None = None
etag: str | None = None
metadata: Mapping[str, Any] = field(default_factory=dict)
def to_response(self) -> dict[str, Any]:
return {
"kind": self.kind,
"name": self.name,
"path": self.path,
"external_id": self.external_id,
"external_url": self.external_url,
"size_bytes": self.size_bytes,
"content_type": self.content_type,
"modified_at": self.modified_at,
"etag": self.etag,
"metadata": dict(self.metadata),
}
def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = None, library_id: str | None = None) -> list[ConnectorBrowseItem]:
browse_path = normalize_connector_browse_path(path)
static_items = _static_listing(profile, path=browse_path, library_id=library_id)
if static_items is not None:
return static_items
if profile.provider == "seafile":
if _metadata_string(profile, "browse_protocol") == "webdav" or _metadata_string(profile, "webdav_endpoint_url"):
return _browse_webdav(profile, path=browse_path)
return _browse_seafile(profile, path=browse_path, library_id=library_id)
if profile.provider in {"webdav", "nextcloud"} or _metadata_string(profile, "browse_protocol") == "webdav":
return _browse_webdav(profile, path=browse_path)
if profile.provider == "smb":
return _browse_smb(profile, path=browse_path)
raise ConnectorBrowseUnsupported(f"Read-only browsing is not implemented for {profile.provider} connector profiles yet")
def parse_webdav_multistatus(*, root_url: str, current_path: str, payload: str | bytes) -> list[ConnectorBrowseItem]:
try:
root = ET.fromstring(payload)
except ET.ParseError as exc:
raise ConnectorBrowseError("Connector returned invalid WebDAV XML") from exc
root_path = _url_path_root(root_url)
current = normalize_connector_browse_path(current_path)
items: list[ConnectorBrowseItem] = []
for response in root.findall("{DAV:}response"):
href = response.findtext("{DAV:}href")
if not href:
continue
relative_path = _relative_href_path(root_path, href)
if relative_path == current:
continue
if current and not relative_path.startswith(current.rstrip("/") + "/"):
continue
parent = relative_path.rsplit("/", 1)[0] if "/" in relative_path else ""
if parent != current:
continue
prop = _webdav_prop(response)
is_collection = prop.find("{DAV:}resourcetype/{DAV:}collection") is not None if prop is not None else False
name = _webdav_display_name(prop) or _path_name(relative_path)
if not name:
continue
items.append(
ConnectorBrowseItem(
kind="folder" if is_collection else "file",
name=name,
path=relative_path,
external_id=_text(prop, "{DAV:}getetag") or relative_path,
size_bytes=None if is_collection else _int(_text(prop, "{DAV:}getcontentlength")),
content_type=None if is_collection else _text(prop, "{DAV:}getcontenttype"),
modified_at=_http_date(_text(prop, "{DAV:}getlastmodified")),
etag=_text(prop, "{DAV:}getetag"),
)
)
return sorted(items, key=lambda item: (item.kind != "library", item.kind != "folder", item.name.casefold(), item.path.casefold()))
def seafile_libraries_from_payload(payload: object) -> list[ConnectorBrowseItem]:
if not isinstance(payload, list):
raise ConnectorBrowseError("Seafile library response must be a list")
libraries = [_seafile_library_item(item) for item in payload if isinstance(item, Mapping)]
return sorted(libraries, key=lambda item: item.name.casefold())
def seafile_directory_items_from_payload(payload: object, *, path: str | None = None) -> list[ConnectorBrowseItem]:
if payload == "uptodate":
return []
if not isinstance(payload, list):
raise ConnectorBrowseError("Seafile directory response must be a list")
browse_path = normalize_connector_browse_path(path)
items = [_seafile_directory_item(item, parent_path=browse_path) for item in payload if isinstance(item, Mapping)]
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
def _browse_seafile(profile: ConnectorProfile, *, path: str, library_id: str | None) -> list[ConnectorBrowseItem]:
repo_id, dir_path = _seafile_repo_and_path(path=path, library_id=library_id)
headers = _seafile_headers(profile)
if not repo_id:
payload = _request_json("GET", _seafile_url(profile, "api2/repos/"), headers=headers)
return seafile_libraries_from_payload(payload)
payload = _request_json(
"GET",
_seafile_url(profile, f"api2/repos/{quote(repo_id, safe='')}/dir/"),
headers=headers,
params={"p": "/" + dir_path if dir_path else "/"},
)
return seafile_directory_items_from_payload(payload, path=dir_path)
def _browse_webdav(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowseItem]:
root_url = _metadata_string(profile, "webdav_endpoint_url") or profile.endpoint_url
if not root_url:
raise ConnectorBrowseError("Connector profile does not define an endpoint URL")
url = _webdav_url(root_url, path)
headers = {"Depth": "1", "Content-Type": "application/xml; charset=utf-8"}
auth: tuple[str, str] | None = None
password = _profile_password(profile)
token = _profile_token(profile)
if profile.username and password:
auth = (profile.username, password)
elif token:
headers["Authorization"] = f"Bearer {token}"
elif profile.credential_mode.casefold() not in {"", "none", "anonymous"} and profile.secret_ref:
raise ConnectorBrowseError("Secret-ref connector credentials need a runtime secret resolver before live browsing")
body = """<?xml version="1.0" encoding="utf-8"?>
<propfind xmlns="DAV:">
<prop>
<displayname />
<resourcetype />
<getcontentlength />
<getcontenttype />
<getlastmodified />
<getetag />
</prop>
</propfind>"""
try:
response = httpx.request("PROPFIND", url, headers=headers, content=body, auth=auth, timeout=15.0)
except httpx.HTTPError as exc:
raise ConnectorBrowseError(f"Connector browse failed: {exc}") from exc
if response.status_code in {401, 403}:
raise ConnectorBrowseError("Connector credentials were rejected")
if response.status_code not in {200, 207}:
raise ConnectorBrowseError(f"Connector browse failed with HTTP {response.status_code}")
return parse_webdav_multistatus(root_url=root_url, current_path=path, payload=response.text)
def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowseItem]:
location = _smb_location(profile)
unc_path = _smb_unc_path(location, path)
smbclient = _smbclient_module()
try:
entries = smbclient.scandir(unc_path, **_smb_client_kwargs(profile, location))
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"SMB connector browse failed: {exc}") from exc
items: list[ConnectorBrowseItem] = []
try:
with entries as iterator:
for entry in iterator:
name = _clean(getattr(entry, "name", None))
if not name or name in {".", ".."}:
continue
try:
is_dir = bool(entry.is_dir())
except Exception:
is_dir = False
stat_result = _smb_entry_stat(entry)
item_path = _join_browse_path(path, name)
items.append(
ConnectorBrowseItem(
kind="folder" if is_dir else "file",
name=name,
path=item_path,
external_id=f"{location.share}:{item_path}",
size_bytes=None if is_dir else _smb_stat_size(stat_result),
content_type=None if is_dir else mimetypes.guess_type(name)[0],
modified_at=_smb_stat_modified_at(stat_result),
etag=_smb_stat_revision(stat_result),
metadata={
"share": location.share,
**({"server": location.server} if _metadata_bool(profile, "expose_server_metadata", default=False) else {}),
},
)
)
except ConnectorBrowseError:
raise
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"SMB connector browse failed: {exc}") from exc
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
def _seafile_headers(profile: ConnectorProfile) -> dict[str, str]:
token = _seafile_token(profile)
return {"Authorization": f"Token {token}", "Accept": "application/json"}
def _seafile_token(profile: ConnectorProfile) -> str:
token = _profile_token(profile)
if token:
return token
password = _profile_password(profile)
if profile.username and password:
payload = _request_json(
"POST",
_seafile_url(profile, "api2/auth-token/"),
data={"username": profile.username, "password": password},
)
if not isinstance(payload, Mapping) or not _clean(payload.get("token")):
raise ConnectorBrowseError("Seafile did not return an account token")
return _clean(payload.get("token")) or ""
if profile.secret_ref:
raise ConnectorBrowseError("Secret-ref Seafile credentials need a runtime secret resolver before live browsing")
raise ConnectorBrowseError("Seafile connector profiles require token credentials or username plus password credentials")
def _seafile_url(profile: ConnectorProfile, suffix: str) -> str:
if not profile.endpoint_url:
raise ConnectorBrowseError("Seafile connector profile does not define endpoint_url")
return urljoin(profile.endpoint_url.rstrip("/") + "/", suffix.lstrip("/"))
def _request_json(
method: str,
url: str,
*,
headers: Mapping[str, str] | None = None,
params: Mapping[str, str] | None = None,
data: Mapping[str, str] | None = None,
) -> object:
try:
response = httpx.request(method, url, headers=dict(headers or {}), params=params, data=data, timeout=15.0)
except httpx.HTTPError as exc:
raise ConnectorBrowseError(f"Connector browse failed: {exc}") from exc
if response.status_code in {401, 403}:
raise ConnectorBrowseError("Connector credentials were rejected")
if response.status_code not in {200, 201}:
raise ConnectorBrowseError(f"Connector browse failed with HTTP {response.status_code}")
try:
return response.json()
except ValueError as exc:
raise ConnectorBrowseError("Connector returned invalid JSON") from exc
def _seafile_repo_and_path(*, path: str, library_id: str | None) -> tuple[str | None, str]:
repo_id = _clean(library_id)
if repo_id:
return repo_id, path
if not path:
return None, ""
first, _, rest = path.partition("/")
return first, rest
def _seafile_library_item(value: Mapping[str, Any]) -> ConnectorBrowseItem:
repo_id = _clean(value.get("id") or value.get("repo_id"))
name = _clean(value.get("name") or value.get("repo_name") or repo_id)
if not repo_id or not name:
raise ConnectorBrowseError("Seafile library entries require id and name")
return ConnectorBrowseItem(
kind="library",
name=name,
path=repo_id,
external_id=repo_id,
size_bytes=_int(value.get("size") or value.get("repo_size")),
modified_at=_timestamp(value.get("mtime")),
metadata={
key: value[key]
for key in ("type", "permission", "encrypted", "owner", "file_count")
if key in value
},
)
def _seafile_directory_item(value: Mapping[str, Any], *, parent_path: str) -> ConnectorBrowseItem:
name = _clean(value.get("name") or value.get("obj_name"))
if not name:
raise ConnectorBrowseError("Seafile directory entries require name")
item_type = str(value.get("type") or ("dir" if value.get("is_dir") else "file")).casefold()
kind = "folder" if item_type in {"dir", "folder"} else "file"
path = _join_browse_path(parent_path, name)
content_type = None if kind == "folder" else mimetypes.guess_type(name)[0]
return ConnectorBrowseItem(
kind=kind,
name=name,
path=path,
external_id=_clean(value.get("id") or value.get("obj_id")),
size_bytes=None if kind == "folder" else _int(value.get("size")),
content_type=content_type,
modified_at=_timestamp(value.get("mtime") or value.get("modified")),
etag=_clean(value.get("id") or value.get("obj_id")),
metadata={
key: value[key]
for key in ("permission", "modifier_email", "modifier_name")
if key in value
},
)
def _static_listing(profile: ConnectorProfile, *, path: str, library_id: str | None) -> list[ConnectorBrowseItem] | None:
listing = profile.metadata.get("static_listing")
if listing is None:
return None
if isinstance(listing, list):
raw_items = listing if path == "" else []
elif isinstance(listing, Mapping):
key = library_id or path
raw_items = listing.get(key)
if raw_items is None and key == "":
raw_items = listing.get("/")
else:
raise ConnectorBrowseError("Connector static_listing metadata must be a list or object")
if raw_items is None:
return []
if not isinstance(raw_items, list):
raise ConnectorBrowseError("Connector static_listing entries must be lists")
return sorted(
[_static_item(item, parent_path=path) for item in raw_items if isinstance(item, Mapping)],
key=lambda item: (item.kind != "library", item.kind != "folder", item.name.casefold(), item.path.casefold()),
)
def _static_item(value: Mapping[str, Any], *, parent_path: str) -> ConnectorBrowseItem:
kind = str(value.get("kind") or "file").strip().casefold()
if kind not in {"library", "folder", "file"}:
raise ConnectorBrowseError("Connector browse items must be library, folder or file")
name = str(value.get("name") or value.get("path") or "").strip()
if not name:
raise ConnectorBrowseError("Connector browse items require name")
path = normalize_connector_browse_path(value.get("path"))
if not path:
path = _join_browse_path(parent_path, name)
return ConnectorBrowseItem(
kind=kind,
name=name,
path=path,
external_id=_clean(value.get("external_id")),
external_url=_clean(value.get("external_url")),
size_bytes=_int(value.get("size_bytes")),
content_type=_clean(value.get("content_type")),
modified_at=_clean(value.get("modified_at")),
etag=_clean(value.get("etag")),
metadata=value.get("metadata") if isinstance(value.get("metadata"), Mapping) else {},
)
def _webdav_url(root_url: str, path: str) -> str:
base = root_url if root_url.endswith("/") else root_url + "/"
if not path:
return base
quoted = "/".join(quote(part, safe="") for part in path.split("/") if part)
return urljoin(base, quoted + "/")
def _url_path_root(root_url: str) -> str:
path = unquote(urlsplit(root_url).path or "/")
return path if path.endswith("/") else path + "/"
def _relative_href_path(root_path: str, href: str) -> str:
href_path = unquote(urlsplit(href).path or href).strip()
if href_path.startswith(root_path):
return normalize_connector_browse_path(href_path[len(root_path):])
return normalize_connector_browse_path(href_path.rsplit("/", 1)[-1])
def _webdav_prop(response: ET.Element) -> ET.Element:
prop = response.find("{DAV:}propstat/{DAV:}prop")
if prop is not None:
return prop
prop = response.find(".//{DAV:}prop")
return prop if prop is not None else ET.Element("prop")
def _webdav_display_name(prop: ET.Element | None) -> str | None:
return _clean(_text(prop, "{DAV:}displayname"))
def _text(prop: ET.Element | None, tag: str) -> str | None:
if prop is None:
return None
child = prop.find(tag)
return child.text.strip() if child is not None and child.text else None
def _http_date(value: str | None) -> str | None:
if not value:
return None
try:
return parsedate_to_datetime(value).isoformat()
except (TypeError, ValueError):
return value
def _timestamp(value: object) -> str | None:
number = _int(value)
if number is None:
return _clean(value)
return datetime.fromtimestamp(number, tz=timezone.utc).isoformat()
def _metadata_string(profile: ConnectorProfile, key: str) -> str | None:
return _clean(profile.metadata.get(key))
def _metadata_bool(profile: ConnectorProfile, key: str, *, default: bool = False) -> bool:
value = profile.metadata.get(key)
if value is None:
return default
if isinstance(value, bool):
return value
return str(value).strip().casefold() in {"1", "true", "yes", "on"}
def _env_required(name: str, profile_id: str) -> str:
value = _clean(os.environ.get(name))
if not value:
raise ConnectorBrowseError(f"Connector profile {profile_id} is missing required credential environment")
return value
def normalize_connector_browse_path(value: object) -> str:
if value is None:
return ""
path = str(value).replace("\\", "/").strip().strip("/")
parts = [part for part in path.split("/") if part and part not in {"."}]
if any(part == ".." for part in parts):
raise ConnectorBrowseError("Connector browse paths cannot contain parent directory segments")
return "/".join(parts)
def _join_browse_path(parent: str, name: str) -> str:
child = normalize_connector_browse_path(name)
if not parent:
return child
return f"{parent.rstrip('/')}/{child}"
def _path_name(path: str) -> str:
return path.rstrip("/").rsplit("/", 1)[-1]
def _int(value: object) -> int | None:
if value is None or value == "":
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _clean(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
@dataclass(frozen=True, slots=True)
class _SmbLocation:
server: str
share: str
port: int
root_path: str = ""
def _smb_location(profile: ConnectorProfile) -> _SmbLocation:
if not profile.endpoint_url:
raise ConnectorBrowseError("SMB connector profile does not define endpoint_url")
parsed = urlsplit(profile.endpoint_url)
if parsed.scheme.casefold() != "smb":
raise ConnectorBrowseError("SMB connector endpoint_url must use smb://")
server = _clean(parsed.hostname)
if not server:
raise ConnectorBrowseError("SMB connector endpoint_url must include a server")
path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part]
if not path_parts:
raise ConnectorBrowseError("SMB connector endpoint_url must include a share name")
root_parts = path_parts[1:]
if profile.base_path:
root_parts.extend(normalize_connector_browse_path(profile.base_path).split("/"))
return _SmbLocation(
server=server,
share=path_parts[0],
port=parsed.port or _int(profile.metadata.get("port")) or 445,
root_path=normalize_connector_browse_path("/".join(root_parts)),
)
def _smb_unc_path(location: _SmbLocation, path: str) -> str:
parts = [part for part in (location.root_path, normalize_connector_browse_path(path)) if part]
suffix = "\\".join(part.replace("/", "\\") for part in parts)
base = f"\\\\{location.server}\\{location.share}"
return f"{base}\\{suffix}" if suffix else base
def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]:
kwargs: dict[str, object] = {
"port": location.port,
"require_signing": _metadata_bool(profile, "require_signing", default=True),
"auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm",
}
if "encrypt" in profile.metadata:
kwargs["encrypt"] = _metadata_bool(profile, "encrypt", default=False)
password = _profile_password(profile)
token = _profile_token(profile)
if profile.username and password:
kwargs["username"] = profile.username
kwargs["password"] = password
elif token:
raise ConnectorBrowseError("SMB connector profiles do not support bearer-token credentials")
elif profile.credential_mode.casefold() not in {"", "none", "anonymous"} and profile.secret_ref:
raise ConnectorBrowseError("Secret-ref SMB credentials need a runtime secret resolver before live browsing")
return kwargs
def _profile_password(profile: ConnectorProfile) -> str | None:
if profile.password_value:
return profile.password_value
if profile.password_env:
return _env_required(profile.password_env, profile.id)
return None
def _profile_token(profile: ConnectorProfile) -> str | None:
if profile.token_value:
return profile.token_value
if profile.token_env:
return _env_required(profile.token_env, profile.id)
return None
def _smbclient_module() -> Any:
try:
return import_module("smbclient")
except ImportError as exc:
raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc
def _smb_entry_stat(entry: object) -> object | None:
try:
return entry.stat() # type: ignore[attr-defined]
except Exception:
return None
def _smb_stat_size(stat_result: object | None) -> int | None:
return _int(getattr(stat_result, "st_size", None))
def _smb_stat_modified_at(stat_result: object | None) -> str | None:
value = getattr(stat_result, "st_mtime", None)
if value is None:
return None
try:
return datetime.fromtimestamp(float(value), tz=timezone.utc).isoformat()
except (TypeError, ValueError, OSError, OverflowError):
return None
def _smb_stat_revision(stat_result: object | None) -> str | None:
mtime_ns = getattr(stat_result, "st_mtime_ns", None)
size = getattr(stat_result, "st_size", None)
if mtime_ns is not None:
return f"{mtime_ns}:{size or 0}"
modified = getattr(stat_result, "st_mtime", None)
if modified is not None:
return f"{modified}:{size or 0}"
return None

340
src/govoplan_files/backend/storage/connector_credential_store.py Normal file
View File

@@ -0,0 +1,340 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
from sqlalchemy.orm import Session
from govoplan_core.core.policy import normalize_policy_scope_type, policy_source_path
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
from govoplan_files.backend.db.models import FileConnectorCredential
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_policy import ConnectorPolicySource, connector_policy_sources_from_payload
from govoplan_files.backend.storage.connector_profiles import supported_connector_providers
@dataclass(frozen=True, slots=True)
class ConnectorCredential:
id: str
label: str
scope_type: str
scope_id: str | None = None
provider: str | None = None
enabled: bool = True
credential_mode: str = "none"
username: str | None = None
password_env: str | None = None
token_env: str | None = None
secret_ref: str | None = None
password_value: str | None = None
token_value: str | None = None
policy_sources: tuple[ConnectorPolicySource, ...] = ()
metadata: Mapping[str, Any] | None = None
source_kind: str = "database"
@property
def source_path(self) -> str:
return policy_source_path(self.scope_type, self.scope_id)
@property
def credential_secret_source(self) -> str | None:
if self.secret_ref:
return "secret_ref"
if self.password_value or self.token_value:
return "stored"
if self.token_env:
return "token_env"
if self.password_env:
return "password_env"
return None
@property
def credentials_configured(self) -> bool:
if self.credential_mode.casefold() in {"", "none", "anonymous"}:
return True
return bool(self.secret_ref or self.password_value or self.token_value or self.password_env or self.token_env)
def to_response(self) -> dict[str, Any]:
return {
"id": self.id,
"label": self.label,
"provider": self.provider,
"enabled": self.enabled,
"scope_type": self.scope_type,
"scope_id": self.scope_id,
"source_path": self.source_path,
"credential_mode": self.credential_mode,
"credential_secret_source": self.credential_secret_source,
"credentials_configured": self.credentials_configured,
"username": self.username,
"policy_sources": [_policy_source_response(source) for source in self.policy_sources],
"metadata": dict(self.metadata or {}),
"source_kind": self.source_kind,
}
def list_database_connector_credentials(
session: Session,
*,
tenant_id: str,
include_disabled: bool = False,
) -> list[ConnectorCredential]:
return [connector_credential_from_row(row) for row in list_connector_credential_rows(session, tenant_id=tenant_id, include_disabled=include_disabled)]
def list_connector_credential_rows(
session: Session,
*,
tenant_id: str,
include_disabled: bool = False,
) -> list[FileConnectorCredential]:
query = session.query(FileConnectorCredential).filter(
(FileConnectorCredential.scope_type == "system")
| (FileConnectorCredential.tenant_id == tenant_id)
)
if not include_disabled:
query = query.filter(FileConnectorCredential.enabled.is_(True))
return query.order_by(FileConnectorCredential.scope_type.asc(), FileConnectorCredential.label.asc()).all()
def connector_credential_from_row(row: FileConnectorCredential) -> ConnectorCredential:
policy_sources = []
if row.policy:
policy_sources = connector_policy_sources_from_payload({
"scope_type": row.scope_type,
"scope_id": row.scope_id,
"label": row.label,
"policy": row.policy,
})
return ConnectorCredential(
id=row.id,
label=row.label,
scope_type=row.scope_type,
scope_id=row.scope_id,
provider=row.provider,
enabled=row.enabled,
credential_mode=row.credential_mode,
username=row.username,
password_env=row.password_env,
token_env=row.token_env,
secret_ref=row.secret_ref,
password_value=decrypt_secret(row.password_encrypted),
token_value=decrypt_secret(row.token_encrypted),
policy_sources=tuple(policy_sources),
metadata=dict(row.metadata_ or {}),
)
def get_connector_credential_row(
session: Session,
*,
tenant_id: str,
credential_id: str,
include_disabled: bool = False,
) -> FileConnectorCredential:
row = session.get(FileConnectorCredential, credential_id)
if row is None or (row.scope_type != "system" and row.tenant_id != tenant_id):
raise FileStorageError("Connector credential not found")
if not include_disabled and not row.enabled:
raise FileStorageError("Connector credential not found")
return row
def credential_rows_by_id(
session: Session,
*,
tenant_id: str,
credential_ids: set[str],
include_disabled: bool = False,
) -> dict[str, FileConnectorCredential]:
if not credential_ids:
return {}
rows = session.query(FileConnectorCredential).filter(FileConnectorCredential.id.in_(credential_ids)).all()
result: dict[str, FileConnectorCredential] = {}
for row in rows:
if row.scope_type != "system" and row.tenant_id != tenant_id:
continue
if not include_disabled and not row.enabled:
continue
result[row.id] = row
return result
def create_connector_credential_row(
session: Session,
*,
tenant_id: str,
user_id: str | None,
credential_id: str,
label: str,
scope_type: str,
scope_id: str | None = None,
provider: str | None = None,
enabled: bool = True,
credential_mode: str = "none",
username: str | None = None,
password: str | None = None,
token: str | None = None,
password_env: str | None = None,
token_env: str | None = None,
secret_ref: str | None = None,
policy: Mapping[str, Any] | None = None,
metadata: Mapping[str, Any] | None = None,
) -> FileConnectorCredential:
clean_id = _normalize_id(credential_id)
if session.get(FileConnectorCredential, clean_id) is not None:
raise FileStorageError(f"Connector credential already exists: {clean_id}")
clean_scope_type, clean_scope_id, row_tenant_id = _normalize_scope(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
row = FileConnectorCredential(
id=clean_id,
tenant_id=row_tenant_id,
scope_type=clean_scope_type,
scope_id=clean_scope_id,
label=_normalize_label(label),
provider=_normalize_provider(provider),
enabled=bool(enabled),
credential_mode=_normalize_credential_mode(credential_mode),
username=_clean(username),
password_encrypted=encrypt_secret(_clean(password)),
token_encrypted=encrypt_secret(_clean(token)),
password_env=_clean(password_env),
token_env=_clean(token_env),
secret_ref=_clean(secret_ref),
policy=dict(policy or {}),
metadata_=dict(metadata or {}),
created_by_user_id=user_id,
updated_by_user_id=user_id,
)
session.add(row)
session.flush()
return row
def update_connector_credential_row(
session: Session,
row: FileConnectorCredential,
*,
user_id: str | None,
label: str | None = None,
provider: str | None = None,
enabled: bool | None = None,
credential_mode: str | None = None,
username: str | None = None,
password: str | None = None,
token: str | None = None,
password_env: str | None = None,
token_env: str | None = None,
secret_ref: str | None = None,
policy: Mapping[str, Any] | None = None,
metadata: Mapping[str, Any] | None = None,
clear_password: bool = False,
clear_token: bool = False,
) -> FileConnectorCredential:
if label is not None:
row.label = _normalize_label(label)
if provider is not None:
row.provider = _normalize_provider(provider)
if enabled is not None:
row.enabled = bool(enabled)
if credential_mode is not None:
row.credential_mode = _normalize_credential_mode(credential_mode)
if username is not None:
row.username = _clean(username)
if password is not None:
row.password_encrypted = encrypt_secret(_clean(password))
elif clear_password:
row.password_encrypted = None
if token is not None:
row.token_encrypted = encrypt_secret(_clean(token))
elif clear_token:
row.token_encrypted = None
if password_env is not None:
row.password_env = _clean(password_env)
if token_env is not None:
row.token_env = _clean(token_env)
if secret_ref is not None:
row.secret_ref = _clean(secret_ref)
if policy is not None:
row.policy = dict(policy)
if metadata is not None:
row.metadata_ = dict(metadata)
row.updated_by_user_id = user_id
session.add(row)
session.flush()
return row
def deactivate_connector_credential_row(session: Session, row: FileConnectorCredential, *, user_id: str | None) -> FileConnectorCredential:
row.enabled = False
row.updated_by_user_id = user_id
session.add(row)
session.flush()
return row
def _normalize_scope(*, tenant_id: str, scope_type: str, scope_id: str | None) -> tuple[str, str | None, str | None]:
clean_scope_type = normalize_policy_scope_type(scope_type)
clean_scope_id = _clean(scope_id)
if clean_scope_type == "system":
return "system", None, None
if clean_scope_type == "tenant":
return "tenant", tenant_id, tenant_id
if clean_scope_type in {"user", "group", "campaign"}:
if not clean_scope_id:
raise FileStorageError(f"{clean_scope_type.capitalize()} connector credentials require scope_id")
return clean_scope_type, clean_scope_id, tenant_id
raise FileStorageError("Unsupported connector credential scope")
def _normalize_id(value: str) -> str:
clean = _clean(value)
if not clean:
raise FileStorageError("Connector credential id is required")
if len(clean) > 255:
raise FileStorageError("Connector credential id is too long")
if any(char.isspace() for char in clean):
raise FileStorageError("Connector credential id cannot contain whitespace")
return clean
def _normalize_label(value: str) -> str:
clean = value.strip()
if not clean:
raise FileStorageError("Connector credential label is required")
if len(clean) > 255:
raise FileStorageError("Connector credential label is too long")
return clean
def _normalize_provider(value: str | None) -> str | None:
clean = _clean(value)
if clean is None:
return None
clean = clean.casefold()
if clean not in supported_connector_providers():
raise FileStorageError(f"Unsupported connector credential provider: {value}")
return clean
def _normalize_credential_mode(value: str) -> str:
clean = value.strip().casefold() or "none"
if clean not in {"none", "anonymous", "basic", "token", "secret_ref"}:
raise FileStorageError(f"Unsupported connector credential mode: {value}")
return clean
def _policy_source_response(source: ConnectorPolicySource) -> dict[str, Any]:
return {
"scope_type": source.scope_type,
"scope_id": source.scope_id,
"label": source.label,
"policy": dict(source.policy or {}),
}
def _clean(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None

222
src/govoplan_files/backend/storage/connector_imports.py Normal file
View File

@@ -0,0 +1,222 @@
from __future__ import annotations
from dataclasses import dataclass, field
import mimetypes
from typing import Any
import httpx
from govoplan_files.backend.storage.connector_browse import (
ConnectorBrowseError,
ConnectorBrowseUnsupported,
_clean,
_metadata_string,
_profile_password,
_profile_token,
_request_json,
_smb_client_kwargs,
_smb_location,
_smb_stat_modified_at,
_smb_stat_revision,
_smb_stat_size,
_smb_unc_path,
_smbclient_module,
_seafile_headers,
_seafile_url,
_webdav_url,
normalize_connector_browse_path,
)
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
from govoplan_files.backend.storage.paths import filename_from_path
class ConnectorImportError(RuntimeError):
pass
class ConnectorImportUnsupported(ConnectorImportError):
pass
@dataclass(frozen=True, slots=True)
class ConnectorDownloadedFile:
filename: str
data: bytes
content_type: str | None = None
revision: str | None = None
external_id: str | None = None
external_url: str | None = None
metadata: dict[str, Any] = field(default_factory=dict)
def read_connector_file(
profile: ConnectorProfile,
*,
library_id: str,
path: str,
max_bytes: int,
) -> ConnectorDownloadedFile:
if profile.provider == "seafile":
if _metadata_string(profile, "browse_protocol") == "webdav" or _metadata_string(profile, "webdav_endpoint_url"):
return _read_webdav_file(profile, path=path, max_bytes=max_bytes)
return _read_seafile_file(profile, library_id=library_id, path=path, max_bytes=max_bytes)
if profile.provider in {"webdav", "nextcloud"}:
return _read_webdav_file(profile, path=path, max_bytes=max_bytes)
if profile.provider == "smb":
return _read_smb_file(profile, path=path, max_bytes=max_bytes)
raise ConnectorImportUnsupported(f"Connector file import is not implemented for {profile.provider} profiles yet")
def _read_seafile_file(profile: ConnectorProfile, *, library_id: str, path: str, max_bytes: int) -> ConnectorDownloadedFile:
repo_id = _clean(library_id)
if not repo_id:
raise ConnectorImportError("Seafile import requires library_id")
file_path = "/" + normalize_connector_browse_path(path)
if file_path == "/":
raise ConnectorImportError("Seafile import requires a file path")
try:
headers = _seafile_headers(profile)
detail = _request_json(
"GET",
_seafile_url(profile, f"api2/repos/{repo_id}/file/detail/"),
headers=headers,
params={"p": file_path},
)
if isinstance(detail, dict):
size = _int(detail.get("size"))
if size is not None and size > max_bytes:
raise ConnectorImportError(f"Seafile file exceeds limit of {max_bytes} bytes")
download_url = _request_json(
"GET",
_seafile_url(profile, f"api2/repos/{repo_id}/file/"),
headers=headers,
params={"p": file_path, "reuse": "1"},
)
except ConnectorBrowseError as exc:
raise ConnectorImportError(str(exc)) from exc
if not isinstance(download_url, str) or not download_url.strip():
raise ConnectorImportError("Seafile did not return a file download URL")
try:
response = httpx.request("GET", download_url, timeout=30.0)
except httpx.HTTPError as exc:
raise ConnectorImportError(f"Seafile file download failed: {exc}") from exc
if response.status_code != 200:
raise ConnectorImportError(f"Seafile file download failed with HTTP {response.status_code}")
data = response.content
if len(data) > max_bytes:
raise ConnectorImportError(f"Seafile file exceeds limit of {max_bytes} bytes")
detail = detail if isinstance(detail, dict) else {}
filename = filename_from_path(str(detail.get("name") or path))
content_type = response.headers.get("content-type") or mimetypes.guess_type(filename)[0]
external_id = f"{repo_id}:{normalize_connector_browse_path(path)}"
return ConnectorDownloadedFile(
filename=filename,
data=data,
content_type=content_type,
revision=_clean(detail.get("id") or detail.get("mtime") or detail.get("last_modified")),
external_id=external_id,
external_url=download_url,
metadata={
"library_id": repo_id,
"library_path": normalize_connector_browse_path(path),
"size": len(data),
**({key: detail[key] for key in ("permission", "last_modified", "mtime", "uploader_email", "last_modifier_email") if key in detail}),
},
)
def _read_webdav_file(profile: ConnectorProfile, *, path: str, max_bytes: int) -> ConnectorDownloadedFile:
root_url = _metadata_string(profile, "webdav_endpoint_url") or profile.endpoint_url
if not root_url:
raise ConnectorImportError("WebDAV connector profile does not define endpoint_url")
file_path = normalize_connector_browse_path(path)
if not file_path:
raise ConnectorImportError("WebDAV import requires a file path")
url = _webdav_url(root_url, file_path).rstrip("/")
headers: dict[str, str] = {}
auth: tuple[str, str] | None = None
password = _profile_password(profile)
token = _profile_token(profile)
if profile.username and password:
auth = (profile.username, password)
elif token:
headers["Authorization"] = f"Bearer {token}"
elif profile.credential_mode.casefold() not in {"", "none", "anonymous"} and profile.secret_ref:
raise ConnectorImportError("Secret-ref WebDAV credentials need a runtime secret resolver before live import")
try:
response = httpx.request("GET", url, headers=headers, auth=auth, timeout=30.0)
except httpx.HTTPError as exc:
raise ConnectorImportError(f"WebDAV file download failed: {exc}") from exc
if response.status_code in {401, 403}:
raise ConnectorImportError("Connector credentials were rejected")
if response.status_code != 200:
raise ConnectorImportError(f"WebDAV file download failed with HTTP {response.status_code}")
length = _int(response.headers.get("content-length"))
if length is not None and length > max_bytes:
raise ConnectorImportError(f"WebDAV file exceeds limit of {max_bytes} bytes")
data = response.content
if len(data) > max_bytes:
raise ConnectorImportError(f"WebDAV file exceeds limit of {max_bytes} bytes")
filename = filename_from_path(file_path)
revision = _clean(response.headers.get("etag") or response.headers.get("last-modified"))
return ConnectorDownloadedFile(
filename=filename,
data=data,
content_type=response.headers.get("content-type") or mimetypes.guess_type(filename)[0],
revision=revision,
external_id=file_path,
external_url=url,
metadata={"path": file_path, "size": len(data)},
)
def _read_smb_file(profile: ConnectorProfile, *, path: str, max_bytes: int) -> ConnectorDownloadedFile:
location = _smb_location(profile)
file_path = normalize_connector_browse_path(path)
if not file_path:
raise ConnectorImportError("SMB import requires a file path")
unc_path = _smb_unc_path(location, file_path)
try:
smbclient = _smbclient_module()
kwargs = _smb_client_kwargs(profile, location)
stat_result = smbclient.stat(unc_path, **kwargs)
size = _smb_stat_size(stat_result)
if size is not None and size > max_bytes:
raise ConnectorImportError(f"SMB file exceeds limit of {max_bytes} bytes")
with smbclient.open_file(unc_path, mode="rb", **kwargs) as handle:
data = handle.read(max_bytes + 1)
except ConnectorBrowseUnsupported as exc:
raise ConnectorImportUnsupported(str(exc)) from exc
except ConnectorBrowseError as exc:
raise ConnectorImportError(str(exc)) from exc
except ConnectorImportError:
raise
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorImportError(f"SMB file download failed: {exc}") from exc
if len(data) > max_bytes:
raise ConnectorImportError(f"SMB file exceeds limit of {max_bytes} bytes")
filename = filename_from_path(file_path)
revision = _smb_stat_revision(stat_result)
return ConnectorDownloadedFile(
filename=filename,
data=data,
content_type=mimetypes.guess_type(filename)[0],
revision=revision,
external_id=f"{location.share}:{file_path}",
external_url=f"smb://{location.server}:{location.port}/{location.share}/{file_path}",
metadata={
"share": location.share,
"path": file_path,
"size": len(data),
"modified_at": _smb_stat_modified_at(stat_result),
},
)
def _int(value: object) -> int | None:
if value is None or value == "":
return None
try:
return int(value)
except (TypeError, ValueError):
return None

310
src/govoplan_files/backend/storage/connector_policy.py Normal file
View File

@@ -0,0 +1,310 @@
from __future__ import annotations
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from fnmatch import fnmatchcase
from typing import Any
from govoplan_core.core.policy import PolicyDecision, PolicySourceStep, normalize_policy_scope_type
_ALLOW_KEYS = ("allow", "allowlist", "whitelist")
_DENY_KEYS = ("deny", "denylist", "blacklist")
_FIELD_ALIASES = {
"connectors": ("connectors", "connector_ids", "connector_id"),
"credentials": ("credentials", "credential_ids", "credential_id"),
"providers": ("providers", "provider"),
"external_ids": ("external_ids", "external_id"),
"external_paths": ("external_paths", "paths", "path_prefixes", "external_path"),
"external_urls": ("external_urls", "urls", "external_url"),
}
CONNECTOR_POLICY_FIELDS = tuple(_FIELD_ALIASES)
class ConnectorPolicyDenied(RuntimeError):
def __init__(self, decision: PolicyDecision) -> None:
super().__init__(decision.reason or "Connector policy denied")
self.decision = decision
@dataclass(frozen=True, slots=True)
class ConnectorAccessRequest:
connector_id: str | None = None
credential_id: str | None = None
provider: str | None = None
external_id: str | None = None
external_path: str | None = None
external_url: str | None = None
operation: str = "access"
@classmethod
def from_provenance(cls, value: Mapping[str, Any] | None, *, operation: str = "access") -> "ConnectorAccessRequest":
payload = value or {}
return cls(
connector_id=_clean(payload.get("connector_id")),
credential_id=_clean(payload.get("credential_id") or payload.get("connector_credential_id")),
provider=_clean(payload.get("provider") or payload.get("source_type")),
external_id=_clean(payload.get("external_id")),
external_path=_clean(payload.get("external_path")),
external_url=_clean(payload.get("external_url")),
operation=_clean(operation) or "access",
)
def to_dict(self) -> dict[str, str | None]:
return {
"connector_id": self.connector_id,
"credential_id": self.credential_id,
"provider": self.provider,
"external_id": self.external_id,
"external_path": self.external_path,
"external_url": self.external_url,
"operation": self.operation,
}
@dataclass(frozen=True, slots=True)
class ConnectorPolicySource:
scope_type: str
label: str
scope_id: str | None = None
policy: Mapping[str, Any] | None = None
def source_step(self, applied_fields: Iterable[str]) -> PolicySourceStep:
return PolicySourceStep(
scope_type=normalize_policy_scope_type(self.scope_type),
scope_id=self.scope_id,
label=self.label,
applied_fields=tuple(dict.fromkeys(applied_fields)),
policy=dict(self.policy or {}),
)
def connector_policy_sources_from_payload(value: object) -> list[ConnectorPolicySource]:
if value is None or value == "":
return []
if isinstance(value, Mapping):
raw_sources = value.get("sources")
if raw_sources is None:
raw_sources = [value]
elif isinstance(value, list):
raw_sources = value
else:
raise ValueError("connector_policy must be a JSON object or list")
if not isinstance(raw_sources, list):
raise ValueError("connector_policy.sources must be a list")
return [_source_from_mapping(item) for item in raw_sources if isinstance(item, Mapping)]
def connector_policy_applied_fields(policy: Mapping[str, Any] | None) -> tuple[str, ...]:
return _applied_fields(_policy_mapping(policy))
def filtered_connector_policy_source(source: ConnectorPolicySource, fields: Iterable[str]) -> ConnectorPolicySource:
allowed_fields = {field for field in fields if field in _FIELD_ALIASES}
if not allowed_fields:
return ConnectorPolicySource(scope_type=source.scope_type, scope_id=source.scope_id, label=source.label, policy={})
policy = _policy_mapping(source.policy)
filtered: dict[str, Any] = {}
for prefix, keys in (("allow", _ALLOW_KEYS), ("deny", _DENY_KEYS)):
rules = _merged_rule(policy, keys)
selected = {field: _string_list(rules.get(field)) for field in allowed_fields}
selected = {field: values for field, values in selected.items() if values}
if selected:
filtered[prefix] = selected
return ConnectorPolicySource(scope_type=source.scope_type, scope_id=source.scope_id, label=source.label, policy=filtered)
def connector_policy_sources_for_fields(
sources: Iterable[ConnectorPolicySource],
fields: Iterable[str],
) -> list[ConnectorPolicySource]:
return [filtered_connector_policy_source(source, fields) for source in sources]
def connector_policy_decision(
request: ConnectorAccessRequest,
sources: Iterable[ConnectorPolicySource],
) -> PolicyDecision:
source_list = list(sources)
source_steps: list[PolicySourceStep] = []
deny_matches: list[dict[str, Any]] = []
allow_misses: list[dict[str, Any]] = []
for source in source_list:
policy = _policy_mapping(source.policy)
applied_fields = _applied_fields(policy)
if applied_fields:
source_steps.append(source.source_step(applied_fields))
deny_policy = _merged_rule(policy, _DENY_KEYS)
for field, patterns in _rules_by_field(deny_policy).items():
if _matches_field(request, field, patterns):
deny_matches.append({
"scope": source.source_step((f"deny.{field}",)).to_dict(),
"field": field,
"patterns": patterns,
})
allow_policy = _merged_rule(policy, _ALLOW_KEYS)
for field, patterns in _rules_by_field(allow_policy).items():
if not _matches_field(request, field, patterns):
allow_misses.append({
"scope": source.source_step((f"allow.{field}",)).to_dict(),
"field": field,
"patterns": patterns,
})
details = {
"request": request.to_dict(),
"deny_matches": deny_matches,
"allow_misses": allow_misses,
}
if deny_matches:
return PolicyDecision(
allowed=False,
reason="Connector access is denied by a blacklist rule.",
source_path=tuple(source_steps),
requirements=("connector_policy_denylist",),
details=details,
)
if allow_misses:
return PolicyDecision(
allowed=False,
reason="Connector access is not allowed by the configured whitelist.",
source_path=tuple(source_steps),
requirements=("connector_policy_allowlist",),
details=details,
)
return PolicyDecision(
allowed=True,
reason=None,
source_path=tuple(source_steps),
requirements=(),
details=details,
)
def ensure_connector_policy_allows(
request: ConnectorAccessRequest,
sources: Iterable[ConnectorPolicySource],
) -> PolicyDecision:
decision = connector_policy_decision(request, sources)
if not decision.allowed:
raise ConnectorPolicyDenied(decision)
return decision
def _source_from_mapping(value: Mapping[str, Any]) -> ConnectorPolicySource:
scope_type = normalize_policy_scope_type(str(value.get("scope_type") or "system"))
scope_id = _clean(value.get("scope_id"))
if scope_type == "system":
scope_id = None
elif not scope_id:
raise ValueError(f"{scope_type} connector policy sources require scope_id")
label = _clean(value.get("label")) or scope_type.capitalize()
policy = value.get("policy")
if policy is None:
policy = {key: value[key] for key in (*_ALLOW_KEYS, *_DENY_KEYS) if key in value}
if policy is not None and not isinstance(policy, Mapping):
raise ValueError("connector policy source policy must be a JSON object")
return ConnectorPolicySource(scope_type=scope_type, scope_id=scope_id, label=label, policy=policy or {})
def _policy_mapping(value: Mapping[str, Any] | None) -> Mapping[str, Any]:
return value if isinstance(value, Mapping) else {}
def _merged_rule(policy: Mapping[str, Any], keys: Iterable[str]) -> dict[str, Any]:
merged: dict[str, Any] = {}
for key in keys:
raw = policy.get(key)
if isinstance(raw, Mapping):
merged.update(raw)
return merged
def _rules_by_field(value: Mapping[str, Any]) -> dict[str, list[str]]:
result: dict[str, list[str]] = {}
for field, aliases in _FIELD_ALIASES.items():
items: list[str] = []
for alias in aliases:
items.extend(_string_list(value.get(alias)))
if items:
result[field] = list(dict.fromkeys(items))
return result
def _applied_fields(policy: Mapping[str, Any]) -> tuple[str, ...]:
fields: list[str] = []
for prefix, keys in (("allow", _ALLOW_KEYS), ("deny", _DENY_KEYS)):
rules = _merged_rule(policy, keys)
fields.extend(f"{prefix}.{field}" for field in _rules_by_field(rules))
return tuple(dict.fromkeys(fields))
def _matches_field(request: ConnectorAccessRequest, field: str, patterns: list[str]) -> bool:
if field == "connectors":
return _matches_exact(request.connector_id, patterns)
if field == "credentials":
return _matches_exact(request.credential_id, patterns)
if field == "providers":
return _matches_exact(request.provider, patterns)
if field == "external_ids":
return _matches_glob(request.external_id, patterns)
if field == "external_paths":
return _matches_path(request.external_path, patterns)
if field == "external_urls":
return _matches_glob(request.external_url, patterns)
return False
def _matches_exact(value: str | None, patterns: list[str]) -> bool:
if value is None:
return False
clean = value.casefold()
return any(pattern == "*" or clean == pattern.casefold() for pattern in patterns)
def _matches_glob(value: str | None, patterns: list[str]) -> bool:
if value is None:
return False
clean = value.casefold()
return any(fnmatchcase(clean, pattern.casefold()) for pattern in patterns)
def _matches_path(value: str | None, patterns: list[str]) -> bool:
if value is None:
return False
clean = value.replace("\\", "/").strip()
for pattern in patterns:
normalized = pattern.replace("\\", "/").strip()
if normalized == "*":
return True
if any(token in normalized for token in "*?[]"):
if fnmatchcase(clean.casefold(), normalized.casefold()):
return True
continue
if clean.casefold() == normalized.casefold() or clean.casefold().startswith(normalized.rstrip("/").casefold() + "/"):
return True
return False
def _string_list(value: object) -> list[str]:
if value is None:
return []
if isinstance(value, str):
values = [value]
elif isinstance(value, Iterable) and not isinstance(value, (bytes, bytearray, Mapping)):
values = [str(item) for item in value]
else:
values = [str(value)]
return [item.strip() for item in values if item.strip()]
def _clean(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None

334
src/govoplan_files/backend/storage/connector_policy_store.py Normal file
View File

@@ -0,0 +1,334 @@
from __future__ import annotations
from collections.abc import Iterable, Mapping
from typing import Any
from sqlalchemy.orm import Session
from govoplan_core.core.policy import normalize_policy_scope_type
from govoplan_files.backend.db.models import FileConnectorPolicy
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_policy import (
CONNECTOR_POLICY_FIELDS,
ConnectorPolicySource,
connector_policy_applied_fields,
)
_RULE_GROUPS = ("allow", "deny")
_FIELD_ALIASES = {
"connectors": ("connectors", "connector_ids", "connector_id"),
"credentials": ("credentials", "credential_ids", "credential_id"),
"providers": ("providers", "provider"),
"external_ids": ("external_ids", "external_id"),
"external_paths": ("external_paths", "paths", "path_prefixes", "external_path"),
"external_urls": ("external_urls", "urls", "external_url"),
}
def get_connector_policy(
session: Session,
*,
tenant_id: str,
scope_type: str,
scope_id: str | None = None,
) -> dict[str, Any]:
row = _connector_policy_row(session, tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
return _normalize_connector_policy(row.policy if row is not None else None)
def set_connector_policy(
session: Session,
*,
tenant_id: str,
user_id: str | None,
scope_type: str,
scope_id: str | None = None,
policy: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
normalized = _normalize_connector_policy(policy)
parent = parent_connector_policy(session, tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
if parent is not None:
_validate_lower_level_limits(parent, normalized)
row_tenant_id, row_scope_type, row_scope_id = _policy_scope_key(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
row = _connector_policy_row(session, tenant_id=tenant_id, scope_type=row_scope_type, scope_id=row_scope_id)
if row is None:
row = FileConnectorPolicy(
tenant_id=row_tenant_id,
scope_type=row_scope_type,
scope_id=row_scope_id,
policy=normalized,
created_by_user_id=user_id,
updated_by_user_id=user_id,
)
else:
row.policy = normalized
row.updated_by_user_id = user_id
session.add(row)
session.flush()
return normalized
def connector_policy_response(
session: Session,
*,
tenant_id: str,
scope_type: str,
scope_id: str | None = None,
) -> dict[str, Any]:
clean_scope_type, clean_scope_id = _policy_scope_ref(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
policy = get_connector_policy(session, tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id)
effective_sources = effective_connector_policy_sources(session, tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id)
parent_policy = parent_connector_policy(session, tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id)
parent_sources = parent_connector_policy_sources(session, tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id)
return {
"scope_type": clean_scope_type,
"scope_id": clean_scope_id,
"policy": policy,
"effective_policy": merge_connector_policies(source.policy for source in effective_sources),
"parent_policy": parent_policy,
"effective_policy_sources": [_source_step(source) for source in effective_sources],
"parent_policy_sources": [_source_step(source) for source in parent_sources],
}
def effective_connector_policy_sources(
session: Session,
*,
tenant_id: str,
scope_type: str,
scope_id: str | None = None,
) -> list[ConnectorPolicySource]:
clean_scope_type, clean_scope_id = _policy_scope_ref(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
sources: list[ConnectorPolicySource] = []
for source_scope_type, source_scope_id in _policy_source_chain(tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id):
row = _connector_policy_row(session, tenant_id=tenant_id, scope_type=source_scope_type, scope_id=source_scope_id)
if row is None:
continue
policy = _normalize_connector_policy(row.policy)
if _policy_is_empty(policy):
continue
sources.append(
ConnectorPolicySource(
scope_type=source_scope_type,
scope_id=source_scope_id,
label=_policy_source_label(source_scope_type),
policy=policy,
)
)
return sources
def parent_connector_policy_sources(
session: Session,
*,
tenant_id: str,
scope_type: str,
scope_id: str | None = None,
) -> list[ConnectorPolicySource]:
clean_scope_type, clean_scope_id = _policy_scope_ref(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
chain = _policy_source_chain(tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id)
parent_chain = chain[:-1] if chain else []
sources: list[ConnectorPolicySource] = []
for source_scope_type, source_scope_id in parent_chain:
row = _connector_policy_row(session, tenant_id=tenant_id, scope_type=source_scope_type, scope_id=source_scope_id)
if row is None:
continue
policy = _normalize_connector_policy(row.policy)
if _policy_is_empty(policy):
continue
sources.append(
ConnectorPolicySource(
scope_type=source_scope_type,
scope_id=source_scope_id,
label=_policy_source_label(source_scope_type),
policy=policy,
)
)
return sources
def parent_connector_policy(
session: Session,
*,
tenant_id: str,
scope_type: str,
scope_id: str | None = None,
) -> dict[str, Any] | None:
sources = parent_connector_policy_sources(session, tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
if not sources:
return None
return merge_connector_policies(source.policy for source in sources)
def validate_connector_policy_allowed_by_parent(parent_policy: Mapping[str, Any] | None, local_policy: Mapping[str, Any] | None) -> None:
if parent_policy is None:
return
_validate_lower_level_limits(_normalize_connector_policy(parent_policy), _normalize_connector_policy(local_policy))
def merge_connector_policies(policies: Iterable[Mapping[str, Any] | None]) -> dict[str, Any]:
result = _empty_connector_policy()
for policy in policies:
normalized = _normalize_connector_policy(policy)
for group in _RULE_GROUPS:
rules = normalized.get(group) or {}
if not isinstance(rules, Mapping):
continue
target = result[group]
for field in CONNECTOR_POLICY_FIELDS:
values = _string_list(rules.get(field))
if values:
target[field] = _unique([*target.get(field, []), *values])
lower = normalized.get("allow_lower_level_limits") or {}
if isinstance(lower, Mapping):
for key, value in lower.items():
if isinstance(value, bool):
result["allow_lower_level_limits"][str(key)] = value
return _compact_connector_policy(result, keep_lower=True)
def _connector_policy_row(
session: Session,
*,
tenant_id: str,
scope_type: str,
scope_id: str | None = None,
) -> FileConnectorPolicy | None:
row_tenant_id, row_scope_type, row_scope_id = _policy_scope_key(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
query = session.query(FileConnectorPolicy).filter(FileConnectorPolicy.scope_type == row_scope_type)
query = query.filter(FileConnectorPolicy.tenant_id.is_(None) if row_tenant_id is None else FileConnectorPolicy.tenant_id == row_tenant_id)
query = query.filter(FileConnectorPolicy.scope_id.is_(None) if row_scope_id is None else FileConnectorPolicy.scope_id == row_scope_id)
return query.order_by(FileConnectorPolicy.created_at.asc()).first()
def _policy_scope_key(*, tenant_id: str, scope_type: str, scope_id: str | None) -> tuple[str | None, str, str | None]:
clean_scope_type, clean_scope_id = _policy_scope_ref(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
if clean_scope_type == "system":
return None, "system", None
return tenant_id, clean_scope_type, clean_scope_id
def _policy_scope_ref(*, tenant_id: str, scope_type: str, scope_id: str | None) -> tuple[str, str | None]:
clean_scope_type = normalize_policy_scope_type(scope_type)
clean_scope_id = _clean(scope_id)
if clean_scope_type == "system":
return "system", None
if clean_scope_type == "tenant":
if clean_scope_id and clean_scope_id != tenant_id:
raise FileStorageError("Tenant connector policy scope does not belong to the active tenant")
return "tenant", tenant_id
if clean_scope_type in {"user", "group", "campaign"}:
if not clean_scope_id:
raise FileStorageError(f"{clean_scope_type.capitalize()} connector policy requires scope_id")
return clean_scope_type, clean_scope_id
raise FileStorageError("Unsupported connector policy scope")
def _policy_source_chain(*, tenant_id: str, scope_type: str, scope_id: str | None) -> list[tuple[str, str | None]]:
chain: list[tuple[str, str | None]] = [("system", None)]
if scope_type == "system":
return chain
chain.append(("tenant", tenant_id))
if scope_type == "tenant":
return chain
chain.append((scope_type, scope_id))
return chain
def _normalize_connector_policy(policy: Mapping[str, Any] | None) -> dict[str, Any]:
if policy is None:
return {}
if not isinstance(policy, Mapping):
raise FileStorageError("Connector policy must be a JSON object")
normalized = _empty_connector_policy()
for group in _RULE_GROUPS:
raw = policy.get(group)
if raw is None:
raw = policy.get(f"{group}list")
if raw is None:
raw = policy.get("whitelist" if group == "allow" else "blacklist")
if raw is not None and not isinstance(raw, Mapping):
raise FileStorageError(f"Connector policy {group} rules must be an object")
for field, aliases in _FIELD_ALIASES.items():
values: list[str] = []
if isinstance(raw, Mapping):
for alias in aliases:
values.extend(_string_list(raw.get(alias)))
if values:
normalized[group][field] = _unique(values)
lower = policy.get("allow_lower_level_limits")
if lower is not None:
if not isinstance(lower, Mapping):
raise FileStorageError("Connector policy allow_lower_level_limits must be an object")
for key, value in lower.items():
clean_key = str(key).strip()
if clean_key and isinstance(value, bool):
normalized["allow_lower_level_limits"][clean_key] = value
return _compact_connector_policy(normalized, keep_lower=True)
def _empty_connector_policy() -> dict[str, Any]:
return {"allow": {}, "deny": {}, "allow_lower_level_limits": {}}
def _compact_connector_policy(policy: dict[str, Any], *, keep_lower: bool = False) -> dict[str, Any]:
compact: dict[str, Any] = {}
for group in _RULE_GROUPS:
rules = {field: _unique(_string_list(values)) for field, values in (policy.get(group) or {}).items()}
rules = {field: values for field, values in rules.items() if values}
if rules:
compact[group] = rules
lower = policy.get("allow_lower_level_limits") or {}
if isinstance(lower, Mapping):
lower_compact = {str(key): bool(value) for key, value in lower.items() if isinstance(value, bool)}
if lower_compact or keep_lower:
compact["allow_lower_level_limits"] = lower_compact
return compact
def _policy_is_empty(policy: Mapping[str, Any]) -> bool:
return not connector_policy_applied_fields(policy) and not bool(policy.get("allow_lower_level_limits"))
def _source_step(source: ConnectorPolicySource) -> dict[str, Any]:
return source.source_step(connector_policy_applied_fields(source.policy)).to_dict()
def _policy_source_label(scope_type: str) -> str:
if scope_type == "system":
return "System connector policy"
if scope_type == "tenant":
return "Tenant connector policy"
return f"{scope_type.capitalize()} connector policy"
def _validate_lower_level_limits(parent_policy: Mapping[str, Any], local_policy: Mapping[str, Any]) -> None:
limits = parent_policy.get("allow_lower_level_limits")
if not isinstance(limits, Mapping):
return
for field in connector_policy_applied_fields(local_policy):
allowed = limits.get(field)
if allowed is False:
raise FileStorageError(f"Parent connector policy does not allow lower-level {field} limits")
def _string_list(value: object) -> list[str]:
if value is None:
return []
if isinstance(value, str):
clean = value.strip()
return [clean] if clean else []
if isinstance(value, Iterable) and not isinstance(value, (str, bytes, bytearray, Mapping)):
return [str(item).strip() for item in value if str(item).strip()]
return [str(value).strip()] if str(value).strip() else []
def _unique(values: Iterable[str]) -> list[str]:
return list(dict.fromkeys(value for value in values if value))
def _clean(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None

298
src/govoplan_files/backend/storage/connector_profile_store.py Normal file
View File

@@ -0,0 +1,298 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from sqlalchemy.orm import Session
from govoplan_core.core.policy import normalize_policy_scope_type
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_credential_store import connector_credential_from_row, credential_rows_by_id
from govoplan_files.backend.storage.connector_policy import connector_policy_sources_from_payload
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, supported_connector_providers
def list_database_connector_profiles(
session: Session,
*,
tenant_id: str,
include_disabled: bool = False,
) -> list[ConnectorProfile]:
query = session.query(FileConnectorProfile).filter(
(FileConnectorProfile.scope_type == "system")
| (FileConnectorProfile.tenant_id == tenant_id)
)
if not include_disabled:
query = query.filter(FileConnectorProfile.enabled.is_(True))
rows = query.order_by(FileConnectorProfile.scope_type.asc(), FileConnectorProfile.label.asc()).all()
credential_ids = {_clean(row.credential_profile_id) for row in rows if _clean(row.credential_profile_id)}
credentials = credential_rows_by_id(session, tenant_id=tenant_id, credential_ids={item for item in credential_ids if item}, include_disabled=include_disabled)
return [connector_profile_from_row(row, credential_row=credentials.get(row.credential_profile_id or "")) for row in rows]
def list_connector_profile_rows(
session: Session,
*,
tenant_id: str,
include_disabled: bool = False,
) -> list[FileConnectorProfile]:
query = session.query(FileConnectorProfile).filter(
(FileConnectorProfile.scope_type == "system")
| (FileConnectorProfile.tenant_id == tenant_id)
)
if not include_disabled:
query = query.filter(FileConnectorProfile.enabled.is_(True))
return query.order_by(FileConnectorProfile.scope_type.asc(), FileConnectorProfile.label.asc()).all()
def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileConnectorCredential | None = None) -> ConnectorProfile:
policy_sources = []
if row.policy:
policy_sources = connector_policy_sources_from_payload({
"scope_type": row.scope_type,
"scope_id": row.scope_id,
"label": row.label,
"policy": row.policy,
})
credential = connector_credential_from_row(credential_row) if credential_row is not None else None
if credential:
policy_sources.extend(credential.policy_sources)
return ConnectorProfile(
id=row.id,
label=row.label,
provider=row.provider,
scope_type=row.scope_type,
scope_id=row.scope_id,
endpoint_url=row.endpoint_url,
base_path=row.base_path,
enabled=row.enabled,
credential_profile_id=row.credential_profile_id,
credential_profile_label=credential.label if credential else None,
credential_mode=credential.credential_mode if credential else row.credential_mode,
username=credential.username if credential else row.username,
password_env=credential.password_env if credential else row.password_env,
token_env=credential.token_env if credential else row.token_env,
secret_ref=credential.secret_ref if credential else row.secret_ref,
password_value=credential.password_value if credential else decrypt_secret(row.password_encrypted),
token_value=credential.token_value if credential else decrypt_secret(row.token_encrypted),
capabilities=tuple(_string_list(row.capabilities)),
policy_sources=tuple(policy_sources),
metadata=dict(row.metadata_ or {}),
source_kind="database",
)
def get_connector_profile_row(
session: Session,
*,
tenant_id: str,
profile_id: str,
include_disabled: bool = False,
) -> FileConnectorProfile:
row = session.get(FileConnectorProfile, profile_id)
if row is None or (row.scope_type != "system" and row.tenant_id != tenant_id):
raise FileStorageError("Connector profile not found")
if not include_disabled and not row.enabled:
raise FileStorageError("Connector profile not found")
return row
def create_connector_profile_row(
session: Session,
*,
tenant_id: str,
user_id: str | None,
profile_id: str,
label: str,
provider: str,
scope_type: str,
scope_id: str | None = None,
endpoint_url: str | None = None,
base_path: str | None = None,
enabled: bool = True,
credential_profile_id: str | None = None,
credential_mode: str = "none",
username: str | None = None,
password: str | None = None,
token: str | None = None,
password_env: str | None = None,
token_env: str | None = None,
secret_ref: str | None = None,
capabilities: list[str] | None = None,
policy: Mapping[str, Any] | None = None,
metadata: Mapping[str, Any] | None = None,
) -> FileConnectorProfile:
clean_id = _normalize_profile_id(profile_id)
if session.get(FileConnectorProfile, clean_id) is not None:
raise FileStorageError(f"Connector profile already exists: {clean_id}")
clean_scope_type, clean_scope_id, row_tenant_id = _normalize_scope(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id)
row = FileConnectorProfile(
id=clean_id,
tenant_id=row_tenant_id,
scope_type=clean_scope_type,
scope_id=clean_scope_id,
label=_normalize_label(label),
provider=_normalize_provider(provider),
endpoint_url=_clean(endpoint_url),
base_path=_clean(base_path),
enabled=bool(enabled),
credential_profile_id=_clean(credential_profile_id),
credential_mode=_normalize_credential_mode(credential_mode),
username=_clean(username),
password_encrypted=encrypt_secret(_clean(password)),
token_encrypted=encrypt_secret(_clean(token)),
password_env=_clean(password_env),
token_env=_clean(token_env),
secret_ref=_clean(secret_ref),
capabilities=_string_list(capabilities),
policy=dict(policy or {}),
metadata_=dict(metadata or {}),
created_by_user_id=user_id,
updated_by_user_id=user_id,
)
session.add(row)
session.flush()
return row
def update_connector_profile_row(
session: Session,
row: FileConnectorProfile,
*,
user_id: str | None,
label: str | None = None,
provider: str | None = None,
endpoint_url: str | None = None,
base_path: str | None = None,
enabled: bool | None = None,
credential_profile_id: str | None = None,
credential_mode: str | None = None,
username: str | None = None,
password: str | None = None,
token: str | None = None,
password_env: str | None = None,
token_env: str | None = None,
secret_ref: str | None = None,
capabilities: list[str] | None = None,
policy: Mapping[str, Any] | None = None,
metadata: Mapping[str, Any] | None = None,
clear_password: bool = False,
clear_token: bool = False,
) -> FileConnectorProfile:
if label is not None:
row.label = _normalize_label(label)
if provider is not None:
row.provider = _normalize_provider(provider)
if endpoint_url is not None:
row.endpoint_url = _clean(endpoint_url)
if base_path is not None:
row.base_path = _clean(base_path)
if enabled is not None:
row.enabled = bool(enabled)
if credential_profile_id is not None:
row.credential_profile_id = _clean(credential_profile_id)
if credential_mode is not None:
row.credential_mode = _normalize_credential_mode(credential_mode)
if username is not None:
row.username = _clean(username)
if password is not None:
row.password_encrypted = encrypt_secret(_clean(password))
elif clear_password:
row.password_encrypted = None
if token is not None:
row.token_encrypted = encrypt_secret(_clean(token))
elif clear_token:
row.token_encrypted = None
if password_env is not None:
row.password_env = _clean(password_env)
if token_env is not None:
row.token_env = _clean(token_env)
if secret_ref is not None:
row.secret_ref = _clean(secret_ref)
if capabilities is not None:
row.capabilities = _string_list(capabilities)
if policy is not None:
row.policy = dict(policy)
if metadata is not None:
row.metadata_ = dict(metadata)
row.updated_by_user_id = user_id
session.add(row)
session.flush()
return row
def deactivate_connector_profile_row(session: Session, row: FileConnectorProfile, *, user_id: str | None) -> FileConnectorProfile:
row.enabled = False
row.updated_by_user_id = user_id
session.add(row)
session.flush()
return row
def _normalize_scope(*, tenant_id: str, scope_type: str, scope_id: str | None) -> tuple[str, str | None, str | None]:
clean_scope_type = normalize_policy_scope_type(scope_type)
clean_scope_id = _clean(scope_id)
if clean_scope_type == "system":
return "system", None, None
if clean_scope_type == "tenant":
return "tenant", tenant_id, tenant_id
if clean_scope_type in {"user", "group", "campaign"}:
if not clean_scope_id:
raise FileStorageError(f"{clean_scope_type.capitalize()} connector profiles require scope_id")
return clean_scope_type, clean_scope_id, tenant_id
raise FileStorageError("Unsupported connector profile scope")
def _normalize_profile_id(value: str) -> str:
clean = _clean(value)
if not clean:
raise FileStorageError("Connector profile id is required")
if len(clean) > 255:
raise FileStorageError("Connector profile id is too long")
if any(char.isspace() for char in clean):
raise FileStorageError("Connector profile id cannot contain whitespace")
return clean
def _normalize_label(value: str) -> str:
clean = value.strip()
if not clean:
raise FileStorageError("Connector profile label is required")
if len(clean) > 255:
raise FileStorageError("Connector profile label is too long")
return clean
def _normalize_provider(value: str) -> str:
clean = value.strip().casefold()
if clean not in supported_connector_providers():
raise FileStorageError(f"Unsupported connector provider: {value}")
return clean
def _normalize_credential_mode(value: str) -> str:
clean = value.strip().casefold() or "none"
if clean not in {"none", "anonymous", "basic", "token", "secret_ref"}:
raise FileStorageError(f"Unsupported connector credential mode: {value}")
return clean
def _string_list(value: object) -> list[str]:
if value is None:
return []
if isinstance(value, str):
values = value.split(",")
elif isinstance(value, (list, tuple, set)):
values = value
else:
values = [value]
return [str(item).strip() for item in values if str(item).strip()]
def _clean(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None

272
src/govoplan_files/backend/storage/connector_profiles.py Normal file
View File

@@ -0,0 +1,272 @@
from __future__ import annotations
import json
import os
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any
from govoplan_core.core.policy import normalize_policy_scope_type, policy_source_path
from govoplan_files.backend.storage.connector_policy import ConnectorPolicySource, connector_policy_sources_from_payload
_ENV_JSON_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON", "FILES_CONNECTOR_PROFILES_JSON")
_ENV_FILE_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE", "FILES_CONNECTOR_PROFILES_FILE")
_SUPPORTED_PROVIDERS = {"seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"}
_INLINE_SECRET_FIELDS = ("password", "token", "api_key", "access_key", "secret_key")
def supported_connector_providers() -> set[str]:
return set(_SUPPORTED_PROVIDERS)
@dataclass(frozen=True, slots=True)
class ConnectorProfile:
id: str
label: str
provider: str
scope_type: str = "system"
scope_id: str | None = None
endpoint_url: str | None = None
base_path: str | None = None
enabled: bool = True
credential_profile_id: str | None = None
credential_profile_label: str | None = None
credential_mode: str = "none"
username: str | None = None
password_env: str | None = None
token_env: str | None = None
secret_ref: str | None = None
password_value: str | None = field(default=None, repr=False)
token_value: str | None = field(default=None, repr=False)
has_inline_secret: bool = False
capabilities: tuple[str, ...] = ()
policy_sources: tuple[ConnectorPolicySource, ...] = ()
metadata: Mapping[str, Any] = field(default_factory=dict)
source_kind: str = "settings"
@property
def source_path(self) -> str:
return policy_source_path(self.scope_type, self.scope_id)
@property
def credential_source(self) -> str | None:
if self.credential_profile_id:
return "credential_profile"
if self.secret_ref:
return "secret_ref"
if self.token_value or self.password_value:
return "stored"
if self.token_env:
return "token_env"
if self.password_env:
return "password_env"
if self.has_inline_secret:
return "inline"
return None
@property
def credentials_configured(self) -> bool:
mode = self.credential_mode.casefold()
if mode in {"", "none", "anonymous"}:
return True
if self.secret_ref or self.has_inline_secret or self.password_value or self.token_value:
return True
if self.token_env:
return bool(os.environ.get(self.token_env))
if self.password_env:
return bool(os.environ.get(self.password_env))
return False
def to_response(self) -> dict[str, Any]:
return {
"id": self.id,
"label": self.label,
"provider": self.provider,
"endpoint_url": self.endpoint_url,
"base_path": self.base_path,
"enabled": self.enabled,
"scope_type": self.scope_type,
"scope_id": self.scope_id,
"source_path": self.source_path,
"credential_profile_id": self.credential_profile_id,
"credential_profile_label": self.credential_profile_label,
"credential_mode": self.credential_mode,
"credential_source": self.credential_source,
"credentials_configured": self.credentials_configured,
"username": self.username,
"capabilities": list(self.capabilities),
"policy_sources": [_policy_source_response(source) for source in self.policy_sources],
"metadata": dict(self.metadata),
"source_kind": self.source_kind,
}
def connector_profiles_from_settings(settings: object | None = None) -> list[ConnectorProfile]:
raw = _raw_profiles_payload(settings)
if raw in (None, ""):
return []
payload = json.loads(raw) if isinstance(raw, str) else raw
return connector_profiles_from_payload(payload)
def connector_profiles_from_payload(value: object) -> list[ConnectorProfile]:
if isinstance(value, Mapping):
raw_profiles = value.get("profiles", [])
elif isinstance(value, list):
raw_profiles = value
else:
raise ValueError("Connector profiles must be a JSON object with profiles or a list")
if not isinstance(raw_profiles, list):
raise ValueError("Connector profiles payload requires a profiles list")
return [_profile_from_mapping(item) for item in raw_profiles if isinstance(item, Mapping)]
def _profile_from_mapping(value: Mapping[str, Any]) -> ConnectorProfile:
profile_id = _clean(value.get("id") or value.get("connector_id") or value.get("name"))
if not profile_id:
raise ValueError("Connector profiles require id")
provider = (_clean(value.get("provider") or value.get("type")) or "generic").casefold()
if provider not in _SUPPORTED_PROVIDERS:
raise ValueError(f"Unsupported connector provider: {provider}")
scope_type = normalize_policy_scope_type(str(value.get("scope_type") or "system"))
scope_id = _clean(value.get("scope_id"))
if scope_type == "system":
scope_id = None
elif not scope_id:
raise ValueError(f"{scope_type} connector profiles require scope_id")
credentials = value.get("credentials")
credentials = credentials if isinstance(credentials, Mapping) else {}
mode = _clean(value.get("credential_mode") or value.get("auth_type") or credentials.get("mode") or credentials.get("type")) or "none"
profile = ConnectorProfile(
id=profile_id,
label=_clean(value.get("label") or value.get("name")) or profile_id,
provider=provider,
endpoint_url=_clean(value.get("endpoint_url") or value.get("base_url") or value.get("url")),
base_path=_clean(value.get("base_path") or value.get("path") or value.get("root")),
enabled=_bool(value.get("enabled"), default=True),
scope_type=scope_type,
scope_id=scope_id,
credential_profile_id=_clean(value.get("credential_profile_id") or value.get("credential_id")),
credential_profile_label=_clean(value.get("credential_profile_label") or value.get("credential_label")),
credential_mode=mode,
username=_clean(value.get("username") or credentials.get("username")),
password_env=_clean(value.get("password_env") or credentials.get("password_env")),
token_env=_clean(value.get("token_env") or credentials.get("token_env")),
secret_ref=_clean(value.get("secret_ref") or credentials.get("secret_ref")),
password_value=_clean(value.get("password") or credentials.get("password")),
token_value=_clean(value.get("token") or credentials.get("token")),
has_inline_secret=any(_clean(value.get(field) or credentials.get(field)) for field in _INLINE_SECRET_FIELDS),
capabilities=tuple(_string_list(value.get("capabilities"))),
metadata=_public_metadata(value.get("metadata")),
)
return ConnectorProfile(
id=profile.id,
label=profile.label,
provider=profile.provider,
scope_type=profile.scope_type,
scope_id=profile.scope_id,
endpoint_url=profile.endpoint_url,
base_path=profile.base_path,
enabled=profile.enabled,
credential_profile_id=profile.credential_profile_id,
credential_profile_label=profile.credential_profile_label,
credential_mode=profile.credential_mode,
username=profile.username,
password_env=profile.password_env,
token_env=profile.token_env,
secret_ref=profile.secret_ref,
password_value=profile.password_value,
token_value=profile.token_value,
has_inline_secret=profile.has_inline_secret,
capabilities=profile.capabilities,
policy_sources=tuple(_policy_sources_from_profile(value, profile)),
metadata=profile.metadata,
source_kind="settings",
)
def _raw_profiles_payload(settings: object | None) -> object:
for key in _ENV_JSON_KEYS:
value = os.environ.get(key)
if value:
return value
for key in _ENV_FILE_KEYS:
path = os.environ.get(key)
if path:
with open(path, encoding="utf-8") as handle:
return handle.read()
if settings is not None:
value = getattr(settings, "files_connector_profiles_json", None)
if value:
return value
value = getattr(settings, "files_connector_profiles", None)
if value:
return value
return None
def _policy_sources_from_profile(value: Mapping[str, Any], profile: ConnectorProfile) -> list[ConnectorPolicySource]:
raw_sources: list[Mapping[str, Any]] = []
policy = value.get("policy")
if isinstance(policy, Mapping):
raw_sources.append({
"scope_type": profile.scope_type,
"scope_id": profile.scope_id,
"label": profile.label,
"policy": policy,
})
configured_sources = value.get("policy_sources") or value.get("connector_policy_sources")
if configured_sources is not None:
raw = connector_policy_sources_from_payload(configured_sources)
raw_sources.extend(_policy_source_response(source) for source in raw)
return connector_policy_sources_from_payload(raw_sources)
def _policy_source_response(source: ConnectorPolicySource) -> dict[str, Any]:
return {
"scope_type": source.scope_type,
"scope_id": source.scope_id,
"label": source.label,
"policy": dict(source.policy or {}),
}
def _public_metadata(value: object) -> Mapping[str, Any]:
if not isinstance(value, Mapping):
return {}
return {
str(key): item
for key, item in value.items()
if str(key).strip().casefold() not in _INLINE_SECRET_FIELDS
}
def _string_list(value: object) -> list[str]:
if value is None:
return []
if isinstance(value, str):
values: Iterable[object] = value.split(",")
elif isinstance(value, Iterable) and not isinstance(value, (bytes, bytearray, Mapping)):
values = value
else:
values = (value,)
return [str(item).strip() for item in values if str(item).strip()]
def _bool(value: object, *, default: bool) -> bool:
if value is None:
return default
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.strip().casefold() not in {"0", "false", "no", "off"}
return bool(value)
def _clean(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None

145
src/govoplan_files/backend/storage/connector_providers.py Normal file
View File

@@ -0,0 +1,145 @@
from __future__ import annotations
from dataclasses import dataclass
from importlib.util import find_spec
CONNECTOR_PROVIDER_CAPABILITY = "files.connector.providers"
@dataclass(frozen=True, slots=True)
class ConnectorProviderDescriptor:
provider: str
label: str
protocol: str
implemented: bool
browse_supported: bool
import_supported: bool
optional_dependency: str | None
permission_model: str
sync_strategy: str
conflict_strategy: str
preview_strategy: str
audit_events: tuple[str, ...]
notes: str | None = None
def to_response(self) -> dict[str, object]:
return {
"provider": self.provider,
"label": self.label,
"protocol": self.protocol,
"implemented": self.implemented,
"installed": self.installed,
"browse_supported": self.browse_supported,
"import_supported": self.import_supported,
"optional_dependency": self.optional_dependency,
"permission_model": self.permission_model,
"sync_strategy": self.sync_strategy,
"conflict_strategy": self.conflict_strategy,
"preview_strategy": self.preview_strategy,
"audit_events": list(self.audit_events),
"notes": self.notes,
}
@property
def installed(self) -> bool:
if self.optional_dependency is None:
return self.implemented
return find_spec(self.optional_dependency) is not None
def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
freeze_model = "Imported or synced connector files become managed GovOPlaN files with frozen blob/version provenance."
governed_permissions = "GovOPlaN profile scope and connector policy are enforced before browse/import/sync; remote ACLs are still evaluated by the upstream service."
return (
ConnectorProviderDescriptor(
provider="seafile",
label="Seafile",
protocol="seafile-api",
implemented=True,
browse_supported=True,
import_supported=True,
optional_dependency=None,
permission_model=governed_permissions,
sync_strategy="On-demand browse/import/sync; sync updates the managed file version when source content changes and never mutates the remote source.",
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling: reject, overwrite, or rename.",
preview_strategy="Previews are generated from the frozen managed file after import or sync.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Native account-token API for libraries, directories, file detail, and download-link import; WebDAV opt-in remains available.",
),
ConnectorProviderDescriptor(
provider="nextcloud",
label="Nextcloud",
protocol="webdav",
implemented=True,
browse_supported=True,
import_supported=True,
optional_dependency=None,
permission_model=governed_permissions,
sync_strategy="On-demand WebDAV browse/import/sync; sync updates the managed file version when source content changes and never mutates the remote source.",
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling: reject, overwrite, or rename.",
preview_strategy="Previews are generated from the frozen managed file after import or sync.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Uses the configured Nextcloud WebDAV endpoint, basic auth, or bearer token profile credentials.",
),
ConnectorProviderDescriptor(
provider="webdav",
label="WebDAV",
protocol="webdav",
implemented=True,
browse_supported=True,
import_supported=True,
optional_dependency=None,
permission_model=governed_permissions,
sync_strategy="On-demand WebDAV browse/import/sync; sync updates the managed file version when source content changes and never mutates the remote source.",
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling: reject, overwrite, or rename.",
preview_strategy="Previews are generated from the frozen managed file after import or sync.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Generic WebDAV provider for standards-compatible stores.",
),
ConnectorProviderDescriptor(
provider="smb",
label="SMB",
protocol="smb",
implemented=True,
browse_supported=True,
import_supported=True,
optional_dependency="smbprotocol",
permission_model=governed_permissions,
sync_strategy="On-demand browse/import/sync over administrator-declared shares; sync updates managed versions and never mutates the remote share.",
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after download.",
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the share.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Requires the optional smbprotocol dependency and an smb://server[:port]/share[/path] profile endpoint.",
),
ConnectorProviderDescriptor(
provider="nfs",
label="NFS",
protocol="nfs",
implemented=False,
browse_supported=False,
import_supported=False,
optional_dependency=None,
permission_model=governed_permissions,
sync_strategy="Planned optional provider over administrator-mounted paths or a sidecar service.",
conflict_strategy="Will use managed import conflict_strategy handling after download.",
preview_strategy="Will preview from frozen managed file after import, not directly from the mount.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Kept optional because NFS availability is deployment and host-kernel dependent.",
),
ConnectorProviderDescriptor(
provider="local",
label="Local filesystem",
protocol="file",
implemented=False,
browse_supported=False,
import_supported=False,
optional_dependency=None,
permission_model="Local connector access should be restricted to administrator-declared roots plus GovOPlaN profile and policy checks.",
sync_strategy="Planned optional provider; managed storage local backend already exists separately.",
conflict_strategy=freeze_model,
preview_strategy="Will preview from frozen managed file after import or sync.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Separate from the existing managed-file local storage backend.",
),
)

248
src/govoplan_files/backend/storage/connector_spaces.py Normal file
View File

@@ -0,0 +1,248 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from sqlalchemy import false, or_
from sqlalchemy.orm import Session
from govoplan_files.backend.db.models import FileConnectorSpace
from govoplan_files.backend.storage.access import ensure_owner_access, user_group_ids
from govoplan_files.backend.storage.common import FileStorageError, utcnow
from govoplan_files.backend.storage.connector_browse import normalize_connector_browse_path
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
SYNC_MODES = {"manual"}
def connector_space_owner_id(space: FileConnectorSpace) -> str:
return space.owner_user_id if space.owner_type == "user" else space.owner_group_id # type: ignore[return-value]
def create_connector_space(
session: Session,
*,
tenant_id: str,
owner_type: str,
owner_id: str,
user_id: str,
label: str,
profile: ConnectorProfile,
library_id: str | None = None,
remote_path: str | None = None,
sync_mode: str = "manual",
read_only: bool = True,
metadata: Mapping[str, Any] | None = None,
is_admin: bool = False,
) -> FileConnectorSpace:
owner_type = owner_type.lower().strip()
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
label = _normalize_label(label)
sync_mode = _normalize_sync_mode(sync_mode)
remote_path = normalize_connector_browse_path(remote_path)
library_id = _clean_optional(library_id)
existing = _owned_query(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id).filter(
FileConnectorSpace.label == label,
FileConnectorSpace.deleted_at.is_(None),
).first()
if existing is not None:
raise FileStorageError(f"Connector space already exists: {label}")
space = FileConnectorSpace(
tenant_id=tenant_id,
owner_type=owner_type,
owner_user_id=owner_id if owner_type == "user" else None,
owner_group_id=owner_id if owner_type == "group" else None,
label=label,
connector_profile_id=profile.id,
provider=profile.provider,
library_id=library_id,
remote_path=remote_path,
sync_mode=sync_mode,
read_only=read_only,
is_active=True,
created_by_user_id=user_id,
metadata_=dict(metadata or {}),
)
session.add(space)
session.flush()
return space
def list_connector_spaces_for_user(
session: Session,
*,
tenant_id: str,
user_id: str,
owner_type: str | None = None,
owner_id: str | None = None,
include_inactive: bool = False,
is_admin: bool = False,
) -> list[FileConnectorSpace]:
query = session.query(FileConnectorSpace).filter(
FileConnectorSpace.tenant_id == tenant_id,
FileConnectorSpace.deleted_at.is_(None),
)
if not include_inactive:
query = query.filter(FileConnectorSpace.is_active.is_(True))
if owner_type:
if not owner_id:
raise FileStorageError("owner_id is required when owner_type is set")
owner_type = owner_type.lower().strip()
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
query = _owner_filter(query, owner_type, owner_id)
elif not is_admin:
group_ids = user_group_ids(session, tenant_id=tenant_id, user_id=user_id)
query = query.filter(
or_(
FileConnectorSpace.owner_user_id == user_id,
FileConnectorSpace.owner_group_id.in_(group_ids) if group_ids else false(),
)
)
return query.order_by(FileConnectorSpace.owner_type.asc(), FileConnectorSpace.label.asc()).all()
def get_connector_space_for_user(
session: Session,
*,
tenant_id: str,
user_id: str,
space_id: str,
include_inactive: bool = False,
is_admin: bool = False,
) -> FileConnectorSpace:
query = session.query(FileConnectorSpace).filter(
FileConnectorSpace.id == space_id,
FileConnectorSpace.tenant_id == tenant_id,
FileConnectorSpace.deleted_at.is_(None),
)
if not include_inactive:
query = query.filter(FileConnectorSpace.is_active.is_(True))
space = query.one_or_none()
if space is None:
raise FileStorageError("Connector space not found")
ensure_owner_access(
session,
tenant_id=tenant_id,
owner_type=space.owner_type,
owner_id=connector_space_owner_id(space),
user_id=user_id,
is_admin=is_admin,
)
return space
def update_connector_space(
session: Session,
space: FileConnectorSpace,
*,
user_id: str,
label: str | None = None,
library_id: str | None = None,
remote_path: str | None = None,
sync_mode: str | None = None,
is_active: bool | None = None,
metadata: Mapping[str, Any] | None = None,
is_admin: bool = False,
) -> FileConnectorSpace:
ensure_owner_access(
session,
tenant_id=space.tenant_id,
owner_type=space.owner_type,
owner_id=connector_space_owner_id(space),
user_id=user_id,
is_admin=is_admin,
)
if label is not None:
new_label = _normalize_label(label)
existing = _owned_query(
session,
tenant_id=space.tenant_id,
owner_type=space.owner_type,
owner_id=connector_space_owner_id(space),
).filter(
FileConnectorSpace.id != space.id,
FileConnectorSpace.label == new_label,
FileConnectorSpace.deleted_at.is_(None),
).first()
if existing is not None:
raise FileStorageError(f"Connector space already exists: {new_label}")
space.label = new_label
if library_id is not None:
space.library_id = _clean_optional(library_id)
if remote_path is not None:
space.remote_path = normalize_connector_browse_path(remote_path)
if sync_mode is not None:
space.sync_mode = _normalize_sync_mode(sync_mode)
if is_active is not None:
space.is_active = bool(is_active)
if metadata is not None:
space.metadata_ = dict(metadata)
session.add(space)
session.flush()
return space
def soft_delete_connector_space(
session: Session,
space: FileConnectorSpace,
*,
user_id: str,
is_admin: bool = False,
) -> FileConnectorSpace:
ensure_owner_access(
session,
tenant_id=space.tenant_id,
owner_type=space.owner_type,
owner_id=connector_space_owner_id(space),
user_id=user_id,
is_admin=is_admin,
)
space.deleted_at = utcnow()
space.is_active = False
session.add(space)
session.flush()
return space
def _owned_query(session: Session, *, tenant_id: str, owner_type: str, owner_id: str):
query = session.query(FileConnectorSpace).filter(
FileConnectorSpace.tenant_id == tenant_id,
FileConnectorSpace.owner_type == owner_type,
)
return _owner_filter(query, owner_type, owner_id)
def _owner_filter(query, owner_type: str, owner_id: str):
if owner_type == "user":
return query.filter(FileConnectorSpace.owner_user_id == owner_id)
if owner_type == "group":
return query.filter(FileConnectorSpace.owner_group_id == owner_id)
raise FileStorageError("Files must be owned by a user or group")
def _normalize_label(value: str) -> str:
label = value.strip()
if not label:
raise FileStorageError("Connector space label is required")
if len(label) > 255:
raise FileStorageError("Connector space label is too long")
return label
def _normalize_sync_mode(value: str) -> str:
mode = value.strip().casefold()
if mode not in SYNC_MODES:
raise FileStorageError(f"Unsupported connector space sync mode: {value}")
return mode
def _clean_optional(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None

233
src/govoplan_files/backend/storage/files.py
View File

@@ -6,7 +6,7 @@ from pathlib import PurePosixPath
from typing import Any, Iterable
from uuid import uuid4
from sqlalchemy import or_
from sqlalchemy import and_, or_
from sqlalchemy.orm import Session
from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_ACCESS, CampaignAccessProvider
@@ -16,6 +16,7 @@ from govoplan_files.backend.storage.access import ensure_owner_access, ensure_sh
from govoplan_files.backend.storage.backends import StorageBackendError, get_storage_backend
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile, utcnow
from govoplan_files.backend.storage.paths import filename_from_path, join_folder_filename, normalize_folder, normalize_logical_path, safe_storage_component
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata
def _campaign_access_provider() -> CampaignAccessProvider:
@@ -181,6 +182,136 @@ def create_file_asset(
return UploadedStoredFile(asset=asset, version=version, blob=blob)
def sync_file_asset_from_source(
session: Session,
*,
tenant_id: str,
owner_type: str,
owner_id: str,
user_id: str,
filename: str,
data: bytes,
metadata: dict[str, Any],
folder: str | None = None,
display_path: str | None = None,
content_type: str | None = None,
campaign_id: str | None = None,
conflict_strategy: str = "rename",
is_admin: bool = False,
) -> tuple[UploadedStoredFile, str, str | None]:
owner_type = owner_type.lower().strip()
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
provenance = source_provenance_from_metadata(metadata)
if not provenance:
raise FileStorageError("Connector sync requires source provenance")
existing = find_asset_by_source(
session,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
source_provenance=provenance,
)
if existing is None:
stored = create_file_asset(
session,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
filename=filename,
data=data,
folder=folder,
display_path=display_path,
content_type=content_type,
metadata=metadata,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
is_admin=is_admin,
)
return stored, "created", None
previous_version_id = existing.current_version_id
stored, action = update_file_asset_content(
session,
existing,
tenant_id=tenant_id,
user_id=user_id,
filename=filename,
data=data,
content_type=content_type,
metadata=metadata,
)
if campaign_id:
share_file(session, tenant_id=tenant_id, asset=existing, target_type="campaign", target_id=campaign_id, permission="read", user_id=user_id)
return stored, action, previous_version_id
def find_asset_by_source(
session: Session,
*,
tenant_id: str,
owner_type: str,
owner_id: str,
source_provenance: dict[str, Any],
) -> FileAsset | None:
wanted = _source_identity(source_provenance)
if wanted is None:
return None
assets = (
_asset_query_for_owner(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id)
.filter(FileAsset.deleted_at.is_(None))
.order_by(FileAsset.updated_at.desc())
.all()
)
for asset in assets:
if _source_identity(source_provenance_from_metadata(asset.metadata_ or {})) == wanted:
return asset
return None
def update_file_asset_content(
session: Session,
asset: FileAsset,
*,
tenant_id: str,
user_id: str,
filename: str,
data: bytes,
content_type: str | None,
metadata: dict[str, Any],
) -> tuple[UploadedStoredFile, str]:
if asset.tenant_id != tenant_id or asset.deleted_at is not None:
raise FileStorageError("File not found")
safe_filename = filename_from_path(normalize_logical_path(filename, fallback_filename="file"))
if not content_type:
content_type = mimetypes.guess_type(safe_filename)[0] or "application/octet-stream"
current_version, current_blob = current_version_and_blob(session, asset)
checksum = hashlib.sha256(data).hexdigest()
asset.metadata_ = metadata
session.add(asset)
if current_blob.checksum_sha256 == checksum and current_blob.size_bytes == len(data):
return UploadedStoredFile(asset=asset, version=current_version, blob=current_blob), "unchanged"
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
version = FileVersion(
tenant_id=tenant_id,
file_asset_id=asset.id,
blob_id=blob.id,
version_number=_next_version_number(session, asset.id),
filename_at_upload=safe_filename,
display_path_at_upload=asset.display_path,
content_type=content_type,
size_bytes=blob.size_bytes,
checksum_sha256=blob.checksum_sha256,
created_by_user_id=user_id,
)
session.add(version)
session.flush()
asset.current_version_id = version.id
session.add(asset)
return UploadedStoredFile(asset=asset, version=version, blob=blob), "updated"
def get_asset_for_user(session: Session, *, tenant_id: str, user_id: str, asset_id: str, require_write: bool = False, is_admin: bool = False) -> FileAsset:
asset = session.get(FileAsset, asset_id)
if not asset or asset.tenant_id != tenant_id or asset.deleted_at is not None:
@@ -224,6 +355,71 @@ def list_assets_for_user(
include_deleted: bool = False,
is_admin: bool = False,
) -> list[FileAsset]:
query = _asset_visibility_query_for_user(
session,
tenant_id=tenant_id,
user_id=user_id,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
include_deleted=include_deleted,
is_admin=is_admin,
)
return query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc(), FileAsset.id.asc()).all()
def list_assets_for_user_window(
session: Session,
*,
tenant_id: str,
user_id: str,
owner_type: str | None = None,
owner_id: str | None = None,
campaign_id: str | None = None,
path_prefix: str | None = None,
include_deleted: bool = False,
is_admin: bool = False,
page_size: int,
after_display_path: str | None = None,
after_updated_at=None,
after_id: str | None = None,
) -> tuple[list[FileAsset], bool]:
query = _asset_visibility_query_for_user(
session,
tenant_id=tenant_id,
user_id=user_id,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
include_deleted=include_deleted,
is_admin=is_admin,
)
if after_display_path is not None and after_updated_at is not None and after_id:
query = query.filter(
or_(
FileAsset.display_path > after_display_path,
and_(FileAsset.display_path == after_display_path, FileAsset.updated_at < after_updated_at),
and_(FileAsset.display_path == after_display_path, FileAsset.updated_at == after_updated_at, FileAsset.id > after_id),
)
)
rows = query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc(), FileAsset.id.asc()).limit(page_size + 1).all()
return rows[:page_size], len(rows) > page_size
def _asset_visibility_query_for_user(
session: Session,
*,
tenant_id: str,
user_id: str,
owner_type: str | None = None,
owner_id: str | None = None,
campaign_id: str | None = None,
path_prefix: str | None = None,
include_deleted: bool = False,
is_admin: bool = False,
):
query = session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id)
if not include_deleted:
query = query.filter(FileAsset.deleted_at.is_(None))
@@ -255,7 +451,7 @@ def list_assets_for_user(
prefix = normalize_folder(path_prefix)
if prefix:
query = query.filter(FileAsset.display_path.like(f"{prefix}/%"))
return query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc()).all()
return query
def current_version_and_blob(session: Session, asset: FileAsset) -> tuple[FileVersion, FileBlob]:
@@ -553,6 +749,39 @@ def _normalize_conflict_strategy(strategy: str | None) -> str:
return normalized
def _next_version_number(session: Session, asset_id: str) -> int:
row = (
session.query(FileVersion.version_number)
.filter(FileVersion.file_asset_id == asset_id)
.order_by(FileVersion.version_number.desc())
.first()
)
return (int(row[0]) if row else 0) + 1
def _source_identity(provenance: dict[str, Any] | None) -> tuple[object, ...] | None:
if not provenance:
return None
connector_id = _clean_identity(provenance.get("connector_id"))
provider = _clean_identity(provenance.get("provider"))
external_id = _clean_identity(provenance.get("external_id"))
if connector_id and external_id:
return ("external_id", connector_id, provider, external_id)
external_path = _clean_identity(provenance.get("external_path"))
metadata = provenance.get("metadata") if isinstance(provenance.get("metadata"), dict) else {}
library_id = _clean_identity(metadata.get("library_id") or metadata.get("profile_id") or metadata.get("share"))
if connector_id and external_path:
return ("external_path", connector_id, provider, library_id, external_path)
return None
def _clean_identity(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
def _copy_asset_to_path(
session: Session,
asset: FileAsset,

52
src/govoplan_files/backend/storage/folders.py
View File

@@ -68,13 +68,63 @@ def list_folders_for_user(
include_deleted: bool = False,
is_admin: bool = False,
) -> list[FileFolder]:
query = _folder_visibility_query_for_user(
session,
tenant_id=tenant_id,
user_id=user_id,
owner_type=owner_type,
owner_id=owner_id,
include_deleted=include_deleted,
is_admin=is_admin,
)
return query.order_by(FileFolder.path.asc(), FileFolder.id.asc()).all()
def list_folders_for_user_window(
session: Session,
*,
tenant_id: str,
user_id: str,
owner_type: str,
owner_id: str,
include_deleted: bool = False,
is_admin: bool = False,
page_size: int,
after_path: str | None = None,
after_id: str | None = None,
) -> tuple[list[FileFolder], bool]:
query = _folder_visibility_query_for_user(
session,
tenant_id=tenant_id,
user_id=user_id,
owner_type=owner_type,
owner_id=owner_id,
include_deleted=include_deleted,
is_admin=is_admin,
)
if after_path is not None and after_id:
query = query.filter(or_(FileFolder.path > after_path, (FileFolder.path == after_path) & (FileFolder.id > after_id)))
rows = query.order_by(FileFolder.path.asc(), FileFolder.id.asc()).limit(page_size + 1).all()
return rows[:page_size], len(rows) > page_size
def _folder_visibility_query_for_user(
session: Session,
*,
tenant_id: str,
user_id: str,
owner_type: str,
owner_id: str,
include_deleted: bool = False,
is_admin: bool = False,
):
owner_type = owner_type.lower().strip()
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
query = session.query(FileFolder).filter(FileFolder.tenant_id == tenant_id, FileFolder.owner_type == owner_type)
query = _owner_filter(query, owner_type, owner_id)
if not include_deleted:
query = query.filter(FileFolder.deleted_at.is_(None))
return query.order_by(FileFolder.path.asc()).all()
return query
def soft_delete_folder(

90
src/govoplan_files/backend/storage/provenance.py Normal file
View File

@@ -0,0 +1,90 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
SOURCE_PROVENANCE_METADATA_KEY = "source_provenance"
SOURCE_REVISION_METADATA_KEY = "source_revision"
_PROVENANCE_STRING_FIELDS = {
"source_type",
"connector_id",
"provider",
"external_id",
"external_path",
"external_url",
"revision",
"revision_label",
"observed_at",
"imported_at",
}
def normalize_source_revision(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
def normalize_source_provenance(value: object, *, source_revision: object = None) -> dict[str, Any] | None:
if value is None:
revision = normalize_source_revision(source_revision)
return {"revision": revision} if revision else None
if not isinstance(value, Mapping):
raise ValueError("source_provenance must be a JSON object")
payload: dict[str, Any] = {}
for field in _PROVENANCE_STRING_FIELDS:
normalized = normalize_source_revision(value.get(field))
if normalized:
payload[field] = normalized
extra = value.get("metadata")
if isinstance(extra, Mapping) and extra:
payload["metadata"] = dict(extra)
elif extra is not None and extra != "":
raise ValueError("source_provenance.metadata must be a JSON object")
revision = normalize_source_revision(source_revision)
if revision:
payload["revision"] = revision
return payload or None
def source_metadata(
*,
existing: Mapping[str, Any] | None = None,
source_provenance: object = None,
source_revision: object = None,
) -> dict[str, Any] | None:
payload = dict(existing or {})
normalized_provenance = normalize_source_provenance(source_provenance, source_revision=source_revision)
normalized_revision = normalize_source_revision(source_revision)
if normalized_provenance:
payload[SOURCE_PROVENANCE_METADATA_KEY] = normalized_provenance
if normalized_revision:
payload[SOURCE_REVISION_METADATA_KEY] = normalized_revision
elif normalized_provenance and normalized_provenance.get("revision"):
payload[SOURCE_REVISION_METADATA_KEY] = str(normalized_provenance["revision"])
return payload or None
def source_provenance_from_metadata(metadata: Mapping[str, Any] | None) -> dict[str, Any] | None:
if not isinstance(metadata, Mapping):
return None
value = metadata.get(SOURCE_PROVENANCE_METADATA_KEY)
if isinstance(value, Mapping):
return dict(value)
return None
def source_revision_from_metadata(metadata: Mapping[str, Any] | None) -> str | None:
if not isinstance(metadata, Mapping):
return None
revision = normalize_source_revision(metadata.get(SOURCE_REVISION_METADATA_KEY))
if revision:
return revision
provenance = source_provenance_from_metadata(metadata)
return normalize_source_revision(provenance.get("revision") if provenance else None)

3
src/govoplan_files/backend/storage/services.py
View File

@@ -51,7 +51,7 @@ from govoplan_files.backend.storage.folders import (
soft_delete_folder,
)
from govoplan_files.backend.storage.search import match_assets, resolve_patterns
from govoplan_files.backend.storage.transfers import build_rename_preview, rename_selection, transfer_selection
from govoplan_files.backend.storage.transfers import build_rename_preview, legacy_rename_selection_without_owner_context, rename_selection, transfer_selection
__all__ = [
"FileConflictResolution",
@@ -76,6 +76,7 @@ __all__ = [
"match_assets",
"read_asset_bytes",
"record_campaign_attachment_uses_for_job",
"legacy_rename_selection_without_owner_context",
"rename_asset",
"rename_selection",
"resolve_patterns",

84
src/govoplan_files/backend/storage/transfers.py
View File

@@ -341,7 +341,7 @@ def _file_new_path_for_root(path: str, root: str, new_root: str, *, recursive: b
return normalize_logical_path(f"{new_root}/{relative}")
def rename_selection(
def _rename_selection(
session: Session,
*,
tenant_id: str,
@@ -359,6 +359,7 @@ def rename_selection(
recursive: bool = False,
dry_run: bool = True,
is_admin: bool = False,
allow_legacy_without_owner_context: bool = False,
) -> list[RenamePlanItem]:
mode = mode.lower().strip()
if mode not in {"direct", "prefix", "suffix", "replace"}:
@@ -387,10 +388,12 @@ def rename_selection(
file_ids=selected_file_ids,
)
)
else:
elif allow_legacy_without_owner_context:
for file_id in selected_file_ids:
asset = get_asset_for_user(session, tenant_id=tenant_id, user_id=user_id, asset_id=file_id, require_write=True, is_admin=is_admin)
assets[asset.id] = asset
else:
raise FileStorageError("Bulk rename requires an owner file space")
folder_rows_by_path: dict[str, FileFolder] = {}
affected_folder_paths: set[str] = set()
@@ -480,3 +483,80 @@ def rename_selection(
rename_asset(assets[asset_id], new_path=new_path)
session.add(assets[asset_id])
return plan
def rename_selection(
session: Session,
*,
tenant_id: str,
user_id: str,
file_ids: list[str],
folder_paths: list[str],
owner_type: str,
owner_id: str,
mode: str,
new_name: str | None = None,
find: str | None = None,
replacement: str = "",
prefix: str = "",
suffix: str = "",
recursive: bool = False,
dry_run: bool = True,
is_admin: bool = False,
) -> list[RenamePlanItem]:
return _rename_selection(
session,
tenant_id=tenant_id,
user_id=user_id,
file_ids=file_ids,
folder_paths=folder_paths,
owner_type=owner_type,
owner_id=owner_id,
mode=mode,
new_name=new_name,
find=find,
replacement=replacement,
prefix=prefix,
suffix=suffix,
recursive=recursive,
dry_run=dry_run,
is_admin=is_admin,
)
def legacy_rename_selection_without_owner_context(
session: Session,
*,
tenant_id: str,
user_id: str,
file_ids: list[str],
mode: str,
new_name: str | None = None,
find: str | None = None,
replacement: str = "",
prefix: str = "",
suffix: str = "",
dry_run: bool = True,
is_admin: bool = False,
) -> list[RenamePlanItem]:
"""Compatibility path for historical file-only callers that lack owner scope."""
return _rename_selection(
session,
tenant_id=tenant_id,
user_id=user_id,
file_ids=file_ids,
folder_paths=[],
owner_type=None,
owner_id=None,
mode=mode,
new_name=new_name,
find=find,
replacement=replacement,
prefix=prefix,
suffix=suffix,
recursive=False,
dry_run=dry_run,
is_admin=is_admin,
allow_legacy_without_owner_context=True,
)