add-ideas/govoplan-files
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

intermittent commit

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-14 13:22:11 +02:00
parent b8b395e8b5
commit f3210234d3
23 changed files with 2027 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dev/connectors/.env.example
View File

@@ -17,3 +17,9 @@ SMB_HOST_PORT=1445
SMB_SHARE_NAME=files SMB_SHARE_NAME=files
SMB_USER=govoplan SMB_USER=govoplan
SMB_PASSWORD=govoplan-smb SMB_PASSWORD=govoplan-smb
MINIO_API_HOST_PORT=9000
MINIO_CONSOLE_HOST_PORT=9001
MINIO_ROOT_USER=govoplan
MINIO_ROOT_PASSWORD=govoplan-minio
MINIO_BUCKET=govoplan

38
dev/connectors/README.md
View File

@@ -10,7 +10,7 @@ binds services to localhost high ports.
cd /mnt/DATA/git/govoplan-files/dev/connectors cd /mnt/DATA/git/govoplan-files/dev/connectors
cp .env.example .env cp .env.example .env
mkdir -p data/smb data/webdav mkdir -p data/smb data/webdav
docker compose up -d nextcloud nextcloud-db webdav smb docker compose up -d nextcloud nextcloud-db webdav smb minio
docker compose up -d seafile-db seafile-memcached seafile docker compose up -d seafile-db seafile-memcached seafile
``` ```
@@ -30,6 +30,8 @@ Endpoints:
`http://127.0.0.1:9082/seafdav/` `http://127.0.0.1:9082/seafdav/`
- WebDAV: `http://127.0.0.1:9083` - WebDAV: `http://127.0.0.1:9083`
- SMB: `smb://127.0.0.1:1445/files` - SMB: `smb://127.0.0.1:1445/files`
- MinIO/S3 API: `http://127.0.0.1:9000`, console
`http://127.0.0.1:9001`
The local fixture data under `data/` is ignored by git. The local fixture data under `data/` is ignored by git.
@@ -93,6 +95,25 @@ Example local profile file:
"password_env": "SMB_PASSWORD", "password_env": "SMB_PASSWORD",
"capabilities": ["browse", "import"], "capabilities": ["browse", "import"],
"policy": { "allow": { "providers": ["smb"] } } "policy": { "allow": { "providers": ["smb"] } }
},
{
"id": "dev-s3",
"label": "Dev MinIO",
"provider": "s3",
"endpoint_url": "http://127.0.0.1:9000",
"base_path": "",
"scope_type": "system",
"credential_mode": "basic",
"username": "govoplan",
"password_env": "MINIO_ROOT_PASSWORD",
"capabilities": ["browse", "import"],
"metadata": {
"bucket": "govoplan",
"region": "us-east-1",
"path_style": true,
"verify_tls": false
},
"policy": { "allow": { "providers": ["s3"] } }
} }
] ]
} }
@@ -115,9 +136,11 @@ cd /mnt/DATA/git/govoplan-files/dev/connectors
``` ```
The script reads `.env` from this directory when present, seeds tiny WebDAV, The script reads `.env` from this directory when present, seeds tiny WebDAV,
Nextcloud, and SMB fixtures, then browses and imports them through the connector Nextcloud, SMB, and MinIO fixtures, then browses and imports them through the
helper layer. Pass `--require-smb` after recreating the SMB container to fail on connector helper layer. Pass `--require-smb` after recreating the SMB container
SMB access errors instead of reporting them as an optional skip. to fail on SMB access errors instead of reporting them as an optional skip. Pass
`--require-s3` after starting MinIO and installing `govoplan-files[s3]` to fail
on S3 access errors instead of reporting them as an optional skip.
SMB smoke checks need the optional Python dependency in the environment running SMB smoke checks need the optional Python dependency in the environment running
the script: the script:
@@ -126,6 +149,13 @@ the script:
/mnt/DATA/git/govoplan-core/.venv/bin/python -m pip install -e /mnt/DATA/git/govoplan-files[smb] /mnt/DATA/git/govoplan-core/.venv/bin/python -m pip install -e /mnt/DATA/git/govoplan-files[smb]
``` ```
S3 smoke checks need the optional boto3 dependency in the environment running
the script:
```bash
/mnt/DATA/git/govoplan-core/.venv/bin/python -m pip install -e /mnt/DATA/git/govoplan-files[s3]
```
The SMB service is built from `dev/connectors/smb/` so the development share is The SMB service is built from `dev/connectors/smb/` so the development share is
deterministic: one `files` share backed by `data/smb`, with the credentials from deterministic: one `files` share backed by `data/smb`, with the credentials from
`.env`. `.env`.

14
dev/connectors/docker-compose.yml
View File

@@ -99,8 +99,22 @@ services:
volumes: volumes:
- ./data/smb:/storage - ./data/smb:/storage
minio:
image: minio/minio:latest
restart: unless-stopped
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-govoplan}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-govoplan-minio}
ports:
- "127.0.0.1:${MINIO_API_HOST_PORT:-9000}:9000"
- "127.0.0.1:${MINIO_CONSOLE_HOST_PORT:-9001}:9001"
volumes:
- minio:/data
volumes: volumes:
nextcloud-db: nextcloud-db:
nextcloud: nextcloud:
seafile-db: seafile-db:
seafile-data: seafile-data:
minio:

73
dev/connectors/smoke.py
View File

@@ -18,16 +18,17 @@ ROOT = Path(__file__).resolve().parent
def main() -> int: def main() -> int:
parser = argparse.ArgumentParser(description="Smoke-test GovOPlaN connector helpers against the local dev compose stack.") parser = argparse.ArgumentParser(description="Smoke-test GovOPlaN connector helpers against the local dev compose stack.")
parser.add_argument("--require-smb", action="store_true", help="Fail if the SMB connector cannot browse/import.") parser.add_argument("--require-smb", action="store_true", help="Fail if the SMB connector cannot browse/import.")
parser.add_argument("--require-s3", action="store_true", help="Fail if the S3 connector cannot browse/import.")
parser.add_argument("--debug-smb", action="store_true", help="Print direct smbclient probes before the connector smoke check.") parser.add_argument("--debug-smb", action="store_true", help="Print direct smbclient probes before the connector smoke check.")
args = parser.parse_args() args = parser.parse_args()
_load_dotenv() _load_dotenv()
_default_env() _default_env()
preflight_failures = _preflight_services(require_smb=args.require_smb) preflight_failures = _preflight_services(require_smb=args.require_smb, require_s3=args.require_s3)
if preflight_failures: if preflight_failures:
for failure in preflight_failures: for failure in preflight_failures:
print(f"FAIL {failure}") print(f"FAIL {failure}")
print("Start or recreate the connector stack from this directory with: docker compose up -d nextcloud nextcloud-db webdav smb") print("Start or recreate the connector stack from this directory with: docker compose up -d nextcloud nextcloud-db webdav smb minio")
return 1 return 1
_seed_webdav_fixture() _seed_webdav_fixture()
_seed_smb_fixture() _seed_smb_fixture()
@@ -36,6 +37,13 @@ def main() -> int:
except httpx.HTTPError as exc: except httpx.HTTPError as exc:
print(f"FAIL dev-nextcloud seed failed: {exc}") print(f"FAIL dev-nextcloud seed failed: {exc}")
return 1 return 1
try:
_seed_s3_fixture()
except Exception as exc:
if args.require_s3:
print(f"FAIL dev-s3 seed failed: {exc}")
return 1
print(f"SKIP dev-s3 seed failed: {exc}")
profiles = {profile.id: profile for profile in connector_profiles_from_payload({"profiles": _profile_payloads()})} profiles = {profile.id: profile for profile in connector_profiles_from_payload({"profiles": _profile_payloads()})}
if args.debug_smb: if args.debug_smb:
@@ -59,6 +67,15 @@ def main() -> int:
else: else:
print(f"SKIP {message}") print(f"SKIP {message}")
try:
_exercise_profile(profiles["dev-s3"], folder="GovOPlaN", file_path="GovOPlaN/s3-live.txt", expected="s3 live fixture")
except Exception as exc:
message = f"dev-s3: {exc}"
if args.require_s3:
failures.append(message)
else:
print(f"SKIP {message}")
if failures: if failures:
for failure in failures: for failure in failures:
print(f"FAIL {failure}") print(f"FAIL {failure}")
@@ -76,6 +93,9 @@ def _default_env() -> None:
"SMB_SHARE_NAME": "files", "SMB_SHARE_NAME": "files",
"SMB_USER": "govoplan", "SMB_USER": "govoplan",
"SMB_PASSWORD": "govoplan-smb", "SMB_PASSWORD": "govoplan-smb",
"MINIO_ROOT_USER": "govoplan",
"MINIO_ROOT_PASSWORD": "govoplan-minio",
"MINIO_BUCKET": "govoplan",
} }
for key, value in defaults.items(): for key, value in defaults.items():
os.environ.setdefault(key, value) os.environ.setdefault(key, value)
@@ -96,7 +116,7 @@ def _load_dotenv() -> None:
os.environ[key] = value.strip().strip("\"'") os.environ[key] = value.strip().strip("\"'")
def _preflight_services(*, require_smb: bool) -> list[str]: def _preflight_services(*, require_smb: bool, require_s3: bool) -> list[str]:
failures: list[str] = [] failures: list[str] = []
nextcloud_url = f"http://127.0.0.1:{os.getenv('NEXTCLOUD_HOST_PORT', '9081')}/status.php" nextcloud_url = f"http://127.0.0.1:{os.getenv('NEXTCLOUD_HOST_PORT', '9081')}/status.php"
try: try:
@@ -121,6 +141,14 @@ def _preflight_services(*, require_smb: bool) -> list[str]:
pass pass
except OSError as exc: except OSError as exc:
failures.append(f"dev-smb is not reachable at 127.0.0.1:{smb_port}: {exc}") failures.append(f"dev-smb is not reachable at 127.0.0.1:{smb_port}: {exc}")
if require_s3:
minio_url = f"http://127.0.0.1:{os.getenv('MINIO_API_HOST_PORT', '9000')}/minio/health/live"
try:
response = httpx.get(minio_url, timeout=3.0)
if response.status_code != 200:
failures.append(f"dev-s3 expected HTTP 200 at {minio_url}, got HTTP {response.status_code}")
except httpx.HTTPError as exc:
failures.append(f"dev-s3 is not reachable at {minio_url}: {exc}")
return failures return failures
@@ -150,6 +178,20 @@ def _profile_payloads() -> list[dict[str, object]]:
"username": os.getenv("SMB_USER", "govoplan"), "username": os.getenv("SMB_USER", "govoplan"),
"password_env": "SMB_PASSWORD", "password_env": "SMB_PASSWORD",
}, },
{
"id": "dev-s3",
"provider": "s3",
"endpoint_url": f"http://127.0.0.1:{os.getenv('MINIO_API_HOST_PORT', '9000')}",
"credential_mode": "basic",
"username": os.getenv("MINIO_ROOT_USER", "govoplan"),
"password_env": "MINIO_ROOT_PASSWORD",
"metadata": {
"bucket": os.getenv("MINIO_BUCKET", "govoplan"),
"region": "us-east-1",
"path_style": True,
"verify_tls": False,
},
},
] ]
@@ -211,5 +253,30 @@ def _seed_nextcloud_fixture() -> None:
raise RuntimeError(f"Nextcloud PUT failed with HTTP {response.status_code}: {response.text[:200]}") raise RuntimeError(f"Nextcloud PUT failed with HTTP {response.status_code}: {response.text[:200]}")
def _seed_s3_fixture() -> None:
try:
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
except ImportError as exc:
raise RuntimeError("boto3 is not installed; install govoplan-files[s3]") from exc
bucket = os.getenv("MINIO_BUCKET", "govoplan")
client = boto3.client(
"s3",
endpoint_url=f"http://127.0.0.1:{os.getenv('MINIO_API_HOST_PORT', '9000')}",
aws_access_key_id=os.getenv("MINIO_ROOT_USER", "govoplan"),
aws_secret_access_key=os.getenv("MINIO_ROOT_PASSWORD", "govoplan-minio"),
region_name="us-east-1",
config=Config(s3={"addressing_style": "path"}),
)
try:
client.create_bucket(Bucket=bucket)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code")
if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
raise
client.put_object(Bucket=bucket, Key="GovOPlaN/s3-live.txt", Body=b"s3 live fixture\n", ContentType="text/plain")
if __name__ == "__main__": if __name__ == "__main__":
raise SystemExit(main()) raise SystemExit(main())

3
pyproject.toml
View File

@@ -17,6 +17,9 @@ dependencies = [
] ]
[project.optional-dependencies] [project.optional-dependencies]
s3 = [
"boto3>=1.34,<2",
]
smb = [ smb = [
"smbprotocol>=1.13", "smbprotocol>=1.13",
] ]

60
src/govoplan_files/backend/change_tracking.py
View File

@@ -1,12 +1,18 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime from datetime import datetime
from typing import Any
from sqlalchemy import event, inspect from sqlalchemy import event
from sqlalchemy.orm import Session as OrmSession from sqlalchemy.orm import Session as OrmSession
from govoplan_core.core.change_sequence import record_change from govoplan_core.core.change_sequence import record_change
from govoplan_core.core.sqlalchemy_change_tracking import (
ensure_object_id,
has_attr_changes,
object_state,
operation_for_soft_deletable,
previous_value,
)
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare, new_uuid from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare, new_uuid
FILES_MODULE_ID = "files" FILES_MODULE_ID = "files"
@@ -39,7 +45,7 @@ def _record_files_changes(session: OrmSession, _flush_context: object, _instance
def _record_asset_change(session: OrmSession, asset: FileAsset) -> None: def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
operation = _operation_for_soft_deletable( operation = operation_for_soft_deletable(
asset, asset,
changed_attrs=( changed_attrs=(
"owner_type", "owner_type",
@@ -70,7 +76,7 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
"owner_type": asset.owner_type, "owner_type": asset.owner_type,
"owner_id": _owner_id(asset.owner_type, asset.owner_user_id, asset.owner_group_id), "owner_id": _owner_id(asset.owner_type, asset.owner_user_id, asset.owner_group_id),
"path": asset.display_path, "path": asset.display_path,
"previous_path": _previous_value(asset, "display_path"), "previous_path": previous_value(asset, "display_path"),
"filename": asset.filename, "filename": asset.filename,
"deleted_at": _isoformat(asset.deleted_at), "deleted_at": _isoformat(asset.deleted_at),
}, },
@@ -78,7 +84,7 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
def _record_folder_change(session: OrmSession, folder: FileFolder) -> None: def _record_folder_change(session: OrmSession, folder: FileFolder) -> None:
operation = _operation_for_soft_deletable( operation = operation_for_soft_deletable(
folder, folder,
changed_attrs=("owner_type", "owner_user_id", "owner_group_id", "path", "deleted_at", "metadata_"), changed_attrs=("owner_type", "owner_user_id", "owner_group_id", "path", "deleted_at", "metadata_"),
) )
@@ -99,17 +105,17 @@ def _record_folder_change(session: OrmSession, folder: FileFolder) -> None:
"owner_type": folder.owner_type, "owner_type": folder.owner_type,
"owner_id": _owner_id(folder.owner_type, folder.owner_user_id, folder.owner_group_id), "owner_id": _owner_id(folder.owner_type, folder.owner_user_id, folder.owner_group_id),
"path": folder.path, "path": folder.path,
"previous_path": _previous_value(folder, "path"), "previous_path": previous_value(folder, "path"),
"deleted_at": _isoformat(folder.deleted_at), "deleted_at": _isoformat(folder.deleted_at),
}, },
) )
def _record_share_visibility_change(session: OrmSession, share: FileShare) -> None: def _record_share_visibility_change(session: OrmSession, share: FileShare) -> None:
state = inspect(share) state = object_state(share)
if not share.file_asset_id: if not share.file_asset_id:
return return
if not state.pending and not _has_attr_changes( if not state.pending and not has_attr_changes(
state, state,
("file_asset_id", "target_type", "target_id", "permission", "revoked_at"), ("file_asset_id", "target_type", "target_id", "permission", "revoked_at"),
): ):
@@ -135,44 +141,8 @@ def _record_share_visibility_change(session: OrmSession, share: FileShare) -> No
) )
def _operation_for_soft_deletable(obj: object, *, changed_attrs: tuple[str, ...]) -> str | None:
state = inspect(obj)
if state.pending:
return "created"
if not _has_attr_changes(state, changed_attrs):
return None
if "deleted_at" in state.attrs:
history = state.attrs.deleted_at.history
if history.has_changes():
if any(value is not None for value in history.added):
return "deleted"
if any(value is not None for value in history.deleted):
return "created"
return "updated"
def _has_attr_changes(state: Any, attrs: tuple[str, ...]) -> bool:
return any(name in state.attrs and state.attrs[name].history.has_changes() for name in attrs)
def _previous_value(obj: object, attr_name: str) -> str | None:
state = inspect(obj)
if attr_name not in state.attrs:
return None
history = state.attrs[attr_name].history
if not history.has_changes() or not history.deleted:
return None
value = history.deleted[0]
return str(value) if value is not None else None
def _ensure_id(obj: object) -> str: def _ensure_id(obj: object) -> str:
resource_id = getattr(obj, "id", None) return ensure_object_id(obj, new_uuid)
if resource_id:
return str(resource_id)
resource_id = new_uuid()
setattr(obj, "id", resource_id)
return resource_id
def _owner_id(owner_type: str, owner_user_id: str | None, owner_group_id: str | None) -> str | None: def _owner_id(owner_type: str, owner_user_id: str | None, owner_group_id: str | None) -> str | None:

245
src/govoplan_files/backend/router.py
View File

@@ -105,6 +105,7 @@ from govoplan_files.backend.storage.access import ensure_group_access, group_ref
from govoplan_files.backend.storage.archives import create_zip_file, extract_zip_upload from govoplan_files.backend.storage.archives import create_zip_file, extract_zip_upload
from govoplan_files.backend.storage.common import FileStorageError from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_credential_store import ( from govoplan_files.backend.storage.connector_credential_store import (
ConnectorCredential,
connector_credential_from_row, connector_credential_from_row,
create_connector_credential_row, create_connector_credential_row,
deactivate_connector_credential_row, deactivate_connector_credential_row,
@@ -802,6 +803,14 @@ def _download_connector_payload(
return source_path, downloaded, metadata or {} return source_path, downloaded, metadata or {}
def _connector_browse_next_token(items: list[Any]) -> str | None:
for item in items:
token = item.metadata.get("next_continuation_token") if hasattr(item, "metadata") else None
if token:
return str(token)
return None
def _connector_audit_details(asset: FileAsset, version: FileVersion, blob: FileBlob, *, operation: str) -> dict[str, object] | None: def _connector_audit_details(asset: FileAsset, version: FileVersion, blob: FileBlob, *, operation: str) -> dict[str, object] | None:
metadata = asset.metadata_ or {} metadata = asset.metadata_ or {}
provenance = source_provenance_from_metadata(metadata) provenance = source_provenance_from_metadata(metadata)
@@ -2148,6 +2157,148 @@ def _full_file_connector_settings_delta_response(
) )
def _changed_file_connector_setting_ids(entries: list[ChangeSequenceEntry]) -> tuple[set[str], set[str], set[str], bool]:
changed_profile_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
}
changed_credential_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
}
changed_space_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
}
policy_changed = any(entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries)
return changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed
def _file_connector_settings_changed_sections(
*,
changed_profile_ids: set[str],
changed_credential_ids: set[str],
changed_space_ids: set[str],
policy_changed: bool,
profiles: list[ConnectorProfile],
) -> list[str]:
changed_sections = []
if changed_profile_ids or any(profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids for profile in profiles):
changed_sections.append("profiles")
if changed_credential_ids:
changed_sections.append("credentials")
if changed_space_ids:
changed_sections.append("spaces")
if policy_changed:
changed_sections.append("policy")
return changed_sections
def _file_connector_settings_deleted_items(
entries: list[ChangeSequenceEntry],
*,
visible_profiles: dict[str, ConnectorProfile],
visible_credentials: dict[str, ConnectorCredential],
visible_spaces: dict[str, FileConnectorSpace],
) -> list[DeltaDeletedItem]:
return [
_connector_deleted_item(entry)
for entry in entries
if (
entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
and entry.resource_id not in visible_profiles
)
or (
entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
and entry.resource_id not in visible_credentials
)
or (
entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
and entry.resource_id not in visible_spaces
)
]
def _incremental_file_connector_settings_delta_response(
session: Session,
principal: ApiPrincipal,
*,
entries: list[ChangeSequenceEntry],
has_more: bool,
scope_type: str,
scope_id: str | None,
provider: str | None,
campaign_id: str | None,
include_disabled: bool,
include_inactive: bool,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
) -> FileConnectorSettingsDeltaResponse:
changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed = _changed_file_connector_setting_ids(entries)
profiles = _visible_connector_profiles(
session,
principal,
provider=provider,
campaign_id=campaign_id,
include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal),
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
include_effective_policy=False,
)
visible_profiles = {
profile.id: profile
for profile in profiles
if profile.id in changed_profile_ids or (profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids)
}
credentials = _visible_connector_credentials(
session,
principal,
provider=provider,
include_disabled=include_disabled,
)
visible_credentials = {
credential.id: credential
for credential in credentials
if credential.id in changed_credential_ids
}
spaces = _visible_connector_spaces(
session,
principal,
owner_type=owner_type,
owner_id=owner_id,
include_inactive=include_inactive,
)
visible_spaces = {
space.id: space
for space in spaces
if space.id in changed_space_ids
}
return FileConnectorSettingsDeltaResponse(
profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in visible_profiles.values()],
credentials=[FileConnectorCredentialResponse(**credential.to_response()) for credential in visible_credentials.values()],
spaces=[_connector_space_response(space) for space in visible_spaces.values()],
policy=FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) if policy_changed else None,
changed_sections=_file_connector_settings_changed_sections(
changed_profile_ids=changed_profile_ids,
changed_credential_ids=changed_credential_ids,
changed_space_ids=changed_space_ids,
policy_changed=policy_changed,
profiles=profiles,
),
deleted=_file_connector_settings_deleted_items(
entries,
visible_profiles=visible_profiles,
visible_credentials=visible_credentials,
visible_spaces=visible_spaces,
),
watermark=_file_connector_settings_response_watermark(session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more),
has_more=has_more,
full=False,
)
@router.get("/connectors/settings/delta", response_model=FileConnectorSettingsDeltaResponse) @router.get("/connectors/settings/delta", response_model=FileConnectorSettingsDeltaResponse)
def connector_settings_delta( def connector_settings_delta(
scope_type: str = Query(default="tenant"), scope_type: str = Query(default="tenant"),
@@ -2193,94 +2344,19 @@ def connector_settings_delta(
owner_type=owner_type, owner_type=owner_type,
owner_id=owner_id, owner_id=owner_id,
) )
changed_profile_ids = { return _incremental_file_connector_settings_delta_response(
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
}
changed_credential_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
}
changed_space_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
}
policy_changed = any(entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries)
profiles = _visible_connector_profiles(
session, session,
principal, principal,
entries=entries,
has_more=has_more,
scope_type=scope_type,
scope_id=scope_id,
provider=provider, provider=provider,
campaign_id=campaign_id, campaign_id=campaign_id,
include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal),
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
include_effective_policy=False,
)
visible_profiles = {
profile.id: profile
for profile in profiles
if profile.id in changed_profile_ids or (profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids)
}
credentials = _visible_connector_credentials(
session,
principal,
provider=provider,
include_disabled=include_disabled, include_disabled=include_disabled,
) include_inactive=include_inactive,
visible_credentials = {
credential.id: credential
for credential in credentials
if credential.id in changed_credential_ids
}
spaces = _visible_connector_spaces(
session,
principal,
owner_type=owner_type, owner_type=owner_type,
owner_id=owner_id, owner_id=owner_id,
include_inactive=include_inactive,
)
visible_spaces = {
space.id: space
for space in spaces
if space.id in changed_space_ids
}
changed_sections = []
if changed_profile_ids or any(profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids for profile in profiles):
changed_sections.append("profiles")
if changed_credential_ids:
changed_sections.append("credentials")
if changed_space_ids:
changed_sections.append("spaces")
if policy_changed:
changed_sections.append("policy")
deleted = [
_connector_deleted_item(entry)
for entry in entries
if (
entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
and entry.resource_id not in visible_profiles
)
or (
entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
and entry.resource_id not in visible_credentials
)
or (
entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
and entry.resource_id not in visible_spaces
)
]
return FileConnectorSettingsDeltaResponse(
profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in visible_profiles.values()],
credentials=[FileConnectorCredentialResponse(**credential.to_response()) for credential in visible_credentials.values()],
spaces=[_connector_space_response(space) for space in visible_spaces.values()],
policy=FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) if policy_changed else None,
changed_sections=changed_sections,
deleted=deleted,
watermark=_file_connector_settings_response_watermark(session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more),
has_more=has_more,
full=False,
) )
except (FileStorageError, ValueError, json.JSONDecodeError) as exc: except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
raise _http_error(exc) from exc raise _http_error(exc) from exc
@@ -2819,6 +2895,7 @@ def browse_connector_profile_items(
profile_id: str, profile_id: str,
path: str | None = None, path: str | None = None,
library_id: str | None = None, library_id: str | None = None,
continuation_token: str | None = None,
campaign_id: str | None = None, campaign_id: str | None = None,
session: Session = Depends(get_session), session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_any_scope("files:file:read", "files:file:upload", "files:file:download", "files:file:admin", "system:settings:read", "admin:settings:read")), principal: ApiPrincipal = Depends(require_any_scope("files:file:read", "files:file:upload", "files:file:download", "files:file:admin", "system:settings:read", "admin:settings:read")),
@@ -2839,7 +2916,7 @@ def browse_connector_profile_items(
) )
if not decision.allowed: if not decision.allowed:
raise ConnectorPolicyDenied(decision) raise ConnectorPolicyDenied(decision)
items = browse_connector_profile(profile, path=browse_path, library_id=library_id) items = browse_connector_profile(profile, path=browse_path, library_id=library_id, continuation_token=continuation_token)
except ConnectorPolicyDenied as exc: except ConnectorPolicyDenied as exc:
raise _connector_policy_error(exc) from exc raise _connector_policy_error(exc) from exc
except ConnectorBrowseUnsupported as exc: except ConnectorBrowseUnsupported as exc:
@@ -2851,6 +2928,8 @@ def browse_connector_profile_items(
provider=profile.provider, provider=profile.provider,
path=browse_path, path=browse_path,
library_id=library_id, library_id=library_id,
next_continuation_token=_connector_browse_next_token(items),
has_more=any(bool(item.metadata.get("listing_truncated")) for item in items),
decision=decision.to_dict(), decision=decision.to_dict(),
items=[FileConnectorBrowseItem(**item.to_response()) for item in items], items=[FileConnectorBrowseItem(**item.to_response()) for item in items],
) )
@@ -3230,7 +3309,7 @@ def download_archive(
get_asset_for_user(session, tenant_id=principal.tenant_id, user_id=principal.user.id, asset_id=file_id, is_admin=_is_admin(principal)) get_asset_for_user(session, tenant_id=principal.tenant_id, user_id=principal.user.id, asset_id=file_id, is_admin=_is_admin(principal))
for file_id in payload.file_ids for file_id in payload.file_ids
] ]
tmp = tempfile.NamedTemporaryFile(prefix="multimailer-files-", suffix=".zip", delete=False) tmp = tempfile.NamedTemporaryFile(prefix="govoplan-files-", suffix=".zip", delete=False)
tmp_path = tmp.name tmp_path = tmp.name
tmp.close() tmp.close()
try: try:

38
src/govoplan_files/backend/runtime.py
View File

@@ -1,36 +1,10 @@
from __future__ import annotations from __future__ import annotations
from typing import Any from govoplan_core.core.runtime import ModuleRuntimeState
_runtime_registry: object | None = None _runtime = ModuleRuntimeState("Files")
_runtime_settings: object | None = None
configure_runtime = _runtime.configure_runtime
def configure_runtime(*, registry: object | None = None, settings: object | None = None) -> None: get_registry = _runtime.get_registry
global _runtime_registry, _runtime_settings get_settings = _runtime.get_settings
if registry is not None: settings = _runtime.settings
_runtime_registry = registry
if settings is not None:
_runtime_settings = settings
def get_registry() -> object | None:
return _runtime_registry
def get_settings() -> object:
if _runtime_settings is not None:
return _runtime_settings
try:
from govoplan_core.settings import settings as legacy_settings
except ModuleNotFoundError as exc:
raise RuntimeError("GovOPlaN Files runtime settings are not configured") from exc
return legacy_settings
class SettingsProxy:
def __getattr__(self, name: str) -> Any:
return getattr(get_settings(), name)
settings = SettingsProxy()

12
src/govoplan_files/backend/schemas.py
View File

@@ -287,7 +287,7 @@ class FileConnectorDiscoveryCandidate(BaseModel):
class FileConnectorDiscoveryRequest(BaseModel): class FileConnectorDiscoveryRequest(BaseModel):
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"]
endpoint_url: str endpoint_url: str
base_path: str | None = None base_path: str | None = None
credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] = "none" credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] = "none"
@@ -309,7 +309,7 @@ class FileConnectorDiscoveryResponse(BaseModel):
class FileConnectorProfileCreateRequest(BaseModel): class FileConnectorProfileCreateRequest(BaseModel):
id: str id: str
label: str label: str
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"]
endpoint_url: str | None = None endpoint_url: str | None = None
base_path: str | None = None base_path: str | None = None
enabled: bool = True enabled: bool = True
@@ -325,7 +325,7 @@ class FileConnectorProfileCreateRequest(BaseModel):
class FileConnectorProfileUpdateRequest(BaseModel): class FileConnectorProfileUpdateRequest(BaseModel):
label: str | None = None label: str | None = None
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] | None = None
endpoint_url: str | None = None endpoint_url: str | None = None
base_path: str | None = None base_path: str | None = None
enabled: bool | None = None enabled: bool | None = None
@@ -342,7 +342,7 @@ class FileConnectorProfileUpdateRequest(BaseModel):
class FileConnectorCredentialCreateRequest(BaseModel): class FileConnectorCredentialCreateRequest(BaseModel):
id: str id: str
label: str label: str
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] | None = None
enabled: bool = True enabled: bool = True
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "tenant" scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "tenant"
scope_id: str | None = None scope_id: str | None = None
@@ -354,7 +354,7 @@ class FileConnectorCredentialCreateRequest(BaseModel):
class FileConnectorCredentialUpdateRequest(BaseModel): class FileConnectorCredentialUpdateRequest(BaseModel):
label: str | None = None label: str | None = None
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] | None = None
enabled: bool | None = None enabled: bool | None = None
credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] | None = None credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] | None = None
credentials: FileConnectorProfileCredentialsRequest | None = None credentials: FileConnectorProfileCredentialsRequest | None = None
@@ -404,6 +404,8 @@ class FileConnectorBrowseResponse(BaseModel):
path: str = "" path: str = ""
library_id: str | None = None library_id: str | None = None
read_only: bool = True read_only: bool = True
next_continuation_token: str | None = None
has_more: bool = False
decision: dict[str, Any] decision: dict[str, Any]
items: list[FileConnectorBrowseItem] items: list[FileConnectorBrowseItem]

2
src/govoplan_files/backend/storage/campaign_attachments.py
View File

@@ -273,7 +273,7 @@ def prepared_campaign_snapshot(
campaign_id: str, campaign_id: str,
raw_json: dict[str, Any], raw_json: dict[str, Any],
include_bytes: bool, include_bytes: bool,
prefix: str = "multimailer-managed-campaign-", prefix: str = "govoplan-managed-campaign-",
) -> Iterator[PreparedCampaignSnapshot]: ) -> Iterator[PreparedCampaignSnapshot]:
temp_dir = Path(tempfile.mkdtemp(prefix=prefix)) temp_dir = Path(tempfile.mkdtemp(prefix=prefix))
try: try:

175
src/govoplan_files/backend/storage/campaign_usage.py
View File

@@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
from collections import defaultdict from collections import defaultdict
from dataclasses import dataclass, field
from pathlib import PurePosixPath from pathlib import PurePosixPath
from typing import Any, Iterable from typing import Any, Iterable
@@ -24,6 +25,15 @@ def _candidate_match_keys(raw_match: str) -> set[str]:
AttachmentUseKey = tuple[str, str, str, str] AttachmentUseKey = tuple[str, str, str, str]
@dataclass(slots=True)
class _AttachmentBatchRefs:
managed_by_job: dict[str, list[dict[str, object]]] = field(default_factory=lambda: defaultdict(list))
fallback_attachments_by_job: dict[str, list[dict[str, object]]] = field(default_factory=lambda: defaultdict(list))
asset_ids: set[str] = field(default_factory=set)
version_ids: set[str] = field(default_factory=set)
blob_ids: set[str] = field(default_factory=set)
def _attachment_use_key(*, job_id: str, file_version_id: str, filename_used: str, stage: str) -> AttachmentUseKey: def _attachment_use_key(*, job_id: str, file_version_id: str, filename_used: str, stage: str) -> AttachmentUseKey:
return job_id, file_version_id, filename_used, stage return job_id, file_version_id, filename_used, stage
@@ -134,22 +144,48 @@ def record_campaign_attachment_uses_for_jobs(
if not job_list: if not job_list:
return return
managed_by_job: dict[str, list[dict[str, object]]] = defaultdict(list) refs = _collect_attachment_batch_refs(job_list)
fallback_attachments_by_job: dict[str, list[dict[str, object]]] = defaultdict(list)
job_by_id = {job.id: job for job in job_list} job_by_id = {job.id: job for job in job_list}
asset_ids: set[str] = set() known_keys = _known_use_keys_for_jobs(session, job_list, stage=stage)
version_ids: set[str] = set() assets_by_id, versions_by_id, blobs_by_id = _load_managed_attachment_entities(session, refs)
blob_ids: set[str] = set()
for job in job_list: _record_managed_attachment_uses(
session,
job_by_id=job_by_id,
managed_by_job=refs.managed_by_job,
assets_by_id=assets_by_id,
versions_by_id=versions_by_id,
blobs_by_id=blobs_by_id,
stage=stage,
known_keys=known_keys,
)
_record_fallback_attachment_uses(
session,
job_by_id=job_by_id,
fallback_attachments_by_job=refs.fallback_attachments_by_job,
stage=stage,
known_keys=known_keys,
)
def _collect_attachment_batch_refs(jobs: list[CampaignJobLike]) -> _AttachmentBatchRefs:
refs = _AttachmentBatchRefs()
for job in jobs:
attachments = job.resolved_attachments or [] attachments = job.resolved_attachments or []
if not isinstance(attachments, list): if not isinstance(attachments, list):
continue continue
for attachment in attachments: for attachment in attachments:
if not isinstance(attachment, dict): if not isinstance(attachment, dict):
continue continue
if not _collect_managed_attachment_refs(refs, job.id, attachment):
refs.fallback_attachments_by_job[job.id].append(attachment)
return refs
def _collect_managed_attachment_refs(refs: _AttachmentBatchRefs, job_id: str, attachment: dict[str, object]) -> bool:
managed_matches = attachment.get("managed_matches") managed_matches = attachment.get("managed_matches")
if isinstance(managed_matches, list) and managed_matches: if not isinstance(managed_matches, list) or not managed_matches:
return False
for item in managed_matches: for item in managed_matches:
if not isinstance(item, dict): if not isinstance(item, dict):
continue continue
@@ -158,38 +194,41 @@ def record_campaign_attachment_uses_for_jobs(
blob_id = str(item.get("blob_id") or "") blob_id = str(item.get("blob_id") or "")
if not asset_id or not version_id or not blob_id: if not asset_id or not version_id or not blob_id:
continue continue
managed_by_job[job.id].append(item) refs.managed_by_job[job_id].append(item)
asset_ids.add(asset_id) refs.asset_ids.add(asset_id)
version_ids.add(version_id) refs.version_ids.add(version_id)
blob_ids.add(blob_id) refs.blob_ids.add(blob_id)
else: return True
fallback_attachments_by_job[job.id].append(attachment)
assets_by_id = {
item.id: item
for item in session.query(FileAsset).filter(FileAsset.id.in_(asset_ids)).all()
} if asset_ids else {}
versions_by_id = {
item.id: item
for item in session.query(FileVersion).filter(FileVersion.id.in_(version_ids)).all()
} if version_ids else {}
blobs_by_id = {
item.id: item
for item in session.query(FileBlob).filter(FileBlob.id.in_(blob_ids)).all()
} if blob_ids else {}
known_keys = _known_use_keys_for_jobs(session, job_list, stage=stage)
def _load_managed_attachment_entities(
session: Session,
refs: _AttachmentBatchRefs,
) -> tuple[dict[str, FileAsset], dict[str, FileVersion], dict[str, FileBlob]]:
assets_by_id = {item.id: item for item in session.query(FileAsset).filter(FileAsset.id.in_(refs.asset_ids)).all()} if refs.asset_ids else {}
versions_by_id = {item.id: item for item in session.query(FileVersion).filter(FileVersion.id.in_(refs.version_ids)).all()} if refs.version_ids else {}
blobs_by_id = {item.id: item for item in session.query(FileBlob).filter(FileBlob.id.in_(refs.blob_ids)).all()} if refs.blob_ids else {}
return assets_by_id, versions_by_id, blobs_by_id
def _record_managed_attachment_uses(
session: Session,
*,
job_by_id: dict[str, CampaignJobLike],
managed_by_job: dict[str, list[dict[str, object]]],
assets_by_id: dict[str, FileAsset],
versions_by_id: dict[str, FileVersion],
blobs_by_id: dict[str, FileBlob],
stage: str,
known_keys: set[AttachmentUseKey],
) -> None:
for job_id, items in managed_by_job.items(): for job_id, items in managed_by_job.items():
job = job_by_id[job_id] job = job_by_id[job_id]
for item in items: for item in items:
asset = assets_by_id.get(str(item.get("asset_id") or "")) asset = assets_by_id.get(str(item.get("asset_id") or ""))
version = versions_by_id.get(str(item.get("version_id") or "")) version = versions_by_id.get(str(item.get("version_id") or ""))
blob = blobs_by_id.get(str(item.get("blob_id") or "")) blob = blobs_by_id.get(str(item.get("blob_id") or ""))
if not asset or not version or not blob: if not _managed_attachment_entities_match_job(job, asset, version, blob):
continue
if asset.tenant_id != job.tenant_id or version.tenant_id != job.tenant_id or blob.tenant_id != job.tenant_id:
continue
if version.file_asset_id != asset.id or version.blob_id != blob.id:
continue continue
_add_use( _add_use(
session, session,
@@ -202,29 +241,79 @@ def record_campaign_attachment_uses_for_jobs(
known_keys=known_keys, known_keys=known_keys,
) )
def _managed_attachment_entities_match_job(
job: CampaignJobLike,
asset: FileAsset | None,
version: FileVersion | None,
blob: FileBlob | None,
) -> bool:
if not asset or not version or not blob:
return False
if asset.tenant_id != job.tenant_id or version.tenant_id != job.tenant_id or blob.tenant_id != job.tenant_id:
return False
return version.file_asset_id == asset.id and version.blob_id == blob.id
def _record_fallback_attachment_uses(
session: Session,
*,
job_by_id: dict[str, CampaignJobLike],
fallback_attachments_by_job: dict[str, list[dict[str, object]]],
stage: str,
known_keys: set[AttachmentUseKey],
) -> None:
assets_by_campaign: dict[tuple[str, str], dict[str, FileAsset]] = {} assets_by_campaign: dict[tuple[str, str], dict[str, FileAsset]] = {}
version_blobs_by_campaign: dict[tuple[str, str], dict[str, tuple[FileVersion, FileBlob]]] = {} version_blobs_by_campaign: dict[tuple[str, str], dict[str, tuple[FileVersion, FileBlob]]] = {}
for job_id, attachments in fallback_attachments_by_job.items(): for job_id, attachments in fallback_attachments_by_job.items():
job = job_by_id[job_id] job = job_by_id[job_id]
campaign_key = (job.tenant_id, job.campaign_id) campaign_key = (job.tenant_id, job.campaign_id)
by_key, version_blobs = _fallback_campaign_assets(
session,
job,
campaign_key=campaign_key,
assets_by_campaign=assets_by_campaign,
version_blobs_by_campaign=version_blobs_by_campaign,
)
for attachment in attachments:
_record_fallback_attachment(session, job, attachment, by_key=by_key, version_blobs=version_blobs, stage=stage, known_keys=known_keys)
def _fallback_campaign_assets(
session: Session,
job: CampaignJobLike,
*,
campaign_key: tuple[str, str],
assets_by_campaign: dict[tuple[str, str], dict[str, FileAsset]],
version_blobs_by_campaign: dict[tuple[str, str], dict[str, tuple[FileVersion, FileBlob]]],
) -> tuple[dict[str, FileAsset], dict[str, tuple[FileVersion, FileBlob]]]:
by_key = assets_by_campaign.get(campaign_key) by_key = assets_by_campaign.get(campaign_key)
if by_key is None: if by_key is None:
assets = list_assets_for_user( assets = list_assets_for_user(session, tenant_id=job.tenant_id, user_id="", campaign_id=job.campaign_id, is_admin=True)
session, by_key = _asset_lookup_by_path_and_filename(assets)
tenant_id=job.tenant_id, assets_by_campaign[campaign_key] = by_key
user_id="", version_blobs_by_campaign[campaign_key] = current_versions_and_blobs(session, assets)
campaign_id=job.campaign_id, return by_key, version_blobs_by_campaign[campaign_key]
is_admin=True,
)
by_key = {} def _asset_lookup_by_path_and_filename(assets: list[FileAsset]) -> dict[str, FileAsset]:
by_key: dict[str, FileAsset] = {}
for asset in assets: for asset in assets:
by_key[asset.display_path.strip("/")] = asset by_key[asset.display_path.strip("/")] = asset
by_key.setdefault(asset.filename, asset) by_key.setdefault(asset.filename, asset)
assets_by_campaign[campaign_key] = by_key return by_key
version_blobs_by_campaign[campaign_key] = current_versions_and_blobs(session, assets)
version_blobs = version_blobs_by_campaign[campaign_key]
for attachment in attachments:
def _record_fallback_attachment(
session: Session,
job: CampaignJobLike,
attachment: dict[str, object],
*,
by_key: dict[str, FileAsset],
version_blobs: dict[str, tuple[FileVersion, FileBlob]],
stage: str,
known_keys: set[AttachmentUseKey],
) -> None:
matches = attachment.get("matches") if isinstance(attachment.get("matches"), list) else [] matches = attachment.get("matches") if isinstance(attachment.get("matches"), list) else []
for raw in matches: for raw in matches:
if not isinstance(raw, str): if not isinstance(raw, str):

232
src/govoplan_files/backend/storage/connector_browse.py
View File

@@ -53,7 +53,7 @@ class ConnectorBrowseItem:
} }
def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = None, library_id: str | None = None) -> list[ConnectorBrowseItem]: def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = None, library_id: str | None = None, continuation_token: str | None = None) -> list[ConnectorBrowseItem]:
browse_path = normalize_connector_browse_path(path) browse_path = normalize_connector_browse_path(path)
static_items = _static_listing(profile, path=browse_path, library_id=library_id) static_items = _static_listing(profile, path=browse_path, library_id=library_id)
if static_items is not None: if static_items is not None:
@@ -66,6 +66,8 @@ def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = No
return _browse_webdav(profile, path=browse_path) return _browse_webdav(profile, path=browse_path)
if profile.provider == "smb": if profile.provider == "smb":
return _browse_smb(profile, path=browse_path) return _browse_smb(profile, path=browse_path)
if profile.provider == "s3":
return _browse_s3(profile, path=browse_path, library_id=library_id, continuation_token=continuation_token)
raise ConnectorBrowseUnsupported(f"Read-only browsing is not implemented for {profile.provider} connector profiles yet") raise ConnectorBrowseUnsupported(f"Read-only browsing is not implemented for {profile.provider} connector profiles yet")
@@ -222,6 +224,227 @@ def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowse
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold())) return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]:
client = _s3_client(profile)
bucket = _s3_bucket(profile, library_id)
if not bucket:
try:
payload = client.list_buckets()
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"S3 connector browse failed: {exc}") from exc
buckets = payload.get("Buckets") if isinstance(payload, Mapping) else None
if not isinstance(buckets, list):
raise ConnectorBrowseError("S3 connector returned invalid bucket list")
return sorted(
(
ConnectorBrowseItem(
kind="library",
name=_clean(item.get("Name")) or "",
path=_clean(item.get("Name")) or "",
external_id=_clean(item.get("Name")),
modified_at=_timestamp(item.get("CreationDate")),
metadata={"bucket": _clean(item.get("Name"))},
)
for item in buckets
if isinstance(item, Mapping) and _clean(item.get("Name"))
),
key=lambda item: item.name.casefold(),
)
prefix = _s3_object_key(profile, path, directory=True)
params: dict[str, object] = {
"Bucket": bucket,
"Prefix": prefix,
"Delimiter": "/",
"MaxKeys": _s3_max_keys(profile),
}
next_page_request = _clean(continuation_token) or _metadata_string(profile, "continuation_token")
if next_page_request:
params["ContinuationToken"] = next_page_request
try:
payload = client.list_objects_v2(**params)
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"S3 connector browse failed: {exc}") from exc
if not isinstance(payload, Mapping):
raise ConnectorBrowseError("S3 connector returned invalid object listing")
items = [
*_s3_prefix_items(bucket=bucket, browse_path=path, base_prefix=prefix, prefixes=payload.get("CommonPrefixes")),
*_s3_object_items(bucket=bucket, browse_path=path, base_prefix=prefix, objects=payload.get("Contents")),
]
next_token = _clean(payload.get("NextContinuationToken"))
if next_token and items:
last = items[-1]
items[-1] = ConnectorBrowseItem(
kind=last.kind,
name=last.name,
path=last.path,
external_id=last.external_id,
external_url=last.external_url,
size_bytes=last.size_bytes,
content_type=last.content_type,
modified_at=last.modified_at,
etag=last.etag,
metadata={**dict(last.metadata), "next_continuation_token": next_token, "listing_truncated": True},
)
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
def _s3_client(profile: ConnectorProfile) -> Any:
if profile.secret_ref:
raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing")
try:
boto3 = import_module("boto3")
config_module = import_module("botocore.config")
except ImportError as exc:
raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
kwargs: dict[str, object] = {}
if profile.endpoint_url:
kwargs["endpoint_url"] = profile.endpoint_url
region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region")
if region:
kwargs["region_name"] = region
access_key = profile.username or _metadata_env(profile, "access_key_id_env") or _metadata_string(profile, "access_key_id")
secret_key = _profile_password(profile) or _metadata_env(profile, "secret_access_key_env")
session_token = _profile_token(profile) or _metadata_env(profile, "session_token_env")
if access_key:
kwargs["aws_access_key_id"] = access_key
if secret_key:
kwargs["aws_secret_access_key"] = secret_key
if session_token:
kwargs["aws_session_token"] = session_token
verify = _s3_verify(profile)
if verify is not None:
kwargs["verify"] = verify
addressing_style = _s3_addressing_style(profile)
if addressing_style:
kwargs["config"] = config_module.Config(s3={"addressing_style": addressing_style})
try:
return boto3.client("s3", **kwargs)
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc
def _s3_bucket(profile: ConnectorProfile, library_id: str | None = None) -> str | None:
return _metadata_string(profile, "bucket") or _metadata_string(profile, "bucket_name") or _clean(library_id)
def _s3_object_key(profile: ConnectorProfile, path: str, *, directory: bool = False) -> str:
base_prefix = normalize_connector_browse_path(profile.base_path or _metadata_string(profile, "base_prefix"))
browse_path = normalize_connector_browse_path(path)
key = "/".join(part for part in (base_prefix, browse_path) if part)
if directory and key:
return key.rstrip("/") + "/"
return key
def _s3_prefix_items(
*,
bucket: str,
browse_path: str,
base_prefix: str,
prefixes: object,
) -> list[ConnectorBrowseItem]:
if not isinstance(prefixes, list):
return []
items: list[ConnectorBrowseItem] = []
for item in prefixes:
if not isinstance(item, Mapping):
continue
key = _clean(item.get("Prefix"))
if not key:
continue
relative = _s3_relative_key(key, base_prefix=base_prefix)
name = _path_name(relative)
if not name:
continue
path = _join_browse_path(browse_path, name)
items.append(
ConnectorBrowseItem(
kind="folder",
name=name,
path=path,
external_id=f"{bucket}:{key}",
metadata={"bucket": bucket, "key": key},
)
)
return items
def _s3_object_items(
*,
bucket: str,
browse_path: str,
base_prefix: str,
objects: object,
) -> list[ConnectorBrowseItem]:
if not isinstance(objects, list):
return []
items: list[ConnectorBrowseItem] = []
for item in objects:
if not isinstance(item, Mapping):
continue
key = _clean(item.get("Key"))
if not key or key == base_prefix:
continue
relative = _s3_relative_key(key, base_prefix=base_prefix)
name = _path_name(relative)
if not name:
continue
path = _join_browse_path(browse_path, name)
items.append(
ConnectorBrowseItem(
kind="file",
name=name,
path=path,
external_id=f"{bucket}:{key}",
size_bytes=_int(item.get("Size")),
content_type=mimetypes.guess_type(name)[0],
modified_at=_timestamp(item.get("LastModified")),
etag=_clean(item.get("ETag")),
metadata={
"bucket": bucket,
"key": key,
**({"storage_class": item["StorageClass"]} if "StorageClass" in item else {}),
},
)
)
return items
def _s3_relative_key(key: str, *, base_prefix: str) -> str:
clean_key = key.rstrip("/")
clean_base = base_prefix.rstrip("/")
if clean_base and clean_key.startswith(f"{clean_base}/"):
return clean_key[len(clean_base) + 1 :]
return clean_key
def _s3_max_keys(profile: ConnectorProfile) -> int:
configured = _int(profile.metadata.get("max_keys"))
if configured is None:
return 1000
return max(1, min(configured, 1000))
def _s3_verify(profile: ConnectorProfile) -> bool | str | None:
ca_bundle = _metadata_string(profile, "ca_bundle")
if ca_bundle:
return ca_bundle
if "verify_tls" in profile.metadata:
return _metadata_bool(profile, "verify_tls", default=True)
if "tls_verify" in profile.metadata:
return _metadata_bool(profile, "tls_verify", default=True)
return None
def _s3_addressing_style(profile: ConnectorProfile) -> str | None:
style = _metadata_string(profile, "addressing_style")
if style in {"path", "virtual", "auto"}:
return style
if _metadata_bool(profile, "path_style", default=False):
return "path"
return None
def _seafile_headers(profile: ConnectorProfile) -> dict[str, str]: def _seafile_headers(profile: ConnectorProfile) -> dict[str, str]:
token = _seafile_token(profile) token = _seafile_token(profile)
return {"Authorization": f"Token {token}", "Accept": "application/json"} return {"Authorization": f"Token {token}", "Accept": "application/json"}
@@ -444,6 +667,13 @@ def _metadata_bool(profile: ConnectorProfile, key: str, *, default: bool = False
return str(value).strip().casefold() in {"1", "true", "yes", "on"} return str(value).strip().casefold() in {"1", "true", "yes", "on"}
def _metadata_env(profile: ConnectorProfile, key: str) -> str | None:
env_name = _metadata_string(profile, key)
if not env_name:
return None
return _env_required(env_name, profile.id)
def _env_required(name: str, profile_id: str) -> str: def _env_required(name: str, profile_id: str) -> str:
value = _clean(os.environ.get(name)) value = _clean(os.environ.get(name))
if not value: if not value:

62
src/govoplan_files/backend/storage/connector_imports.py
View File

@@ -21,6 +21,9 @@ from govoplan_files.backend.storage.connector_browse import (
_smb_stat_size, _smb_stat_size,
_smb_unc_path, _smb_unc_path,
_smbclient_module, _smbclient_module,
_s3_bucket,
_s3_client,
_s3_object_key,
_seafile_headers, _seafile_headers,
_seafile_url, _seafile_url,
_webdav_url, _webdav_url,
@@ -64,6 +67,8 @@ def read_connector_file(
return _read_webdav_file(profile, path=path, max_bytes=max_bytes) return _read_webdav_file(profile, path=path, max_bytes=max_bytes)
if profile.provider == "smb": if profile.provider == "smb":
return _read_smb_file(profile, path=path, max_bytes=max_bytes) return _read_smb_file(profile, path=path, max_bytes=max_bytes)
if profile.provider == "s3":
return _read_s3_file(profile, library_id=library_id, path=path, max_bytes=max_bytes)
raise ConnectorImportUnsupported(f"Connector file import is not implemented for {profile.provider} profiles yet") raise ConnectorImportUnsupported(f"Connector file import is not implemented for {profile.provider} profiles yet")
@@ -213,6 +218,63 @@ def _read_smb_file(profile: ConnectorProfile, *, path: str, max_bytes: int) -> C
) )
def _read_s3_file(profile: ConnectorProfile, *, library_id: str, path: str, max_bytes: int) -> ConnectorDownloadedFile:
bucket = _s3_bucket(profile, library_id)
if not bucket:
raise ConnectorImportError("S3 import requires a bucket in library_id or profile metadata")
key = _s3_object_key(profile, path)
if not key:
raise ConnectorImportError("S3 import requires an object key")
try:
client = _s3_client(profile)
except ConnectorBrowseUnsupported as exc:
raise ConnectorImportUnsupported(str(exc)) from exc
except ConnectorBrowseError as exc:
raise ConnectorImportError(str(exc)) from exc
try:
detail = client.head_object(Bucket=bucket, Key=key)
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorImportError(f"S3 object metadata lookup failed: {exc}") from exc
if not isinstance(detail, dict):
raise ConnectorImportError("S3 connector returned invalid object metadata")
size = _int(detail.get("ContentLength"))
if size is not None and size > max_bytes:
raise ConnectorImportError(f"S3 object exceeds limit of {max_bytes} bytes")
version_id = _clean(detail.get("VersionId"))
request: dict[str, object] = {"Bucket": bucket, "Key": key}
if version_id:
request["VersionId"] = version_id
try:
response = client.get_object(**request)
body = response.get("Body")
data = body.read(max_bytes + 1) if hasattr(body, "read") else bytes(response.get("Body") or b"")
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorImportError(f"S3 object download failed: {exc}") from exc
if len(data) > max_bytes:
raise ConnectorImportError(f"S3 object exceeds limit of {max_bytes} bytes")
content_type = _clean(response.get("ContentType") if isinstance(response, dict) else None) or _clean(detail.get("ContentType")) or mimetypes.guess_type(key)[0]
etag = _clean(response.get("ETag") if isinstance(response, dict) else None) or _clean(detail.get("ETag"))
filename = filename_from_path(key)
return ConnectorDownloadedFile(
filename=filename,
data=data,
content_type=content_type,
revision=version_id or etag or _clean(detail.get("LastModified")),
external_id=f"{bucket}:{key}",
external_url=f"s3://{bucket}/{key}",
metadata={
"bucket": bucket,
"key": key,
"version_id": version_id,
"etag": etag,
"size": len(data),
**({"checksum_sha256": detail["ChecksumSHA256"]} if "ChecksumSHA256" in detail else {}),
**({"checksum_crc32": detail["ChecksumCRC32"]} if "ChecksumCRC32" in detail else {}),
**({"storage_class": detail["StorageClass"]} if "StorageClass" in detail else {}),
},
)
def _int(value: object) -> int | None: def _int(value: object) -> int | None:
if value is None or value == "": if value is None or value == "":
return None return None

116
src/govoplan_files/backend/storage/connector_profiles.py
View File

@@ -12,8 +12,17 @@ from govoplan_files.backend.storage.connector_policy import ConnectorPolicySourc
_ENV_JSON_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON", "FILES_CONNECTOR_PROFILES_JSON") _ENV_JSON_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON", "FILES_CONNECTOR_PROFILES_JSON")
_ENV_FILE_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE", "FILES_CONNECTOR_PROFILES_FILE") _ENV_FILE_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE", "FILES_CONNECTOR_PROFILES_FILE")
_SUPPORTED_PROVIDERS = {"seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"} _SUPPORTED_PROVIDERS = {"seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"}
_INLINE_SECRET_FIELDS = ("password", "token", "api_key", "access_key", "secret_key") _INLINE_SECRET_FIELDS = (
"password",
"token",
"api_key",
"access_key",
"access_key_id",
"secret_key",
"secret_access_key",
"session_token",
)
def supported_connector_providers() -> set[str]: def supported_connector_providers() -> set[str]:
@@ -123,43 +132,19 @@ def connector_profiles_from_payload(value: object) -> list[ConnectorProfile]:
def _profile_from_mapping(value: Mapping[str, Any]) -> ConnectorProfile: def _profile_from_mapping(value: Mapping[str, Any]) -> ConnectorProfile:
profile_id = _clean(value.get("id") or value.get("connector_id") or value.get("name")) profile_id = _profile_id_from_mapping(value)
if not profile_id: provider = _provider_from_mapping(value)
raise ValueError("Connector profiles require id") scope_type, scope_id = _scope_from_mapping(value)
provider = (_clean(value.get("provider") or value.get("type")) or "generic").casefold() credentials = _credentials_from_mapping(value)
if provider not in _SUPPORTED_PROVIDERS:
raise ValueError(f"Unsupported connector provider: {provider}")
scope_type = normalize_policy_scope_type(str(value.get("scope_type") or "system"))
scope_id = _clean(value.get("scope_id"))
if scope_type == "system":
scope_id = None
elif not scope_id:
raise ValueError(f"{scope_type} connector profiles require scope_id")
credentials = value.get("credentials")
credentials = credentials if isinstance(credentials, Mapping) else {}
mode = _clean(value.get("credential_mode") or value.get("auth_type") or credentials.get("mode") or credentials.get("type")) or "none" mode = _clean(value.get("credential_mode") or value.get("auth_type") or credentials.get("mode") or credentials.get("type")) or "none"
profile = ConnectorProfile( profile = _profile_from_normalized_mapping(
id=profile_id, value,
label=_clean(value.get("label") or value.get("name")) or profile_id, profile_id=profile_id,
provider=provider, provider=provider,
endpoint_url=_clean(value.get("endpoint_url") or value.get("base_url") or value.get("url")),
base_path=_clean(value.get("base_path") or value.get("path") or value.get("root")),
enabled=_bool(value.get("enabled"), default=True),
scope_type=scope_type, scope_type=scope_type,
scope_id=scope_id, scope_id=scope_id,
credential_profile_id=_clean(value.get("credential_profile_id") or value.get("credential_id")),
credential_profile_label=_clean(value.get("credential_profile_label") or value.get("credential_label")),
credential_mode=mode, credential_mode=mode,
username=_clean(value.get("username") or credentials.get("username")), credentials=credentials,
password_env=_clean(value.get("password_env") or credentials.get("password_env")),
token_env=_clean(value.get("token_env") or credentials.get("token_env")),
secret_ref=_clean(value.get("secret_ref") or credentials.get("secret_ref")),
password_value=_clean(value.get("password") or credentials.get("password")),
token_value=_clean(value.get("token") or credentials.get("token")),
has_inline_secret=any(_clean(value.get(field) or credentials.get(field)) for field in _INLINE_SECRET_FIELDS),
capabilities=tuple(_string_list(value.get("capabilities"))),
metadata=_public_metadata(value.get("metadata")),
) )
return ConnectorProfile( return ConnectorProfile(
id=profile.id, id=profile.id,
@@ -187,6 +172,69 @@ def _profile_from_mapping(value: Mapping[str, Any]) -> ConnectorProfile:
) )
def _profile_id_from_mapping(value: Mapping[str, Any]) -> str:
profile_id = _clean(value.get("id") or value.get("connector_id") or value.get("name"))
if not profile_id:
raise ValueError("Connector profiles require id")
return profile_id
def _provider_from_mapping(value: Mapping[str, Any]) -> str:
provider = (_clean(value.get("provider") or value.get("type")) or "generic").casefold()
if provider not in _SUPPORTED_PROVIDERS:
raise ValueError(f"Unsupported connector provider: {provider}")
return provider
def _scope_from_mapping(value: Mapping[str, Any]) -> tuple[str, str | None]:
scope_type = normalize_policy_scope_type(str(value.get("scope_type") or "system"))
scope_id = _clean(value.get("scope_id"))
if scope_type == "system":
return scope_type, None
if not scope_id:
raise ValueError(f"{scope_type} connector profiles require scope_id")
return scope_type, scope_id
def _credentials_from_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]:
credentials = value.get("credentials")
return credentials if isinstance(credentials, Mapping) else {}
def _profile_from_normalized_mapping(
value: Mapping[str, Any],
*,
profile_id: str,
provider: str,
scope_type: str,
scope_id: str | None,
credential_mode: str,
credentials: Mapping[str, Any],
) -> ConnectorProfile:
return ConnectorProfile(
id=profile_id,
label=_clean(value.get("label") or value.get("name")) or profile_id,
provider=provider,
endpoint_url=_clean(value.get("endpoint_url") or value.get("base_url") or value.get("url")),
base_path=_clean(value.get("base_path") or value.get("path") or value.get("root")),
enabled=_bool(value.get("enabled"), default=True),
scope_type=scope_type,
scope_id=scope_id,
credential_profile_id=_clean(value.get("credential_profile_id") or value.get("credential_id")),
credential_profile_label=_clean(value.get("credential_profile_label") or value.get("credential_label")),
credential_mode=credential_mode,
username=_clean(value.get("username") or credentials.get("username")),
password_env=_clean(value.get("password_env") or credentials.get("password_env")),
token_env=_clean(value.get("token_env") or credentials.get("token_env")),
secret_ref=_clean(value.get("secret_ref") or credentials.get("secret_ref")),
password_value=_clean(value.get("password") or credentials.get("password")),
token_value=_clean(value.get("token") or credentials.get("token")),
has_inline_secret=any(_clean(value.get(field) or credentials.get(field)) for field in _INLINE_SECRET_FIELDS),
capabilities=tuple(_string_list(value.get("capabilities"))),
metadata=_public_metadata(value.get("metadata")),
)
def _raw_profiles_payload(settings: object | None) -> object: def _raw_profiles_payload(settings: object | None) -> object:
for key in _ENV_JSON_KEYS: for key in _ENV_JSON_KEYS:
value = os.environ.get(key) value = os.environ.get(key)

45
src/govoplan_files/backend/storage/connector_providers.py
View File

@@ -112,6 +112,51 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Requires the optional smbprotocol dependency and an smb://server[:port]/share[/path] profile endpoint.", notes="Requires the optional smbprotocol dependency and an smb://server[:port]/share[/path] profile endpoint.",
), ),
ConnectorProviderDescriptor(
provider="s3",
label="S3-compatible object storage",
protocol="s3-api",
implemented=True,
browse_supported=True,
import_supported=True,
optional_dependency="boto3",
permission_model=governed_permissions,
sync_strategy="On-demand bucket/prefix browse and object import/sync; sync updates managed versions and never mutates the remote bucket.",
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.",
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Configure endpoint_url for MinIO or other S3-compatible stores; use profile metadata for bucket, region, addressing_style, verify_tls, and ca_bundle.",
),
ConnectorProviderDescriptor(
provider="sharepoint",
label="SharePoint",
protocol="microsoft-graph/sharepoint",
implemented=False,
browse_supported=False,
import_supported=False,
optional_dependency=None,
permission_model=governed_permissions,
sync_strategy="Planned read-only browse/import through deployment-managed Microsoft Graph or SharePoint credentials.",
conflict_strategy=freeze_model,
preview_strategy="Will preview from frozen managed file after import, not directly from SharePoint.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Provider/profile key is reserved; live browsing/import still needs Graph/SharePoint authentication and paging implementation.",
),
ConnectorProviderDescriptor(
provider="onedrive",
label="OneDrive",
protocol="microsoft-graph/onedrive",
implemented=False,
browse_supported=False,
import_supported=False,
optional_dependency=None,
permission_model=governed_permissions,
sync_strategy="Planned read-only browse/import through deployment-managed Microsoft Graph credentials.",
conflict_strategy=freeze_model,
preview_strategy="Will preview from frozen managed file after import, not directly from OneDrive.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Provider/profile key is reserved; live browsing/import still needs Graph drive selection, OAuth/app registration, and paging implementation.",
),
ConnectorProviderDescriptor( ConnectorProviderDescriptor(
provider="nfs", provider="nfs",
label="NFS", label="NFS",

863
src/govoplan_files/backend/storage/transfers.py
View File

File diff suppressed because it is too large Load Diff

157
tests/test_connector_providers.py Normal file
View File

@@ -0,0 +1,157 @@
from __future__ import annotations
import unittest
from datetime import UTC, datetime
from unittest.mock import patch
from govoplan_files.backend.storage.connector_browse import browse_connector_profile
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseUnsupported
from govoplan_files.backend.storage.connector_imports import read_connector_file
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload
from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors
class FakeBody:
def __init__(self, data: bytes) -> None:
self.data = data
def read(self, _limit: int) -> bytes:
return self.data
class FakeS3Client:
def __init__(self) -> None:
self.list_objects_request: dict[str, object] | None = None
self.head_request: dict[str, object] | None = None
self.get_request: dict[str, object] | None = None
def list_objects_v2(self, **kwargs: object) -> dict[str, object]:
self.list_objects_request = dict(kwargs)
return {
"CommonPrefixes": [{"Prefix": "root/reports/"}],
"Contents": [
{
"Key": "root/report.xlsx",
"Size": 123,
"LastModified": datetime(2026, 7, 12, 10, 0, tzinfo=UTC),
"ETag": '"etag-1"',
"StorageClass": "STANDARD",
}
],
"NextContinuationToken": "next-page",
}
def list_buckets(self) -> dict[str, object]:
return {"Buckets": [{"Name": "archive", "CreationDate": datetime(2026, 7, 12, 9, 0, tzinfo=UTC)}]}
def head_object(self, **kwargs: object) -> dict[str, object]:
self.head_request = dict(kwargs)
return {
"ContentLength": 13,
"ContentType": "text/plain",
"ETag": '"etag-2"',
"VersionId": "version-1",
"ChecksumSHA256": "checksum",
}
def get_object(self, **kwargs: object) -> dict[str, object]:
self.get_request = dict(kwargs)
return {"Body": FakeBody(b"hello s3\n"), "ContentType": "text/plain", "ETag": '"etag-2"'}
def s3_profile(**overrides: object) -> ConnectorProfile:
values = {
"id": "dev-s3",
"label": "Dev S3",
"provider": "s3",
"endpoint_url": "http://127.0.0.1:9000",
"base_path": "root",
"credential_mode": "basic",
"username": "access-key",
"password_value": "secret-key",
"metadata": {"bucket": "govoplan", "region": "eu-central-1", "path_style": True},
}
values.update(overrides)
return ConnectorProfile(**values)
class ConnectorProviderTests(unittest.TestCase):
def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None:
descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()}
self.assertTrue(descriptors["s3"].implemented)
self.assertTrue(descriptors["s3"].browse_supported)
self.assertEqual("boto3", descriptors["s3"].optional_dependency)
self.assertFalse(descriptors["sharepoint"].implemented)
self.assertFalse(descriptors["onedrive"].implemented)
def test_profile_payload_accepts_new_providers_and_redacts_secret_metadata(self) -> None:
profiles = connector_profiles_from_payload(
{
"profiles": [
{
"id": "dev-s3",
"label": "Dev S3",
"provider": "s3",
"username": "access-key",
"password": "secret-key",
"metadata": {"bucket": "govoplan", "secret_access_key": "do-not-return"},
},
{"id": "sharepoint", "provider": "sharepoint"},
{"id": "onedrive", "provider": "onedrive"},
]
}
)
by_id = {profile.id: profile for profile in profiles}
self.assertEqual("s3", by_id["dev-s3"].provider)
self.assertTrue(by_id["dev-s3"].credentials_configured)
self.assertEqual({"bucket": "govoplan"}, dict(by_id["dev-s3"].metadata))
self.assertEqual("sharepoint", by_id["sharepoint"].provider)
self.assertEqual("onedrive", by_id["onedrive"].provider)
def test_s3_browse_lists_prefixes_and_objects_with_provenance_metadata(self) -> None:
client = FakeS3Client()
with patch("govoplan_files.backend.storage.connector_browse._s3_client", return_value=client):
items = browse_connector_profile(s3_profile(), path="")
self.assertEqual({"Bucket": "govoplan", "Prefix": "root/", "Delimiter": "/", "MaxKeys": 1000}, client.list_objects_request)
self.assertEqual(["folder", "file"], [item.kind for item in items])
self.assertEqual("reports", items[0].path)
self.assertEqual("report.xlsx", items[1].path)
self.assertEqual("govoplan:root/report.xlsx", items[1].external_id)
self.assertEqual("root/report.xlsx", items[1].metadata["key"])
self.assertEqual("next-page", items[1].metadata["next_continuation_token"])
def test_s3_browse_lists_buckets_when_profile_has_no_bucket(self) -> None:
client = FakeS3Client()
with patch("govoplan_files.backend.storage.connector_browse._s3_client", return_value=client):
items = browse_connector_profile(s3_profile(metadata={}), path="")
self.assertEqual(["archive"], [item.path for item in items])
def test_s3_browse_reports_missing_optional_dependency(self) -> None:
with patch("govoplan_files.backend.storage.connector_browse.import_module", side_effect=ImportError):
with self.assertRaisesRegex(ConnectorBrowseUnsupported, "boto3"):
browse_connector_profile(s3_profile(), path="")
def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None:
client = FakeS3Client()
with patch("govoplan_files.backend.storage.connector_imports._s3_client", return_value=client):
downloaded = read_connector_file(s3_profile(), library_id="", path="report.txt", max_bytes=1024)
self.assertEqual({"Bucket": "govoplan", "Key": "root/report.txt"}, client.head_request)
self.assertEqual({"Bucket": "govoplan", "Key": "root/report.txt", "VersionId": "version-1"}, client.get_request)
self.assertEqual("report.txt", downloaded.filename)
self.assertEqual(b"hello s3\n", downloaded.data)
self.assertEqual("version-1", downloaded.revision)
self.assertEqual("govoplan:root/report.txt", downloaded.external_id)
self.assertEqual("s3://govoplan/root/report.txt", downloaded.external_url)
self.assertEqual("checksum", downloaded.metadata["checksum_sha256"])
if __name__ == "__main__":
unittest.main()

36
tests/test_router_contract.py Normal file
View File

@@ -0,0 +1,36 @@
from __future__ import annotations
import unittest
from govoplan_files.backend.router import router
class FilesRouterContractTests(unittest.TestCase):
def test_connector_routes_keep_existing_api_paths(self) -> None:
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
expected = {
(("GET",), "/files/connectors/providers"),
(("GET",), "/files/connectors/settings/delta"),
(("GET",), "/files/connectors/profiles"),
(("POST",), "/files/connectors/profiles"),
(("GET",), "/files/connectors/profiles/{profile_id}/browse"),
(("POST",), "/files/connectors/profiles/{profile_id}/import"),
(("POST",), "/files/connectors/profiles/{profile_id}/sync"),
(("GET",), "/files/connectors/credentials"),
(("POST",), "/files/connectors/credentials"),
(("GET",), "/files/connector-spaces"),
(("POST",), "/files/connector-spaces"),
}
self.assertTrue(expected.issubset(routes))
def test_bulk_organize_routes_keep_existing_api_paths(self) -> None:
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
self.assertIn((("POST",), "/files/bulk-rename"), routes)
self.assertIn((("POST",), "/files/transfer"), routes)
if __name__ == "__main__":
unittest.main()

67
tests/test_transfer_helpers.py Normal file
View File

@@ -0,0 +1,67 @@
from __future__ import annotations
import unittest
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError
from govoplan_files.backend.storage.transfers import _normalize_rename_request, _normalize_transfer_request
class TransferHelperTests(unittest.TestCase):
def test_transfer_normalization_deduplicates_folder_paths_and_conflicts(self) -> None:
normalized = _normalize_transfer_request(
operation=" COPY ",
source_owner_type=" USER ",
target_owner_type=" GROUP ",
folder_paths=["Reports", "Reports/2026", "", "./Archive"],
target_folder=" Target ",
conflict_strategy="rename",
conflict_resolutions=[FileConflictResolution(target_path="Target/a.txt", action="skip")],
)
self.assertEqual("copy", normalized.operation)
self.assertEqual("user", normalized.source_owner_type)
self.assertEqual("group", normalized.target_owner_type)
self.assertEqual(["Reports", "Reports/2026", "Archive"], normalized.source_folder_paths)
self.assertEqual("Target", normalized.target_folder)
self.assertEqual("rename", normalized.conflict_strategy)
self.assertEqual("skip", normalized.conflict_resolution_map["Target/a.txt"].action)
def test_transfer_normalization_rejects_unknown_operation(self) -> None:
with self.assertRaisesRegex(FileStorageError, "Unsupported transfer operation"):
_normalize_transfer_request(
operation="link",
source_owner_type="user",
target_owner_type="group",
folder_paths=[],
target_folder="",
conflict_strategy="reject",
conflict_resolutions=None,
)
def test_rename_normalization_collapses_nested_folder_roots(self) -> None:
normalized = _normalize_rename_request(
file_ids=["file-1", "file-1", "file-2"],
folder_paths=["Reports", "Reports/2026", "Archive"],
owner_type=" USER ",
owner_id="owner-1",
mode=" prefix ",
)
self.assertEqual("prefix", normalized.mode)
self.assertEqual(["file-1", "file-2"], normalized.selected_file_ids)
self.assertEqual(["Archive", "Reports"], normalized.selected_folder_roots)
self.assertEqual("user", normalized.owner_type)
def test_rename_normalization_rejects_invalid_direct_multi_select(self) -> None:
with self.assertRaisesRegex(FileStorageError, "Direct rename requires exactly one selected item"):
_normalize_rename_request(
file_ids=["file-1", "file-2"],
folder_paths=[],
owner_type="user",
owner_id="owner-1",
mode="direct",
)
if __name__ == "__main__":
unittest.main()

9
webui/src/api/files.ts
View File

@@ -150,7 +150,7 @@ export type FileConnectorDiscoveryResponse = {
export type FileConnectorProfilePayload = { export type FileConnectorProfilePayload = {
id: string; id: string;
label: string; label: string;
provider: "seafile" | "nextcloud" | "webdav" | "smb" | "nfs" | "dms" | "generic"; provider: "seafile" | "nextcloud" | "webdav" | "smb" | "s3" | "sharepoint" | "onedrive" | "nfs" | "dms" | "generic";
endpoint_url?: string | null; endpoint_url?: string | null;
base_path?: string | null; base_path?: string | null;
enabled?: boolean; enabled?: boolean;
@@ -172,7 +172,7 @@ export type FileConnectorProfileUpdatePayload = Partial<Omit<FileConnectorProfil
export type FileConnectorCredentialPayload = { export type FileConnectorCredentialPayload = {
id: string; id: string;
label: string; label: string;
provider?: "seafile" | "nextcloud" | "webdav" | "smb" | "nfs" | "dms" | "generic" | null; provider?: "seafile" | "nextcloud" | "webdav" | "smb" | "s3" | "sharepoint" | "onedrive" | "nfs" | "dms" | "generic" | null;
enabled?: boolean; enabled?: boolean;
scope_type?: "system" | "tenant" | "user" | "group" | "campaign"; scope_type?: "system" | "tenant" | "user" | "group" | "campaign";
scope_id?: string | null; scope_id?: string | null;
@@ -206,6 +206,8 @@ export type FileConnectorBrowseResponse = {
path: string; path: string;
library_id?: string | null; library_id?: string | null;
read_only: boolean; read_only: boolean;
next_continuation_token?: string | null;
has_more: boolean;
decision: FileConnectorPolicyDecision; decision: FileConnectorPolicyDecision;
items: FileConnectorBrowseItem[]; items: FileConnectorBrowseItem[];
}; };
@@ -808,11 +810,12 @@ export function deactivateFileConnectorProfile(settings: ApiSettings, profileId:
export function browseFileConnectorProfile( export function browseFileConnectorProfile(
settings: ApiSettings, settings: ApiSettings,
profileId: string, profileId: string,
params: {path?: string;library_id?: string;campaign_id?: string;} = {}) params: {path?: string;library_id?: string;continuation_token?: string;campaign_id?: string;} = {})
: Promise<FileConnectorBrowseResponse> { : Promise<FileConnectorBrowseResponse> {
const search = new URLSearchParams(); const search = new URLSearchParams();
if (params.path) search.set("path", params.path); if (params.path) search.set("path", params.path);
if (params.library_id) search.set("library_id", params.library_id); if (params.library_id) search.set("library_id", params.library_id);
if (params.continuation_token) search.set("continuation_token", params.continuation_token);
if (params.campaign_id) search.set("campaign_id", params.campaign_id); if (params.campaign_id) search.set("campaign_id", params.campaign_id);
const suffix = search.toString() ? `?${search.toString()}` : ""; const suffix = search.toString() ? `?${search.toString()}` : "";
return apiFetch<FileConnectorBrowseResponse>(settings, `/api/v1/files/connectors/profiles/${encodeURIComponent(profileId)}/browse${suffix}`); return apiFetch<FileConnectorBrowseResponse>(settings, `/api/v1/files/connectors/profiles/${encodeURIComponent(profileId)}/browse${suffix}`);

71
webui/src/features/files/FileConnectorSettingsPanel.tsx
View File

@@ -5,6 +5,8 @@ import {
Card, Card,
ConnectionTree, ConnectionTree,
ConfirmDialog, ConfirmDialog,
CredentialPanel,
DisabledActionTooltip,
DismissibleAlert, DismissibleAlert,
Dialog, Dialog,
FormField, FormField,
@@ -123,6 +125,9 @@ const PROVIDERS: Array<{id: Provider;label: string;}> = [
{ id: "nextcloud", label: "i18n:govoplan-files.nextcloud.aab73a3d" }, { id: "nextcloud", label: "i18n:govoplan-files.nextcloud.aab73a3d" },
{ id: "seafile", label: "i18n:govoplan-files.seafile.600192cc" }, { id: "seafile", label: "i18n:govoplan-files.seafile.600192cc" },
{ id: "smb", label: "i18n:govoplan-files.smb.515d2f88" }, { id: "smb", label: "i18n:govoplan-files.smb.515d2f88" },
{ id: "s3", label: "S3" },
{ id: "sharepoint", label: "SharePoint" },
{ id: "onedrive", label: "OneDrive" },
{ id: "nfs", label: "i18n:govoplan-files.nfs.db121865" }, { id: "nfs", label: "i18n:govoplan-files.nfs.db121865" },
{ id: "dms", label: "i18n:govoplan-files.dms.477e5652" }, { id: "dms", label: "i18n:govoplan-files.dms.477e5652" },
{ id: "generic", label: "i18n:govoplan-files.generic.ff7613e5" }]; { id: "generic", label: "i18n:govoplan-files.generic.ff7613e5" }];
@@ -133,6 +138,9 @@ const ENDPOINT_PLACEHOLDERS: Record<Provider, string> = {
nextcloud: "http://127.0.0.1:9081/remote.php/dav/files/admin/", nextcloud: "http://127.0.0.1:9081/remote.php/dav/files/admin/",
seafile: "http://127.0.0.1:9082/", seafile: "http://127.0.0.1:9082/",
smb: "smb://127.0.0.1:1445/files", smb: "smb://127.0.0.1:1445/files",
s3: "http://127.0.0.1:9000",
sharepoint: "https://graph.microsoft.com/v1.0/sites/{site-id}/drives/{drive-id}",
onedrive: "https://graph.microsoft.com/v1.0/drives/{drive-id}",
nfs: "nfs://127.0.0.1/export", nfs: "nfs://127.0.0.1/export",
dms: "https://dms.example.test", dms: "https://dms.example.test",
generic: "https://files.example.test" generic: "https://files.example.test"
@@ -340,6 +348,17 @@ export default function FileConnectorSettingsPanel({
setCredentialDraft((current) => current ? { ...current, ...patch } : current); setCredentialDraft((current) => current ? { ...current, ...patch } : current);
} }
function patchCredentialValues(patch: {username?: string | null;password?: string | null;}) {
const next: Partial<ConnectorCredentialDraft> = {};
if (patch.username !== undefined) next.username = String(patch.username ?? "");
if (patch.password !== undefined) next.password = String(patch.password ?? "");
patchCredentialDraft(next);
}
function patchCredentialToken(patch: {password?: string | null;}) {
if (patch.password !== undefined) patchCredentialDraft({ token: String(patch.password ?? "") });
}
function patchPolicyDraft(patch: Partial<CentralConnectorPolicyDraft>) { function patchPolicyDraft(patch: Partial<CentralConnectorPolicyDraft>) {
setPolicyDraft((current) => ({ ...current, ...patch })); setPolicyDraft((current) => ({ ...current, ...patch }));
} }
@@ -906,11 +925,11 @@ export default function FileConnectorSettingsPanel({
footer={credentialDraft ? footer={credentialDraft ?
<> <>
<Button onClick={closeCredentialDialog} disabled={saving}>i18n:govoplan-files.cancel.77dfd213</Button> <Button onClick={closeCredentialDialog} disabled={saving}>i18n:govoplan-files.cancel.77dfd213</Button>
<span className="disabled-action-tooltip" data-tooltip={credentialSaveTooltip}> <DisabledActionTooltip reason={credentialSaveTooltip}>
<Button variant="primary" onClick={() => void saveCredentialDraft()} disabled={credentialSaveDisabled}> <Button variant="primary" onClick={() => void saveCredentialDraft()} disabled={credentialSaveDisabled}>
<Save size={16} aria-hidden="true" /> {saving ? "i18n:govoplan-files.saving.ae7e8875" : "i18n:govoplan-files.save_credential.2c02a6d2"} <Save size={16} aria-hidden="true" /> {saving ? "i18n:govoplan-files.saving.ae7e8875" : "i18n:govoplan-files.save_credential.2c02a6d2"}
</Button> </Button>
</span> </DisabledActionTooltip>
</> : </> :
null}> null}>
@@ -983,35 +1002,47 @@ export default function FileConnectorSettingsPanel({
actor: "Connector administrator", actor: "Connector administrator",
target: "Credential mode" target: "Credential mode"
}} /> : }} /> :
<div className="form-grid two-column-form-grid"> <div className="file-connector-secret-fields">
{(credentialDraft.credentialMode === "basic" || credentialDraft.credentialMode === "secret_ref") &&
<FormField label="i18n:govoplan-files.username.84c29015">
<input value={credentialDraft.username} disabled={saving} onChange={(event) => patchCredentialDraft({ username: event.target.value })} autoComplete="username" />
</FormField>
}
{credentialDraft.credentialMode === "basic" && {credentialDraft.credentialMode === "basic" &&
<FormField label="i18n:govoplan-files.password.8be3c943"> <CredentialPanel
<input value={credentialDraft.password} disabled={saving} onChange={(event) => patchCredentialDraft({ password: event.target.value })} type="password" autoComplete="new-password" placeholder={editingCredentialId ? "i18n:govoplan-files.leave_empty_to_keep_saved_password.6ec39f5e" : ""} /> values={{ username: credentialDraft.username, password: credentialDraft.password }}
</FormField> onChange={patchCredentialValues}
} disabled={saving}
{credentialDraft.credentialMode === "token" && savedPassword={Boolean(editingCredentialId) && !credentialDraft.clearPassword}
<FormField label="i18n:govoplan-files.token.a1141eb9"> savedPasswordPlaceholder="i18n:govoplan-files.leave_empty_to_keep_saved_password.6ec39f5e" />
<input value={credentialDraft.token} disabled={saving} onChange={(event) => patchCredentialDraft({ token: event.target.value })} type="password" placeholder={editingCredentialId ? "i18n:govoplan-files.leave_empty_to_keep_saved_token.e725857b" : ""} />
</FormField>
} }
{credentialDraft.credentialMode === "secret_ref" && {credentialDraft.credentialMode === "secret_ref" &&
<>
<CredentialPanel
values={{ username: credentialDraft.username }}
onChange={patchCredentialValues}
disabled={saving}
showPassword={false} />
<div className="form-grid two-column-form-grid">
<FormField label="i18n:govoplan-files.secret_reference.04ed2221"> <FormField label="i18n:govoplan-files.secret_reference.04ed2221">
<input value={credentialDraft.secretRef} disabled={saving} onChange={(event) => patchCredentialDraft({ secretRef: event.target.value })} /> <input value={credentialDraft.secretRef} disabled={saving} onChange={(event) => patchCredentialDraft({ secretRef: event.target.value })} />
</FormField> </FormField>
</div>
</>
}
{credentialDraft.credentialMode === "token" &&
<CredentialPanel
values={{ password: credentialDraft.token }}
onChange={patchCredentialToken}
disabled={saving}
showUsername={false}
passwordLabel="i18n:govoplan-files.token.a1141eb9"
savedPassword={Boolean(editingCredentialId) && !credentialDraft.clearToken}
savedPasswordPlaceholder="i18n:govoplan-files.leave_empty_to_keep_saved_token.e725857b" />
} }
</div> </div>
} }
<div className="button-row compact-actions"> <div className="button-row compact-actions">
<span className="disabled-action-tooltip" data-tooltip={credentialTestDisabled ? credentialTestTooltip : ""}> <DisabledActionTooltip reason={credentialTestDisabled ? credentialTestTooltip : ""}>
<Button type="button" onClick={() => void testCredentialLogin()} disabled={credentialTestDisabled}> <Button type="button" onClick={() => void testCredentialLogin()} disabled={credentialTestDisabled}>
<RefreshCw size={16} aria-hidden="true" /> {testingCredentialLogin ? "Testing login" : "Test login"} <RefreshCw size={16} aria-hidden="true" /> {testingCredentialLogin ? "Testing login" : "Test login"}
</Button> </Button>
</span> </DisabledActionTooltip>
</div> </div>
{credentialLoginResult && {credentialLoginResult &&
<DismissibleAlert tone={credentialLoginResult.ok ? "success" : "warning"} resetKey={credentialLoginResult.message}> <DismissibleAlert tone={credentialLoginResult.ok ? "success" : "warning"} resetKey={credentialLoginResult.message}>
@@ -1072,11 +1103,11 @@ export default function FileConnectorSettingsPanel({
footer={draft ? footer={draft ?
<> <>
<Button onClick={closeProfileDialog} disabled={saving}>i18n:govoplan-files.cancel.77dfd213</Button> <Button onClick={closeProfileDialog} disabled={saving}>i18n:govoplan-files.cancel.77dfd213</Button>
<span className="disabled-action-tooltip" data-tooltip={profileSaveTooltip}> <DisabledActionTooltip reason={profileSaveTooltip}>
<Button variant="primary" onClick={() => void saveDraft()} disabled={profileSaveDisabled}> <Button variant="primary" onClick={() => void saveDraft()} disabled={profileSaveDisabled}>
<Save size={16} aria-hidden="true" /> {saving ? "i18n:govoplan-files.saving.ae7e8875" : "i18n:govoplan-files.save_connection.07796d38"} <Save size={16} aria-hidden="true" /> {saving ? "i18n:govoplan-files.saving.ae7e8875" : "i18n:govoplan-files.save_connection.07796d38"}
</Button> </Button>
</span> </DisabledActionTooltip>
</> : </> :
null}> null}>

11
webui/src/features/files/FilesPage.tsx
View File

@@ -182,6 +182,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
}, [visibleFiles, folders, currentFolder, searchActive, sortColumn, sortDirection]); }, [visibleFiles, folders, currentFolder, searchActive, sortColumn, sortDirection]);
const fileListViewportRef = useRef<HTMLDivElement | null>(null); const fileListViewportRef = useRef<HTMLDivElement | null>(null);
const fileListMeasureRowRef = useRef<HTMLDivElement | null>(null); const fileListMeasureRowRef = useRef<HTMLDivElement | null>(null);
const managedSpaceLoadsInFlightRef = useRef<Set<string>>(new Set());
const [fileListViewportHeight, setFileListViewportHeight] = useState(0); const [fileListViewportHeight, setFileListViewportHeight] = useState(0);
const [fileListScrollTop, setFileListScrollTop] = useState(0); const [fileListScrollTop, setFileListScrollTop] = useState(0);
const [fileListRowHeight, setFileListRowHeight] = useState(DEFAULT_FILE_LIST_ROW_HEIGHT); const [fileListRowHeight, setFileListRowHeight] = useState(DEFAULT_FILE_LIST_ROW_HEIGHT);
@@ -291,8 +292,9 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
try { try {
const response = await listFileSpaces(settings); const response = await listFileSpaces(settings);
setSpaces(response.spaces); setSpaces(response.spaces);
setActiveSpaceId((current) => current || response.spaces[0]?.id || ""); const nextActiveSpace = response.spaces.find((space) => space.id === activeSpaceId) ?? response.spaces[0] ?? null;
await Promise.all(response.spaces.filter((space) => !isConnectorSpace(space)).map((space) => loadSpaceContents(space, { silent: true }))); setActiveSpaceId(nextActiveSpace?.id || "");
if (nextActiveSpace && !isConnectorSpace(nextActiveSpace)) await loadSpaceContents(nextActiveSpace, { silent: true });
} catch (err) { } catch (err) {
setError(err instanceof Error ? err.message : String(err)); setError(err instanceof Error ? err.message : String(err));
} finally { } finally {
@@ -331,6 +333,8 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
await loadConnectorSpaceContents(space, { silent: options.silent, folderPath: "" }); await loadConnectorSpaceContents(space, { silent: options.silent, folderPath: "" });
return; return;
} }
if (managedSpaceLoadsInFlightRef.current.has(space.id)) return;
managedSpaceLoadsInFlightRef.current.add(space.id);
if (!options.silent) { if (!options.silent) {
setBusy(true); setBusy(true);
setError(""); setError("");
@@ -372,6 +376,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
} catch (err) { } catch (err) {
setError(err instanceof Error ? err.message : String(err)); setError(err instanceof Error ? err.message : String(err));
} finally { } finally {
managedSpaceLoadsInFlightRef.current.delete(space.id);
if (!options.silent) setBusy(false); if (!options.silent) setBusy(false);
} }
} }
@@ -411,6 +416,8 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
const nextSpace = spaces.find((space) => space.id === activeSpaceId); const nextSpace = spaces.find((space) => space.id === activeSpaceId);
if (nextSpace && isConnectorSpace(nextSpace)) { if (nextSpace && isConnectorSpace(nextSpace)) {
void loadConnectorSpaceContents(nextSpace, { folderPath: "", silent: true }); void loadConnectorSpaceContents(nextSpace, { folderPath: "", silent: true });
} else if (nextSpace && filesBySpace[nextSpace.id] === undefined && foldersBySpace[nextSpace.id] === undefined) {
void loadSpaceContents(nextSpace, { silent: true });
} }
// eslint-disable-next-line react-hooks/exhaustive-deps // eslint-disable-next-line react-hooks/exhaustive-deps
}, [activeSpaceId, spaces]); }, [activeSpaceId, spaces]);

73
webui/src/styles/file-manager.css
View File

@@ -14,7 +14,7 @@
gap: 8px; gap: 8px;
flex-wrap: wrap; flex-wrap: wrap;
padding: 10px 14px; padding: 10px 14px;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
background: var(--panel-header); background: var(--panel-header);
} }
@@ -29,7 +29,7 @@
grid-template-columns: minmax(240px, 300px) minmax(0, 1fr); grid-template-columns: minmax(240px, 300px) minmax(0, 1fr);
min-height: 0; min-height: 0;
height: 100%; height: 100%;
border: 1px solid var(--line); border: var(--border-line);
border-radius: 0; border-radius: 0;
overflow: hidden; overflow: hidden;
background: var(--panel); background: var(--panel);
@@ -46,7 +46,7 @@
.file-list-sticky { .file-list-sticky {
flex: 0 0 auto; flex: 0 0 auto;
z-index: 2; z-index: 2;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
background: var(--panel-soft); background: var(--panel-soft);
} }
@@ -97,7 +97,7 @@
gap: 12px; gap: 12px;
flex-wrap: wrap; flex-wrap: wrap;
padding: 8px 14px; padding: 8px 14px;
border-top: 1px solid var(--line); border-top: var(--border-line);
color: var(--muted); color: var(--muted);
font-size: 12px; font-size: 12px;
} }
@@ -130,7 +130,7 @@
.file-list-table-head { .file-list-table-head {
padding: 10px 14px; padding: 10px 14px;
border-top: 1px solid var(--line); border-top: var(--border-line);
background: var(--panel); background: var(--panel);
color: var(--muted); color: var(--muted);
font-size: 12px; font-size: 12px;
@@ -161,7 +161,7 @@
.file-list-row { .file-list-row {
min-height: 58px; min-height: 58px;
padding: 9px 14px; padding: 9px 14px;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
cursor: default; cursor: default;
user-select: none; user-select: none;
} }
@@ -244,7 +244,7 @@
} }
.folder-row .file-row-icon { .folder-row .file-row-icon {
color: #b6791d; color: var(--warning-deep);
} }
.file-row-tail { .file-row-tail {
@@ -274,7 +274,7 @@
display: grid; display: grid;
min-width: 190px; min-width: 190px;
overflow: hidden; overflow: hidden;
border: 1px solid var(--line-dark); border: var(--border-line-dark);
border-radius: 12px; border-radius: 12px;
background: var(--panel); background: var(--panel);
box-shadow: var(--shadow-popover); box-shadow: var(--shadow-popover);
@@ -299,7 +299,7 @@
.file-context-menu button:hover, .file-context-menu button:hover,
.file-context-menu button:focus-visible { .file-context-menu button:focus-visible {
background: rgba(13, 110, 253, .08); background: var(--primary-soft);
outline: none; outline: none;
} }
@@ -315,14 +315,14 @@
display: grid; display: grid;
place-items: center; place-items: center;
padding: 22px; padding: 22px;
background: rgba(28, 25, 22, .38); background: var(--dialog-backdrop);
} }
.file-dialog { .file-dialog {
width: min(620px, 100%); width: min(620px, 100%);
max-height: min(720px, calc(100vh - 44px)); max-height: min(720px, calc(100vh - 44px));
overflow: auto; overflow: auto;
border: 1px solid var(--line-dark); border: var(--border-line-dark);
border-radius: 16px; border-radius: 16px;
background: var(--panel); background: var(--panel);
box-shadow: var(--shadow-popover); box-shadow: var(--shadow-popover);
@@ -334,7 +334,7 @@
justify-content: space-between; justify-content: space-between;
gap: 12px; gap: 12px;
padding: 15px 18px; padding: 15px 18px;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
background: var(--panel-soft); background: var(--panel-soft);
} }
@@ -397,7 +397,7 @@
overflow-y: auto; overflow-y: auto;
scrollbar-gutter: stable; scrollbar-gutter: stable;
padding: 8px; padding: 8px;
border: 1px solid var(--line); border: var(--border-line);
border-radius: 12px; border-radius: 12px;
background: var(--panel-soft); background: var(--panel-soft);
} }
@@ -428,13 +428,13 @@
.file-folder-selector-node:hover:not(:disabled), .file-folder-selector-node:hover:not(:disabled),
.file-folder-selector-node:focus-visible, .file-folder-selector-node:focus-visible,
.file-folder-selector-node.is-selected { .file-folder-selector-node.is-selected {
background: rgba(13, 110, 253, .09); background: var(--primary-soft);
outline: none; outline: none;
} }
.file-folder-selector-node.is-selected { .file-folder-selector-node.is-selected {
color: var(--text-strong); color: var(--text-strong);
box-shadow: inset 0 0 0 1px rgba(13, 110, 253, .25); box-shadow: var(--primary-inset-ring);
} }
.file-folder-selector-empty { .file-folder-selector-empty {
@@ -462,7 +462,7 @@
grid-template-rows: auto auto minmax(0, 1fr); grid-template-rows: auto auto minmax(0, 1fr);
min-height: 320px; min-height: 320px;
overflow: hidden; overflow: hidden;
border: 1px solid var(--line); border: var(--border-line);
border-radius: 12px; border-radius: 12px;
background: var(--panel); background: var(--panel);
} }
@@ -473,7 +473,7 @@
gap: 10px; gap: 10px;
align-items: center; align-items: center;
padding: 10px; padding: 10px;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
background: var(--panel-soft); background: var(--panel-soft);
} }
@@ -577,7 +577,7 @@
gap: 3px; gap: 3px;
min-width: 0; min-width: 0;
padding: 10px 12px; padding: 10px 12px;
border: 1px solid var(--line); border: var(--border-line);
border-radius: 10px; border-radius: 10px;
background: var(--panel-soft); background: var(--panel-soft);
} }
@@ -625,7 +625,7 @@
gap: 12px; gap: 12px;
align-items: center; align-items: center;
padding: 10px 12px; padding: 10px 12px;
border: 1px solid var(--line); border: var(--border-line);
border-radius: 8px; border-radius: 8px;
background: var(--panel-soft); background: var(--panel-soft);
} }
@@ -698,6 +698,11 @@
white-space: nowrap; white-space: nowrap;
} }
.file-connector-secret-fields {
display: grid;
gap: 18px;
}
@media (max-width: 760px) { @media (max-width: 760px) {
.file-connector-profile-row { .file-connector-profile-row {
grid-template-columns: 1fr; grid-template-columns: 1fr;
@@ -726,7 +731,7 @@
.file-tree-panel { .file-tree-panel {
max-height: 260px; max-height: 260px;
border-right: 0; border-right: 0;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
} }
.file-list-table-head { .file-list-table-head {
@@ -760,7 +765,7 @@
z-index: 35; z-index: 35;
display: grid; display: grid;
place-items: center; place-items: center;
background: rgba(255, 255, 255, .12); background: var(--panel-glass);
backdrop-filter: blur(1px); backdrop-filter: blur(1px);
} }
@@ -768,7 +773,7 @@
display: grid; display: grid;
gap: 3px; gap: 3px;
padding: 12px 14px; padding: 12px 14px;
border: 1px solid var(--line); border: var(--border-line);
border-radius: 12px; border-radius: 12px;
background: var(--panel-soft); background: var(--panel-soft);
} }
@@ -791,7 +796,7 @@
gap: 10px; gap: 10px;
align-items: center; align-items: center;
padding: 10px; padding: 10px;
border: 1px solid var(--line); border: var(--border-line);
border-radius: 12px; border-radius: 12px;
background: var(--panel-soft); background: var(--panel-soft);
} }
@@ -842,8 +847,8 @@
.rename-preview-more:hover, .rename-preview-more:hover,
.rename-preview-more:focus-visible { .rename-preview-more:focus-visible {
border-color: rgba(13, 110, 253, .45); border-color: var(--primary-border);
background: rgba(13, 110, 253, .08); background: var(--primary-soft);
color: var(--text-strong); color: var(--text-strong);
outline: none; outline: none;
} }
@@ -860,7 +865,7 @@
.managed-file-chooser-dialog > .dialog-header { .managed-file-chooser-dialog > .dialog-header {
padding: 16px 18px; padding: 16px 18px;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
background: var(--panel); background: var(--panel);
} }
@@ -891,9 +896,9 @@
.managed-file-chooser-footer { .managed-file-chooser-footer {
flex: 0 0 auto; flex: 0 0 auto;
padding: 12px 18px; padding: 12px 18px;
border-top: 1px solid var(--line); border-top: var(--border-line);
background: var(--panel); background: var(--panel);
box-shadow: 0 -8px 24px rgba(15, 23, 42, .06); box-shadow: var(--shadow-footer);
} }
.managed-file-chooser-layout { .managed-file-chooser-layout {
@@ -911,7 +916,7 @@
.managed-file-chooser-spaces { .managed-file-chooser-spaces {
min-width: 0; min-width: 0;
padding: 16px; padding: 16px;
border-right: 1px solid var(--line); border-right: var(--border-line);
background: var(--panel-soft); background: var(--panel-soft);
overflow: auto; overflow: auto;
} }
@@ -1014,7 +1019,7 @@
gap: 12px; gap: 12px;
align-items: center; align-items: center;
padding: 12px 14px; padding: 12px 14px;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
} }
.managed-file-breadcrumb { .managed-file-breadcrumb {
@@ -1046,7 +1051,7 @@
display: grid; display: grid;
gap: 8px; gap: 8px;
padding: 12px 14px; padding: 12px 14px;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
background: var(--panel-soft); background: var(--panel-soft);
} }
@@ -1075,7 +1080,7 @@
align-items: center; align-items: center;
min-height: 38px; min-height: 38px;
padding: 0 20px; padding: 0 20px;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
background: var(--panel-soft); background: var(--panel-soft);
color: var(--muted); color: var(--muted);
font-size: 12px; font-size: 12px;
@@ -1206,7 +1211,7 @@
.managed-file-chooser-note { .managed-file-chooser-note {
margin: 0; margin: 0;
padding: 10px 14px; padding: 10px 14px;
border-top: 1px solid var(--line); border-top: var(--border-line);
background: var(--panel-soft); background: var(--panel-soft);
} }
@@ -1313,7 +1318,7 @@
.managed-file-chooser-spaces { .managed-file-chooser-spaces {
border-right: 0; border-right: 0;
border-bottom: 1px solid var(--line); border-bottom: var(--border-line);
max-height: 160px; max-height: 160px;
} }