intermittent commit

This commit is contained in:
2026-07-14 13:22:11 +02:00
parent b8b395e8b5
commit f3210234d3
23 changed files with 2027 additions and 555 deletions

View File

@@ -17,3 +17,9 @@ SMB_HOST_PORT=1445
SMB_SHARE_NAME=files
SMB_USER=govoplan
SMB_PASSWORD=govoplan-smb
MINIO_API_HOST_PORT=9000
MINIO_CONSOLE_HOST_PORT=9001
MINIO_ROOT_USER=govoplan
MINIO_ROOT_PASSWORD=govoplan-minio
MINIO_BUCKET=govoplan

View File

@@ -10,7 +10,7 @@ binds services to localhost high ports.
cd /mnt/DATA/git/govoplan-files/dev/connectors
cp .env.example .env
mkdir -p data/smb data/webdav
docker compose up -d nextcloud nextcloud-db webdav smb
docker compose up -d nextcloud nextcloud-db webdav smb minio
docker compose up -d seafile-db seafile-memcached seafile
```
@@ -30,6 +30,8 @@ Endpoints:
`http://127.0.0.1:9082/seafdav/`
- WebDAV: `http://127.0.0.1:9083`
- SMB: `smb://127.0.0.1:1445/files`
- MinIO/S3 API: `http://127.0.0.1:9000`, console
`http://127.0.0.1:9001`
The local fixture data under `data/` is ignored by git.
@@ -93,6 +95,25 @@ Example local profile file:
"password_env": "SMB_PASSWORD",
"capabilities": ["browse", "import"],
"policy": { "allow": { "providers": ["smb"] } }
},
{
"id": "dev-s3",
"label": "Dev MinIO",
"provider": "s3",
"endpoint_url": "http://127.0.0.1:9000",
"base_path": "",
"scope_type": "system",
"credential_mode": "basic",
"username": "govoplan",
"password_env": "MINIO_ROOT_PASSWORD",
"capabilities": ["browse", "import"],
"metadata": {
"bucket": "govoplan",
"region": "us-east-1",
"path_style": true,
"verify_tls": false
},
"policy": { "allow": { "providers": ["s3"] } }
}
]
}
@@ -115,9 +136,11 @@ cd /mnt/DATA/git/govoplan-files/dev/connectors
```
The script reads `.env` from this directory when present, seeds tiny WebDAV,
Nextcloud, and SMB fixtures, then browses and imports them through the connector
helper layer. Pass `--require-smb` after recreating the SMB container to fail on
SMB access errors instead of reporting them as an optional skip.
Nextcloud, SMB, and MinIO fixtures, then browses and imports them through the
connector helper layer. Pass `--require-smb` after recreating the SMB container
to fail on SMB access errors instead of reporting them as an optional skip. Pass
`--require-s3` after starting MinIO and installing `govoplan-files[s3]` to fail
on S3 access errors instead of reporting them as an optional skip.
SMB smoke checks need the optional Python dependency in the environment running
the script:
@@ -126,6 +149,13 @@ the script:
/mnt/DATA/git/govoplan-core/.venv/bin/python -m pip install -e /mnt/DATA/git/govoplan-files[smb]
```
S3 smoke checks need the optional boto3 dependency in the environment running
the script:
```bash
/mnt/DATA/git/govoplan-core/.venv/bin/python -m pip install -e /mnt/DATA/git/govoplan-files[s3]
```
The SMB service is built from `dev/connectors/smb/` so the development share is
deterministic: one `files` share backed by `data/smb`, with the credentials from
`.env`.

View File

@@ -99,8 +99,22 @@ services:
volumes:
- ./data/smb:/storage
minio:
image: minio/minio:latest
restart: unless-stopped
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-govoplan}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-govoplan-minio}
ports:
- "127.0.0.1:${MINIO_API_HOST_PORT:-9000}:9000"
- "127.0.0.1:${MINIO_CONSOLE_HOST_PORT:-9001}:9001"
volumes:
- minio:/data
volumes:
nextcloud-db:
nextcloud:
seafile-db:
seafile-data:
minio:

View File

@@ -18,16 +18,17 @@ ROOT = Path(__file__).resolve().parent
def main() -> int:
parser = argparse.ArgumentParser(description="Smoke-test GovOPlaN connector helpers against the local dev compose stack.")
parser.add_argument("--require-smb", action="store_true", help="Fail if the SMB connector cannot browse/import.")
parser.add_argument("--require-s3", action="store_true", help="Fail if the S3 connector cannot browse/import.")
parser.add_argument("--debug-smb", action="store_true", help="Print direct smbclient probes before the connector smoke check.")
args = parser.parse_args()
_load_dotenv()
_default_env()
preflight_failures = _preflight_services(require_smb=args.require_smb)
preflight_failures = _preflight_services(require_smb=args.require_smb, require_s3=args.require_s3)
if preflight_failures:
for failure in preflight_failures:
print(f"FAIL {failure}")
print("Start or recreate the connector stack from this directory with: docker compose up -d nextcloud nextcloud-db webdav smb")
print("Start or recreate the connector stack from this directory with: docker compose up -d nextcloud nextcloud-db webdav smb minio")
return 1
_seed_webdav_fixture()
_seed_smb_fixture()
@@ -36,6 +37,13 @@ def main() -> int:
except httpx.HTTPError as exc:
print(f"FAIL dev-nextcloud seed failed: {exc}")
return 1
try:
_seed_s3_fixture()
except Exception as exc:
if args.require_s3:
print(f"FAIL dev-s3 seed failed: {exc}")
return 1
print(f"SKIP dev-s3 seed failed: {exc}")
profiles = {profile.id: profile for profile in connector_profiles_from_payload({"profiles": _profile_payloads()})}
if args.debug_smb:
@@ -59,6 +67,15 @@ def main() -> int:
else:
print(f"SKIP {message}")
try:
_exercise_profile(profiles["dev-s3"], folder="GovOPlaN", file_path="GovOPlaN/s3-live.txt", expected="s3 live fixture")
except Exception as exc:
message = f"dev-s3: {exc}"
if args.require_s3:
failures.append(message)
else:
print(f"SKIP {message}")
if failures:
for failure in failures:
print(f"FAIL {failure}")
@@ -76,6 +93,9 @@ def _default_env() -> None:
"SMB_SHARE_NAME": "files",
"SMB_USER": "govoplan",
"SMB_PASSWORD": "govoplan-smb",
"MINIO_ROOT_USER": "govoplan",
"MINIO_ROOT_PASSWORD": "govoplan-minio",
"MINIO_BUCKET": "govoplan",
}
for key, value in defaults.items():
os.environ.setdefault(key, value)
@@ -96,7 +116,7 @@ def _load_dotenv() -> None:
os.environ[key] = value.strip().strip("\"'")
def _preflight_services(*, require_smb: bool) -> list[str]:
def _preflight_services(*, require_smb: bool, require_s3: bool) -> list[str]:
failures: list[str] = []
nextcloud_url = f"http://127.0.0.1:{os.getenv('NEXTCLOUD_HOST_PORT', '9081')}/status.php"
try:
@@ -121,6 +141,14 @@ def _preflight_services(*, require_smb: bool) -> list[str]:
pass
except OSError as exc:
failures.append(f"dev-smb is not reachable at 127.0.0.1:{smb_port}: {exc}")
if require_s3:
minio_url = f"http://127.0.0.1:{os.getenv('MINIO_API_HOST_PORT', '9000')}/minio/health/live"
try:
response = httpx.get(minio_url, timeout=3.0)
if response.status_code != 200:
failures.append(f"dev-s3 expected HTTP 200 at {minio_url}, got HTTP {response.status_code}")
except httpx.HTTPError as exc:
failures.append(f"dev-s3 is not reachable at {minio_url}: {exc}")
return failures
@@ -150,6 +178,20 @@ def _profile_payloads() -> list[dict[str, object]]:
"username": os.getenv("SMB_USER", "govoplan"),
"password_env": "SMB_PASSWORD",
},
{
"id": "dev-s3",
"provider": "s3",
"endpoint_url": f"http://127.0.0.1:{os.getenv('MINIO_API_HOST_PORT', '9000')}",
"credential_mode": "basic",
"username": os.getenv("MINIO_ROOT_USER", "govoplan"),
"password_env": "MINIO_ROOT_PASSWORD",
"metadata": {
"bucket": os.getenv("MINIO_BUCKET", "govoplan"),
"region": "us-east-1",
"path_style": True,
"verify_tls": False,
},
},
]
@@ -211,5 +253,30 @@ def _seed_nextcloud_fixture() -> None:
raise RuntimeError(f"Nextcloud PUT failed with HTTP {response.status_code}: {response.text[:200]}")
def _seed_s3_fixture() -> None:
try:
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
except ImportError as exc:
raise RuntimeError("boto3 is not installed; install govoplan-files[s3]") from exc
bucket = os.getenv("MINIO_BUCKET", "govoplan")
client = boto3.client(
"s3",
endpoint_url=f"http://127.0.0.1:{os.getenv('MINIO_API_HOST_PORT', '9000')}",
aws_access_key_id=os.getenv("MINIO_ROOT_USER", "govoplan"),
aws_secret_access_key=os.getenv("MINIO_ROOT_PASSWORD", "govoplan-minio"),
region_name="us-east-1",
config=Config(s3={"addressing_style": "path"}),
)
try:
client.create_bucket(Bucket=bucket)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code")
if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
raise
client.put_object(Bucket=bucket, Key="GovOPlaN/s3-live.txt", Body=b"s3 live fixture\n", ContentType="text/plain")
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -17,6 +17,9 @@ dependencies = [
]
[project.optional-dependencies]
s3 = [
"boto3>=1.34,<2",
]
smb = [
"smbprotocol>=1.13",
]

View File

@@ -1,12 +1,18 @@
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import event, inspect
from sqlalchemy import event
from sqlalchemy.orm import Session as OrmSession
from govoplan_core.core.change_sequence import record_change
from govoplan_core.core.sqlalchemy_change_tracking import (
ensure_object_id,
has_attr_changes,
object_state,
operation_for_soft_deletable,
previous_value,
)
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare, new_uuid
FILES_MODULE_ID = "files"
@@ -39,7 +45,7 @@ def _record_files_changes(session: OrmSession, _flush_context: object, _instance
def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
operation = _operation_for_soft_deletable(
operation = operation_for_soft_deletable(
asset,
changed_attrs=(
"owner_type",
@@ -70,7 +76,7 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
"owner_type": asset.owner_type,
"owner_id": _owner_id(asset.owner_type, asset.owner_user_id, asset.owner_group_id),
"path": asset.display_path,
"previous_path": _previous_value(asset, "display_path"),
"previous_path": previous_value(asset, "display_path"),
"filename": asset.filename,
"deleted_at": _isoformat(asset.deleted_at),
},
@@ -78,7 +84,7 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
def _record_folder_change(session: OrmSession, folder: FileFolder) -> None:
operation = _operation_for_soft_deletable(
operation = operation_for_soft_deletable(
folder,
changed_attrs=("owner_type", "owner_user_id", "owner_group_id", "path", "deleted_at", "metadata_"),
)
@@ -99,17 +105,17 @@ def _record_folder_change(session: OrmSession, folder: FileFolder) -> None:
"owner_type": folder.owner_type,
"owner_id": _owner_id(folder.owner_type, folder.owner_user_id, folder.owner_group_id),
"path": folder.path,
"previous_path": _previous_value(folder, "path"),
"previous_path": previous_value(folder, "path"),
"deleted_at": _isoformat(folder.deleted_at),
},
)
def _record_share_visibility_change(session: OrmSession, share: FileShare) -> None:
state = inspect(share)
state = object_state(share)
if not share.file_asset_id:
return
if not state.pending and not _has_attr_changes(
if not state.pending and not has_attr_changes(
state,
("file_asset_id", "target_type", "target_id", "permission", "revoked_at"),
):
@@ -135,44 +141,8 @@ def _record_share_visibility_change(session: OrmSession, share: FileShare) -> No
)
def _operation_for_soft_deletable(obj: object, *, changed_attrs: tuple[str, ...]) -> str | None:
state = inspect(obj)
if state.pending:
return "created"
if not _has_attr_changes(state, changed_attrs):
return None
if "deleted_at" in state.attrs:
history = state.attrs.deleted_at.history
if history.has_changes():
if any(value is not None for value in history.added):
return "deleted"
if any(value is not None for value in history.deleted):
return "created"
return "updated"
def _has_attr_changes(state: Any, attrs: tuple[str, ...]) -> bool:
return any(name in state.attrs and state.attrs[name].history.has_changes() for name in attrs)
def _previous_value(obj: object, attr_name: str) -> str | None:
state = inspect(obj)
if attr_name not in state.attrs:
return None
history = state.attrs[attr_name].history
if not history.has_changes() or not history.deleted:
return None
value = history.deleted[0]
return str(value) if value is not None else None
def _ensure_id(obj: object) -> str:
resource_id = getattr(obj, "id", None)
if resource_id:
return str(resource_id)
resource_id = new_uuid()
setattr(obj, "id", resource_id)
return resource_id
return ensure_object_id(obj, new_uuid)
def _owner_id(owner_type: str, owner_user_id: str | None, owner_group_id: str | None) -> str | None:

View File

@@ -105,6 +105,7 @@ from govoplan_files.backend.storage.access import ensure_group_access, group_ref
from govoplan_files.backend.storage.archives import create_zip_file, extract_zip_upload
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_credential_store import (
ConnectorCredential,
connector_credential_from_row,
create_connector_credential_row,
deactivate_connector_credential_row,
@@ -802,6 +803,14 @@ def _download_connector_payload(
return source_path, downloaded, metadata or {}
def _connector_browse_next_token(items: list[Any]) -> str | None:
for item in items:
token = item.metadata.get("next_continuation_token") if hasattr(item, "metadata") else None
if token:
return str(token)
return None
def _connector_audit_details(asset: FileAsset, version: FileVersion, blob: FileBlob, *, operation: str) -> dict[str, object] | None:
metadata = asset.metadata_ or {}
provenance = source_provenance_from_metadata(metadata)
@@ -2148,6 +2157,148 @@ def _full_file_connector_settings_delta_response(
)
def _changed_file_connector_setting_ids(entries: list[ChangeSequenceEntry]) -> tuple[set[str], set[str], set[str], bool]:
changed_profile_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
}
changed_credential_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
}
changed_space_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
}
policy_changed = any(entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries)
return changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed
def _file_connector_settings_changed_sections(
*,
changed_profile_ids: set[str],
changed_credential_ids: set[str],
changed_space_ids: set[str],
policy_changed: bool,
profiles: list[ConnectorProfile],
) -> list[str]:
changed_sections = []
if changed_profile_ids or any(profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids for profile in profiles):
changed_sections.append("profiles")
if changed_credential_ids:
changed_sections.append("credentials")
if changed_space_ids:
changed_sections.append("spaces")
if policy_changed:
changed_sections.append("policy")
return changed_sections
def _file_connector_settings_deleted_items(
entries: list[ChangeSequenceEntry],
*,
visible_profiles: dict[str, ConnectorProfile],
visible_credentials: dict[str, ConnectorCredential],
visible_spaces: dict[str, FileConnectorSpace],
) -> list[DeltaDeletedItem]:
return [
_connector_deleted_item(entry)
for entry in entries
if (
entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
and entry.resource_id not in visible_profiles
)
or (
entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
and entry.resource_id not in visible_credentials
)
or (
entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
and entry.resource_id not in visible_spaces
)
]
def _incremental_file_connector_settings_delta_response(
session: Session,
principal: ApiPrincipal,
*,
entries: list[ChangeSequenceEntry],
has_more: bool,
scope_type: str,
scope_id: str | None,
provider: str | None,
campaign_id: str | None,
include_disabled: bool,
include_inactive: bool,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
) -> FileConnectorSettingsDeltaResponse:
changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed = _changed_file_connector_setting_ids(entries)
profiles = _visible_connector_profiles(
session,
principal,
provider=provider,
campaign_id=campaign_id,
include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal),
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
include_effective_policy=False,
)
visible_profiles = {
profile.id: profile
for profile in profiles
if profile.id in changed_profile_ids or (profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids)
}
credentials = _visible_connector_credentials(
session,
principal,
provider=provider,
include_disabled=include_disabled,
)
visible_credentials = {
credential.id: credential
for credential in credentials
if credential.id in changed_credential_ids
}
spaces = _visible_connector_spaces(
session,
principal,
owner_type=owner_type,
owner_id=owner_id,
include_inactive=include_inactive,
)
visible_spaces = {
space.id: space
for space in spaces
if space.id in changed_space_ids
}
return FileConnectorSettingsDeltaResponse(
profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in visible_profiles.values()],
credentials=[FileConnectorCredentialResponse(**credential.to_response()) for credential in visible_credentials.values()],
spaces=[_connector_space_response(space) for space in visible_spaces.values()],
policy=FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) if policy_changed else None,
changed_sections=_file_connector_settings_changed_sections(
changed_profile_ids=changed_profile_ids,
changed_credential_ids=changed_credential_ids,
changed_space_ids=changed_space_ids,
policy_changed=policy_changed,
profiles=profiles,
),
deleted=_file_connector_settings_deleted_items(
entries,
visible_profiles=visible_profiles,
visible_credentials=visible_credentials,
visible_spaces=visible_spaces,
),
watermark=_file_connector_settings_response_watermark(session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more),
has_more=has_more,
full=False,
)
@router.get("/connectors/settings/delta", response_model=FileConnectorSettingsDeltaResponse)
def connector_settings_delta(
scope_type: str = Query(default="tenant"),
@@ -2193,94 +2344,19 @@ def connector_settings_delta(
owner_type=owner_type,
owner_id=owner_id,
)
changed_profile_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
}
changed_credential_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
}
changed_space_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
}
policy_changed = any(entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries)
profiles = _visible_connector_profiles(
return _incremental_file_connector_settings_delta_response(
session,
principal,
entries=entries,
has_more=has_more,
scope_type=scope_type,
scope_id=scope_id,
provider=provider,
campaign_id=campaign_id,
include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal),
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
include_effective_policy=False,
)
visible_profiles = {
profile.id: profile
for profile in profiles
if profile.id in changed_profile_ids or (profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids)
}
credentials = _visible_connector_credentials(
session,
principal,
provider=provider,
include_disabled=include_disabled,
)
visible_credentials = {
credential.id: credential
for credential in credentials
if credential.id in changed_credential_ids
}
spaces = _visible_connector_spaces(
session,
principal,
include_inactive=include_inactive,
owner_type=owner_type,
owner_id=owner_id,
include_inactive=include_inactive,
)
visible_spaces = {
space.id: space
for space in spaces
if space.id in changed_space_ids
}
changed_sections = []
if changed_profile_ids or any(profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids for profile in profiles):
changed_sections.append("profiles")
if changed_credential_ids:
changed_sections.append("credentials")
if changed_space_ids:
changed_sections.append("spaces")
if policy_changed:
changed_sections.append("policy")
deleted = [
_connector_deleted_item(entry)
for entry in entries
if (
entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
and entry.resource_id not in visible_profiles
)
or (
entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
and entry.resource_id not in visible_credentials
)
or (
entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
and entry.resource_id not in visible_spaces
)
]
return FileConnectorSettingsDeltaResponse(
profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in visible_profiles.values()],
credentials=[FileConnectorCredentialResponse(**credential.to_response()) for credential in visible_credentials.values()],
spaces=[_connector_space_response(space) for space in visible_spaces.values()],
policy=FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) if policy_changed else None,
changed_sections=changed_sections,
deleted=deleted,
watermark=_file_connector_settings_response_watermark(session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more),
has_more=has_more,
full=False,
)
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
raise _http_error(exc) from exc
@@ -2819,6 +2895,7 @@ def browse_connector_profile_items(
profile_id: str,
path: str | None = None,
library_id: str | None = None,
continuation_token: str | None = None,
campaign_id: str | None = None,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_any_scope("files:file:read", "files:file:upload", "files:file:download", "files:file:admin", "system:settings:read", "admin:settings:read")),
@@ -2839,7 +2916,7 @@ def browse_connector_profile_items(
)
if not decision.allowed:
raise ConnectorPolicyDenied(decision)
items = browse_connector_profile(profile, path=browse_path, library_id=library_id)
items = browse_connector_profile(profile, path=browse_path, library_id=library_id, continuation_token=continuation_token)
except ConnectorPolicyDenied as exc:
raise _connector_policy_error(exc) from exc
except ConnectorBrowseUnsupported as exc:
@@ -2851,6 +2928,8 @@ def browse_connector_profile_items(
provider=profile.provider,
path=browse_path,
library_id=library_id,
next_continuation_token=_connector_browse_next_token(items),
has_more=any(bool(item.metadata.get("listing_truncated")) for item in items),
decision=decision.to_dict(),
items=[FileConnectorBrowseItem(**item.to_response()) for item in items],
)
@@ -3230,7 +3309,7 @@ def download_archive(
get_asset_for_user(session, tenant_id=principal.tenant_id, user_id=principal.user.id, asset_id=file_id, is_admin=_is_admin(principal))
for file_id in payload.file_ids
]
tmp = tempfile.NamedTemporaryFile(prefix="multimailer-files-", suffix=".zip", delete=False)
tmp = tempfile.NamedTemporaryFile(prefix="govoplan-files-", suffix=".zip", delete=False)
tmp_path = tmp.name
tmp.close()
try:

View File

@@ -1,36 +1,10 @@
from __future__ import annotations
from typing import Any
from govoplan_core.core.runtime import ModuleRuntimeState
_runtime_registry: object | None = None
_runtime_settings: object | None = None
_runtime = ModuleRuntimeState("Files")
def configure_runtime(*, registry: object | None = None, settings: object | None = None) -> None:
global _runtime_registry, _runtime_settings
if registry is not None:
_runtime_registry = registry
if settings is not None:
_runtime_settings = settings
def get_registry() -> object | None:
return _runtime_registry
def get_settings() -> object:
if _runtime_settings is not None:
return _runtime_settings
try:
from govoplan_core.settings import settings as legacy_settings
except ModuleNotFoundError as exc:
raise RuntimeError("GovOPlaN Files runtime settings are not configured") from exc
return legacy_settings
class SettingsProxy:
def __getattr__(self, name: str) -> Any:
return getattr(get_settings(), name)
settings = SettingsProxy()
configure_runtime = _runtime.configure_runtime
get_registry = _runtime.get_registry
get_settings = _runtime.get_settings
settings = _runtime.settings

View File

@@ -287,7 +287,7 @@ class FileConnectorDiscoveryCandidate(BaseModel):
class FileConnectorDiscoveryRequest(BaseModel):
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"]
provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"]
endpoint_url: str
base_path: str | None = None
credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] = "none"
@@ -309,7 +309,7 @@ class FileConnectorDiscoveryResponse(BaseModel):
class FileConnectorProfileCreateRequest(BaseModel):
id: str
label: str
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"]
provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"]
endpoint_url: str | None = None
base_path: str | None = None
enabled: bool = True
@@ -325,7 +325,7 @@ class FileConnectorProfileCreateRequest(BaseModel):
class FileConnectorProfileUpdateRequest(BaseModel):
label: str | None = None
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None
provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] | None = None
endpoint_url: str | None = None
base_path: str | None = None
enabled: bool | None = None
@@ -342,7 +342,7 @@ class FileConnectorProfileUpdateRequest(BaseModel):
class FileConnectorCredentialCreateRequest(BaseModel):
id: str
label: str
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None
provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] | None = None
enabled: bool = True
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "tenant"
scope_id: str | None = None
@@ -354,7 +354,7 @@ class FileConnectorCredentialCreateRequest(BaseModel):
class FileConnectorCredentialUpdateRequest(BaseModel):
label: str | None = None
provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None
provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] | None = None
enabled: bool | None = None
credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] | None = None
credentials: FileConnectorProfileCredentialsRequest | None = None
@@ -404,6 +404,8 @@ class FileConnectorBrowseResponse(BaseModel):
path: str = ""
library_id: str | None = None
read_only: bool = True
next_continuation_token: str | None = None
has_more: bool = False
decision: dict[str, Any]
items: list[FileConnectorBrowseItem]

View File

@@ -273,7 +273,7 @@ def prepared_campaign_snapshot(
campaign_id: str,
raw_json: dict[str, Any],
include_bytes: bool,
prefix: str = "multimailer-managed-campaign-",
prefix: str = "govoplan-managed-campaign-",
) -> Iterator[PreparedCampaignSnapshot]:
temp_dir = Path(tempfile.mkdtemp(prefix=prefix))
try:

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass, field
from pathlib import PurePosixPath
from typing import Any, Iterable
@@ -24,6 +25,15 @@ def _candidate_match_keys(raw_match: str) -> set[str]:
AttachmentUseKey = tuple[str, str, str, str]
@dataclass(slots=True)
class _AttachmentBatchRefs:
managed_by_job: dict[str, list[dict[str, object]]] = field(default_factory=lambda: defaultdict(list))
fallback_attachments_by_job: dict[str, list[dict[str, object]]] = field(default_factory=lambda: defaultdict(list))
asset_ids: set[str] = field(default_factory=set)
version_ids: set[str] = field(default_factory=set)
blob_ids: set[str] = field(default_factory=set)
def _attachment_use_key(*, job_id: str, file_version_id: str, filename_used: str, stage: str) -> AttachmentUseKey:
return job_id, file_version_id, filename_used, stage
@@ -134,62 +144,91 @@ def record_campaign_attachment_uses_for_jobs(
if not job_list:
return
managed_by_job: dict[str, list[dict[str, object]]] = defaultdict(list)
fallback_attachments_by_job: dict[str, list[dict[str, object]]] = defaultdict(list)
refs = _collect_attachment_batch_refs(job_list)
job_by_id = {job.id: job for job in job_list}
asset_ids: set[str] = set()
version_ids: set[str] = set()
blob_ids: set[str] = set()
known_keys = _known_use_keys_for_jobs(session, job_list, stage=stage)
assets_by_id, versions_by_id, blobs_by_id = _load_managed_attachment_entities(session, refs)
for job in job_list:
_record_managed_attachment_uses(
session,
job_by_id=job_by_id,
managed_by_job=refs.managed_by_job,
assets_by_id=assets_by_id,
versions_by_id=versions_by_id,
blobs_by_id=blobs_by_id,
stage=stage,
known_keys=known_keys,
)
_record_fallback_attachment_uses(
session,
job_by_id=job_by_id,
fallback_attachments_by_job=refs.fallback_attachments_by_job,
stage=stage,
known_keys=known_keys,
)
def _collect_attachment_batch_refs(jobs: list[CampaignJobLike]) -> _AttachmentBatchRefs:
refs = _AttachmentBatchRefs()
for job in jobs:
attachments = job.resolved_attachments or []
if not isinstance(attachments, list):
continue
for attachment in attachments:
if not isinstance(attachment, dict):
continue
managed_matches = attachment.get("managed_matches")
if isinstance(managed_matches, list) and managed_matches:
for item in managed_matches:
if not isinstance(item, dict):
continue
asset_id = str(item.get("asset_id") or "")
version_id = str(item.get("version_id") or "")
blob_id = str(item.get("blob_id") or "")
if not asset_id or not version_id or not blob_id:
continue
managed_by_job[job.id].append(item)
asset_ids.add(asset_id)
version_ids.add(version_id)
blob_ids.add(blob_id)
else:
fallback_attachments_by_job[job.id].append(attachment)
if not _collect_managed_attachment_refs(refs, job.id, attachment):
refs.fallback_attachments_by_job[job.id].append(attachment)
return refs
assets_by_id = {
item.id: item
for item in session.query(FileAsset).filter(FileAsset.id.in_(asset_ids)).all()
} if asset_ids else {}
versions_by_id = {
item.id: item
for item in session.query(FileVersion).filter(FileVersion.id.in_(version_ids)).all()
} if version_ids else {}
blobs_by_id = {
item.id: item
for item in session.query(FileBlob).filter(FileBlob.id.in_(blob_ids)).all()
} if blob_ids else {}
known_keys = _known_use_keys_for_jobs(session, job_list, stage=stage)
def _collect_managed_attachment_refs(refs: _AttachmentBatchRefs, job_id: str, attachment: dict[str, object]) -> bool:
managed_matches = attachment.get("managed_matches")
if not isinstance(managed_matches, list) or not managed_matches:
return False
for item in managed_matches:
if not isinstance(item, dict):
continue
asset_id = str(item.get("asset_id") or "")
version_id = str(item.get("version_id") or "")
blob_id = str(item.get("blob_id") or "")
if not asset_id or not version_id or not blob_id:
continue
refs.managed_by_job[job_id].append(item)
refs.asset_ids.add(asset_id)
refs.version_ids.add(version_id)
refs.blob_ids.add(blob_id)
return True
def _load_managed_attachment_entities(
session: Session,
refs: _AttachmentBatchRefs,
) -> tuple[dict[str, FileAsset], dict[str, FileVersion], dict[str, FileBlob]]:
assets_by_id = {item.id: item for item in session.query(FileAsset).filter(FileAsset.id.in_(refs.asset_ids)).all()} if refs.asset_ids else {}
versions_by_id = {item.id: item for item in session.query(FileVersion).filter(FileVersion.id.in_(refs.version_ids)).all()} if refs.version_ids else {}
blobs_by_id = {item.id: item for item in session.query(FileBlob).filter(FileBlob.id.in_(refs.blob_ids)).all()} if refs.blob_ids else {}
return assets_by_id, versions_by_id, blobs_by_id
def _record_managed_attachment_uses(
session: Session,
*,
job_by_id: dict[str, CampaignJobLike],
managed_by_job: dict[str, list[dict[str, object]]],
assets_by_id: dict[str, FileAsset],
versions_by_id: dict[str, FileVersion],
blobs_by_id: dict[str, FileBlob],
stage: str,
known_keys: set[AttachmentUseKey],
) -> None:
for job_id, items in managed_by_job.items():
job = job_by_id[job_id]
for item in items:
asset = assets_by_id.get(str(item.get("asset_id") or ""))
version = versions_by_id.get(str(item.get("version_id") or ""))
blob = blobs_by_id.get(str(item.get("blob_id") or ""))
if not asset or not version or not blob:
continue
if asset.tenant_id != job.tenant_id or version.tenant_id != job.tenant_id or blob.tenant_id != job.tenant_id:
continue
if version.file_asset_id != asset.id or version.blob_id != blob.id:
if not _managed_attachment_entities_match_job(job, asset, version, blob):
continue
_add_use(
session,
@@ -202,50 +241,100 @@ def record_campaign_attachment_uses_for_jobs(
known_keys=known_keys,
)
def _managed_attachment_entities_match_job(
job: CampaignJobLike,
asset: FileAsset | None,
version: FileVersion | None,
blob: FileBlob | None,
) -> bool:
if not asset or not version or not blob:
return False
if asset.tenant_id != job.tenant_id or version.tenant_id != job.tenant_id or blob.tenant_id != job.tenant_id:
return False
return version.file_asset_id == asset.id and version.blob_id == blob.id
def _record_fallback_attachment_uses(
session: Session,
*,
job_by_id: dict[str, CampaignJobLike],
fallback_attachments_by_job: dict[str, list[dict[str, object]]],
stage: str,
known_keys: set[AttachmentUseKey],
) -> None:
assets_by_campaign: dict[tuple[str, str], dict[str, FileAsset]] = {}
version_blobs_by_campaign: dict[tuple[str, str], dict[str, tuple[FileVersion, FileBlob]]] = {}
for job_id, attachments in fallback_attachments_by_job.items():
job = job_by_id[job_id]
campaign_key = (job.tenant_id, job.campaign_id)
by_key = assets_by_campaign.get(campaign_key)
if by_key is None:
assets = list_assets_for_user(
session,
tenant_id=job.tenant_id,
user_id="",
campaign_id=job.campaign_id,
is_admin=True,
)
by_key = {}
for asset in assets:
by_key[asset.display_path.strip("/")] = asset
by_key.setdefault(asset.filename, asset)
assets_by_campaign[campaign_key] = by_key
version_blobs_by_campaign[campaign_key] = current_versions_and_blobs(session, assets)
version_blobs = version_blobs_by_campaign[campaign_key]
by_key, version_blobs = _fallback_campaign_assets(
session,
job,
campaign_key=campaign_key,
assets_by_campaign=assets_by_campaign,
version_blobs_by_campaign=version_blobs_by_campaign,
)
for attachment in attachments:
matches = attachment.get("matches") if isinstance(attachment.get("matches"), list) else []
for raw in matches:
if not isinstance(raw, str):
continue
asset = next((by_key[key] for key in _candidate_match_keys(raw) if key in by_key), None)
if not asset:
continue
version_blob = version_blobs.get(asset.id)
if not version_blob:
continue
version, blob = version_blob
_add_use(
session,
job,
asset=asset,
version=version,
blob=blob,
filename_used=asset.filename,
stage=stage,
known_keys=known_keys,
)
_record_fallback_attachment(session, job, attachment, by_key=by_key, version_blobs=version_blobs, stage=stage, known_keys=known_keys)
def _fallback_campaign_assets(
session: Session,
job: CampaignJobLike,
*,
campaign_key: tuple[str, str],
assets_by_campaign: dict[tuple[str, str], dict[str, FileAsset]],
version_blobs_by_campaign: dict[tuple[str, str], dict[str, tuple[FileVersion, FileBlob]]],
) -> tuple[dict[str, FileAsset], dict[str, tuple[FileVersion, FileBlob]]]:
by_key = assets_by_campaign.get(campaign_key)
if by_key is None:
assets = list_assets_for_user(session, tenant_id=job.tenant_id, user_id="", campaign_id=job.campaign_id, is_admin=True)
by_key = _asset_lookup_by_path_and_filename(assets)
assets_by_campaign[campaign_key] = by_key
version_blobs_by_campaign[campaign_key] = current_versions_and_blobs(session, assets)
return by_key, version_blobs_by_campaign[campaign_key]
def _asset_lookup_by_path_and_filename(assets: list[FileAsset]) -> dict[str, FileAsset]:
by_key: dict[str, FileAsset] = {}
for asset in assets:
by_key[asset.display_path.strip("/")] = asset
by_key.setdefault(asset.filename, asset)
return by_key
def _record_fallback_attachment(
session: Session,
job: CampaignJobLike,
attachment: dict[str, object],
*,
by_key: dict[str, FileAsset],
version_blobs: dict[str, tuple[FileVersion, FileBlob]],
stage: str,
known_keys: set[AttachmentUseKey],
) -> None:
matches = attachment.get("matches") if isinstance(attachment.get("matches"), list) else []
for raw in matches:
if not isinstance(raw, str):
continue
asset = next((by_key[key] for key in _candidate_match_keys(raw) if key in by_key), None)
if not asset:
continue
version_blob = version_blobs.get(asset.id)
if not version_blob:
continue
version, blob = version_blob
_add_use(
session,
job,
asset=asset,
version=version,
blob=blob,
filename_used=asset.filename,
stage=stage,
known_keys=known_keys,
)
def record_campaign_attachment_uses_for_job(session: Session, job: CampaignJobLike, *, stage: str = "built") -> None:

View File

@@ -53,7 +53,7 @@ class ConnectorBrowseItem:
}
def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = None, library_id: str | None = None) -> list[ConnectorBrowseItem]:
def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = None, library_id: str | None = None, continuation_token: str | None = None) -> list[ConnectorBrowseItem]:
browse_path = normalize_connector_browse_path(path)
static_items = _static_listing(profile, path=browse_path, library_id=library_id)
if static_items is not None:
@@ -66,6 +66,8 @@ def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = No
return _browse_webdav(profile, path=browse_path)
if profile.provider == "smb":
return _browse_smb(profile, path=browse_path)
if profile.provider == "s3":
return _browse_s3(profile, path=browse_path, library_id=library_id, continuation_token=continuation_token)
raise ConnectorBrowseUnsupported(f"Read-only browsing is not implemented for {profile.provider} connector profiles yet")
@@ -222,6 +224,227 @@ def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowse
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]:
client = _s3_client(profile)
bucket = _s3_bucket(profile, library_id)
if not bucket:
try:
payload = client.list_buckets()
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"S3 connector browse failed: {exc}") from exc
buckets = payload.get("Buckets") if isinstance(payload, Mapping) else None
if not isinstance(buckets, list):
raise ConnectorBrowseError("S3 connector returned invalid bucket list")
return sorted(
(
ConnectorBrowseItem(
kind="library",
name=_clean(item.get("Name")) or "",
path=_clean(item.get("Name")) or "",
external_id=_clean(item.get("Name")),
modified_at=_timestamp(item.get("CreationDate")),
metadata={"bucket": _clean(item.get("Name"))},
)
for item in buckets
if isinstance(item, Mapping) and _clean(item.get("Name"))
),
key=lambda item: item.name.casefold(),
)
prefix = _s3_object_key(profile, path, directory=True)
params: dict[str, object] = {
"Bucket": bucket,
"Prefix": prefix,
"Delimiter": "/",
"MaxKeys": _s3_max_keys(profile),
}
next_page_request = _clean(continuation_token) or _metadata_string(profile, "continuation_token")
if next_page_request:
params["ContinuationToken"] = next_page_request
try:
payload = client.list_objects_v2(**params)
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"S3 connector browse failed: {exc}") from exc
if not isinstance(payload, Mapping):
raise ConnectorBrowseError("S3 connector returned invalid object listing")
items = [
*_s3_prefix_items(bucket=bucket, browse_path=path, base_prefix=prefix, prefixes=payload.get("CommonPrefixes")),
*_s3_object_items(bucket=bucket, browse_path=path, base_prefix=prefix, objects=payload.get("Contents")),
]
next_token = _clean(payload.get("NextContinuationToken"))
if next_token and items:
last = items[-1]
items[-1] = ConnectorBrowseItem(
kind=last.kind,
name=last.name,
path=last.path,
external_id=last.external_id,
external_url=last.external_url,
size_bytes=last.size_bytes,
content_type=last.content_type,
modified_at=last.modified_at,
etag=last.etag,
metadata={**dict(last.metadata), "next_continuation_token": next_token, "listing_truncated": True},
)
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
def _s3_client(profile: ConnectorProfile) -> Any:
if profile.secret_ref:
raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing")
try:
boto3 = import_module("boto3")
config_module = import_module("botocore.config")
except ImportError as exc:
raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
kwargs: dict[str, object] = {}
if profile.endpoint_url:
kwargs["endpoint_url"] = profile.endpoint_url
region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region")
if region:
kwargs["region_name"] = region
access_key = profile.username or _metadata_env(profile, "access_key_id_env") or _metadata_string(profile, "access_key_id")
secret_key = _profile_password(profile) or _metadata_env(profile, "secret_access_key_env")
session_token = _profile_token(profile) or _metadata_env(profile, "session_token_env")
if access_key:
kwargs["aws_access_key_id"] = access_key
if secret_key:
kwargs["aws_secret_access_key"] = secret_key
if session_token:
kwargs["aws_session_token"] = session_token
verify = _s3_verify(profile)
if verify is not None:
kwargs["verify"] = verify
addressing_style = _s3_addressing_style(profile)
if addressing_style:
kwargs["config"] = config_module.Config(s3={"addressing_style": addressing_style})
try:
return boto3.client("s3", **kwargs)
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc
def _s3_bucket(profile: ConnectorProfile, library_id: str | None = None) -> str | None:
return _metadata_string(profile, "bucket") or _metadata_string(profile, "bucket_name") or _clean(library_id)
def _s3_object_key(profile: ConnectorProfile, path: str, *, directory: bool = False) -> str:
base_prefix = normalize_connector_browse_path(profile.base_path or _metadata_string(profile, "base_prefix"))
browse_path = normalize_connector_browse_path(path)
key = "/".join(part for part in (base_prefix, browse_path) if part)
if directory and key:
return key.rstrip("/") + "/"
return key
def _s3_prefix_items(
*,
bucket: str,
browse_path: str,
base_prefix: str,
prefixes: object,
) -> list[ConnectorBrowseItem]:
if not isinstance(prefixes, list):
return []
items: list[ConnectorBrowseItem] = []
for item in prefixes:
if not isinstance(item, Mapping):
continue
key = _clean(item.get("Prefix"))
if not key:
continue
relative = _s3_relative_key(key, base_prefix=base_prefix)
name = _path_name(relative)
if not name:
continue
path = _join_browse_path(browse_path, name)
items.append(
ConnectorBrowseItem(
kind="folder",
name=name,
path=path,
external_id=f"{bucket}:{key}",
metadata={"bucket": bucket, "key": key},
)
)
return items
def _s3_object_items(
*,
bucket: str,
browse_path: str,
base_prefix: str,
objects: object,
) -> list[ConnectorBrowseItem]:
if not isinstance(objects, list):
return []
items: list[ConnectorBrowseItem] = []
for item in objects:
if not isinstance(item, Mapping):
continue
key = _clean(item.get("Key"))
if not key or key == base_prefix:
continue
relative = _s3_relative_key(key, base_prefix=base_prefix)
name = _path_name(relative)
if not name:
continue
path = _join_browse_path(browse_path, name)
items.append(
ConnectorBrowseItem(
kind="file",
name=name,
path=path,
external_id=f"{bucket}:{key}",
size_bytes=_int(item.get("Size")),
content_type=mimetypes.guess_type(name)[0],
modified_at=_timestamp(item.get("LastModified")),
etag=_clean(item.get("ETag")),
metadata={
"bucket": bucket,
"key": key,
**({"storage_class": item["StorageClass"]} if "StorageClass" in item else {}),
},
)
)
return items
def _s3_relative_key(key: str, *, base_prefix: str) -> str:
clean_key = key.rstrip("/")
clean_base = base_prefix.rstrip("/")
if clean_base and clean_key.startswith(f"{clean_base}/"):
return clean_key[len(clean_base) + 1 :]
return clean_key
def _s3_max_keys(profile: ConnectorProfile) -> int:
configured = _int(profile.metadata.get("max_keys"))
if configured is None:
return 1000
return max(1, min(configured, 1000))
def _s3_verify(profile: ConnectorProfile) -> bool | str | None:
ca_bundle = _metadata_string(profile, "ca_bundle")
if ca_bundle:
return ca_bundle
if "verify_tls" in profile.metadata:
return _metadata_bool(profile, "verify_tls", default=True)
if "tls_verify" in profile.metadata:
return _metadata_bool(profile, "tls_verify", default=True)
return None
def _s3_addressing_style(profile: ConnectorProfile) -> str | None:
style = _metadata_string(profile, "addressing_style")
if style in {"path", "virtual", "auto"}:
return style
if _metadata_bool(profile, "path_style", default=False):
return "path"
return None
def _seafile_headers(profile: ConnectorProfile) -> dict[str, str]:
token = _seafile_token(profile)
return {"Authorization": f"Token {token}", "Accept": "application/json"}
@@ -444,6 +667,13 @@ def _metadata_bool(profile: ConnectorProfile, key: str, *, default: bool = False
return str(value).strip().casefold() in {"1", "true", "yes", "on"}
def _metadata_env(profile: ConnectorProfile, key: str) -> str | None:
env_name = _metadata_string(profile, key)
if not env_name:
return None
return _env_required(env_name, profile.id)
def _env_required(name: str, profile_id: str) -> str:
value = _clean(os.environ.get(name))
if not value:

View File

@@ -21,6 +21,9 @@ from govoplan_files.backend.storage.connector_browse import (
_smb_stat_size,
_smb_unc_path,
_smbclient_module,
_s3_bucket,
_s3_client,
_s3_object_key,
_seafile_headers,
_seafile_url,
_webdav_url,
@@ -64,6 +67,8 @@ def read_connector_file(
return _read_webdav_file(profile, path=path, max_bytes=max_bytes)
if profile.provider == "smb":
return _read_smb_file(profile, path=path, max_bytes=max_bytes)
if profile.provider == "s3":
return _read_s3_file(profile, library_id=library_id, path=path, max_bytes=max_bytes)
raise ConnectorImportUnsupported(f"Connector file import is not implemented for {profile.provider} profiles yet")
@@ -213,6 +218,63 @@ def _read_smb_file(profile: ConnectorProfile, *, path: str, max_bytes: int) -> C
)
def _read_s3_file(profile: ConnectorProfile, *, library_id: str, path: str, max_bytes: int) -> ConnectorDownloadedFile:
bucket = _s3_bucket(profile, library_id)
if not bucket:
raise ConnectorImportError("S3 import requires a bucket in library_id or profile metadata")
key = _s3_object_key(profile, path)
if not key:
raise ConnectorImportError("S3 import requires an object key")
try:
client = _s3_client(profile)
except ConnectorBrowseUnsupported as exc:
raise ConnectorImportUnsupported(str(exc)) from exc
except ConnectorBrowseError as exc:
raise ConnectorImportError(str(exc)) from exc
try:
detail = client.head_object(Bucket=bucket, Key=key)
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorImportError(f"S3 object metadata lookup failed: {exc}") from exc
if not isinstance(detail, dict):
raise ConnectorImportError("S3 connector returned invalid object metadata")
size = _int(detail.get("ContentLength"))
if size is not None and size > max_bytes:
raise ConnectorImportError(f"S3 object exceeds limit of {max_bytes} bytes")
version_id = _clean(detail.get("VersionId"))
request: dict[str, object] = {"Bucket": bucket, "Key": key}
if version_id:
request["VersionId"] = version_id
try:
response = client.get_object(**request)
body = response.get("Body")
data = body.read(max_bytes + 1) if hasattr(body, "read") else bytes(response.get("Body") or b"")
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorImportError(f"S3 object download failed: {exc}") from exc
if len(data) > max_bytes:
raise ConnectorImportError(f"S3 object exceeds limit of {max_bytes} bytes")
content_type = _clean(response.get("ContentType") if isinstance(response, dict) else None) or _clean(detail.get("ContentType")) or mimetypes.guess_type(key)[0]
etag = _clean(response.get("ETag") if isinstance(response, dict) else None) or _clean(detail.get("ETag"))
filename = filename_from_path(key)
return ConnectorDownloadedFile(
filename=filename,
data=data,
content_type=content_type,
revision=version_id or etag or _clean(detail.get("LastModified")),
external_id=f"{bucket}:{key}",
external_url=f"s3://{bucket}/{key}",
metadata={
"bucket": bucket,
"key": key,
"version_id": version_id,
"etag": etag,
"size": len(data),
**({"checksum_sha256": detail["ChecksumSHA256"]} if "ChecksumSHA256" in detail else {}),
**({"checksum_crc32": detail["ChecksumCRC32"]} if "ChecksumCRC32" in detail else {}),
**({"storage_class": detail["StorageClass"]} if "StorageClass" in detail else {}),
},
)
def _int(value: object) -> int | None:
if value is None or value == "":
return None

View File

@@ -12,8 +12,17 @@ from govoplan_files.backend.storage.connector_policy import ConnectorPolicySourc
_ENV_JSON_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON", "FILES_CONNECTOR_PROFILES_JSON")
_ENV_FILE_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE", "FILES_CONNECTOR_PROFILES_FILE")
_SUPPORTED_PROVIDERS = {"seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"}
_INLINE_SECRET_FIELDS = ("password", "token", "api_key", "access_key", "secret_key")
_SUPPORTED_PROVIDERS = {"seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"}
_INLINE_SECRET_FIELDS = (
"password",
"token",
"api_key",
"access_key",
"access_key_id",
"secret_key",
"secret_access_key",
"session_token",
)
def supported_connector_providers() -> set[str]:
@@ -123,43 +132,19 @@ def connector_profiles_from_payload(value: object) -> list[ConnectorProfile]:
def _profile_from_mapping(value: Mapping[str, Any]) -> ConnectorProfile:
profile_id = _clean(value.get("id") or value.get("connector_id") or value.get("name"))
if not profile_id:
raise ValueError("Connector profiles require id")
provider = (_clean(value.get("provider") or value.get("type")) or "generic").casefold()
if provider not in _SUPPORTED_PROVIDERS:
raise ValueError(f"Unsupported connector provider: {provider}")
scope_type = normalize_policy_scope_type(str(value.get("scope_type") or "system"))
scope_id = _clean(value.get("scope_id"))
if scope_type == "system":
scope_id = None
elif not scope_id:
raise ValueError(f"{scope_type} connector profiles require scope_id")
credentials = value.get("credentials")
credentials = credentials if isinstance(credentials, Mapping) else {}
profile_id = _profile_id_from_mapping(value)
provider = _provider_from_mapping(value)
scope_type, scope_id = _scope_from_mapping(value)
credentials = _credentials_from_mapping(value)
mode = _clean(value.get("credential_mode") or value.get("auth_type") or credentials.get("mode") or credentials.get("type")) or "none"
profile = ConnectorProfile(
id=profile_id,
label=_clean(value.get("label") or value.get("name")) or profile_id,
profile = _profile_from_normalized_mapping(
value,
profile_id=profile_id,
provider=provider,
endpoint_url=_clean(value.get("endpoint_url") or value.get("base_url") or value.get("url")),
base_path=_clean(value.get("base_path") or value.get("path") or value.get("root")),
enabled=_bool(value.get("enabled"), default=True),
scope_type=scope_type,
scope_id=scope_id,
credential_profile_id=_clean(value.get("credential_profile_id") or value.get("credential_id")),
credential_profile_label=_clean(value.get("credential_profile_label") or value.get("credential_label")),
credential_mode=mode,
username=_clean(value.get("username") or credentials.get("username")),
password_env=_clean(value.get("password_env") or credentials.get("password_env")),
token_env=_clean(value.get("token_env") or credentials.get("token_env")),
secret_ref=_clean(value.get("secret_ref") or credentials.get("secret_ref")),
password_value=_clean(value.get("password") or credentials.get("password")),
token_value=_clean(value.get("token") or credentials.get("token")),
has_inline_secret=any(_clean(value.get(field) or credentials.get(field)) for field in _INLINE_SECRET_FIELDS),
capabilities=tuple(_string_list(value.get("capabilities"))),
metadata=_public_metadata(value.get("metadata")),
credentials=credentials,
)
return ConnectorProfile(
id=profile.id,
@@ -187,6 +172,69 @@ def _profile_from_mapping(value: Mapping[str, Any]) -> ConnectorProfile:
)
def _profile_id_from_mapping(value: Mapping[str, Any]) -> str:
profile_id = _clean(value.get("id") or value.get("connector_id") or value.get("name"))
if not profile_id:
raise ValueError("Connector profiles require id")
return profile_id
def _provider_from_mapping(value: Mapping[str, Any]) -> str:
provider = (_clean(value.get("provider") or value.get("type")) or "generic").casefold()
if provider not in _SUPPORTED_PROVIDERS:
raise ValueError(f"Unsupported connector provider: {provider}")
return provider
def _scope_from_mapping(value: Mapping[str, Any]) -> tuple[str, str | None]:
scope_type = normalize_policy_scope_type(str(value.get("scope_type") or "system"))
scope_id = _clean(value.get("scope_id"))
if scope_type == "system":
return scope_type, None
if not scope_id:
raise ValueError(f"{scope_type} connector profiles require scope_id")
return scope_type, scope_id
def _credentials_from_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]:
credentials = value.get("credentials")
return credentials if isinstance(credentials, Mapping) else {}
def _profile_from_normalized_mapping(
value: Mapping[str, Any],
*,
profile_id: str,
provider: str,
scope_type: str,
scope_id: str | None,
credential_mode: str,
credentials: Mapping[str, Any],
) -> ConnectorProfile:
return ConnectorProfile(
id=profile_id,
label=_clean(value.get("label") or value.get("name")) or profile_id,
provider=provider,
endpoint_url=_clean(value.get("endpoint_url") or value.get("base_url") or value.get("url")),
base_path=_clean(value.get("base_path") or value.get("path") or value.get("root")),
enabled=_bool(value.get("enabled"), default=True),
scope_type=scope_type,
scope_id=scope_id,
credential_profile_id=_clean(value.get("credential_profile_id") or value.get("credential_id")),
credential_profile_label=_clean(value.get("credential_profile_label") or value.get("credential_label")),
credential_mode=credential_mode,
username=_clean(value.get("username") or credentials.get("username")),
password_env=_clean(value.get("password_env") or credentials.get("password_env")),
token_env=_clean(value.get("token_env") or credentials.get("token_env")),
secret_ref=_clean(value.get("secret_ref") or credentials.get("secret_ref")),
password_value=_clean(value.get("password") or credentials.get("password")),
token_value=_clean(value.get("token") or credentials.get("token")),
has_inline_secret=any(_clean(value.get(field) or credentials.get(field)) for field in _INLINE_SECRET_FIELDS),
capabilities=tuple(_string_list(value.get("capabilities"))),
metadata=_public_metadata(value.get("metadata")),
)
def _raw_profiles_payload(settings: object | None) -> object:
for key in _ENV_JSON_KEYS:
value = os.environ.get(key)

View File

@@ -112,6 +112,51 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Requires the optional smbprotocol dependency and an smb://server[:port]/share[/path] profile endpoint.",
),
ConnectorProviderDescriptor(
provider="s3",
label="S3-compatible object storage",
protocol="s3-api",
implemented=True,
browse_supported=True,
import_supported=True,
optional_dependency="boto3",
permission_model=governed_permissions,
sync_strategy="On-demand bucket/prefix browse and object import/sync; sync updates managed versions and never mutates the remote bucket.",
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.",
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Configure endpoint_url for MinIO or other S3-compatible stores; use profile metadata for bucket, region, addressing_style, verify_tls, and ca_bundle.",
),
ConnectorProviderDescriptor(
provider="sharepoint",
label="SharePoint",
protocol="microsoft-graph/sharepoint",
implemented=False,
browse_supported=False,
import_supported=False,
optional_dependency=None,
permission_model=governed_permissions,
sync_strategy="Planned read-only browse/import through deployment-managed Microsoft Graph or SharePoint credentials.",
conflict_strategy=freeze_model,
preview_strategy="Will preview from frozen managed file after import, not directly from SharePoint.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Provider/profile key is reserved; live browsing/import still needs Graph/SharePoint authentication and paging implementation.",
),
ConnectorProviderDescriptor(
provider="onedrive",
label="OneDrive",
protocol="microsoft-graph/onedrive",
implemented=False,
browse_supported=False,
import_supported=False,
optional_dependency=None,
permission_model=governed_permissions,
sync_strategy="Planned read-only browse/import through deployment-managed Microsoft Graph credentials.",
conflict_strategy=freeze_model,
preview_strategy="Will preview from frozen managed file after import, not directly from OneDrive.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Provider/profile key is reserved; live browsing/import still needs Graph drive selection, OAuth/app registration, and paging implementation.",
),
ConnectorProviderDescriptor(
provider="nfs",
label="NFS",

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,157 @@
from __future__ import annotations
import unittest
from datetime import UTC, datetime
from unittest.mock import patch
from govoplan_files.backend.storage.connector_browse import browse_connector_profile
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseUnsupported
from govoplan_files.backend.storage.connector_imports import read_connector_file
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload
from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors
class FakeBody:
def __init__(self, data: bytes) -> None:
self.data = data
def read(self, _limit: int) -> bytes:
return self.data
class FakeS3Client:
def __init__(self) -> None:
self.list_objects_request: dict[str, object] | None = None
self.head_request: dict[str, object] | None = None
self.get_request: dict[str, object] | None = None
def list_objects_v2(self, **kwargs: object) -> dict[str, object]:
self.list_objects_request = dict(kwargs)
return {
"CommonPrefixes": [{"Prefix": "root/reports/"}],
"Contents": [
{
"Key": "root/report.xlsx",
"Size": 123,
"LastModified": datetime(2026, 7, 12, 10, 0, tzinfo=UTC),
"ETag": '"etag-1"',
"StorageClass": "STANDARD",
}
],
"NextContinuationToken": "next-page",
}
def list_buckets(self) -> dict[str, object]:
return {"Buckets": [{"Name": "archive", "CreationDate": datetime(2026, 7, 12, 9, 0, tzinfo=UTC)}]}
def head_object(self, **kwargs: object) -> dict[str, object]:
self.head_request = dict(kwargs)
return {
"ContentLength": 13,
"ContentType": "text/plain",
"ETag": '"etag-2"',
"VersionId": "version-1",
"ChecksumSHA256": "checksum",
}
def get_object(self, **kwargs: object) -> dict[str, object]:
self.get_request = dict(kwargs)
return {"Body": FakeBody(b"hello s3\n"), "ContentType": "text/plain", "ETag": '"etag-2"'}
def s3_profile(**overrides: object) -> ConnectorProfile:
values = {
"id": "dev-s3",
"label": "Dev S3",
"provider": "s3",
"endpoint_url": "http://127.0.0.1:9000",
"base_path": "root",
"credential_mode": "basic",
"username": "access-key",
"password_value": "secret-key",
"metadata": {"bucket": "govoplan", "region": "eu-central-1", "path_style": True},
}
values.update(overrides)
return ConnectorProfile(**values)
class ConnectorProviderTests(unittest.TestCase):
def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None:
descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()}
self.assertTrue(descriptors["s3"].implemented)
self.assertTrue(descriptors["s3"].browse_supported)
self.assertEqual("boto3", descriptors["s3"].optional_dependency)
self.assertFalse(descriptors["sharepoint"].implemented)
self.assertFalse(descriptors["onedrive"].implemented)
def test_profile_payload_accepts_new_providers_and_redacts_secret_metadata(self) -> None:
profiles = connector_profiles_from_payload(
{
"profiles": [
{
"id": "dev-s3",
"label": "Dev S3",
"provider": "s3",
"username": "access-key",
"password": "secret-key",
"metadata": {"bucket": "govoplan", "secret_access_key": "do-not-return"},
},
{"id": "sharepoint", "provider": "sharepoint"},
{"id": "onedrive", "provider": "onedrive"},
]
}
)
by_id = {profile.id: profile for profile in profiles}
self.assertEqual("s3", by_id["dev-s3"].provider)
self.assertTrue(by_id["dev-s3"].credentials_configured)
self.assertEqual({"bucket": "govoplan"}, dict(by_id["dev-s3"].metadata))
self.assertEqual("sharepoint", by_id["sharepoint"].provider)
self.assertEqual("onedrive", by_id["onedrive"].provider)
def test_s3_browse_lists_prefixes_and_objects_with_provenance_metadata(self) -> None:
client = FakeS3Client()
with patch("govoplan_files.backend.storage.connector_browse._s3_client", return_value=client):
items = browse_connector_profile(s3_profile(), path="")
self.assertEqual({"Bucket": "govoplan", "Prefix": "root/", "Delimiter": "/", "MaxKeys": 1000}, client.list_objects_request)
self.assertEqual(["folder", "file"], [item.kind for item in items])
self.assertEqual("reports", items[0].path)
self.assertEqual("report.xlsx", items[1].path)
self.assertEqual("govoplan:root/report.xlsx", items[1].external_id)
self.assertEqual("root/report.xlsx", items[1].metadata["key"])
self.assertEqual("next-page", items[1].metadata["next_continuation_token"])
def test_s3_browse_lists_buckets_when_profile_has_no_bucket(self) -> None:
client = FakeS3Client()
with patch("govoplan_files.backend.storage.connector_browse._s3_client", return_value=client):
items = browse_connector_profile(s3_profile(metadata={}), path="")
self.assertEqual(["archive"], [item.path for item in items])
def test_s3_browse_reports_missing_optional_dependency(self) -> None:
with patch("govoplan_files.backend.storage.connector_browse.import_module", side_effect=ImportError):
with self.assertRaisesRegex(ConnectorBrowseUnsupported, "boto3"):
browse_connector_profile(s3_profile(), path="")
def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None:
client = FakeS3Client()
with patch("govoplan_files.backend.storage.connector_imports._s3_client", return_value=client):
downloaded = read_connector_file(s3_profile(), library_id="", path="report.txt", max_bytes=1024)
self.assertEqual({"Bucket": "govoplan", "Key": "root/report.txt"}, client.head_request)
self.assertEqual({"Bucket": "govoplan", "Key": "root/report.txt", "VersionId": "version-1"}, client.get_request)
self.assertEqual("report.txt", downloaded.filename)
self.assertEqual(b"hello s3\n", downloaded.data)
self.assertEqual("version-1", downloaded.revision)
self.assertEqual("govoplan:root/report.txt", downloaded.external_id)
self.assertEqual("s3://govoplan/root/report.txt", downloaded.external_url)
self.assertEqual("checksum", downloaded.metadata["checksum_sha256"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,36 @@
from __future__ import annotations
import unittest
from govoplan_files.backend.router import router
class FilesRouterContractTests(unittest.TestCase):
def test_connector_routes_keep_existing_api_paths(self) -> None:
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
expected = {
(("GET",), "/files/connectors/providers"),
(("GET",), "/files/connectors/settings/delta"),
(("GET",), "/files/connectors/profiles"),
(("POST",), "/files/connectors/profiles"),
(("GET",), "/files/connectors/profiles/{profile_id}/browse"),
(("POST",), "/files/connectors/profiles/{profile_id}/import"),
(("POST",), "/files/connectors/profiles/{profile_id}/sync"),
(("GET",), "/files/connectors/credentials"),
(("POST",), "/files/connectors/credentials"),
(("GET",), "/files/connector-spaces"),
(("POST",), "/files/connector-spaces"),
}
self.assertTrue(expected.issubset(routes))
def test_bulk_organize_routes_keep_existing_api_paths(self) -> None:
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
self.assertIn((("POST",), "/files/bulk-rename"), routes)
self.assertIn((("POST",), "/files/transfer"), routes)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,67 @@
from __future__ import annotations
import unittest
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError
from govoplan_files.backend.storage.transfers import _normalize_rename_request, _normalize_transfer_request
class TransferHelperTests(unittest.TestCase):
def test_transfer_normalization_deduplicates_folder_paths_and_conflicts(self) -> None:
normalized = _normalize_transfer_request(
operation=" COPY ",
source_owner_type=" USER ",
target_owner_type=" GROUP ",
folder_paths=["Reports", "Reports/2026", "", "./Archive"],
target_folder=" Target ",
conflict_strategy="rename",
conflict_resolutions=[FileConflictResolution(target_path="Target/a.txt", action="skip")],
)
self.assertEqual("copy", normalized.operation)
self.assertEqual("user", normalized.source_owner_type)
self.assertEqual("group", normalized.target_owner_type)
self.assertEqual(["Reports", "Reports/2026", "Archive"], normalized.source_folder_paths)
self.assertEqual("Target", normalized.target_folder)
self.assertEqual("rename", normalized.conflict_strategy)
self.assertEqual("skip", normalized.conflict_resolution_map["Target/a.txt"].action)
def test_transfer_normalization_rejects_unknown_operation(self) -> None:
with self.assertRaisesRegex(FileStorageError, "Unsupported transfer operation"):
_normalize_transfer_request(
operation="link",
source_owner_type="user",
target_owner_type="group",
folder_paths=[],
target_folder="",
conflict_strategy="reject",
conflict_resolutions=None,
)
def test_rename_normalization_collapses_nested_folder_roots(self) -> None:
normalized = _normalize_rename_request(
file_ids=["file-1", "file-1", "file-2"],
folder_paths=["Reports", "Reports/2026", "Archive"],
owner_type=" USER ",
owner_id="owner-1",
mode=" prefix ",
)
self.assertEqual("prefix", normalized.mode)
self.assertEqual(["file-1", "file-2"], normalized.selected_file_ids)
self.assertEqual(["Archive", "Reports"], normalized.selected_folder_roots)
self.assertEqual("user", normalized.owner_type)
def test_rename_normalization_rejects_invalid_direct_multi_select(self) -> None:
with self.assertRaisesRegex(FileStorageError, "Direct rename requires exactly one selected item"):
_normalize_rename_request(
file_ids=["file-1", "file-2"],
folder_paths=[],
owner_type="user",
owner_id="owner-1",
mode="direct",
)
if __name__ == "__main__":
unittest.main()

View File

@@ -150,7 +150,7 @@ export type FileConnectorDiscoveryResponse = {
export type FileConnectorProfilePayload = {
id: string;
label: string;
provider: "seafile" | "nextcloud" | "webdav" | "smb" | "nfs" | "dms" | "generic";
provider: "seafile" | "nextcloud" | "webdav" | "smb" | "s3" | "sharepoint" | "onedrive" | "nfs" | "dms" | "generic";
endpoint_url?: string | null;
base_path?: string | null;
enabled?: boolean;
@@ -172,7 +172,7 @@ export type FileConnectorProfileUpdatePayload = Partial<Omit<FileConnectorProfil
export type FileConnectorCredentialPayload = {
id: string;
label: string;
provider?: "seafile" | "nextcloud" | "webdav" | "smb" | "nfs" | "dms" | "generic" | null;
provider?: "seafile" | "nextcloud" | "webdav" | "smb" | "s3" | "sharepoint" | "onedrive" | "nfs" | "dms" | "generic" | null;
enabled?: boolean;
scope_type?: "system" | "tenant" | "user" | "group" | "campaign";
scope_id?: string | null;
@@ -206,6 +206,8 @@ export type FileConnectorBrowseResponse = {
path: string;
library_id?: string | null;
read_only: boolean;
next_continuation_token?: string | null;
has_more: boolean;
decision: FileConnectorPolicyDecision;
items: FileConnectorBrowseItem[];
};
@@ -808,11 +810,12 @@ export function deactivateFileConnectorProfile(settings: ApiSettings, profileId:
export function browseFileConnectorProfile(
settings: ApiSettings,
profileId: string,
params: {path?: string;library_id?: string;campaign_id?: string;} = {})
params: {path?: string;library_id?: string;continuation_token?: string;campaign_id?: string;} = {})
: Promise<FileConnectorBrowseResponse> {
const search = new URLSearchParams();
if (params.path) search.set("path", params.path);
if (params.library_id) search.set("library_id", params.library_id);
if (params.continuation_token) search.set("continuation_token", params.continuation_token);
if (params.campaign_id) search.set("campaign_id", params.campaign_id);
const suffix = search.toString() ? `?${search.toString()}` : "";
return apiFetch<FileConnectorBrowseResponse>(settings, `/api/v1/files/connectors/profiles/${encodeURIComponent(profileId)}/browse${suffix}`);

View File

@@ -5,6 +5,8 @@ import {
Card,
ConnectionTree,
ConfirmDialog,
CredentialPanel,
DisabledActionTooltip,
DismissibleAlert,
Dialog,
FormField,
@@ -123,6 +125,9 @@ const PROVIDERS: Array<{id: Provider;label: string;}> = [
{ id: "nextcloud", label: "i18n:govoplan-files.nextcloud.aab73a3d" },
{ id: "seafile", label: "i18n:govoplan-files.seafile.600192cc" },
{ id: "smb", label: "i18n:govoplan-files.smb.515d2f88" },
{ id: "s3", label: "S3" },
{ id: "sharepoint", label: "SharePoint" },
{ id: "onedrive", label: "OneDrive" },
{ id: "nfs", label: "i18n:govoplan-files.nfs.db121865" },
{ id: "dms", label: "i18n:govoplan-files.dms.477e5652" },
{ id: "generic", label: "i18n:govoplan-files.generic.ff7613e5" }];
@@ -133,6 +138,9 @@ const ENDPOINT_PLACEHOLDERS: Record<Provider, string> = {
nextcloud: "http://127.0.0.1:9081/remote.php/dav/files/admin/",
seafile: "http://127.0.0.1:9082/",
smb: "smb://127.0.0.1:1445/files",
s3: "http://127.0.0.1:9000",
sharepoint: "https://graph.microsoft.com/v1.0/sites/{site-id}/drives/{drive-id}",
onedrive: "https://graph.microsoft.com/v1.0/drives/{drive-id}",
nfs: "nfs://127.0.0.1/export",
dms: "https://dms.example.test",
generic: "https://files.example.test"
@@ -340,6 +348,17 @@ export default function FileConnectorSettingsPanel({
setCredentialDraft((current) => current ? { ...current, ...patch } : current);
}
function patchCredentialValues(patch: {username?: string | null;password?: string | null;}) {
const next: Partial<ConnectorCredentialDraft> = {};
if (patch.username !== undefined) next.username = String(patch.username ?? "");
if (patch.password !== undefined) next.password = String(patch.password ?? "");
patchCredentialDraft(next);
}
function patchCredentialToken(patch: {password?: string | null;}) {
if (patch.password !== undefined) patchCredentialDraft({ token: String(patch.password ?? "") });
}
function patchPolicyDraft(patch: Partial<CentralConnectorPolicyDraft>) {
setPolicyDraft((current) => ({ ...current, ...patch }));
}
@@ -906,11 +925,11 @@ export default function FileConnectorSettingsPanel({
footer={credentialDraft ?
<>
<Button onClick={closeCredentialDialog} disabled={saving}>i18n:govoplan-files.cancel.77dfd213</Button>
<span className="disabled-action-tooltip" data-tooltip={credentialSaveTooltip}>
<DisabledActionTooltip reason={credentialSaveTooltip}>
<Button variant="primary" onClick={() => void saveCredentialDraft()} disabled={credentialSaveDisabled}>
<Save size={16} aria-hidden="true" /> {saving ? "i18n:govoplan-files.saving.ae7e8875" : "i18n:govoplan-files.save_credential.2c02a6d2"}
</Button>
</span>
</DisabledActionTooltip>
</> :
null}>
@@ -983,35 +1002,47 @@ export default function FileConnectorSettingsPanel({
actor: "Connector administrator",
target: "Credential mode"
}} /> :
<div className="form-grid two-column-form-grid">
{(credentialDraft.credentialMode === "basic" || credentialDraft.credentialMode === "secret_ref") &&
<FormField label="i18n:govoplan-files.username.84c29015">
<input value={credentialDraft.username} disabled={saving} onChange={(event) => patchCredentialDraft({ username: event.target.value })} autoComplete="username" />
</FormField>
}
<div className="file-connector-secret-fields">
{credentialDraft.credentialMode === "basic" &&
<FormField label="i18n:govoplan-files.password.8be3c943">
<input value={credentialDraft.password} disabled={saving} onChange={(event) => patchCredentialDraft({ password: event.target.value })} type="password" autoComplete="new-password" placeholder={editingCredentialId ? "i18n:govoplan-files.leave_empty_to_keep_saved_password.6ec39f5e" : ""} />
</FormField>
}
{credentialDraft.credentialMode === "token" &&
<FormField label="i18n:govoplan-files.token.a1141eb9">
<input value={credentialDraft.token} disabled={saving} onChange={(event) => patchCredentialDraft({ token: event.target.value })} type="password" placeholder={editingCredentialId ? "i18n:govoplan-files.leave_empty_to_keep_saved_token.e725857b" : ""} />
</FormField>
<CredentialPanel
values={{ username: credentialDraft.username, password: credentialDraft.password }}
onChange={patchCredentialValues}
disabled={saving}
savedPassword={Boolean(editingCredentialId) && !credentialDraft.clearPassword}
savedPasswordPlaceholder="i18n:govoplan-files.leave_empty_to_keep_saved_password.6ec39f5e" />
}
{credentialDraft.credentialMode === "secret_ref" &&
<FormField label="i18n:govoplan-files.secret_reference.04ed2221">
<input value={credentialDraft.secretRef} disabled={saving} onChange={(event) => patchCredentialDraft({ secretRef: event.target.value })} />
</FormField>
<>
<CredentialPanel
values={{ username: credentialDraft.username }}
onChange={patchCredentialValues}
disabled={saving}
showPassword={false} />
<div className="form-grid two-column-form-grid">
<FormField label="i18n:govoplan-files.secret_reference.04ed2221">
<input value={credentialDraft.secretRef} disabled={saving} onChange={(event) => patchCredentialDraft({ secretRef: event.target.value })} />
</FormField>
</div>
</>
}
{credentialDraft.credentialMode === "token" &&
<CredentialPanel
values={{ password: credentialDraft.token }}
onChange={patchCredentialToken}
disabled={saving}
showUsername={false}
passwordLabel="i18n:govoplan-files.token.a1141eb9"
savedPassword={Boolean(editingCredentialId) && !credentialDraft.clearToken}
savedPasswordPlaceholder="i18n:govoplan-files.leave_empty_to_keep_saved_token.e725857b" />
}
</div>
}
<div className="button-row compact-actions">
<span className="disabled-action-tooltip" data-tooltip={credentialTestDisabled ? credentialTestTooltip : ""}>
<DisabledActionTooltip reason={credentialTestDisabled ? credentialTestTooltip : ""}>
<Button type="button" onClick={() => void testCredentialLogin()} disabled={credentialTestDisabled}>
<RefreshCw size={16} aria-hidden="true" /> {testingCredentialLogin ? "Testing login" : "Test login"}
</Button>
</span>
</DisabledActionTooltip>
</div>
{credentialLoginResult &&
<DismissibleAlert tone={credentialLoginResult.ok ? "success" : "warning"} resetKey={credentialLoginResult.message}>
@@ -1072,11 +1103,11 @@ export default function FileConnectorSettingsPanel({
footer={draft ?
<>
<Button onClick={closeProfileDialog} disabled={saving}>i18n:govoplan-files.cancel.77dfd213</Button>
<span className="disabled-action-tooltip" data-tooltip={profileSaveTooltip}>
<DisabledActionTooltip reason={profileSaveTooltip}>
<Button variant="primary" onClick={() => void saveDraft()} disabled={profileSaveDisabled}>
<Save size={16} aria-hidden="true" /> {saving ? "i18n:govoplan-files.saving.ae7e8875" : "i18n:govoplan-files.save_connection.07796d38"}
</Button>
</span>
</DisabledActionTooltip>
</> :
null}>

View File

@@ -182,6 +182,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
}, [visibleFiles, folders, currentFolder, searchActive, sortColumn, sortDirection]);
const fileListViewportRef = useRef<HTMLDivElement | null>(null);
const fileListMeasureRowRef = useRef<HTMLDivElement | null>(null);
const managedSpaceLoadsInFlightRef = useRef<Set<string>>(new Set());
const [fileListViewportHeight, setFileListViewportHeight] = useState(0);
const [fileListScrollTop, setFileListScrollTop] = useState(0);
const [fileListRowHeight, setFileListRowHeight] = useState(DEFAULT_FILE_LIST_ROW_HEIGHT);
@@ -291,8 +292,9 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
try {
const response = await listFileSpaces(settings);
setSpaces(response.spaces);
setActiveSpaceId((current) => current || response.spaces[0]?.id || "");
await Promise.all(response.spaces.filter((space) => !isConnectorSpace(space)).map((space) => loadSpaceContents(space, { silent: true })));
const nextActiveSpace = response.spaces.find((space) => space.id === activeSpaceId) ?? response.spaces[0] ?? null;
setActiveSpaceId(nextActiveSpace?.id || "");
if (nextActiveSpace && !isConnectorSpace(nextActiveSpace)) await loadSpaceContents(nextActiveSpace, { silent: true });
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
} finally {
@@ -331,6 +333,8 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
await loadConnectorSpaceContents(space, { silent: options.silent, folderPath: "" });
return;
}
if (managedSpaceLoadsInFlightRef.current.has(space.id)) return;
managedSpaceLoadsInFlightRef.current.add(space.id);
if (!options.silent) {
setBusy(true);
setError("");
@@ -372,6 +376,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
} finally {
managedSpaceLoadsInFlightRef.current.delete(space.id);
if (!options.silent) setBusy(false);
}
}
@@ -411,6 +416,8 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
const nextSpace = spaces.find((space) => space.id === activeSpaceId);
if (nextSpace && isConnectorSpace(nextSpace)) {
void loadConnectorSpaceContents(nextSpace, { folderPath: "", silent: true });
} else if (nextSpace && filesBySpace[nextSpace.id] === undefined && foldersBySpace[nextSpace.id] === undefined) {
void loadSpaceContents(nextSpace, { silent: true });
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [activeSpaceId, spaces]);

View File

@@ -14,7 +14,7 @@
gap: 8px;
flex-wrap: wrap;
padding: 10px 14px;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
background: var(--panel-header);
}
@@ -29,7 +29,7 @@
grid-template-columns: minmax(240px, 300px) minmax(0, 1fr);
min-height: 0;
height: 100%;
border: 1px solid var(--line);
border: var(--border-line);
border-radius: 0;
overflow: hidden;
background: var(--panel);
@@ -46,7 +46,7 @@
.file-list-sticky {
flex: 0 0 auto;
z-index: 2;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
background: var(--panel-soft);
}
@@ -97,7 +97,7 @@
gap: 12px;
flex-wrap: wrap;
padding: 8px 14px;
border-top: 1px solid var(--line);
border-top: var(--border-line);
color: var(--muted);
font-size: 12px;
}
@@ -130,7 +130,7 @@
.file-list-table-head {
padding: 10px 14px;
border-top: 1px solid var(--line);
border-top: var(--border-line);
background: var(--panel);
color: var(--muted);
font-size: 12px;
@@ -161,7 +161,7 @@
.file-list-row {
min-height: 58px;
padding: 9px 14px;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
cursor: default;
user-select: none;
}
@@ -244,7 +244,7 @@
}
.folder-row .file-row-icon {
color: #b6791d;
color: var(--warning-deep);
}
.file-row-tail {
@@ -274,7 +274,7 @@
display: grid;
min-width: 190px;
overflow: hidden;
border: 1px solid var(--line-dark);
border: var(--border-line-dark);
border-radius: 12px;
background: var(--panel);
box-shadow: var(--shadow-popover);
@@ -299,7 +299,7 @@
.file-context-menu button:hover,
.file-context-menu button:focus-visible {
background: rgba(13, 110, 253, .08);
background: var(--primary-soft);
outline: none;
}
@@ -315,14 +315,14 @@
display: grid;
place-items: center;
padding: 22px;
background: rgba(28, 25, 22, .38);
background: var(--dialog-backdrop);
}
.file-dialog {
width: min(620px, 100%);
max-height: min(720px, calc(100vh - 44px));
overflow: auto;
border: 1px solid var(--line-dark);
border: var(--border-line-dark);
border-radius: 16px;
background: var(--panel);
box-shadow: var(--shadow-popover);
@@ -334,7 +334,7 @@
justify-content: space-between;
gap: 12px;
padding: 15px 18px;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
background: var(--panel-soft);
}
@@ -397,7 +397,7 @@
overflow-y: auto;
scrollbar-gutter: stable;
padding: 8px;
border: 1px solid var(--line);
border: var(--border-line);
border-radius: 12px;
background: var(--panel-soft);
}
@@ -428,13 +428,13 @@
.file-folder-selector-node:hover:not(:disabled),
.file-folder-selector-node:focus-visible,
.file-folder-selector-node.is-selected {
background: rgba(13, 110, 253, .09);
background: var(--primary-soft);
outline: none;
}
.file-folder-selector-node.is-selected {
color: var(--text-strong);
box-shadow: inset 0 0 0 1px rgba(13, 110, 253, .25);
box-shadow: var(--primary-inset-ring);
}
.file-folder-selector-empty {
@@ -462,7 +462,7 @@
grid-template-rows: auto auto minmax(0, 1fr);
min-height: 320px;
overflow: hidden;
border: 1px solid var(--line);
border: var(--border-line);
border-radius: 12px;
background: var(--panel);
}
@@ -473,7 +473,7 @@
gap: 10px;
align-items: center;
padding: 10px;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
background: var(--panel-soft);
}
@@ -577,7 +577,7 @@
gap: 3px;
min-width: 0;
padding: 10px 12px;
border: 1px solid var(--line);
border: var(--border-line);
border-radius: 10px;
background: var(--panel-soft);
}
@@ -625,7 +625,7 @@
gap: 12px;
align-items: center;
padding: 10px 12px;
border: 1px solid var(--line);
border: var(--border-line);
border-radius: 8px;
background: var(--panel-soft);
}
@@ -698,6 +698,11 @@
white-space: nowrap;
}
.file-connector-secret-fields {
display: grid;
gap: 18px;
}
@media (max-width: 760px) {
.file-connector-profile-row {
grid-template-columns: 1fr;
@@ -726,7 +731,7 @@
.file-tree-panel {
max-height: 260px;
border-right: 0;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
}
.file-list-table-head {
@@ -760,7 +765,7 @@
z-index: 35;
display: grid;
place-items: center;
background: rgba(255, 255, 255, .12);
background: var(--panel-glass);
backdrop-filter: blur(1px);
}
@@ -768,7 +773,7 @@
display: grid;
gap: 3px;
padding: 12px 14px;
border: 1px solid var(--line);
border: var(--border-line);
border-radius: 12px;
background: var(--panel-soft);
}
@@ -791,7 +796,7 @@
gap: 10px;
align-items: center;
padding: 10px;
border: 1px solid var(--line);
border: var(--border-line);
border-radius: 12px;
background: var(--panel-soft);
}
@@ -842,8 +847,8 @@
.rename-preview-more:hover,
.rename-preview-more:focus-visible {
border-color: rgba(13, 110, 253, .45);
background: rgba(13, 110, 253, .08);
border-color: var(--primary-border);
background: var(--primary-soft);
color: var(--text-strong);
outline: none;
}
@@ -860,7 +865,7 @@
.managed-file-chooser-dialog > .dialog-header {
padding: 16px 18px;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
background: var(--panel);
}
@@ -891,9 +896,9 @@
.managed-file-chooser-footer {
flex: 0 0 auto;
padding: 12px 18px;
border-top: 1px solid var(--line);
border-top: var(--border-line);
background: var(--panel);
box-shadow: 0 -8px 24px rgba(15, 23, 42, .06);
box-shadow: var(--shadow-footer);
}
.managed-file-chooser-layout {
@@ -911,7 +916,7 @@
.managed-file-chooser-spaces {
min-width: 0;
padding: 16px;
border-right: 1px solid var(--line);
border-right: var(--border-line);
background: var(--panel-soft);
overflow: auto;
}
@@ -1014,7 +1019,7 @@
gap: 12px;
align-items: center;
padding: 12px 14px;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
}
.managed-file-breadcrumb {
@@ -1046,7 +1051,7 @@
display: grid;
gap: 8px;
padding: 12px 14px;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
background: var(--panel-soft);
}
@@ -1075,7 +1080,7 @@
align-items: center;
min-height: 38px;
padding: 0 20px;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
background: var(--panel-soft);
color: var(--muted);
font-size: 12px;
@@ -1206,7 +1211,7 @@
.managed-file-chooser-note {
margin: 0;
padding: 10px 14px;
border-top: 1px solid var(--line);
border-top: var(--border-line);
background: var(--panel-soft);
}
@@ -1313,7 +1318,7 @@
.managed-file-chooser-spaces {
border-right: 0;
border-bottom: 1px solid var(--line);
border-bottom: var(--border-line);
max-height: 160px;
}